<commit_before>from sh import git
from requests.auth import HTTPBasicAuth
from globalvars import GlobalVars
import requests
import time
import json

# noinspection PyRedundantParentheses,PyClassHasNoInit,PyBroadException
class GitManager:
    @staticmethod
    def add_to_blacklist(**kwargs):
        blacklist = kwargs.get("blacklist", "website")
        items_to_blacklist = kwargs.get("items_to_blacklist", [])
        username = kwargs.get("username", "")
        chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
        code_permissions = kwargs.get("code_permissions", False)

        for index, item in enumerate(items_to_blacklist):
            items_to_blacklist[index] = item.replace("\s", " ")
            items_to_blacklist[index] = items_to_blacklist[index].replace("\W", "[^A-Za-z0-9_]")

        if blacklist == "website":
            blacklist_file_name = "blacklisted_websites.txt"
        elif blacklist == "keyword":
            blacklist_file_name = "bad_keywords.txt"
        elif blacklist == "username":
            blacklist_file_name = "blacklisted_usernames.txt"
        else:
            # Just checking all bases, but blacklist_file_name *might* have empty value
            # if we don't address it here.
            return (False, "Invalid blacklist type specified, something has broken badly!")

        git.checkout("master")
        try:
            git.pull()
        except:
            pass

        # Check that we're up-to-date with origin (GitHub)
        git.remote.update()
        if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
            return (False, "HEAD isn't at tip of origin's master branch")

        # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
        if blacklist_file_name in git.status():  # Also ugly
            return (False, "{0} modified locally. This is probably bad.".format(blacklist_file_name))

        # Add items to file
        with open(blacklist_file_name, "a+") as blacklist_file:
            last_character = blacklist_file.read()[-1:]
            if last_character != "\n":
                blacklist_file.write("\n")
            blacklist_file.write("\n".join(items_to_blacklist) + "\n")

        # Checkout a new branch (for PRs for non-code-privileged people)
        branch = "auto-blacklist-{0}".format(str(time.time()))
        git.checkout("-b", branch)

        # Clear HEAD just in case
        git.reset("HEAD")

        git.add(blacklist_file_name)
        git.commit("-m", u"Auto blacklist of {0} by {1} --autopull".format(", ".join(items_to_blacklist), username))

        if code_permissions:
            git.checkout("master")
            git.merge(branch)
            git.push()
        else:
            git.push("origin", branch)
            git.checkout("master")

            if GlobalVars.github_username is None or GlobalVars.github_password is None:
                return (False, "Tell someone to set a GH password")

            list_of_domains = ""

            for domain in range(len(items_to_blacklist)):
                list_of_domains += "\n - {0} - [MS search](https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93&body_is_regex=1&body={1})".format(items_to_blacklist[domain], items_to_blacklist[domain].replace(" ", "+"))

            payload = {"title": "{0}: Blacklist {1}".format(username, ", ".join(items_to_blacklist)),
                       "body": "[{0}]({1}) requests the blacklist of the following {2}(s): \n{3}\n<!-- METASMOKE-BLACKLIST {4} -->".format(username, chat_profile_link, blacklist, list_of_domains, "|".join(items_to_blacklist)),
                       "head": branch,
                       "base": "master"}
            response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload))
            print(response.json())
            try:
                git.checkout("deploy")  # Return to deploy, pending the accept of the PR in Master.
                return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(response.json()["html_url"]))
            except KeyError:
                # Error capture/checking for any "invalid" GH reply without an 'html_url' item, which will throw a KeyError.
                if "bad credentials" in str(response.json()['message']).lower():
                    # Capture the case when GH credentials are bad or invalid
                    return (False, "Something is wrong with the GH credentials, tell someone to check them.")
                else:
                    # Capture any other invalid response cases.
                    return (False, "A bad or invalid reply was received from GH, the message was: %s" % response.json()['message'])

        git.checkout("deploy")  # Return to deploy to await CI.

        return (True, "Blacklisted {0} on master - you may need to merge to deploy.".format(", ".join(items_to_blacklist)))

    @staticmethod
    def current_git_status():
        return git.status()
<commit_msg>Revert changes --autopull<commit_after>from sh import git
from requests.auth import HTTPBasicAuth
from globalvars import GlobalVars
import requests
import time
import json

# noinspection PyRedundantParentheses,PyClassHasNoInit,PyBroadException
class GitManager:
    @staticmethod
    def add_to_blacklist(**kwargs):
        blacklist = kwargs.get("blacklist", "website")
        items_to_blacklist = kwargs.get("items_to_blacklist", [])
        username = kwargs.get("username", "")
        chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
        code_permissions = kwargs.get("code_permissions", False)

        for index, item in enumerate(items_to_blacklist):
            items_to_blacklist[index] = item.replace("\s", " ")

        if blacklist == "website":
            blacklist_file_name = "blacklisted_websites.txt"
        elif blacklist == "keyword":
            blacklist_file_name = "bad_keywords.txt"
        elif blacklist == "username":
            blacklist_file_name = "blacklisted_usernames.txt"
        else:
            # Just checking all bases, but blacklist_file_name *might* have empty value
            # if we don't address it here.
            return (False, "Invalid blacklist type specified, something has broken badly!")

        git.checkout("master")
        try:
            git.pull()
        except:
            pass

        # Check that we're up-to-date with origin (GitHub)
        git.remote.update()
        if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
            return (False, "HEAD isn't at tip of origin's master branch")

        # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
        if blacklist_file_name in git.status():  # Also ugly
            return (False, "{0} modified locally. This is probably bad.".format(blacklist_file_name))

        # Add items to file
        with open(blacklist_file_name, "a+") as blacklist_file:
            last_character = blacklist_file.read()[-1:]
            if last_character != "\n":
                blacklist_file.write("\n")
            blacklist_file.write("\n".join(items_to_blacklist) + "\n")

        # Checkout a new branch (for PRs for non-code-privileged people)
        branch = "auto-blacklist-{0}".format(str(time.time()))
        git.checkout("-b", branch)

        # Clear HEAD just in case
        git.reset("HEAD")

        git.add(blacklist_file_name)
        git.commit("-m", u"Auto blacklist of {0} by {1} --autopull".format(", ".join(items_to_blacklist), username))

        if code_permissions:
            git.checkout("master")
            git.merge(branch)
            git.push()
        else:
            git.push("origin", branch)
            git.checkout("master")

            if GlobalVars.github_username is None or GlobalVars.github_password is None:
                return (False, "Tell someone to set a GH password")

            list_of_domains = ""

            for domain in range(len(items_to_blacklist)):
                list_of_domains += "\n - {0} - [MS search](https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93&body_is_regex=1&body={1})".format(items_to_blacklist[domain], items_to_blacklist[domain].replace(" ", "+"))

            payload = {"title": "{0}: Blacklist {1}".format(username, ", ".join(items_to_blacklist)),
                       "body": "[{0}]({1}) requests the blacklist of the following {2}(s): \n{3}\n<!-- METASMOKE-BLACKLIST {4} -->".format(username, chat_profile_link, blacklist, list_of_domains, "|".join(items_to_blacklist)),
                       "head": branch,
                       "base": "master"}
            response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload))
            print(response.json())
            try:
                git.checkout("deploy")  # Return to deploy, pending the accept of the PR in Master.
                return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(response.json()["html_url"]))
            except KeyError:
                # Error capture/checking for any "invalid" GH reply without an 'html_url' item, which will throw a KeyError.
                if "bad credentials" in str(response.json()['message']).lower():
                    # Capture the case when GH credentials are bad or invalid
                    return (False, "Something is wrong with the GH credentials, tell someone to check them.")
                else:
                    # Capture any other invalid response cases.
                    return (False, "A bad or invalid reply was received from GH, the message was: %s" % response.json()['message'])

        git.checkout("deploy")  # Return to deploy to await CI.

        return (True, "Blacklisted {0} on master - you may need to merge to deploy.".format(", ".join(items_to_blacklist)))

    @staticmethod
    def current_git_status():
        return git.status()
<|endoftext|>


<commit_before>from sh import git
from requests.auth import HTTPBasicAuth
from globalvars import GlobalVars
import requests
import time
import json

class GitManager:
    @classmethod
    def add_to_blacklist(self, items_to_blacklist, username, code_permissions):
        # Check if we're on master
        if git("rev-parse", "--abbrev-ref", "HEAD").strip() != "master":
            return (False, "Not currently on master.")

        # Check that we're up-to-date with origin (GitHub)
        git.remote.update()
        if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
            return (False, "HEAD isn't at tip of origin's master branch")

        # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
        if "blacklisted_websites.txt" in git.status():  # Also ugly
            return (False, "blacklisted_websites.txt modified locally. This is probably bad.")

        # Store current commit hash
        current_commit = git("rev-parse", "HEAD").strip()

        # Add items to file

        with open("blacklisted_websites.txt", "a") as blacklisted_websites:
            blacklisted_websites.write("\n" + "\n".join(items_to_blacklist))

        # Checkout a new branch (mostly unnecessary, but may help if we create PRs in the future
        branch = "auto-blacklist-{0}".format(str(time.time()))
        git.checkout("-b", branch)

        # Clear HEAD just in case
        git.reset("HEAD")

        git.add("blacklisted_websites.txt")
        git.commit("-m", "Auto blacklist of {0} by {1} --autopull".format(", ".join(items_to_blacklist), username))

        if code_permissions:
            git.checkout("master")
            git.merge(branch)
            git.push()
        else:
            git.push("origin", branch)
            git.checkout("master")

            if GlobalVars.github_username is None or GlobalVars.github_password is None:
                return (False, "tell someone to set a GH password")

            payload = {"title": "{0}: Blacklist {1}".format(username, ", ".join(items_to_blacklist)),
                       "body": "{0} requests blacklist of domains: \n\n - {1}".format(username, "\n - ".join(items_to_blacklist)),
                       "head": branch,
                       "base": "master"}
            response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload))
            print(response.json())
            return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(response.json()["html_url"]))

        git.checkout(current_commit)  # Return to old commit to await CI. This will make Smokey think it's in reverted mode if it restarts

        if not code_permissions:
            return (False, "Unable to perform action due to lack of code-level permissions. [Branch pushed](https://github.com/Charcoal-SE/SmokeDetector/tree/{0}), PR at your leisure.".format(branch))

        return (True, "Blacklisted {0} - the entry will be applied via autopull if CI succeeds.".format(", ".join(items_to_blacklist)))

    def current_git_status(self):
        return git.status()
<commit_msg>Make that a class method<commit_after>from sh import git
from requests.auth import HTTPBasicAuth
from globalvars import GlobalVars
import requests
import time
import json

class GitManager:
    @classmethod
    def add_to_blacklist(self, items_to_blacklist, username, code_permissions):
        # Check if we're on master
        if git("rev-parse", "--abbrev-ref", "HEAD").strip() != "master":
            return (False, "Not currently on master.")

        # Check that we're up-to-date with origin (GitHub)
        git.remote.update()
        if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
            return (False, "HEAD isn't at tip of origin's master branch")

        # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
        if "blacklisted_websites.txt" in git.status():  # Also ugly
            return (False, "blacklisted_websites.txt modified locally. This is probably bad.")

        # Store current commit hash
        current_commit = git("rev-parse", "HEAD").strip()

        # Add items to file

        with open("blacklisted_websites.txt", "a") as blacklisted_websites:
            blacklisted_websites.write("\n" + "\n".join(items_to_blacklist))

        # Checkout a new branch (mostly unnecessary, but may help if we create PRs in the future
        branch = "auto-blacklist-{0}".format(str(time.time()))
        git.checkout("-b", branch)

        # Clear HEAD just in case
        git.reset("HEAD")

        git.add("blacklisted_websites.txt")
        git.commit("-m", "Auto blacklist of {0} by {1} --autopull".format(", ".join(items_to_blacklist), username))

        if code_permissions:
            git.checkout("master")
            git.merge(branch)
            git.push()
        else:
            git.push("origin", branch)
            git.checkout("master")

            if GlobalVars.github_username is None or GlobalVars.github_password is None:
                return (False, "tell someone to set a GH password")

            payload = {"title": "{0}: Blacklist {1}".format(username, ", ".join(items_to_blacklist)),
                       "body": "{0} requests blacklist of domains: \n\n - {1}".format(username, "\n - ".join(items_to_blacklist)),
                       "head": branch,
                       "base": "master"}
            response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload))
            print(response.json())
            return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(response.json()["html_url"]))

        git.checkout(current_commit)  # Return to old commit to await CI. This will make Smokey think it's in reverted mode if it restarts

        if not code_permissions:
            return (False, "Unable to perform action due to lack of code-level permissions. [Branch pushed](https://github.com/Charcoal-SE/SmokeDetector/tree/{0}), PR at your leisure.".format(branch))

        return (True, "Blacklisted {0} - the entry will be applied via autopull if CI succeeds.".format(", ".join(items_to_blacklist)))

    @classmethod
    def current_git_status(self):
        return git.status()
<|endoftext|>


<commit_before>from sh import git
from requests.auth import HTTPBasicAuth
from globalvars import GlobalVars
import requests
import time
import json

class GitManager:
    @classmethod
    def add_to_blacklist(self, items_to_blacklist, username, code_permissions):
        # Check if we're on master
        if git("rev-parse", "--abbrev-ref", "HEAD").strip() != "master":
            return (False, "Not currently on master.")

        # Check that we're up-to-date with origin (GitHub)
        git.remote.update()
        if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
            return (False, "HEAD isn't at tip of origin's master branch")

        # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
        if "blacklisted_websites.txt" in git.status():  # Also ugly
            return (False, "blacklisted_websites.txt modified locally. This is probably bad.")

        # Store current commit hash
        current_commit = git("rev-parse", "HEAD").strip()

        # Add items to file
        with open("blacklisted_websites.txt", "a+") as blacklisted_websites:
            last_character = blacklisted_websites.read()[-1:]
            if last_character != "\n":
                blacklisted_websites.write("\n")
            blacklisted_websites.write("\n".join(items_to_blacklist) + "\n")

        # Checkout a new branch (mostly unnecessary, but may help if we create PRs in the future
        branch = "auto-blacklist-{0}".format(str(time.time()))
        git.checkout("-b", branch)

        # Clear HEAD just in case
        git.reset("HEAD")

        git.add("blacklisted_websites.txt")
        git.commit("-m", u"Auto blacklist of {0} by {1} --autopull".format(", ".join(items_to_blacklist), username))

        if code_permissions:
            git.checkout("master")
            git.merge(branch)
            git.push()
        else:
            git.push("origin", branch)
            git.checkout("master")

            if GlobalVars.github_username is None or GlobalVars.github_password is None:
                return (False, "tell someone to set a GH password")

            payload = {"title": "{0}: Blacklist {1}".format(username, ", ".join(items_to_blacklist)),
                       "body": "{0} requests blacklist of domains: \n\n - {1}".format(username, "\n - ".join(items_to_blacklist)),
                       "head": branch,
                       "base": "master"}
            response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload))
            print(response.json())
            return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(response.json()["html_url"]))

        git.checkout(current_commit)  # Return to old commit to await CI. This will make Smokey think it's in reverted mode if it restarts

        if not code_permissions:
            return (False, "Unable to perform action due to lack of code-level permissions. [Branch pushed](https://github.com/Charcoal-SE/SmokeDetector/tree/{0}), PR at your leisure.".format(branch))

        return (True, "Blacklisted {0} - the entry will be applied via autopull if CI succeeds.".format(", ".join(items_to_blacklist)))

    @classmethod
    def current_git_status(self):
        return git.status()
<commit_msg>Added MS search for blacklisted domains<commit_after>from sh import git
from requests.auth import HTTPBasicAuth
from globalvars import GlobalVars
import requests
import time
import json

class GitManager:
    @classmethod
    def add_to_blacklist(self, items_to_blacklist, username, code_permissions):
        # Check if we're on master
        if git("rev-parse", "--abbrev-ref", "HEAD").strip() != "master":
            return (False, "Not currently on master.")

        # Check that we're up-to-date with origin (GitHub)
        git.remote.update()
        if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
            return (False, "HEAD isn't at tip of origin's master branch")

        # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
        if "blacklisted_websites.txt" in git.status():  # Also ugly
            return (False, "blacklisted_websites.txt modified locally. This is probably bad.")

        # Store current commit hash
        current_commit = git("rev-parse", "HEAD").strip()

        # Add items to file
        with open("blacklisted_websites.txt", "a+") as blacklisted_websites:
            last_character = blacklisted_websites.read()[-1:]
            if last_character != "\n":
                blacklisted_websites.write("\n")
            blacklisted_websites.write("\n".join(items_to_blacklist) + "\n")

        # Checkout a new branch (mostly unnecessary, but may help if we create PRs in the future
        branch = "auto-blacklist-{0}".format(str(time.time()))
        git.checkout("-b", branch)

        # Clear HEAD just in case
        git.reset("HEAD")

        git.add("blacklisted_websites.txt")
        git.commit("-m", u"Auto blacklist of {0} by {1} --autopull".format(", ".join(items_to_blacklist), username))

        if code_permissions:
            git.checkout("master")
            git.merge(branch)
            git.push()
        else:
            git.push("origin", branch)
            git.checkout("master")

            if GlobalVars.github_username is None or GlobalVars.github_password is None:
                return (False, "tell someone to set a GH password")

            list_of_domains = ""

            for domain in range(len(items_to_blacklist)):
	            list_of_domains += "\n - {0} - [MS search](https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93&body_is_regex=1&body={0})".format(items_to_blacklist[domain])

            payload = {"title": "{0}: Blacklist {1}".format(username, ", ".join(items_to_blacklist)),
                       "body": "{0} requests blacklist of domains: \n{1}".format(username, list_of_domains),
                       "head": branch,
                       "base": "master"}
            response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload))
            print(response.json())
            return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(response.json()["html_url"]))

        git.checkout(current_commit)  # Return to old commit to await CI. This will make Smokey think it's in reverted mode if it restarts

        if not code_permissions:
            return (False, "Unable to perform action due to lack of code-level permissions. [Branch pushed](https://github.com/Charcoal-SE/SmokeDetector/tree/{0}), PR at your leisure.".format(branch))

        return (True, "Blacklisted {0} - the entry will be applied via autopull if CI succeeds.".format(", ".join(items_to_blacklist)))

    @classmethod
    def current_git_status(self):
        return git.status()
<|endoftext|>


<commit_before># Kaggel Auxillary Functions
# AGC - 2013
import sys
from datetime import timedelta
import numpy as np
import scipy as sp
import matplotlib as plt
from pandas import DataFrame, Series, qcut
from pandas.core.common import adjoin
from pandas.io.data import DataReader
from patsy import dmatrices

def get_dataframe_intersection(df, comparator1,comparator2):
    """
    Return a dataframe with only the columns found in a comparative dataframe.

    Parameters
    ----------
    comparator1: DataFrame
        DataFrame to preform comparison on.
    comparator2: DataFrame
        DataFrame to compare with.

    Returns
    -------
    DataFrame:
        Data frame with columns not found in comparator dropped.

    """
    to_drop = list((c for c in comparator1 if c not in comparator2))
    return df.drop(to_drop, axis=1)

def get_dataframes_intersections(df1, comparator1, df2, comparator2):
    """
    Return DataFrames with the intersection of their column values.

    Parameters
    ----------
    comparator1: DataFrame
        DataFrame to preform comparison on.
    comparator2: DataFrame
        DataFrame to compare with.

    Returns
    -------
    Tuple:
        The resultingDataframe with columns not found in comparator dropped.

    """
    comparator1 = get_dataframe_intersection(df1, comparator1, comparator2)
    comparator2 = get_dataframe_intersection(df2, comparator2, comparator1)
    return comparator1, comparator2

def predict(test_data, results, model_name):
    """
    Return predictions of based on model resutls.

    Parameters
    ----------
    test_data: DataFrame
        should be test data you are trying to predict
    results: Dict
        should be dict of your models results wrapper and the formula used
        to produce it.
            ie.
            results['Model_Name'] = {[<statsmodels.regression.linear_model.RegressionResultsWrapper> , "Price ~ I(Supply, Demand)] }
    model_name: Str
        should be the name of your model. You can iterate through the results dict.

    Returns
    -------
    NumPy array
        Predictions in a flat NumPy array.

    Example
    -------
    results = {'Logit': [<statsmodels.discrete.discrete_model.BinaryResultsWrapper at 0x117896650>,
               'survived ~ C(pclass) + C(sex) + age + sibsp  + C(embarked)']}
    compared_resuts = predict(test_data, results, 'Logit')
    """
    model_params = DataFrame(results[model_name][0].params)
    formula = results[model_name][1]

    # Create regression friendly test DataFrame
    yt, xt = dmatrices(formula, data=test_data, return_type='dataframe')
    xt, model_params = get_dataframes_intersections(xt, xt.columns,
                                                    model_params, model_params.index)
    # Convert to NumPy arrays for performance
    model_params = np.asarray(model_params)
    yt = np.asarray(yt)
    yt = yt.ravel()
    xt = np.asarray(xt)

    # Use our models to create predictions
    row, col = xt.shape
    model_parameters = model_params.ravel()
    model_array = list((model_parameters for parameter in xrange(row)))
    model_array = np.asarray(model_array)

    # Multiply matrix together
    predictions = np.multiply(xt, model_array)
    predictions = np.sum(predictions, axis=1)
    return predictions

def cross_validate_df(df, percent):
    '''
    Return a randomly shuffled supbsets of a DataFrame cross validation
    or for down sampleing.

    Parameters
    ----------
    df : DataFrame
        DataFrame to be sampled. Expects an ordinal index; ie. 0 - 100.
    percent: Int
         the percentage to split of the returned subsets.

    Returns
    -------
    Tuple:
        (df_h1, df_h2), both parts of the split randomly shuffled DataFrame

    Example
    -------
    small_df_half, large_df_half  = cross_validate_df(df, 33)
    '''
    sample_percentage_of_dataframe = int(np.round(((percent * df.index.size) / float(100))))
    rows = np.random.randint(0, df.index.size, sample_percentage_of_dataframe)
    return (df.ix[rows], df.drop(rows))

def dataframe_kfolds(df):
    '''
    Standard kfolds cross cross_validate method.

    Parameters
    ----------
    df : DataFrame
        A pandas dataframe to be operated on.

    Returns
    -------
    Tuple :
        dataframe 10%, dataframe 90% -- random splits using python's random.choice()
    '''
    return cross_validate_df(df, 90)

def dataframe_welch_ttest(df, described_frame, boolean_feature):
    '''
    Parameters
    ----------
    df : DataFrame
       A DataFrame to perfrom welch_ttest on.
    described_frame : DataFrame
       A described DataFrame from the pandas desribe_frame() method.
    boolean_feature: Str
       Name of boolean feature to conduct test on.

    Returns
    -------
    DataFrame :
        t-statistic and p-value for each feature in a pandas dataframe.
    '''

    described_frame['t-statistic'] = np.nan
    described_frame['p-value'] = np.nan

    for name, item in df.iteritems():
        result = sp.stats.ttest_ind(df[name][df[boolean_feature] == 0].dropna(),
                                        df[name][df[boolean_feature] == 1].dropna(),
                                        equal_var=False)
        described_frame.ix[name, 't-statistic'], described_frame.ix[name, 'p-value'] = result
    return described_frame

def category_boolean_maker(series):
    '''
    A funtction for to designate missing records from observed ones.

    When used with the pandas df.series.apply() method, it will create
    a boolean category variable. If values exist the bool will register
    as 1, if nan values exist the bool will register as 0.

    Parameters
    ----------
    series : Series
        A pandas series to perform comparision on.

    Returns
    -------
    Int :
        0 or 1 for missing values.

    '''
    return 0 if np.isnan(series) == True else 1

def columns_to_str(column_list, return_list=0):
    '''
    takes the pandas df. columns method and returns a list with items as strings for easy implementaiton into pandas fucntions

    ins
    --
    column_list -- ussually from pandas df.columns function but can be any list

    return_list -- optional param to return the str list ( use 1 to return)

    out
    --
    a list with str elements

    ex.

    columns_to_str(df.columns)

    Changes

    Index([x3yv_E, x3yv_D, x1yv_E, x1yv_D], dtype=object)

    to

    Index(['x3yv_E', 'x3yv_D', 'x1yv_E', 'x1yv_D'], dtype=object)
    '''
    if return_list == 1:
        output = []
        for i in column_list:
            output.append(str(i))
        return output

    print "[%s]" % "', '".join(map(str, column_list))

def columns_to_plus(column_list, return_list=0):
    '''
    takes the pandas df. columns method and returns a list with items connected with a plus for easy implementaiton into patys's dmatrices function

    ins
    --
    column_list -- ussually from pandas df.columns function but can be any list

    return_list -- optional param to return the str list ( use 1 to return)

    out
    --
    a list with elements conncted with a plus

    ex.

    columns_to_str(df.columns)

    Changes

    Index([x3yv_E, x3yv_D, x1yv_E, x1yv_D], dtype=object)

    to

    Index([x3yv_E + x3yv_D + x1yv_E + x1yv_D], dtype=object)
    '''
    if return_list == 1:
        output = []
        for i in column_list:
            output.append(str(i))
        return output

    print "[%s]" % " + ".join(map(str, column_list))

def add_to_model_subspace(subspace_def, var):
    '''
    come on, it does what it says.

    basic check to easily add vars to model subspace definitions.

    ins
    --
    var - list, that thing(s) your trying to add
    subspace_def - list defining your models subspace

    out
    --
    subspace_def - list defining your models subspace w/ var added
    '''
    for i in range(0, len(var)):
        if (var[i] in subspace_def) == False:
            var[i] = str(var[i])
            assert isinstance(var[i], str)
            subspace_def.append(var[i])

    #return subspace_def

def ml_formula(y, df):
    '''
    a simple function to create a formula using all available features for patsy dmatrices function.

    ins
    --
    y = a string of the variable your trying to predict

    df = the data frame your trying to make your predctions on

    outs
    ---
    a string of the desired formula
    '''
    formula = y +' ~ '
    for i, val in enumerate(df.columns):
        if i == 0 and val != y:
            formula += val
        if i != 0 and val != y:
            formula += ' + ' + val
    return formula

def progress(i, num_tasks):
    '''
    A simple textual progress bar

    Ins
    --
    i = should be an iterable value to measure the progress of your taks
    num_tasks = total number of tasks

    outs
    --
    A progress bar like [#########  ]
    '''
    progress = "\r["

    for _ in range (0,i):
        progress += "#"

    for _ in range (i, num_tasks):
        progress += " "

    progress += "]"

    sys.stdout.write(progress)
    sys.stdout.flush()

def cat_clean(s):
    """
    Cleans categorical variables of NaNs by setting the NaN to 0 so it can be processed by a patsy dmatrices function.

    Inputs
    --
    s = cat value

    output
    --
    cat value or int 0 in place of a NaN val
    """
    if isinstance(s, str) == False and np.isnan(s) == True:
        s = 0
    return s

def quater_maker(d):
    """
    Pareses dates and returns the appropriate quarter.
    --
    Parameters
    d: a python date time object
    --
    Returns:
    The quarter in a string

    AGC 2013
    """
    r = ''
    Q1 = [1, 2, 3]
    Q2 = [4, 5, 6]
    Q3 = [7, 8, 9]
    Q4 = [10, 11, 12]
    if d.month in Q1:
        r = 'Q1'
    if d.month in Q2:
        r = 'Q2'
    if d.month in Q3:
        r = 'Q3'
    if d.month in Q4:
        r = 'Q4'
    return r

      #if date is within daterange, Q1 ect. then output ex: 2011 Q1
      #manipulate months. say if  0<m4: return YYYY + Q1 ect.
      # then apply to dateline

def score_rmsle(y, df, df2, p = 0):
    '''
    ins
    --
    y =  what your trying to predict. must be a string. ie. 'SalesPrice'
    df =  your predictions
    df2 = the solutions set
    p = option to print rmsle as string; 0 = dont print, 1 = print
    outs
    --
    prints rmsle
    rmsle as a float
    '''
    prediction = np.asarray(df[y])
    actual = np.asarray(df2[y])

    rsmle = np.sqrt(np.mean(np.power(np.log(actual + 1) - np.log(prediction + 1), 2)))

    if p == 1 :
        print "rsmle: " + str(rsmle)
    return rsmle

def score_rmse(y, df, df2, p=0 ):
    """
    ins
    --
    y =  what your trying to predict. must be a string. ie. 'SalesPrice'
    df =  your predictions
    df2 = the solutions set
    p = option to print rmsle as string; 0 = don't print, 1 = print

    outs
    --
    prints rmse
    rmse as a float
    """
    prediction = np.asarray(df[y])
    actual = np.asarray(df2[y])

    rsme = np.sqrt(np.mean(np.power((actual - prediction), 2)))

    if p == 1:
        print "rsme: " + str(rsme)

    return rsme

def unwanted_pals(x, s = .1):
    '''
    Inputs
    --
    x should be a pandas series of the results.pvalues of your model
    s = significance level, default at 90% confidence

    Outs
    --
    returns a list of columns below significance level
    '''
    dropl = list()
    for i,z in enumerate(x):
        if z>s:
             dropl.append(x.index[i])
    return dropl

def stock_price_at_date(x, ticker, lag=0):
    '''
    ins
    --
    x should be the date your looking for
    ticker should be the stock ticker
    lag should be # of days to lag stock price

    outs
    --
    stock price.
    '''
    x = (x - timedelta(days = lag))
    r = DataReader(ticker,start=x, end=x, data_source='yahoo')
    r = r.ix[0,5]
    return r

def side_by_side(*objs, **kwds):
    '''
    created by wes mickinney, it only exists here becuase I use this function all the time.
    '''
    space = kwds.get('space', 4)
    reprs = [repr(obj).split('\n') for obj in objs]
    print adjoin(space, *reprs)

def describe_frame(df):
    """
    ins
    ---
    df = dataframe you want to describe

    outs
    ---
    descriptive stats on your dataframe, in dataframe.

    agc2013
    """
    sum_stats = []
    for i in df.columns:
        x = Series(df[i].describe())
        x.name = i
        sum_stats.append(x)
    stats = DataFrame(sum_stats)
    stats
    return stats

def bin_residuals(resid, var, bins):
    '''
    Compute average residuals within bins of a variable.

    Returns a dataframe indexed by the bins, with the bin midpoint,
    the residual average within the bin, and the confidence interval
    bounds.

    ins
    --
    resid, var, bins

    out
    --
    bin DataFrame

    '''
    resid_df = DataFrame({'var': var, 'resid': resid})
    resid_df['bins'] = qcut(var, bins)
    bin_group = resid_df.groupby('bins')
    bin_df = bin_group['var', 'resid'].mean()
    bin_df['count'] = bin_group['resid'].count()
    bin_df['lower_ci'] = -2 * (bin_group['resid'].std() /
                               np.sqrt(bin_group['resid'].count()))
    bin_df['upper_ci'] =  2 * (bin_group['resid'].std() /
                               np.sqrt(bin_df['count']))
    bin_df = bin_df.sort('var')
    return(bin_df)

def plot_binned_residuals(bin_df):
    '''
    Plotted binned residual averages and confidence intervals.

    ins
    --
    bin_df ie from bin_residuals(resid, var, bins)
    outs
    --
    pretty plots
    '''
    plt.plot(bin_df['var'], bin_df['resid'], '.')
    plt.plot(bin_df['var'], bin_df['lower_ci'], '-r')
    plt.plot(bin_df['var'], bin_df['upper_ci'], '-r')
    plt.axhline(0, color = 'gray', lw = .5)
<commit_msg>feat(KaggleAux): reworked columns_to_str and model_subspace<commit_after># Kaggel Auxillary Functions
# AGC - 2013
import sys
from datetime import timedelta
import numpy as np
import scipy as sp
import matplotlib as plt
from pandas import DataFrame, Series, qcut
from pandas.core.common import adjoin
from pandas.io.data import DataReader
from patsy import dmatrices

def get_dataframe_intersection(df, comparator1,comparator2):
    """
    Return a dataframe with only the columns found in a comparative dataframe.

    Parameters
    ----------
    comparator1: DataFrame
        DataFrame to preform comparison on.
    comparator2: DataFrame
        DataFrame to compare with.

    Returns
    -------
    DataFrame:
        Data frame with columns not found in comparator dropped.

    """
    to_drop = list((c for c in comparator1 if c not in comparator2))
    return df.drop(to_drop, axis=1)

def get_dataframes_intersections(df1, comparator1, df2, comparator2):
    """
    Return DataFrames with the intersection of their column values.

    Parameters
    ----------
    comparator1: DataFrame
        DataFrame to preform comparison on.
    comparator2: DataFrame
        DataFrame to compare with.

    Returns
    -------
    Tuple:
        The resultingDataframe with columns not found in comparator dropped.

    """
    comparator1 = get_dataframe_intersection(df1, comparator1, comparator2)
    comparator2 = get_dataframe_intersection(df2, comparator2, comparator1)
    return comparator1, comparator2

def predict(test_data, results, model_name):
    """
    Return predictions of based on model resutls.

    Parameters
    ----------
    test_data: DataFrame
        should be test data you are trying to predict
    results: Dict
        should be dict of your models results wrapper and the formula used
        to produce it.
            ie.
            results['Model_Name'] = {[<statsmodels.regression.linear_model.RegressionResultsWrapper> , "Price ~ I(Supply, Demand)] }
    model_name: Str
        should be the name of your model. You can iterate through the results dict.

    Returns
    -------
    NumPy array
        Predictions in a flat NumPy array.

    Example
    -------
    results = {'Logit': [<statsmodels.discrete.discrete_model.BinaryResultsWrapper at 0x117896650>,
               'survived ~ C(pclass) + C(sex) + age + sibsp  + C(embarked)']}
    compared_resuts = predict(test_data, results, 'Logit')

    """
    model_params = DataFrame(results[model_name][0].params)
    formula = results[model_name][1]

    # Create regression friendly test DataFrame
    yt, xt = dmatrices(formula, data=test_data, return_type='dataframe')
    xt, model_params = get_dataframes_intersections(xt, xt.columns,
                                                    model_params, model_params.index)
    # Convert to NumPy arrays for performance
    model_params = np.asarray(model_params)
    yt = np.asarray(yt)
    yt = yt.ravel()
    xt = np.asarray(xt)

    # Use our models to create predictions
    row, col = xt.shape
    model_parameters = model_params.ravel()
    model_array = list((model_parameters for parameter in xrange(row)))
    model_array = np.asarray(model_array)

    # Multiply matrix together
    predictions = np.multiply(xt, model_array)
    predictions = np.sum(predictions, axis=1)
    return predictions

def cross_validate_df(df, percent):
    '''
    Return a randomly shuffled supbsets of a DataFrame cross validation
    or for down sampleing.

    Parameters
    ----------
    df : DataFrame
        DataFrame to be sampled. Expects an ordinal index; ie. 0 - 100.
    percent: Int
         the percentage to split of the returned subsets.

    Returns
    -------
    Tuple:
        (df_h1, df_h2), both parts of the split randomly shuffled DataFrame

    Example
    -------
    small_df_half, large_df_half  = cross_validate_df(df, 33)

    '''
    sample_percentage_of_dataframe = int(np.round(((percent * df.index.size) / float(100))))
    rows = np.random.randint(0, df.index.size, sample_percentage_of_dataframe)
    return (df.ix[rows], df.drop(rows))

def dataframe_kfolds(df):
    '''
    Standard kfolds cross cross_validate method.

    Parameters
    ----------
    df : DataFrame
        A pandas dataframe to be operated on.

    Returns
    -------
    Tuple :
        dataframe 10%, dataframe 90% -- random splits using python's random.choice()

    '''
    return cross_validate_df(df, 90)

def dataframe_welch_ttest(df, described_frame, boolean_feature):
    '''
    Parameters
    ----------
    df : DataFrame
       A DataFrame to perfrom welch_ttest on.
    described_frame : DataFrame
       A described DataFrame from the pandas desribe_frame() method.
    boolean_feature: Str
       Name of boolean feature to conduct test on.

    Returns
    -------
    DataFrame :
        t-statistic and p-value for each feature in a pandas dataframe.

    '''

    described_frame['t-statistic'] = np.nan
    described_frame['p-value'] = np.nan

    for name, item in df.iteritems():
        result = sp.stats.ttest_ind(df[name][df[boolean_feature] == 0].dropna(),
                                        df[name][df[boolean_feature] == 1].dropna(),
                                        equal_var=False)
        described_frame.ix[name, 't-statistic'], described_frame.ix[name, 'p-value'] = result
    return described_frame

def category_boolean_maker(series):
    '''
    A funtction for to designate missing records from observed ones.

    When used with the pandas df.series.apply() method, it will create
    a boolean category variable. If values exist the bool will register
    as 1, if nan values exist the bool will register as 0.

    Parameters
    ----------
    series : Series
        A pandas series to perform comparision on.

    Returns
    -------
    Int :
        0 or 1 for missing values.

    '''
    return 0 if np.isnan(series) == True else 1

def columns_to_str(column_list, operand=', ', return_list=False):
    '''
    Return the list of features as strings for easy implementaiton patsy formulas.

    Parameters
    ----------
    column_list : list
        A list of features, ussually from generated from pandas's df.columns function.

    operand : str
        a sting to join list together by. Default is a comma: ', '. Could be a plus: ' + '
        for patsy equations.

    return_list : boolean
        ( optional ) return the list of features typecasted to str.

    Returns
    -------
    list :
        a list with elements typecasted to str.

    Example
    -------

    df.columns
    >>> Index([x3yv_E, x3yv_D, x1yv_E, x1yv_D], dtype=object)

    columns_to_str(df.columns)
    >>> Index(['x3yv_E', 'x3yv_D', 'x1yv_E', 'x1yv_D'], dtype=object)
    >>> ['x3yv_E', 'x3yv_D', 'x1yv_E', 'x1yv_D']

    '''
    if return_list == True:
        return list((str(feature) for feature in column_list))

    print "[%s]" % operand.join(map(str, column_list))

def add_to_model_subspace(left, right):
    '''
    Returns the intersection between two lists.

    Useful for defining a model's subspace variables with new added variables.

    Parameters
    ----------
    left : list
        a list you'd like to add to.

    right : list
        a list of things your trying to add

    Returns
    -------
    list :
        a list of the intersection between the passed in left and right lists.

    '''
    return list((str(right[i]) for i in xrange(0, len(right))
                if right[i] in left == False ))

def ml_formula(y, df):
    '''
    a simple function to create a formula using all available features for patsy dmatrices function.

    ins
    --
    y = a string of the variable your trying to predict

    df = the data frame your trying to make your predctions on

    outs
    ---
    a string of the desired formula
    '''
    formula = y +' ~ '
    for i, val in enumerate(df.columns):
        if i == 0 and val != y:
            formula += val
        if i != 0 and val != y:
            formula += ' + ' + val
    return formula

def progress(i, num_tasks):
    '''
    A simple textual progress bar

    Ins
    --
    i = should be an iterable value to measure the progress of your taks
    num_tasks = total number of tasks

    outs
    --
    A progress bar like [#########  ]
    '''
    progress = "\r["

    for _ in range (0,i):
        progress += "#"

    for _ in range (i, num_tasks):
        progress += " "

    progress += "]"

    sys.stdout.write(progress)
    sys.stdout.flush()

def cat_clean(s):
    """
    Cleans categorical variables of NaNs by setting the NaN to 0 so it can be processed by a patsy dmatrices function.

    Inputs
    --
    s = cat value

    output
    --
    cat value or int 0 in place of a NaN val
    """
    if isinstance(s, str) == False and np.isnan(s) == True:
        s = 0
    return s

def quater_maker(d):
    """
    Pareses dates and returns the appropriate quarter.
    --
    Parameters
    d: a python date time object
    --
    Returns:
    The quarter in a string

    AGC 2013
    """
    r = ''
    Q1 = [1, 2, 3]
    Q2 = [4, 5, 6]
    Q3 = [7, 8, 9]
    Q4 = [10, 11, 12]
    if d.month in Q1:
        r = 'Q1'
    if d.month in Q2:
        r = 'Q2'
    if d.month in Q3:
        r = 'Q3'
    if d.month in Q4:
        r = 'Q4'
    return r

      #if date is within daterange, Q1 ect. then output ex: 2011 Q1
      #manipulate months. say if  0<m4: return YYYY + Q1 ect.
      # then apply to dateline

def score_rmsle(y, df, df2, p = 0):
    '''
    ins
    --
    y =  what your trying to predict. must be a string. ie. 'SalesPrice'
    df =  your predictions
    df2 = the solutions set
    p = option to print rmsle as string; 0 = dont print, 1 = print
    outs
    --
    prints rmsle
    rmsle as a float
    '''
    prediction = np.asarray(df[y])
    actual = np.asarray(df2[y])

    rsmle = np.sqrt(np.mean(np.power(np.log(actual + 1) - np.log(prediction + 1), 2)))

    if p == 1 :
        print "rsmle: " + str(rsmle)
    return rsmle

def score_rmse(y, df, df2, p=0 ):
    """
    ins
    --
    y =  what your trying to predict. must be a string. ie. 'SalesPrice'
    df =  your predictions
    df2 = the solutions set
    p = option to print rmsle as string; 0 = don't print, 1 = print

    outs
    --
    prints rmse
    rmse as a float
    """
    prediction = np.asarray(df[y])
    actual = np.asarray(df2[y])

    rsme = np.sqrt(np.mean(np.power((actual - prediction), 2)))

    if p == 1:
        print "rsme: " + str(rsme)

    return rsme

def unwanted_pals(x, s = .1):
    '''
    Inputs
    --
    x should be a pandas series of the results.pvalues of your model
    s = significance level, default at 90% confidence

    Outs
    --
    returns a list of columns below significance level
    '''
    dropl = list()
    for i,z in enumerate(x):
        if z>s:
             dropl.append(x.index[i])
    return dropl

def stock_price_at_date(x, ticker, lag=0):
    '''
    ins
    --
    x should be the date your looking for
    ticker should be the stock ticker
    lag should be # of days to lag stock price

    outs
    --
    stock price.
    '''
    x = (x - timedelta(days = lag))
    r = DataReader(ticker,start=x, end=x, data_source='yahoo')
    r = r.ix[0,5]
    return r

def side_by_side(*objs, **kwds):
    '''
    created by wes mickinney, it only exists here becuase I use this function all the time.
    '''
    space = kwds.get('space', 4)
    reprs = [repr(obj).split('\n') for obj in objs]
    print adjoin(space, *reprs)

def describe_frame(df):
    """
    ins
    ---
    df = dataframe you want to describe

    outs
    ---
    descriptive stats on your dataframe, in dataframe.

    agc2013
    """
    sum_stats = []
    for i in df.columns:
        x = Series(df[i].describe())
        x.name = i
        sum_stats.append(x)
    stats = DataFrame(sum_stats)
    stats
    return stats

def bin_residuals(resid, var, bins):
    '''
    Compute average residuals within bins of a variable.

    Returns a dataframe indexed by the bins, with the bin midpoint,
    the residual average within the bin, and the confidence interval
    bounds.

    ins
    --
    resid, var, bins

    out
    --
    bin DataFrame

    '''
    resid_df = DataFrame({'var': var, 'resid': resid})
    resid_df['bins'] = qcut(var, bins)
    bin_group = resid_df.groupby('bins')
    bin_df = bin_group['var', 'resid'].mean()
    bin_df['count'] = bin_group['resid'].count()
    bin_df['lower_ci'] = -2 * (bin_group['resid'].std() /
                               np.sqrt(bin_group['resid'].count()))
    bin_df['upper_ci'] =  2 * (bin_group['resid'].std() /
                               np.sqrt(bin_df['count']))
    bin_df = bin_df.sort('var')
    return(bin_df)

def plot_binned_residuals(bin_df):
    '''
    Plotted binned residual averages and confidence intervals.

    ins
    --
    bin_df ie from bin_residuals(resid, var, bins)
    outs
    --
    pretty plots
    '''
    plt.plot(bin_df['var'], bin_df['resid'], '.')
    plt.plot(bin_df['var'], bin_df['lower_ci'], '-r')
    plt.plot(bin_df['var'], bin_df['upper_ci'], '-r')
    plt.axhline(0, color = 'gray', lw = .5)
<|endoftext|>


<commit_before>from sh import git
from requests.auth import HTTPBasicAuth
from globalvars import GlobalVars
import requests
import time
import json

class GitManager:
    @classmethod
    def add_to_blacklist(self, **kwargs):
        blacklist = kwargs.get("blacklist", "website")
        items_to_blacklist = kwargs.get("items_to_blacklist", [])
        username = kwargs.get("username", "")
        chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
        code_permissions = kwargs.get("code_permissions", False)

        if blacklist == "website":
            blacklist_file_name = "blacklisted_websites.txt"
        elif blacklist == "keyword":
            blacklist_file_name = "bad_keywords.txt"
        elif blacklist == "username":
            blacklist_file_name = "blacklisted_usernames.txt"

        # Check if we're on master
        if git("rev-parse", "--abbrev-ref", "HEAD").strip() != "master":
            return (False, "Not currently on master.")

        # Check that we're up-to-date with origin (GitHub)
        git.remote.update()
        if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
            return (False, "HEAD isn't at tip of origin's master branch")

        # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
        if blacklist_file_name in git.status():  # Also ugly
            return (False, "{0} modified locally. This is probably bad.".format(blacklist_file_name))

        # Store current commit hash
        current_commit = git("rev-parse", "HEAD").strip()

        # Add items to file
        with open(blacklist_file_name, "a+") as blacklist_file:
            last_character = blacklist_file.read()[-1:]
            if last_character != "\n":
                blacklist_file.write("\n")
            blacklist_file.write("\n".join(items_to_blacklist) + "\n")

        # Checkout a new branch (for PRs for non-code-privileged people)
        branch = "auto-blacklist-{0}".format(str(time.time()))
        git.checkout("-b", branch)

        # Clear HEAD just in case
        git.reset("HEAD")

        git.add(blacklist_file_name)
        git.commit("-m", u"Auto blacklist of {0} by {1} --autopull".format(", ".join(items_to_blacklist), username))

        if code_permissions:
            git.checkout("master")
            git.merge(branch)
            git.push()
        else:
            git.push("origin", branch)
            git.checkout("master")

            if GlobalVars.github_username is None or GlobalVars.github_password is None:
                return (False, "Tell someone to set a GH password")

            list_of_domains = ""

            for domain in range(len(items_to_blacklist)):
                list_of_domains += "\n - {0} - [MS search](https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93&body_is_regex=1&body={0})".format(items_to_blacklist[domain])

            payload = {"title": "{0}: Blacklist {1}".format(username, ", ".join(items_to_blacklist)),
                       "body": "[{0}]({1}) requests the blacklist of the following {2}(s): \n{3}\n<!-- METASMOKE-BLACKLIST {4} -->".format(username, chat_profile_link, blacklist, list_of_domains, "|".join(items_to_blacklist)),
                       "head": branch,
                       "base": "master"}
            response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload))
            print(response.json())
            try:
                return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(response.json()["html_url"]))
            except KeyError:
                # Error capture/checking for any "invalid" GH reply without an 'html_url' item, which will throw a KeyError.
                if "Bad credentials" in str(response.json()['message']):
                    # Capture the case when GH credentials are bad or invalid
                    return (False, "Something is wrong with the GH credentials, tell someone to check the GH credentials.")
                else:
                    # Capture any other invalid response cases.
                    return (False, "A bad or invalid reply was received from GH, the message was: %s" % response.json()['message'])

        git.checkout(current_commit)  # Return to old commit to await CI. This will make Smokey think it's in reverted mode if it restarts

        return (True, "Blacklisted {0} - the entry will be applied via autopull if CI succeeds.".format(", ".join(items_to_blacklist)))

    @classmethod
    def current_git_status(self):
        return git.status()
<commit_msg>Replace \s with a space character<commit_after>from sh import git
from requests.auth import HTTPBasicAuth
from globalvars import GlobalVars
import requests
import time
import json

class GitManager:
    @classmethod
    def add_to_blacklist(self, **kwargs):
        blacklist = kwargs.get("blacklist", "website")
        items_to_blacklist = kwargs.get("items_to_blacklist", [])
        username = kwargs.get("username", "")
        chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
        code_permissions = kwargs.get("code_permissions", False)

        for item in items_to_blacklist:
            item.replace("\s", " ")

        if blacklist == "website":
            blacklist_file_name = "blacklisted_websites.txt"
        elif blacklist == "keyword":
            blacklist_file_name = "bad_keywords.txt"
        elif blacklist == "username":
            blacklist_file_name = "blacklisted_usernames.txt"

        # Check if we're on master
        if git("rev-parse", "--abbrev-ref", "HEAD").strip() != "master":
            return (False, "Not currently on master.")

        # Check that we're up-to-date with origin (GitHub)
        git.remote.update()
        if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
            return (False, "HEAD isn't at tip of origin's master branch")

        # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
        if blacklist_file_name in git.status():  # Also ugly
            return (False, "{0} modified locally. This is probably bad.".format(blacklist_file_name))

        # Store current commit hash
        current_commit = git("rev-parse", "HEAD").strip()

        # Add items to file
        with open(blacklist_file_name, "a+") as blacklist_file:
            last_character = blacklist_file.read()[-1:]
            if last_character != "\n":
                blacklist_file.write("\n")
            blacklist_file.write("\n".join(items_to_blacklist) + "\n")

        # Checkout a new branch (for PRs for non-code-privileged people)
        branch = "auto-blacklist-{0}".format(str(time.time()))
        git.checkout("-b", branch)

        # Clear HEAD just in case
        git.reset("HEAD")

        git.add(blacklist_file_name)
        git.commit("-m", u"Auto blacklist of {0} by {1} --autopull".format(", ".join(items_to_blacklist), username))

        if code_permissions:
            git.checkout("master")
            git.merge(branch)
            git.push()
        else:
            git.push("origin", branch)
            git.checkout("master")

            if GlobalVars.github_username is None or GlobalVars.github_password is None:
                return (False, "Tell someone to set a GH password")

            list_of_domains = ""

            for domain in range(len(items_to_blacklist)):
                list_of_domains += "\n - {0} - [MS search](https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93&body_is_regex=1&body={0})".format(items_to_blacklist[domain])

            payload = {"title": "{0}: Blacklist {1}".format(username, ", ".join(items_to_blacklist)),
                       "body": "[{0}]({1}) requests the blacklist of the following {2}(s): \n{3}\n<!-- METASMOKE-BLACKLIST {4} -->".format(username, chat_profile_link, blacklist, list_of_domains, "|".join(items_to_blacklist)),
                       "head": branch,
                       "base": "master"}
            response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload))
            print(response.json())
            try:
                return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(response.json()["html_url"]))
            except KeyError:
                # Error capture/checking for any "invalid" GH reply without an 'html_url' item, which will throw a KeyError.
                if "Bad credentials" in str(response.json()['message']):
                    # Capture the case when GH credentials are bad or invalid
                    return (False, "Something is wrong with the GH credentials, tell someone to check the GH credentials.")
                else:
                    # Capture any other invalid response cases.
                    return (False, "A bad or invalid reply was received from GH, the message was: %s" % response.json()['message'])

        git.checkout(current_commit)  # Return to old commit to await CI. This will make Smokey think it's in reverted mode if it restarts

        return (True, "Blacklisted {0} - the entry will be applied via autopull if CI succeeds.".format(", ".join(items_to_blacklist)))

    @classmethod
    def current_git_status(self):
        return git.status()
<|endoftext|>


<commit_before>import cgi

class FormField(object):
    __slots__ = ['name', 'autocomplete', 'type', 'value',
                 'css_class', 'description', 'error',
                 'has_errors', 'required', 'css_id',
                 'options', 'size']

    def __init__(self, name, **kwargs):
        self.name = name
        self.type = kwargs.get("type")
        self.value = kwargs.get("value")
        self.css_class = kwargs.get("css_class")
        self.css_id = kwargs.get("css_id")
        self.description = kwargs.get("description")
        self.required = kwargs.get("required", False)
        self.autocomplete = kwargs.get("autocomplete", False)
        self.error = None
        self.has_errors = False
        self.options = kwargs.get("options")
        self.size = kwargs.get("size")

    def validate(self):
        if self.required and (self.value is None or self.value == ""):
            self.has_errors = True
            self.error = "Field is required"
            return False
        return True

    def label(self):
        res = []
        if self.description or self.has_errors:
            lbl_txt = "".format(self.name)
            res.append(lbl_txt)
            if self.description:
                res.append(cgi.escape(self.description))

            if self.has_errors:
                res.append("")
                res.append("{0}".format(self.error))

            res.append("\n")
        return ''.join(res)

    def render(self):
        res = []
        if not self.name:
            raise Exception("Field must have name attribute")

        res.append("<input name=\"{0}\" ".format(self.name))

        if self.type:
            res.append("type=\"{0}\" ".format(self.type))

        if self.value:
            if type(self.value) is str:
                res.append("value=\"{0}\" ".format(cgi.escape(self.value)))
            else:
                res.append("value=\"{0}\" ".format(cgi.escape(str(self.value))))

        if self.autocomplete:
            res.append("autocomplete=\"false\" ")

        if self.css_class:
            res.append("class=\"{0}\" ".format(self.css_class))

        if self.css_id:
            res.append("id=\"{0}\" ".format(self.css_id))

        res.append("/>")

        return ''.join(res)

    def __str__(self):
        return self.label() + self.render()

class IntegerFormField(FormField):

    def validate(self):
        if not super(IntegerFormField, self).validate():
            return False
        if type(self.value) is int:
            pass
        elif type(self.value) is str and self.value.isdigit():
            self.value = int(self.value)
        else:
            self.has_errors = True
            self.error = "Value is *must* be an integer"
            return False
        return True

class SelectFormField(FormField):

    def validate(self):
        if not super(SelectFormField, self).validate():
            return False

        for item in self.options:
            if self.value == str(item[0]):
                return True

        self.has_errors = True
        self.error = "Value not within supported range"
        return False

    def render(self):
        res = []

        res.append("<select name=\"{0}\" ".format(self.name))

        if self.autocomplete:
            res.append("autocomplete=\"false\" ")

        if self.css_class:
            res.append("class=\"{0}\" ".format(self.css_class))

        if self.css_id:
            res.append("id=\"{0}\" ".format(self.css_id))

        res.append(">\n")

        for item in self.options:
            res.append("<option ")
            res.append("value=\"{0}\">".format(item[0]))
            res.append("{0}</option>\n".format(item[1]))

        res.append("</select>\n")

        return ''.join(res)

class TextFormField(FormField):

    def render(self):
        res = []
        res.append("<textarea name=\"{0}\" ".format(self.name))

        if self.autocomplete:
            res.append("autocomplete=\"false\" ")

        if self.css_class:
            res.append("class=\"{0}\" ".format(self.css_class))

        if self.css_id:
            res.append("id=\"{0}\" ".format(self.css_id))

        if 'rows' in self.options:
            res.append("rows=\"{0}\" ".format(self.options['rows']))

        if 'cols' in self.options:
            res.append("cols=\"{0}\" ".format(self.options['cols']))

        res.append(">\n")

        if self.value:
            res.append(cgi.escape(self.value))

        res.append("</textarea>\n")

        return ''.join(res)

class WebForm(object):

    def __init__(self, data=None):
        if data is not None:
            items = self.__class__.__dict__.iteritems()
            for item in items:
                k,v = item
                if isinstance(v, FormField):
                    v.value = data.get(v.name)

    def validate(self):
        res = True
        ret = True
        items = self.__class__.__dict__.iteritems()
        for item in items:
            k,v = item
            if isinstance(v, FormField):
                res = v.validate()
                if not res:
                    ret = False
        return ret
<commit_msg>modified select form to default value when available<commit_after>import cgi

class FormField(object):
    __slots__ = ['name', 'autocomplete', 'type', 'value',
                 'css_class', 'description', 'error',
                 'has_errors', 'required', 'css_id',
                 'options', 'size']

    def __init__(self, name, **kwargs):
        self.name = name
        self.type = kwargs.get("type")
        self.value = kwargs.get("value")
        self.css_class = kwargs.get("css_class")
        self.css_id = kwargs.get("css_id")
        self.description = kwargs.get("description")
        self.required = kwargs.get("required", False)
        self.autocomplete = kwargs.get("autocomplete", False)
        self.error = None
        self.has_errors = False
        self.options = kwargs.get("options")
        self.size = kwargs.get("size")

    def validate(self):
        if self.required and (self.value is None or self.value == ""):
            self.has_errors = True
            self.error = "Field is required"
            return False
        return True

    def label(self):
        res = []
        if self.description or self.has_errors:
            lbl_txt = "".format(self.name)
            res.append(lbl_txt)
            if self.description:
                res.append(cgi.escape(self.description))

            if self.has_errors:
                res.append("")
                res.append("{0}".format(self.error))

            res.append("\n")
        return ''.join(res)

    def render(self):
        res = []
        if not self.name:
            raise Exception("Field must have name attribute")

        res.append("<input name=\"{0}\" ".format(self.name))

        if self.type:
            res.append("type=\"{0}\" ".format(self.type))

        if self.value:
            if type(self.value) is str:
                res.append("value=\"{0}\" ".format(cgi.escape(self.value)))
            else:
                res.append("value=\"{0}\" ".format(cgi.escape(str(self.value))))

        if self.autocomplete:
            res.append("autocomplete=\"false\" ")

        if self.css_class:
            res.append("class=\"{0}\" ".format(self.css_class))

        if self.css_id:
            res.append("id=\"{0}\" ".format(self.css_id))

        res.append("/>")

        return ''.join(res)

    def __str__(self):
        return self.label() + self.render()

class IntegerFormField(FormField):

    def validate(self):
        if not super(IntegerFormField, self).validate():
            return False
        if type(self.value) is int:
            pass
        elif type(self.value) is str and self.value.isdigit():
            self.value = int(self.value)
        else:
            self.has_errors = True
            self.error = "Value is *must* be an integer"
            return False
        return True

class SelectFormField(FormField):

    def validate(self):
        if not super(SelectFormField, self).validate():
            return False

        for item in self.options:
            if self.value == str(item[0]):
                return True

        self.has_errors = True
        self.error = "Value not within supported range"
        return False

    def render(self):
        res = []

        res.append("<select name=\"{0}\" ".format(self.name))

        if self.autocomplete:
            res.append("autocomplete=\"false\" ")

        if self.css_class:
            res.append("class=\"{0}\" ".format(self.css_class))

        if self.css_id:
            res.append("id=\"{0}\" ".format(self.css_id))

        res.append(">\n")

        for item in self.options:
            res.append("<option ")
            if self.value is not None:
                if str(item[0]) == str(self.value):
                    res.append(" selected=\"selected\" ")
            res.append("value=\"{0}\">".format(item[0]))
            res.append("{0}</option>\n".format(item[1]))

        res.append("</select>\n")

        return ''.join(res)

class TextFormField(FormField):

    def render(self):
        res = []
        res.append("<textarea name=\"{0}\" ".format(self.name))

        if self.autocomplete:
            res.append("autocomplete=\"false\" ")

        if self.css_class:
            res.append("class=\"{0}\" ".format(self.css_class))

        if self.css_id:
            res.append("id=\"{0}\" ".format(self.css_id))

        if 'rows' in self.options:
            res.append("rows=\"{0}\" ".format(self.options['rows']))

        if 'cols' in self.options:
            res.append("cols=\"{0}\" ".format(self.options['cols']))

        res.append(">\n")

        if self.value:
            res.append(cgi.escape(self.value))

        res.append("</textarea>\n")

        return ''.join(res)

class WebForm(object):

    def __init__(self, data=None):
        if data is not None:
            items = self.__class__.__dict__.iteritems()
            for item in items:
                k,v = item
                if isinstance(v, FormField):
                    v.value = data.get(v.name)

    def validate(self):
        res = True
        ret = True
        items = self.__class__.__dict__.iteritems()
        for item in items:
            k,v = item
            if isinstance(v, FormField):
                res = v.validate()
                if not res:
                    ret = False
        return ret
<|endoftext|>


<commit_before>#!/usr/bin/env python2
# coding=utf-8

"""
git support library
"""

import sys
import re
from gitver.termcolors import err

# check for the sh package
try:
    import sh
    from sh import ErrorReturnCode, CommandNotFound
except ImportError:
    print "A dependency is missing, please install the \"sh\" package and " \
          "run gitver again."
    sys.exit(1)

hash_matcher = r".*-g([a-fA-F0-9]+)"
tag_matcher = r"v{0,1}(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z-.]*))?"

def git_version():
    try:
        ver = sh.git('--version').stdout.replace('\n', '')
    except CommandNotFound:
        return ''
    return ver

def project_root():
    try:
        root = sh.git('rev-parse', '--show-toplevel').stdout.replace('\n', '')
    except ErrorReturnCode:
        return ''
    return root

def describe_hash():
    try:
        tagver = sh.git(
            'describe', '--long', '--match=v*').stdout.replace('\n', '')
        vm = re.match(hash_matcher, tagver).groups()
        if len(vm) != 1:
            raise AttributeError
    except (ErrorReturnCode, AttributeError):
        return False
    return vm[0]

def count_tag_to_head(tag):
    try:
        c = sh.git(
            'rev-list', tag + "..HEAD", '--count').stdout.replace('\n', '')
        return int(c)
    except ErrorReturnCode:
        return False

def get_build_id():
    try:
        full_build_id = str(
            sh.git('rev-parse', 'HEAD').stdout.replace('\n', ''))
    except ErrorReturnCode:
        return False
    return full_build_id

def last_tag():
    try:
        tag = sh.git('describe', '--abbrev=0').stdout.replace('\n', '')
    except ErrorReturnCode:
        return False

    return tag

def data_from_tag(tag):
    try:
        data = re.match(tag_matcher, tag).groups()
        if len(data) < 3:
            raise AttributeError
    except AttributeError:
        return None
    return data

def get_repo_info():
    # retrieve basic repository information
    desc_hash = describe_hash()
    if desc_hash is None or not desc_hash:
        print err("Error, couldn't describe current hash")
        sys.exit(1)

    hashlen = len(desc_hash)
    full_build_id = get_build_id()
    if not full_build_id or hashlen == 0:
        print err("Couldn't retrieve build id information")
        sys.exit(1)

    # sanity check
    if not full_build_id.startswith(desc_hash):
        print err("Hash problem detected: git-describe reports " + desc_hash +
                  ", but HEAD id is " + full_build_id)
        sys.exit(1)

    tag = last_tag()
    if not tag:
        print err("Couldn't retrieve the latest tag")
        sys.exit(1)

    data = data_from_tag(tag)
    if data is None:
        print err("Couldn't retrieve version information from tag \"" + tag +
                  "\"")
        sys.exit(1)

    vmaj = int(data[0])
    vmin = int(data[1])
    vpatch = int(data[2])
    pr = data[3]
    vcount = count_tag_to_head(tag)

    return {'maj': vmaj, 'min': vmin, 'patch': vpatch, 'count': vcount,
            'build-id': full_build_id[:hashlen], 'full-build-id': full_build_id,
            'last-tag': tag, 'pr': pr}
<commit_msg>Refactor git interface, more readable<commit_after>#!/usr/bin/env python2
# coding=utf-8

"""
git support library
"""

import sys
import re
from gitver.termcolors import err

# check for the sh package
try:
    import sh
    from sh import ErrorReturnCode, CommandNotFound
except ImportError:
    print "A dependency is missing, please install the \"sh\" package and " \
          "run gitver again."
    sys.exit(1)

hash_matcher = r".*-g([a-fA-F0-9]+)"
tag_matcher = r"v{0,1}(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z-.]*))?"

def __git(*args):
    """
    Proxies the specified git command+args and returns a cleaned up version
    of the stdout buffer.
    """
    return sh.git(args).stdout.replace('\n', '')

def git_version():
    try:
        ver = __git('--version')
    except (CommandNotFound, ErrorReturnCode):
        return ''
    return ver

def project_root():
    try:
        root = __git('rev-parse', '--show-toplevel')
    except ErrorReturnCode:
        return ''
    return root

def describe_hash():
    try:
        tagver = __git('describe', '--long', '--match=v*')
        vm = re.match(hash_matcher, tagver).groups()
        if len(vm) != 1:
            raise AttributeError
    except (ErrorReturnCode, AttributeError):
        return False
    return vm[0]

def count_tag_to_head(tag):
    try:
        c = __git('rev-list', tag + "..HEAD", '--count')
        return int(c)
    except ErrorReturnCode:
        return False

def get_build_id():
    try:
        full_build_id = str(__git('rev-parse', 'HEAD'))
    except ErrorReturnCode:
        return False
    return full_build_id

def last_tag():
    try:
        tag = __git('describe', '--abbrev=0')
    except ErrorReturnCode:
        return False

    return tag

def data_from_tag(tag):
    try:
        data = re.match(tag_matcher, tag).groups()
        if len(data) < 3:
            raise AttributeError
    except AttributeError:
        return None
    return data

def get_repo_info():
    # retrieve basic repository information
    desc_hash = describe_hash()
    if desc_hash is None or not desc_hash:
        print err("Error, couldn't describe current hash")
        sys.exit(1)

    hashlen = len(desc_hash)
    full_build_id = get_build_id()
    if not full_build_id or hashlen == 0:
        print err("Couldn't retrieve build id information")
        sys.exit(1)

    # sanity check
    if not full_build_id.startswith(desc_hash):
        print err("Hash problem detected: git-describe reports " + desc_hash +
                  ", but HEAD id is " + full_build_id)
        sys.exit(1)

    tag = last_tag()
    if not tag:
        print err("Couldn't retrieve the latest tag")
        sys.exit(1)

    data = data_from_tag(tag)
    if data is None:
        print err("Couldn't retrieve version information from tag \"" + tag +
                  "\"")
        sys.exit(1)

    vmaj = int(data[0])
    vmin = int(data[1])
    vpatch = int(data[2])
    pr = data[3]
    vcount = count_tag_to_head(tag)

    return {'maj': vmaj, 'min': vmin, 'patch': vpatch, 'count': vcount,
            'build-id': full_build_id[:hashlen], 'full-build-id': full_build_id,
            'last-tag': tag, 'pr': pr}
<|endoftext|>