<commit_before>"""Script to plot accelerometer traces."""

import matplotlib
matplotlib.use('Agg')
from accelerometer import accUtils
import argparse
from datetime import datetime, timedelta, time
import matplotlib.pyplot as plt
import matplotlib.lines as mlines
import matplotlib.patches as mpatches
import numpy as np
import pandas as pd
import sys

# http://pandas-docs.github.io/pandas-docs-travis/whatsnew/v0.21.1.html#restore-matplotlib-datetime-converter-registration
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()

DOHERTY_NatComms_COLOURS = {'sleep':'blue', 'sedentary':'red',
    'tasks-light':'darkorange', 'walking':'lightgreen', 'moderate':'green'}

WILLETS_SciReports_COLOURS = {'sleep':'blue', 'sit.stand':'red',
    'vehicle':'darkorange', 'walking':'lightgreen', 'mixed':'green',
    'bicycling':'purple'}

def main():
    """
    Application entry point responsible for parsing command line requests
    """

    parser = argparse.ArgumentParser(
        description="A script to plot acc time series data.", add_help=True)
    # required
    parser.add_argument('timeSeriesFile', metavar='input file', type=str,
                            help="input .csv.gz time series file to plot")
    parser.add_argument('plotFile', metavar='output file', type=str,
                            help="output .png file to plot to")
    parser.add_argument('--activityModel', type=str,
                            default="activityModels/doherty2018-apr20Update.tar",
                            help="""trained activity model .tar file""")
    parser.add_argument('--useRecommendedImputation',
                            metavar='True/False', default=True, type=str2bool,
                            help="""Highly recommended method to show imputed
                            missing data (default : %(default)s)""")
    parser.add_argument('--imputedLabels',
                            metavar='True/False', default=False, type=str2bool,
                            help="""If activity classification during imputed
                            period will be displayed (default : %(default)s)""")

    # check input is ok
    if len(sys.argv) < 3:
        msg = "\nInvalid input, please enter at least 2 parameters, e.g."
        msg += "\npython accPlot.py timeSeries.csv.gz plot.png \n"
        accUtils.toScreen(msg)
        parser.print_help()
        sys.exit(-1)
    args = parser.parse_args()

    # and then call plot function
    plotTimeSeries(args.timeSeriesFile, args.plotFile,
        activityModel=args.activityModel,
        useRecommendedImputation=args.useRecommendedImputation,
        imputedLabels=args.imputedLabels)

def plotTimeSeries(
        tsFile,
        plotFile,
        activityModel="activityModels/doherty2018-apr20Update.tar",
        useRecommendedImputation=True,
        imputedLabels=False):
    """Plot overall activity and classified activity types

    :param str tsFile: Input filename with .csv.gz time series data
    :param str tsFile: Output filename for .png image
    :param str activityModel: Input tar model file used for activity classification
    :param bool useRecommendedImputation: Highly recommended method to show
        imputed values for missing data
    :param bool imputedLabels: If activity classification during imputed period
        will be displayed

    :return: Writes plot to <plotFile>
    :rtype: void

    :Example:
    >>> import accPlot
    >>> accPlot.plotTimeSeries("sample-timeSeries.csv.gz", "sample-plot.png")
    <plot file written to sample-plot.png>
    """

    # read time series file to pandas DataFrame
    d = accUtils.loadTimeSeriesCSV(tsFile)
    d['acc'] = d['acc'].rolling(window=12, min_periods=1).mean() # smoothing
    d['time'] = d.index.time
    ymin = d['acc'].min()
    ymax = d['acc'].max()

    # infer labels
    labels = []
    for col in d.columns.tolist():
        if col not in [accUtils.TIME_SERIES_COL, 'imputed', 'acc', 'MET']:
            labels += [col]
    print(labels)
    if 'doherty' in activityModel:
        labels_as_col = DOHERTY_NatComms_COLOURS
    elif 'willetts' in activityModel:
        labels_as_col = WILLETS_SciReports_COLOURS
    # add imputation label colour
    labels_as_col['imputed'] = '#fafc6f'

    convert_date = np.vectorize(lambda day, x: matplotlib.dates.date2num(datetime.combine(day, x)))

    # number of rows to display in figure (all days + legend)
    d['date'] = d.index.date
    if not useRecommendedImputation:
        d = d[d['imputed']==0] # if requested, do not show imputed values

    if imputedLabels:
        labelsPosition = 0.75
    else:
        labelsPosition = 1

    groupedDays = d[['acc','time','imputed'] + labels].groupby(by=d['date'])
    nrows = len(groupedDays) + 1

    # create overall figure
    fig = plt.figure(1, figsize=(10,nrows), dpi=100)
    fig.canvas.set_window_title(tsFile)

    # create individual plot for each day
    i = 0
    ax_list = []
    for day, group in groupedDays:
        # set start and end to zero to avoid diagonal fill boxes
        group['imputed'].values[0] = 0
        group['imputed'].values[-1] = 0

        # retrieve time series data for this day
        timeSeries = convert_date(day, group['time'])
        # and then plot time series data for this day
        plt.subplot(nrows, 1, i+1)
        plt.plot(timeSeries, group['acc'], c='k')
        if imputedLabels:
            plt.fill_between(timeSeries, y1 = np.multiply(group['imputed'], ymax),
                y2 = np.multiply(group['imputed'], ymax * labelsPosition),
                color = labels_as_col['imputed'], alpha=1.0, where=group['imputed']==1)
        else:
            plt.fill(timeSeries, np.multiply(group['imputed'], ymax),
            labels_as_col['imputed'], alpha=1.0)

        # change display properties of this subplot
        ax = plt.gca()
        if len(labels)>0:
            ax.stackplot(timeSeries, [np.multiply(group[l], ymax * labelsPosition) for l in labels],
                colors=[labels_as_col[l] for l in labels], alpha=0.5, edgecolor="none")
        # add date label to left hand side of each day's activity plot
        plt.title(
            day.strftime("%A,\n%d %B"), weight='bold',
            x=-.2, y=0.5,
            horizontalalignment='left',
            verticalalignment='center',
            rotation='horizontal',
            transform=ax.transAxes,
            fontsize='medium',
            color='k'
        )
        # run gridlines for each hour bar
        ax.get_xaxis().grid(True, which='major', color='grey', alpha=0.5)
        ax.get_xaxis().grid(True, which='minor', color='grey', alpha=0.25)
        # set x and y-axes
        ax.set_xlim((datetime.combine(day,time(0, 0, 0, 0)),
            datetime.combine(day + timedelta(days=1), time(0, 0, 0, 0))))
        ax.set_xticks(pd.date_range(start=datetime.combine(day,time(0, 0, 0, 0)),
            end=datetime.combine(day + timedelta(days=1), time(0, 0, 0, 0)),
            freq='4H'))
        ax.set_xticks(pd.date_range(start=datetime.combine(day,time(0, 0, 0, 0)),
            end=datetime.combine(day + timedelta(days=1), time(0, 0, 0, 0)),
            freq='1H'), minor=True)
        ax.set_ylim((ymin, ymax))
        ax.get_yaxis().set_ticks([]) # hide y-axis lables
        # make border less harsh between subplots
        ax.spines['top'].set_color('#d3d3d3') # lightgray
        ax.spines['right'].set_visible(False)
        ax.spines['left'].set_visible(False)
        # set background colour to lightgray
        ax.set_facecolor('#d3d3d3')

        # append to list and incrament list counter
        ax_list.append(ax)
        i += 1

    # create new subplot to display legend
    plt.subplot(nrows, 1, i+1)
    ax = plt.gca()
    ax.axis('off') # don't display axis information
    # create a 'patch' for each legend entry
    legend_patches = [mpatches.Patch(color=labels_as_col['imputed'],
                                     label='imputed', alpha=1.0),
                      mlines.Line2D([],[],color='k',label='acceleration')]
    # create lengend entry for each label
    for label in labels:
        col = labels_as_col[label]
        legend_patches.append(mpatches.Patch(color=col, label=label, alpha=0.5))
    # create overall legend
    plt.legend(handles=legend_patches, bbox_to_anchor=(0., 0., 1., 1.),
        loc='center', ncol=min(4,len(legend_patches)), mode="best",
        borderaxespad=0, framealpha=0.6, frameon=True, fancybox=True)

    # remove legend border
    ax.spines['left'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.spines['bottom'].set_visible(False)
    ax_list.append(ax)

    # format x-axis to show hours
    plt.gcf().autofmt_xdate()
    # add hour labels to top of plot
    hours2Display = range(0, 24, 4)
    hrLabels = [(str(hr) + 'am') if hr<=12 else (str(hr-12) + 'pm') for hr in hours2Display]
    ax_list[0].set_xticklabels(hrLabels)
    ax_list[0].tick_params(labelbottom=False, labeltop=True, labelleft=False)

    plt.savefig(plotFile, dpi=200, bbox_inches='tight')
    print('Plot file written to:', plotFile)

def str2bool(v):
    """
    Used to parse true/false values from the command line. E.g. "True" -> True
    """

    return v.lower() in ("yes", "true", "t", "1")

if __name__ == '__main__':
    main()  # Standard boilerplate to call the main() function to begin the program.
<commit_msg>Add command line input option to change proportion of plot labels take up if --imputedLabels is set to True. In addition the default option is now set to 0.9. Many thanks to Xiangnan Dang for adding this feature.<commit_after>"""Script to plot accelerometer traces."""

import matplotlib
matplotlib.use('Agg')
from accelerometer import accUtils
import argparse
from datetime import datetime, timedelta, time
import matplotlib.pyplot as plt
import matplotlib.lines as mlines
import matplotlib.patches as mpatches
import numpy as np
import pandas as pd
import sys

# http://pandas-docs.github.io/pandas-docs-travis/whatsnew/v0.21.1.html#restore-matplotlib-datetime-converter-registration
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()

DOHERTY_NatComms_COLOURS = {'sleep':'blue', 'sedentary':'red',
    'tasks-light':'darkorange', 'walking':'lightgreen', 'moderate':'green'}

WILLETS_SciReports_COLOURS = {'sleep':'blue', 'sit.stand':'red',
    'vehicle':'darkorange', 'walking':'lightgreen', 'mixed':'green',
    'bicycling':'purple'}

def main():
    """
    Application entry point responsible for parsing command line requests
    """

    parser = argparse.ArgumentParser(
        description="A script to plot acc time series data.", add_help=True)
    # required
    parser.add_argument('timeSeriesFile', metavar='input file', type=str,
                            help="input .csv.gz time series file to plot")
    parser.add_argument('plotFile', metavar='output file', type=str,
                            help="output .png file to plot to")
    parser.add_argument('--activityModel', type=str,
                            default="activityModels/doherty2018-apr20Update.tar",
                            help="""trained activity model .tar file""")
    parser.add_argument('--useRecommendedImputation',
                            metavar='True/False', default=True, type=str2bool,
                            help="""Highly recommended method to show imputed
                            missing data (default : %(default)s)""")
    parser.add_argument('--imputedLabels',
                            metavar='True/False', default=False, type=str2bool,
                            help="""If activity classification during imputed
                            period will be displayed (default : %(default)s)""")
    parser.add_argument('--imputedLabelsHeight',
                            metavar='Proportion i.e. 0-1.0', default=0.9,
                            type=float, help="""Proportion of plot labels take
                            if activity classification during imputed
                            period will be displayed (default : %(default)s)""")

    # check input is ok
    if len(sys.argv) < 3:
        msg = "\nInvalid input, please enter at least 2 parameters, e.g."
        msg += "\npython accPlot.py timeSeries.csv.gz plot.png \n"
        accUtils.toScreen(msg)
        parser.print_help()
        sys.exit(-1)
    args = parser.parse_args()

    # and then call plot function
    plotTimeSeries(args.timeSeriesFile, args.plotFile,
        activityModel=args.activityModel,
        useRecommendedImputation=args.useRecommendedImputation,
        imputedLabels=args.imputedLabels,
        imputedLabelsHeight=args.imputedLabelsHeight)

def plotTimeSeries(
        tsFile,
        plotFile,
        activityModel="activityModels/doherty2018-apr20Update.tar",
        useRecommendedImputation=True,
        imputedLabels=False,
        imputedLabelsHeight=0.9):
    """Plot overall activity and classified activity types

    :param str tsFile: Input filename with .csv.gz time series data
    :param str tsFile: Output filename for .png image
    :param str activityModel: Input tar model file used for activity classification
    :param bool useRecommendedImputation: Highly recommended method to show
        imputed values for missing data
    :param bool imputedLabels: If activity classification during imputed period
        will be displayed
    :param float imputedLabelsHeight: Proportion of plot labels take up if
        <imputedLabels> is True

    :return: Writes plot to <plotFile>
    :rtype: void

    :Example:
    >>> import accPlot
    >>> accPlot.plotTimeSeries("sample-timeSeries.csv.gz", "sample-plot.png")
    <plot file written to sample-plot.png>
    """

    # read time series file to pandas DataFrame
    d = accUtils.loadTimeSeriesCSV(tsFile)
    d['acc'] = d['acc'].rolling(window=12, min_periods=1).mean() # smoothing
    d['time'] = d.index.time
    ymin = d['acc'].min()
    ymax = d['acc'].max()

    # infer labels
    labels = []
    for col in d.columns.tolist():
        if col not in [accUtils.TIME_SERIES_COL, 'imputed', 'acc', 'MET']:
            labels += [col]
    print(labels)
    if 'doherty' in activityModel:
        labels_as_col = DOHERTY_NatComms_COLOURS
    elif 'willetts' in activityModel:
        labels_as_col = WILLETS_SciReports_COLOURS
    # add imputation label colour
    labels_as_col['imputed'] = '#fafc6f'

    convert_date = np.vectorize(lambda day, x: matplotlib.dates.date2num(datetime.combine(day, x)))

    # number of rows to display in figure (all days + legend)
    d['date'] = d.index.date
    if not useRecommendedImputation:
        d = d[d['imputed']==0] # if requested, do not show imputed values

    if imputedLabels:
        labelsPosition = imputedLabelsHeight
    else:
        labelsPosition = 1

    groupedDays = d[['acc','time','imputed'] + labels].groupby(by=d['date'])
    nrows = len(groupedDays) + 1

    # create overall figure
    fig = plt.figure(1, figsize=(10,nrows), dpi=100)
    fig.canvas.set_window_title(tsFile)

    # create individual plot for each day
    i = 0
    ax_list = []
    for day, group in groupedDays:
        # set start and end to zero to avoid diagonal fill boxes
        group['imputed'].values[0] = 0
        group['imputed'].values[-1] = 0

        # retrieve time series data for this day
        timeSeries = convert_date(day, group['time'])
        # and then plot time series data for this day
        plt.subplot(nrows, 1, i+1)
        plt.plot(timeSeries, group['acc'], c='k')
        if imputedLabels:
            plt.fill_between(timeSeries, y1 = np.multiply(group['imputed'], ymax),
                y2 = np.multiply(group['imputed'], ymax * labelsPosition),
                color = labels_as_col['imputed'], alpha=1.0, where=group['imputed']==1)
        else:
            plt.fill(timeSeries, np.multiply(group['imputed'], ymax),
            labels_as_col['imputed'], alpha=1.0)

        # change display properties of this subplot
        ax = plt.gca()
        if len(labels)>0:
            ax.stackplot(timeSeries, [np.multiply(group[l], ymax * labelsPosition) for l in labels],
                colors=[labels_as_col[l] for l in labels], alpha=0.5, edgecolor="none")
        # add date label to left hand side of each day's activity plot
        plt.title(
            day.strftime("%A,\n%d %B"), weight='bold',
            x=-.2, y=0.5,
            horizontalalignment='left',
            verticalalignment='center',
            rotation='horizontal',
            transform=ax.transAxes,
            fontsize='medium',
            color='k'
        )
        # run gridlines for each hour bar
        ax.get_xaxis().grid(True, which='major', color='grey', alpha=0.5)
        ax.get_xaxis().grid(True, which='minor', color='grey', alpha=0.25)
        # set x and y-axes
        ax.set_xlim((datetime.combine(day,time(0, 0, 0, 0)),
            datetime.combine(day + timedelta(days=1), time(0, 0, 0, 0))))
        ax.set_xticks(pd.date_range(start=datetime.combine(day,time(0, 0, 0, 0)),
            end=datetime.combine(day + timedelta(days=1), time(0, 0, 0, 0)),
            freq='4H'))
        ax.set_xticks(pd.date_range(start=datetime.combine(day,time(0, 0, 0, 0)),
            end=datetime.combine(day + timedelta(days=1), time(0, 0, 0, 0)),
            freq='1H'), minor=True)
        ax.set_ylim((ymin, ymax))
        ax.get_yaxis().set_ticks([]) # hide y-axis lables
        # make border less harsh between subplots
        ax.spines['top'].set_color('#d3d3d3') # lightgray
        ax.spines['right'].set_visible(False)
        ax.spines['left'].set_visible(False)
        # set background colour to lightgray
        ax.set_facecolor('#d3d3d3')

        # append to list and incrament list counter
        ax_list.append(ax)
        i += 1

    # create new subplot to display legend
    plt.subplot(nrows, 1, i+1)
    ax = plt.gca()
    ax.axis('off') # don't display axis information
    # create a 'patch' for each legend entry
    legend_patches = [mpatches.Patch(color=labels_as_col['imputed'],
                                     label='imputed', alpha=1.0),
                      mlines.Line2D([],[],color='k',label='acceleration')]
    # create lengend entry for each label
    for label in labels:
        col = labels_as_col[label]
        legend_patches.append(mpatches.Patch(color=col, label=label, alpha=0.5))
    # create overall legend
    plt.legend(handles=legend_patches, bbox_to_anchor=(0., 0., 1., 1.),
        loc='center', ncol=min(4,len(legend_patches)), mode="best",
        borderaxespad=0, framealpha=0.6, frameon=True, fancybox=True)

    # remove legend border
    ax.spines['left'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.spines['bottom'].set_visible(False)
    ax_list.append(ax)

    # format x-axis to show hours
    plt.gcf().autofmt_xdate()
    # add hour labels to top of plot
    hours2Display = range(0, 24, 4)
    hrLabels = [(str(hr) + 'am') if hr<=12 else (str(hr-12) + 'pm') for hr in hours2Display]
    ax_list[0].set_xticklabels(hrLabels)
    ax_list[0].tick_params(labelbottom=False, labeltop=True, labelleft=False)

    plt.savefig(plotFile, dpi=200, bbox_inches='tight')
    print('Plot file written to:', plotFile)

def str2bool(v):
    """
    Used to parse true/false values from the command line. E.g. "True" -> True
    """

    return v.lower() in ("yes", "true", "t", "1")

if __name__ == '__main__':
    main()  # Standard boilerplate to call the main() function to begin the program.
<|endoftext|>


<commit_before># All the times here are in hours
# These are the times for non-pgo
import requests
import logging
from mozci.mozci import query_repo_url_from_buildername, query_repo_name_from_buildername, \
        trigger_all_talos_jobs, trigger_range, set_query_source
from mozci.query_jobs import TreeherderApi
from mozci.platforms import build_talos_buildernames_for_repo
from thclient import TreeherderClient
from store_alerts import getAlerts, updateAlert
from mozci.utils.misc import setup_logging
from managed_settings import TBPL_TESTS

LOG = setup_logging(logging.INFO)
TIME_TO_BUILD = 2
TIME_TO_TEST = 1
PENDING_TIME = 1
TIME_TO_WAIT = 2
CYCLE_TIME = 0.25
JSON_PUSHES = "%(repo_url)s/json-pushes"
WEEK = 604800
TWO_WEEKS = 1209600
SIXTY_DAYS = 5184000
SIGNATURE_URL = "https://treeherder.mozilla.org/api/project/%s/performance-data/get_performance_series_summary/?interval=%s"
PERFORMANCE_DATA = "https://treeherder.mozilla.org/api/project/%s/performance-data/get_performance_data/?interval_seconds=%s&signatures=%s"
OPTION_COLLECTION_HASH = "https://treeherder.mozilla.org/api/optioncollectionhash/"
SUCCESS = 0
DRY_RUN = False
REVERSE_TESTS = ['dromaeo_css', 'dromaeo_dom', 'v8_7', 'canvasmark']

# TODO: Ask about e10s
TREEHERDER_PLATFORM = {
    'linux32': 'Ubuntu HW 12.04',
    'linux64': 'Ubuntu HW 12.04 x64',
    'windowsxp': 'Windows XP 32-bit',
    'windows7-32': 'Windows 7 32-bit',
    'windows8-64': 'Windows 8 64-bit',
    'osx-10-10': 'Rev5 MacOSX Yosemite 10.10',
    'android-4-0-armv7-api11': 'Android 4.0 Tegra',
}

# Checked
def checkMerge(revision, buildername):
    "This function returns whether an alert is a merge or not."
    # Query Revision on pushlog, and check if all the authors of changesets are same.
    # We are not doing on the number of changesets, because a merge can have just 2 changesets
    # whereas a patch author can have 6-7 changeset.
    repo_url = query_repo_url_from_buildername(buildername)
    url = "%s?changeset=%s&version=2&full=1" % (JSON_PUSHES % {"repo_url": repo_url}, revision)
    req = requests.get(url).json()
    push_id = req["pushes"].keys()[0]
    changesets = req["pushes"][push_id]["changesets"]
    author = ""
    for changeset in changesets:
        if not author:
            author = changeset["author"]
        if changeset["author"] != author:
            return True

    return False

# Checked
def getRevisions(revision, buildername, start=0, end=0):
    "This function returns list of revisions for an alert."
    # Query Revision on pushlog, and get the revision. This should be easy.
    repo_url = query_repo_url_from_buildername(buildername)
    url = "%s?changeset=%s&version=2" % (JSON_PUSHES % {"repo_url": repo_url}, revision)
    req = requests.get(url).json()
    pushid = int(req["pushes"].keys()[0])
    startID = pushid + start - 1
    endID = pushid + end
    url = "%s?startID=%s&endID=%s&version=2&tipsonly=1" % (JSON_PUSHES % {"repo_url": repo_url}, startID, endID)
    req = requests.get(url).json()
    response = req["pushes"]
    revisionList = []
    for push in sorted(response.keys()):
        revisionList.append(response[push]["changesets"][0][0:12])

    return revisionList

# Checked
def getSuccessfulJobs(revision, buildername):
    "This function returns the number of data points for an alert."
    # Query TH client get_jobs method to get all jobs for a particular buildername
    # Then Query mozci function: https://github.com/armenzg/mozilla_ci_tools/blob/master/mozci/query_jobs.py#L187
    # to get the status of job for each job
    treeherder_api = TreeherderApi()
    repo_name = query_repo_name_from_buildername(buildername)
    matching_jobs = treeherder_api.get_matching_jobs(repo_name, revision, buildername)
    successful_jobs = 0
    for job in matching_jobs:
        status = treeherder_api.get_job_status(job)
        if status == SUCCESS:
            successful_jobs += 1
    return successful_jobs

def compare(test, buildername, revision, previous_revision):
    "This function will compare between 2 given revisions and return result as percentage"
    repo_name = query_repo_name_from_buildername(buildername)
    # Using TWO_WEEKS as interval, may change it afterwards
    signature_request_url = SIGNATURE_URL % (repo_name, TWO_WEEKS)
    signatures = requests.get(signature_request_url).json()
    options_collection_hash_list = requests.get(OPTION_COLLECTION_HASH).json()

    for signature, value in signatures.iteritems():
        # Skip processing subtests. They are identified by 'test' key in the dicitonary.
        if 'test' in value:
            continue

        # Ignoring e10s here.
        # TODO: Revisit this later
        if TBPL_TESTS[test]['testname'].lower() == value['suite'].lower() and TREEHERDER_PLATFORM[value["machine_platform"]] in buildername and 'test_options' not in value:
            test_signature = signature
        else:
            continue

        hash_signature = value['option_collection_hash']
        for key in options_collection_hash_list:
            if hash_signature == key["option_collection_hash"]:
                typeOfTest = key["options"][0]["name"]
                break

        if typeOfTest == 'pgo' and typeOfTest not in buildername:
            # if pgo, it should be present in buildername
            continue
        elif typeOfTest == 'opt':
            # if opt, nothing present in buildername
            break
        else:
            # We do not run talos on any branch other than pgo and opt.
            continue

    # Using TWO_WEEKS as interval, may change it afterwards
    req = requests.get(PERFORMANCE_DATA % (repo_name, TWO_WEEKS, test_signature)).json()
    performance_data = req[0]["blob"]
    treeherder_client = TreeherderClient()
    revision_resultset_id = treeherder_client.get_resultsets(repo_name, revision=revision)[0]["id"]
    previous_revision_resultset_id = treeherder_client.get_resultsets(repo_name, revision=previous_revision)[0]["id"]
    revision_perfdata = []
    previous_revision_perfdata = []

    for data in performance_data:
        if data["result_set_id"] == revision_resultset_id:
            revision_perfdata.append(data["geomean"])
        elif data["result_set_id"] == previous_revision_resultset_id:
            previous_revision_perfdata.append(data["geomean"])

    if revision_perfdata and previous_revision_perfdata:
        mean_revision_perfdata = sum(revision_perfdata) / float(len(revision_perfdata))
        mean_previous_revision_perfdata = sum(previous_revision_perfdata) / float(len(previous_revision_perfdata))

    if test in REVERSE_TESTS:
        # lower value results in regression
        return (mean_revision_perfdata - mean_previous_revision_perfdata) * 100.0 / mean_previous_revision_perfdata
    else:
        # higher value results in regression
        return (mean_previous_revision_perfdata - mean_revision_perfdata) * 100.0 / mean_previous_revision_perfdata

def main():
    alerts = getAlerts()
    for alert in alerts:
        # new alert
        if alert['stage'] == 0:
            if checkMerge(alert['revision'], alert['buildername']) or 'pgo' in alert['buildername']:
                LOG.info("We are ignoring this alert since it is either a merge or a pgo job.")
                alert['stage'] = -1 # We need to have manual inspection in this case.
                alert['user'] = 'human'
                updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
            else:
                alert['stage'] = 1

        # trigger jobs for backfill
        if alert['stage'] == 1:
            LOG.info("We are in stage 1, and going to backfill jobs.")
            revisionList = getRevisions(alert['revision'], alert['buildername'], start=-2, end=2)
            # Setting Treeherder as the source for querying.
            set_query_source("treeherder")
            trigger_range(alert['buildername'], revisionList, times=6, dry_run=DRY_RUN)
            alert['stage'] = 2
            # We want some time interval between stage 1 and 2, so we exit.
            updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
            continue

        # verify jobs for backfill
        if alert['stage'] == 2:
            LOG.info("We are in stage 2, and going to verify if jobs are backfilled.")
            revisionList = getRevisions(alert['revision'], alert['buildername'], start=-2, end=2)
            for revision in revisionList:
                dataPoints = getSuccessfulJobs(revision, alert['buildername'])

                # If dataPoints are less than 6, it means that builds/jobs are still running.
                if dataPoints < 6:
                    # We wait for 6 hours for all triggered tests to complete,
                    # And if they don't then we mark them for manual intervention/
                    alert['loop'] += 1
                    if alert['loop'] > (TIME_TO_BUILD + TIME_TO_TEST + PENDING_TIME) / CYCLE_TIME:
                        alert['stage'] = -1
                        alert['user'] = 'human'
                    else:
                        alert['stage'] = 1

                    break

            if alert['stage'] != 2:
                updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
                continue

            badRevisions = []
            # Reset the loop for upcoming stages
            alert['loop'] = 0
            for i in range(1, len(revisionList)):
                results = compare(alert['test'], alert['buildername'], revisionList[i], revisionList[i-1])
                if results < -2.0:
                    badRevisions.append(revisionList[i])

            if len(badRevisions) != 1:
                alert['stage'] = -1 # too noisy, something bad happened
                alert['user'] = 'human'
                updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
                continue

            if checkMerge(badRevisions[0], alert['buildername']):
                alert['stage'] = -1 # A merge revision is a bad revision, manually inspect
                alert['user'] = 'human'

            if alert['revision'] != badRevisions[0]:
                alert['revision'] = badRevisions[0] # we misreported initially, change the actual regression revision

            alert['stage'] = 3

        # Trigger all talos stage
        if alert['stage'] == 3:
            LOG.info("We are in stage 3, and going to trigger all_talos jobs.")
            repo_name = query_repo_name_from_buildername(alert['buildername'])
            # Setting Treeherder as the source for querying.
            set_query_source("treeherder")
            trigger_all_talos_jobs(repo_name, alert['revision'], times=6, dry_run=DRY_RUN)
            previousRevision = getRevisions(alert['revision'], alert['buildername'], start=-1, end=-1)[0]
            trigger_all_talos_jobs(repo_name, previousRevision, times=6, dry_run=DRY_RUN)
            alert['stage'] = 4
            updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
            continue

        # Verify All talos stage is completed
        if alert['stage'] == 4:
            LOG.info("We are in stage 4, and going to verify if all_talos ran successfully.")
            previousRevision = getRevisions(alert['revision'], alert['buildername'], start=-1, end=-1)[0]
            repo_name = query_repo_name_from_buildername(alert['buildername'])
            all_buildernames = build_talos_buildernames_for_repo(repo_name)
            for revision in [alert['revision'], previousRevision]:
                for buildername in all_buildernames:
                    dataPoints = getSuccessfulJobs(revision, buildername)
                    if dataPoints < 6:
                        # We wait for 8 hours for all talos tests to complete,
                        # And if they don't then we mark them for manual intervention
                        alert['loop'] += 1
                        if alert['loop'] > (TIME_TO_BUILD + TIME_TO_TEST + PENDING_TIME + TIME_TO_WAIT) / CYCLE_TIME:
                            alert['stage'] = -1
                            alert['user'] = 'human'
                        else:
                            alert['stage'] = 3
                        break

                if alert['stage'] != 4:
                    break

            if alert['stage'] != 4:
                updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
                continue

            alert['stage'] = 5  # final stage, sheriff will check for this.

        updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user'])

if __name__=="__main__": main()
<commit_msg>Adding logs in alertbot to improve debugging and to notify the reasons for actions.<commit_after># All the times here are in hours
# These are the times for non-pgo
import requests
import logging
from mozci.mozci import query_repo_url_from_buildername, query_repo_name_from_buildername, \
        trigger_all_talos_jobs, trigger_range, set_query_source
from mozci.query_jobs import TreeherderApi
from mozci.platforms import build_talos_buildernames_for_repo
from thclient import TreeherderClient
from store_alerts import getAlerts, updateAlert
from mozci.utils.misc import setup_logging
from managed_settings import TBPL_TESTS

LOG = setup_logging(logging.INFO)
TIME_TO_BUILD = 2
TIME_TO_TEST = 1
PENDING_TIME = 1
TIME_TO_WAIT = 2
CYCLE_TIME = 0.25
JSON_PUSHES = "%(repo_url)s/json-pushes"
WEEK = 604800
TWO_WEEKS = 1209600
SIXTY_DAYS = 5184000
SIGNATURE_URL = "https://treeherder.mozilla.org/api/project/%s/performance-data/get_performance_series_summary/?interval=%s"
PERFORMANCE_DATA = "https://treeherder.mozilla.org/api/project/%s/performance-data/get_performance_data/?interval_seconds=%s&signatures=%s"
OPTION_COLLECTION_HASH = "https://treeherder.mozilla.org/api/optioncollectionhash/"
SUCCESS = 0
DRY_RUN = False
REVERSE_TESTS = ['dromaeo_css', 'dromaeo_dom', 'v8_7', 'canvasmark']

# TODO: Ask about e10s
TREEHERDER_PLATFORM = {
    'linux32': 'Ubuntu HW 12.04',
    'linux64': 'Ubuntu HW 12.04 x64',
    'windowsxp': 'Windows XP 32-bit',
    'windows7-32': 'Windows 7 32-bit',
    'windows8-64': 'Windows 8 64-bit',
    'osx-10-10': 'Rev5 MacOSX Yosemite 10.10',
    'android-4-0-armv7-api11': 'Android 4.0 Tegra',
}

# Checked
def checkMerge(revision, buildername):
    "This function returns whether an alert is a merge or not."
    # Query Revision on pushlog, and check if all the authors of changesets are same.
    # We are not doing on the number of changesets, because a merge can have just 2 changesets
    # whereas a patch author can have 6-7 changeset.
    repo_url = query_repo_url_from_buildername(buildername)
    url = "%s?changeset=%s&version=2&full=1" % (JSON_PUSHES % {"repo_url": repo_url}, revision)
    req = requests.get(url).json()
    push_id = req["pushes"].keys()[0]
    changesets = req["pushes"][push_id]["changesets"]
    author = ""
    for changeset in changesets:
        if not author:
            author = changeset["author"]
        if changeset["author"] != author:
            return True

    return False

# Checked
def getRevisions(revision, buildername, start=0, end=0):
    "This function returns list of revisions for an alert."
    # Query Revision on pushlog, and get the revision. This should be easy.
    repo_url = query_repo_url_from_buildername(buildername)
    url = "%s?changeset=%s&version=2" % (JSON_PUSHES % {"repo_url": repo_url}, revision)
    req = requests.get(url).json()
    pushid = int(req["pushes"].keys()[0])
    startID = pushid + start - 1
    endID = pushid + end
    url = "%s?startID=%s&endID=%s&version=2&tipsonly=1" % (JSON_PUSHES % {"repo_url": repo_url}, startID, endID)
    req = requests.get(url).json()
    response = req["pushes"]
    revisionList = []
    for push in sorted(response.keys()):
        revisionList.append(response[push]["changesets"][0][0:12])

    return revisionList

# Checked
def getSuccessfulJobs(revision, buildername):
    "This function returns the number of data points for an alert."
    # Query TH client get_jobs method to get all jobs for a particular buildername
    # Then Query mozci function: https://github.com/armenzg/mozilla_ci_tools/blob/master/mozci/query_jobs.py#L187
    # to get the status of job for each job
    treeherder_api = TreeherderApi()
    repo_name = query_repo_name_from_buildername(buildername)
    matching_jobs = treeherder_api.get_matching_jobs(repo_name, revision, buildername)
    successful_jobs = 0
    for job in matching_jobs:
        status = treeherder_api.get_job_status(job)
        if status == SUCCESS:
            successful_jobs += 1
    return successful_jobs

def compare(test, buildername, revision, previous_revision):
    "This function will compare between 2 given revisions and return result as percentage"
    repo_name = query_repo_name_from_buildername(buildername)
    # Using TWO_WEEKS as interval, may change it afterwards
    signature_request_url = SIGNATURE_URL % (repo_name, TWO_WEEKS)
    signatures = requests.get(signature_request_url).json()
    options_collection_hash_list = requests.get(OPTION_COLLECTION_HASH).json()

    for signature, value in signatures.iteritems():
        # Skip processing subtests. They are identified by 'test' key in the dicitonary.
        if 'test' in value:
            continue

        # Ignoring e10s here.
        # TODO: Revisit this later
        if TBPL_TESTS[test]['testname'].lower() == value['suite'].lower() and TREEHERDER_PLATFORM[value["machine_platform"]] in buildername and 'test_options' not in value:
            test_signature = signature
        else:
            continue

        hash_signature = value['option_collection_hash']
        for key in options_collection_hash_list:
            if hash_signature == key["option_collection_hash"]:
                typeOfTest = key["options"][0]["name"]
                break

        if typeOfTest == 'pgo' and typeOfTest not in buildername:
            # if pgo, it should be present in buildername
            continue
        elif typeOfTest == 'opt':
            # if opt, nothing present in buildername
            break
        else:
            # We do not run talos on any branch other than pgo and opt.
            continue

    # Using TWO_WEEKS as interval, may change it afterwards
    req = requests.get(PERFORMANCE_DATA % (repo_name, TWO_WEEKS, test_signature)).json()
    performance_data = req[0]["blob"]
    treeherder_client = TreeherderClient()
    revision_resultset_id = treeherder_client.get_resultsets(repo_name, revision=revision)[0]["id"]
    previous_revision_resultset_id = treeherder_client.get_resultsets(repo_name, revision=previous_revision)[0]["id"]
    revision_perfdata = []
    previous_revision_perfdata = []

    for data in performance_data:
        if data["result_set_id"] == revision_resultset_id:
            revision_perfdata.append(data["geomean"])
        elif data["result_set_id"] == previous_revision_resultset_id:
            previous_revision_perfdata.append(data["geomean"])

    if revision_perfdata and previous_revision_perfdata:
        mean_revision_perfdata = sum(revision_perfdata) / float(len(revision_perfdata))
        mean_previous_revision_perfdata = sum(previous_revision_perfdata) / float(len(previous_revision_perfdata))

    if test in REVERSE_TESTS:
        # lower value results in regression
        return (mean_revision_perfdata - mean_previous_revision_perfdata) * 100.0 / mean_previous_revision_perfdata
    else:
        # higher value results in regression
        return (mean_previous_revision_perfdata - mean_revision_perfdata) * 100.0 / mean_previous_revision_perfdata

def main():
    alerts = getAlerts()
    for alert in alerts:
        # new alert
        if alert['stage'] == 0:
            LOG.info("We are in stage 0.")
            if checkMerge(alert['revision'], alert['buildername']) or 'pgo' in alert['buildername']:
                LOG.info("We are ignoring this alert since it is either a merge or a pgo job.")
                alert['stage'] = -1 # We need to have manual inspection in this case.
                alert['user'] = 'human'
                updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
            else:
                alert['stage'] = 1

        # trigger jobs for backfill
        if alert['stage'] == 1:
            LOG.info("We are in stage 1, and going to backfill jobs.")
            revisionList = getRevisions(alert['revision'], alert['buildername'], start=-2, end=2)
            # Setting Treeherder as the source for querying.
            set_query_source("treeherder")
            trigger_range(alert['buildername'], revisionList, times=6, dry_run=DRY_RUN)
            alert['stage'] = 2
            # We want some time interval between stage 1 and 2, so we exit.
            updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
            continue

        # verify jobs for backfill
        if alert['stage'] == 2:
            LOG.info("We are in stage 2, and going to verify if jobs are backfilled.")
            revisionList = getRevisions(alert['revision'], alert['buildername'], start=-2, end=2)
            for revision in revisionList:
                dataPoints = getSuccessfulJobs(revision, alert['buildername'])

                # If dataPoints are less than 6, it means that builds/jobs are still running.
                if dataPoints < 6:
                    # We wait for 6 hours for all triggered tests to complete,
                    # And if they don't then we mark them for manual intervention/
                    alert['loop'] += 1
                    if alert['loop'] > (TIME_TO_BUILD + TIME_TO_TEST + PENDING_TIME) / CYCLE_TIME:
                        LOG.info("The jobs did not complete backfilling in time, assigning for human inspection.")
                        alert['stage'] = -1
                        alert['user'] = 'human'
                    else:
                        alert['stage'] = 1

                    break

            if alert['stage'] != 2:
                updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
                continue

            badRevisions = []
            # Reset the loop for upcoming stages
            alert['loop'] = 0
            for i in range(1, len(revisionList)):
                results = compare(alert['test'], alert['buildername'], revisionList[i], revisionList[i-1])
                if results < -2.0:
                    badRevisions.append(revisionList[i])

            if len(badRevisions) != 1:
                LOG.info("There are too many bad revisions, assigning for human inspection.")
                alert['stage'] = -1 # too noisy, something bad happened
                alert['user'] = 'human'
                updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
                continue

            if checkMerge(badRevisions[0], alert['buildername']):
                LOG.info("The bad revision %s identified for %s buildername is a merge, "
                         "assigning for human inspection" % (badRevisions[0], alert['buildername']))
                alert['stage'] = -1 # A merge revision is a bad revision, manually inspect
                alert['user'] = 'human'

            if alert['revision'] != badRevisions[0]:
                LOG.info("Alert_Manager misreported the bad revision. The actual bad revision is %s "
                         "for regression on %s buildername." % (badRevisions[0], alert['buildername']))
                alert['revision'] = badRevisions[0] # we misreported initially, change the actual regression revision

            alert['stage'] = 3

        # Trigger all talos stage
        if alert['stage'] == 3:
            LOG.info("We are in stage 3, and going to trigger all_talos jobs.")
            repo_name = query_repo_name_from_buildername(alert['buildername'])
            # Setting Treeherder as the source for querying.
            set_query_source("treeherder")
            trigger_all_talos_jobs(repo_name, alert['revision'], times=6, dry_run=DRY_RUN)
            previousRevision = getRevisions(alert['revision'], alert['buildername'], start=-1, end=-1)[0]
            trigger_all_talos_jobs(repo_name, previousRevision, times=6, dry_run=DRY_RUN)
            alert['stage'] = 4
            updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
            continue

        # Verify All talos stage is completed
        if alert['stage'] == 4:
            LOG.info("We are in stage 4, and going to verify if all_talos ran successfully.")
            previousRevision = getRevisions(alert['revision'], alert['buildername'], start=-1, end=-1)[0]
            repo_name = query_repo_name_from_buildername(alert['buildername'])
            all_buildernames = build_talos_buildernames_for_repo(repo_name)
            for revision in [alert['revision'], previousRevision]:
                for buildername in all_buildernames:
                    dataPoints = getSuccessfulJobs(revision, buildername)
                    if dataPoints < 6:
                        # We wait for 8 hours for all talos tests to complete,
                        # And if they don't then we mark them for manual intervention
                        alert['loop'] += 1
                        if alert['loop'] > (TIME_TO_BUILD + TIME_TO_TEST + PENDING_TIME + TIME_TO_WAIT) / CYCLE_TIME:
                            LOG.info("The all talos jobs did not complete in time, assigning for human inspection.")
                            alert['stage'] = -1
                            alert['user'] = 'human'
                        else:
                            alert['stage'] = 3
                        break

                if alert['stage'] != 4:
                    break

            if alert['stage'] != 4:
                updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'],
                            alert['stage'], alert['loop'], alert['user'])
                continue

            alert['stage'] = 5  # final stage, sheriff will check for this.
            alert['user'] = 'human'
            LOG.info("All automated parts are complete.")

        updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user'])

if __name__=="__main__": main()
<|endoftext|>


<commit_before>from __future__ import division

import numpy as np
from numpy.testing import run_module_suite
from scipy.sparse import csr_matrix

from sklearn.utils.testing import (assert_array_equal, assert_almost_equal,
                                   assert_false, assert_true, assert_raises)
from sklearn.feature_selection.mutual_info_ import (
    mutual_info_regression, mutual_info_classif, _compute_mi)

class TestMIComputation(object):
    def test_dd(self):
        # In discrete case computations are straightforward and can be done
        # by hand on given vectors.
        x = np.array([0, 1, 1, 0, 0])
        y = np.array([1, 0, 0, 0, 1])

        H_x = H_y = -(3/5) * np.log(3/5) - (2/5) * np.log(2/5)
        H_xy = -1/5 * np.log(1/5) - 2/5 * np.log(2/5) - 2/5 * np.log(2/5)
        I_xy = H_x + H_y - H_xy

        assert_almost_equal(_compute_mi(x,  y, True, True), I_xy)

    def test_cc(self):
        # For two continuous variables a good approach is to test on bivariate
        # normal distribution, where mutual information is known.

        # Mean of the distribution, irrelevant for mutual information.
        mean = np.zeros(2)

        # Setup covariance matrix with correlation coeff. equal 0.5.
        sigma_1 = 1
        sigma_2 = 10
        corr = 0.5
        cov = np.array([
            [sigma_1**2, corr * sigma_1 * sigma_2],
            [corr * sigma_1 * sigma_2, sigma_2**2]
        ])

        # True theoretical mutual information.
        I_theory = (np.log(sigma_1) + np.log(sigma_2) -
                    0.5 * np.log(np.linalg.det(cov)))

        np.random.seed(0)
        Z = np.random.multivariate_normal(mean, cov, size=1000)

        x, y = Z[:, 0], Z[:, 1]

        # Theory and computed values won't be very close, assert that relative
        # error is less than 10%.
        for n_neighbors in [3, 5, 7]:
            I_computed = _compute_mi(x, y, False, False, n_neighbors)
            assert_true(np.abs(I_computed - I_theory) < 0.1 * I_theory)

    def test_cd(self):
        # To test define a joint distribution as follows:
        # p(x, y) = p(x) p(y | x)
        # X ~ Bernoulli(p)
        # (Y | x = 0) ~ Uniform(-1, 1)
        # (Y | x = 1) ~ Uniform(0, 2)

        # Use the following formula for mutual information:
        # I(X; Y) = H(Y) - H(Y | X)
        # Two entropies can be computed by hand:
        # H(Y) = -(1-p)/2 * ln((1-p)/2) - p/2*log(p/2) - 1/2*log(1/2)
        # H(Y | X) = ln(2)

        # Now we need to implement sampling from out distribution, which is
        # done easily using conditional distribution logic.

        n_samples = 1000
        np.random.seed(0)

        for p in [0.3, 0.5, 0.7]:
            x = np.random.uniform(size=n_samples) > p

            y = np.empty(n_samples)
            mask = x == 0
            y[mask] = np.random.uniform(-1, 1, size=np.sum(mask))
            y[~mask] = np.random.uniform(0, 2, size=np.sum(~mask))

            I_theory = -0.5 * ((1 - p) * np.log(0.5 * (1 - p)) +
                               p * np.log(0.5 * p) + np.log(0.5)) - np.log(2)

            # Again assert that relative error is less than 10%.
            for n_neighbors in [3, 5, 7]:
                I_computed = _compute_mi(x, y, True, False, n_neighbors)
                assert_true(np.abs(I_computed - I_theory) < 0.1 * I_theory)

class TestMutualInfo(object):
    # We are going test that feature ordering by MI matches our expectations.
    def test_discrete(self):
        X = np.array([[0, 0, 0],
                      [1, 1, 0],
                      [2, 0, 1],
                      [2, 0, 1],
                      [2, 0, 1]])
        y = np.array([0, 1, 2, 2, 1])

        # Here X[:, 0] is the most informative feature, and X[:, 1] is weakly
        # informative.
        mi = mutual_info_classif(X, y, discrete_features=True)
        assert_array_equal(np.argsort(-mi), np.array([0, 2, 1]))

    def test_continuous(self):
        # We generate sample from multivariate normal distribution, using
        # transformation from initially uncorrelated variables. The zero
        # variables after transformation is selected as the target vector,
        # it has the strongest correlation with the variable 2, and
        # the weakest correlation with the variable 1.
        T = np.array([
            [1, 0.5, 2, 1],
            [0, 1, 0.1, 0.0],
            [0, 0.1, 1, 0.1],
            [0, 0.1, 0.1, 1]
        ])
        cov = T.dot(T.T)
        mean = np.zeros(4)

        np.random.seed(0)
        Z = np.random.multivariate_normal(mean, cov, size=1000)
        X = Z[:, 1:]
        y = Z[:, 0]

        mi = mutual_info_regression(X, y, random_state=0)
        assert_array_equal(np.argsort(-mi), np.array([1, 2, 0]))

    def test_mixed(self):
        # Here the target is discrete and there are two continuous and one
        # discrete feature. The idea of this test is clear from the code.
        np.random.seed(0)
        X = np.random.rand(1000, 3)
        X[:, 1] += X[:, 0]
        y = ((0.5 * X[:, 0] + X[:, 2]) > 0.5).astype(int)
        X[:, 2] = X[:, 2] > 0.5

        mi = mutual_info_classif(X, y, discrete_features=[2], random_state=0)
        assert_array_equal(np.argsort(-mi), [2, 0, 1])

    def test_discrete_features_option(self):
        X = np.array([[0, 0, 0],
                      [1, 1, 0],
                      [2, 0, 1],
                      [2, 0, 1],
                      [2, 0, 1]], dtype=float)
        y = np.array([0, 1, 2, 2, 1], dtype=float)
        X_csr = csr_matrix(X)

        for mutual_info in (mutual_info_regression, mutual_info_classif):
            assert_raises(ValueError, mutual_info_regression, X_csr, y,
                          discrete_features=False)

            mi_1 = mutual_info(X, y, discrete_features='auto', random_state=0)
            mi_2 = mutual_info(X, y, discrete_features=False, random_state=0)

            mi_3 = mutual_info(X_csr, y, discrete_features='auto',
                               random_state=0)
            mi_4 = mutual_info(X_csr, y, discrete_features=True,
                               random_state=0)

            assert_array_equal(mi_1, mi_2)
            assert_array_equal(mi_3, mi_4)

        assert_false(np.allclose(mi_1, mi_3))

if __name__ == '__main__':
    run_module_suite()
<commit_msg>TST: Change tolerance checks in test_mutual_info.py<commit_after>from __future__ import division

import numpy as np
from numpy.testing import run_module_suite
from scipy.sparse import csr_matrix

from sklearn.utils.testing import (assert_array_equal, assert_almost_equal,
                                   assert_false, assert_true, assert_raises)
from sklearn.feature_selection.mutual_info_ import (
    mutual_info_regression, mutual_info_classif, _compute_mi)

class TestMIComputation(object):
    def test_dd(self):
        # In discrete case computations are straightforward and can be done
        # by hand on given vectors.
        x = np.array([0, 1, 1, 0, 0])
        y = np.array([1, 0, 0, 0, 1])

        H_x = H_y = -(3/5) * np.log(3/5) - (2/5) * np.log(2/5)
        H_xy = -1/5 * np.log(1/5) - 2/5 * np.log(2/5) - 2/5 * np.log(2/5)
        I_xy = H_x + H_y - H_xy

        assert_almost_equal(_compute_mi(x,  y, True, True), I_xy)

    def test_cc(self):
        # For two continuous variables a good approach is to test on bivariate
        # normal distribution, where mutual information is known.

        # Mean of the distribution, irrelevant for mutual information.
        mean = np.zeros(2)

        # Setup covariance matrix with correlation coeff. equal 0.5.
        sigma_1 = 1
        sigma_2 = 10
        corr = 0.5
        cov = np.array([
            [sigma_1**2, corr * sigma_1 * sigma_2],
            [corr * sigma_1 * sigma_2, sigma_2**2]
        ])

        # True theoretical mutual information.
        I_theory = (np.log(sigma_1) + np.log(sigma_2) -
                    0.5 * np.log(np.linalg.det(cov)))

        np.random.seed(0)
        Z = np.random.multivariate_normal(mean, cov, size=1000)

        x, y = Z[:, 0], Z[:, 1]

        # Theory and computed values won't be very close, assert that the
        # first figures after decimal point match.
        for n_neighbors in [3, 5, 7]:
            I_computed = _compute_mi(x, y, False, False, n_neighbors)
            assert_almost_equal(I_computed, I_theory, 1)

    def test_cd(self):
        # To test define a joint distribution as follows:
        # p(x, y) = p(x) p(y | x)
        # X ~ Bernoulli(p)
        # (Y | x = 0) ~ Uniform(-1, 1)
        # (Y | x = 1) ~ Uniform(0, 2)

        # Use the following formula for mutual information:
        # I(X; Y) = H(Y) - H(Y | X)
        # Two entropies can be computed by hand:
        # H(Y) = -(1-p)/2 * ln((1-p)/2) - p/2*log(p/2) - 1/2*log(1/2)
        # H(Y | X) = ln(2)

        # Now we need to implement sampling from out distribution, which is
        # done easily using conditional distribution logic.

        n_samples = 1000
        np.random.seed(0)

        for p in [0.3, 0.5, 0.7]:
            x = np.random.uniform(size=n_samples) > p

            y = np.empty(n_samples)
            mask = x == 0
            y[mask] = np.random.uniform(-1, 1, size=np.sum(mask))
            y[~mask] = np.random.uniform(0, 2, size=np.sum(~mask))

            I_theory = -0.5 * ((1 - p) * np.log(0.5 * (1 - p)) +
                               p * np.log(0.5 * p) + np.log(0.5)) - np.log(2)

            # Assert the same tolerance.
            for n_neighbors in [3, 5, 7]:
                I_computed = _compute_mi(x, y, True, False, n_neighbors)
                assert_almost_equal(I_computed, I_theory, 1)

class TestMutualInfo(object):
    # We are going test that feature ordering by MI matches our expectations.
    def test_discrete(self):
        X = np.array([[0, 0, 0],
                      [1, 1, 0],
                      [2, 0, 1],
                      [2, 0, 1],
                      [2, 0, 1]])
        y = np.array([0, 1, 2, 2, 1])

        # Here X[:, 0] is the most informative feature, and X[:, 1] is weakly
        # informative.
        mi = mutual_info_classif(X, y, discrete_features=True)
        assert_array_equal(np.argsort(-mi), np.array([0, 2, 1]))

    def test_continuous(self):
        # We generate sample from multivariate normal distribution, using
        # transformation from initially uncorrelated variables. The zero
        # variables after transformation is selected as the target vector,
        # it has the strongest correlation with the variable 2, and
        # the weakest correlation with the variable 1.
        T = np.array([
            [1, 0.5, 2, 1],
            [0, 1, 0.1, 0.0],
            [0, 0.1, 1, 0.1],
            [0, 0.1, 0.1, 1]
        ])
        cov = T.dot(T.T)
        mean = np.zeros(4)

        np.random.seed(0)
        Z = np.random.multivariate_normal(mean, cov, size=1000)
        X = Z[:, 1:]
        y = Z[:, 0]

        mi = mutual_info_regression(X, y, random_state=0)
        assert_array_equal(np.argsort(-mi), np.array([1, 2, 0]))

    def test_mixed(self):
        # Here the target is discrete and there are two continuous and one
        # discrete feature. The idea of this test is clear from the code.
        np.random.seed(0)
        X = np.random.rand(1000, 3)
        X[:, 1] += X[:, 0]
        y = ((0.5 * X[:, 0] + X[:, 2]) > 0.5).astype(int)
        X[:, 2] = X[:, 2] > 0.5

        mi = mutual_info_classif(X, y, discrete_features=[2], random_state=0)
        assert_array_equal(np.argsort(-mi), [2, 0, 1])

    def test_discrete_features_option(self):
        X = np.array([[0, 0, 0],
                      [1, 1, 0],
                      [2, 0, 1],
                      [2, 0, 1],
                      [2, 0, 1]], dtype=float)
        y = np.array([0, 1, 2, 2, 1], dtype=float)
        X_csr = csr_matrix(X)

        for mutual_info in (mutual_info_regression, mutual_info_classif):
            assert_raises(ValueError, mutual_info_regression, X_csr, y,
                          discrete_features=False)

            mi_1 = mutual_info(X, y, discrete_features='auto', random_state=0)
            mi_2 = mutual_info(X, y, discrete_features=False, random_state=0)

            mi_3 = mutual_info(X_csr, y, discrete_features='auto',
                               random_state=0)
            mi_4 = mutual_info(X_csr, y, discrete_features=True,
                               random_state=0)

            assert_array_equal(mi_1, mi_2)
            assert_array_equal(mi_3, mi_4)

        assert_false(np.allclose(mi_1, mi_3))

if __name__ == '__main__':
    run_module_suite()
<|endoftext|>


<commit_before>from datetime import datetime
import jwt

from rest_framework_jwt.settings import api_settings

def get_user_model():
    try:
        from django.contrib.auth import get_user_model
    except ImportError:  # django < 1.5
        from django.contrib.auth.models import User
    else:
        User = get_user_model()

    return User

def jwt_payload_handler(user):
    try:
        username = user.get_username()
    except AttributeError:
        username = user.username

    return {
        'user_id': user.pk,
        'email': user.email,
        'username': username,
        'exp': datetime.utcnow() + api_settings.JWT_EXPIRATION_DELTA
    }

def jwt_get_user_id_from_payload_handler(payload):
    """
    Override this function if user_id is formatted differently in payload
    """
    user_id = payload.get('user_id')
    return user_id

def jwt_encode_handler(payload):
    return jwt.encode(
        payload,
        api_settings.JWT_SECRET_KEY,
        api_settings.JWT_ALGORITHM
    ).decode('utf-8')

def jwt_decode_handler(token):
    return jwt.decode(
        token,
        api_settings.JWT_SECRET_KEY,
        api_settings.JWT_VERIFY
    )
<commit_msg>added back verification and leeway -- using proper kwargs for these. updates #40<commit_after>from datetime import datetime
import jwt

from rest_framework_jwt.settings import api_settings

def get_user_model():
    try:
        from django.contrib.auth import get_user_model
    except ImportError:  # django < 1.5
        from django.contrib.auth.models import User
    else:
        User = get_user_model()

    return User

def jwt_payload_handler(user):
    try:
        username = user.get_username()
    except AttributeError:
        username = user.username

    return {
        'user_id': user.pk,
        'email': user.email,
        'username': username,
        'exp': datetime.utcnow() + api_settings.JWT_EXPIRATION_DELTA
    }

def jwt_get_user_id_from_payload_handler(payload):
    """
    Override this function if user_id is formatted differently in payload
    """
    user_id = payload.get('user_id')
    return user_id

def jwt_encode_handler(payload):
    return jwt.encode(
        payload,
        api_settings.JWT_SECRET_KEY,
        api_settings.JWT_ALGORITHM
    ).decode('utf-8')

def jwt_decode_handler(token):
    return jwt.decode(
        token,
        api_settings.JWT_SECRET_KEY,
        api_settings.JWT_VERIFY,
        verify_expiration=api_settings.JWT_VERIFY_EXPIRATION,
        leeway=api_settings.JWT_LEEWAY
    )
<|endoftext|>


<commit_before>import operator
import typing as tp
import abc
import functools

from metafunctions.core._decorators import binary_operation
from metafunctions.core._decorators import inject_call_state
from metafunctions.core._call_state import CallState
from metafunctions.operators import concat

class MetaFunction(metaclass=abc.ABCMeta):

    # Metafunctions will pass call state to any function with this attribute set to true
    _receives_call_state = True
    _function_join_str = ''

    @abc.abstractmethod
    def __init__(self, *args, **kwargs):
        '''A MetaFunction is a function that contains other functions. When executed, it calls the
        functions it contains.
        '''
        self._functions = []

    @abc.abstractmethod
    def __call__(self, *args, call_state=None, **kwargs):
        '''Call the functions contained in this MetaFunction'''

    def __str__(self):
        return f'({f" {self._function_join_str} ".join(str(f) for f in self.functions)})'

    @property
    def functions(self):
        return self._functions

    @staticmethod
    def make_meta(function):
        '''Wrap the given function in a metafunction, unless it's already a metafunction'''
        if not isinstance(function, MetaFunction):
            return SimpleFunction(function)
        return function

    @staticmethod
    def defer_value(value):
        '''Wrap the given value in a DeferredValue object, (which returns its value when called).'''
        return DeferredValue(value)

    @staticmethod
    def new_call_state():
        return CallState()

    ### Operator overloads ###
    @binary_operation
    def __or__(self, other):
        return FunctionChain.combine(self, other)

    @binary_operation
    def __ror__(self, other):
        return FunctionChain.combine(other, self)

    @binary_operation
    def __and__(self, other):
        return FunctionMerge.combine(concat, self, other)

    @binary_operation
    def __rand__(self, other):
        return FunctionMerge.combine(concat, other, self)

    @binary_operation
    def __add__(self, other):
        return FunctionMerge(operator.add, (self, other))

    @binary_operation
    def __radd__(self, other):
        return FunctionMerge(operator.add, (other, self))

    @binary_operation
    def __sub__(self, other):
        return FunctionMerge(operator.sub, (self, other))

    @binary_operation
    def __rsub__(self, other):
        return FunctionMerge(operator.sub, (other, self))

    @binary_operation
    def __mul__(self, other):
        return FunctionMerge(operator.mul, (self, other))

    @binary_operation
    def __rmul__(self, other):
        return FunctionMerge(operator.mul, (other, self))

    @binary_operation
    def __truediv__(self, other):
        return FunctionMerge(operator.truediv, (self, other))

    @binary_operation
    def __rtruediv__(self, other):
        return FunctionMerge(operator.truediv, (other, self))

    @binary_operation
    def __matmul__(self, other):
        return BroadcastChain.combine(self, other)

    @binary_operation
    def __rmatmul__(self, other):
        return BroadcastChain.combine(other, self)

class FunctionChain(MetaFunction):
    _function_join_str = '|'
    def __init__(self, *functions):
        '''A FunctionChain is a metafunction that calls its functions in sequence, passing the
        results of the first function subsequent functions.
        '''
        super().__init__()
        self._functions = functions

    @inject_call_state
    def __call__(self, *args, **kwargs):
        f_iter = iter(self._functions)
        result = next(f_iter)(*args, **kwargs)
        for f in f_iter:
            result = f(result, **kwargs)
        return result

    def __repr__(self):
        return f'{self.__class__.__name__}{self.functions}'

    @classmethod
    def combine(cls, *funcs):
        '''Merge chains; i.e., combine all FunctionChains in `funcs` into a single FunctionChain.
        '''
        new_funcs = []
        for f in funcs:
            if type(f) is cls:
                new_funcs.extend(f.functions)
            else:
                new_funcs.append(f)
        return cls(*new_funcs)

class BroadcastChain(FunctionChain):
    _function_join_str = '@'

    @inject_call_state
    def __call__(self, *args, **kwargs):
        f_iter = iter(self._functions)
        result = next(f_iter)(*args, **kwargs)
        for f in f_iter:
            result = f(*result, **kwargs)
        return result

class FunctionMerge(MetaFunction):
    _character_to_operator = {
        '+': operator.add,
        '-': operator.sub,
        '*': operator.mul,
        '/': operator.truediv,
        '&': concat,
    }
    _operator_to_character = {v: k for k, v in _character_to_operator.items()}

    def __init__(self, merge_func:tp.Callable, functions:tuple, function_join_str=''):
        '''A FunctionMerge merges its functions by executing all of them and passing their results to `merge_func`

        Args:
            function_join_str: If you're using a `merge_func` that is not one of the standard operator
            functions, use this argument to provide a custom character to use in string formatting. If not provided, we default to using str(merge_func).
        '''
        super().__init__()
        self._merge_func = merge_func
        self._functions = functions
        self._function_join_str = function_join_str or self._operator_to_character.get(
                merge_func, str(merge_func))

    @inject_call_state
    def __call__(self, *args, **kwargs):
        results = (f(*args, **kwargs) for f in self.functions)
        return self._merge_func(*results)

    def __repr__(self):
        return f'{self.__class__.__name__}({self._merge_func}, {self.functions})'

    @classmethod
    def combine(cls, merge_func: tp.Callable, *funcs, function_join_str=None):
        '''Combine FunctionMerges. If consecutive FunctionMerges have the same merge_funcs, combine
        them into a single FunctionMerge.

        NOTE: combine does not check to make sure the merge_func can accept the new number of
        arguments.
        '''
        new_funcs = []
        for f in funcs:
            if isinstance(f, cls) and f._merge_func == merge_func:
                new_funcs.extend(f.functions)
            else:
                new_funcs.append(f)
        return cls(merge_func, tuple(new_funcs), function_join_str=function_join_str)

class SimpleFunction(MetaFunction):
    def __init__(self, function, print_location_in_traceback=True):
        '''A MetaFunction-aware wrapper around a single function
        The `bind` parameter causes us to pass a meta object as the first argument to our inherited function, but it is only respected if the wrapped function is not another metafunction.
        '''
        # An interesting side effect of wraps: it causes simplefunctions to collapse into each
        # other. Because calling wraps on a function copies all that function's attributes to the
        # new function, we copy _function, etc from the wrapped function. Essentially
        # absorbing it. I'm not sure if that's good or bad.
        functools.wraps(function)(self)

        super().__init__()
        self._function = function
        self.add_location_to_traceback = print_location_in_traceback

    @inject_call_state
    def __call__(self, *args, call_state, **kwargs):
        call_state._called_functions.append(self)
        if getattr(self._function, '_receives_call_state', False):
            kwargs['call_state'] = call_state
        try:
            return self._function(*args, **kwargs)
        except Exception as e:
            self._handle_exception(call_state, e)

    def __repr__(self):
        return f'{self.__class__.__name__}({self.functions[0]!r})'

    def __str__(self):
        try:
            return self.__name__
        except AttributeError:
            # We're usually wrapping a function, but it's possible we're wrapping another
            # metafunction
            return str(self.functions[0])

    @property
    def functions(self):
        return (self._function, )

    def _handle_exception(self, call_state, e):
        if self.add_location_to_traceback:
            from metafunctions.util import highlight_current_function
            detailed_message = str(e)
            if call_state:
                detailed_message = f"{str(e)} \n\nOccured in the following function: {highlight_current_function(call_state)}"
            raise type(e)(detailed_message).with_traceback(e.__traceback__)
        raise

class DeferredValue(SimpleFunction):
    def __init__(self, value):
        '''A simple Deferred Value. Returns `value` when called. Equivalent to lambda x: x.
        '''
        self._value = value
        self.__name__ = repr(value)

    def __call__(self, *args, **kwargs):
        return self._value

    def __repr__(self):
        return f'{self.__class__.__name__}({self._value!r})'

    @property
    def functions(self):
        return (self, )
<commit_msg>Add some documentation to FunctionMerge<commit_after>import operator
import typing as tp
import abc
import functools

from metafunctions.core._decorators import binary_operation
from metafunctions.core._decorators import inject_call_state
from metafunctions.core._call_state import CallState
from metafunctions.operators import concat

class MetaFunction(metaclass=abc.ABCMeta):

    # Metafunctions will pass call state to any function with this attribute set to true
    _receives_call_state = True
    _function_join_str = ''

    @abc.abstractmethod
    def __init__(self, *args, **kwargs):
        '''A MetaFunction is a function that contains other functions. When executed, it calls the
        functions it contains.
        '''
        self._functions = []

    @abc.abstractmethod
    def __call__(self, *args, call_state=None, **kwargs):
        '''Call the functions contained in this MetaFunction'''

    def __str__(self):
        return f'({f" {self._function_join_str} ".join(str(f) for f in self.functions)})'

    @property
    def functions(self):
        return self._functions

    @staticmethod
    def make_meta(function):
        '''Wrap the given function in a metafunction, unless it's already a metafunction'''
        if not isinstance(function, MetaFunction):
            return SimpleFunction(function)
        return function

    @staticmethod
    def defer_value(value):
        '''Wrap the given value in a DeferredValue object, (which returns its value when called).'''
        return DeferredValue(value)

    @staticmethod
    def new_call_state():
        return CallState()

    ### Operator overloads ###
    @binary_operation
    def __or__(self, other):
        return FunctionChain.combine(self, other)

    @binary_operation
    def __ror__(self, other):
        return FunctionChain.combine(other, self)

    @binary_operation
    def __and__(self, other):
        return FunctionMerge.combine(concat, self, other)

    @binary_operation
    def __rand__(self, other):
        return FunctionMerge.combine(concat, other, self)

    @binary_operation
    def __add__(self, other):
        return FunctionMerge(operator.add, (self, other))

    @binary_operation
    def __radd__(self, other):
        return FunctionMerge(operator.add, (other, self))

    @binary_operation
    def __sub__(self, other):
        return FunctionMerge(operator.sub, (self, other))

    @binary_operation
    def __rsub__(self, other):
        return FunctionMerge(operator.sub, (other, self))

    @binary_operation
    def __mul__(self, other):
        return FunctionMerge(operator.mul, (self, other))

    @binary_operation
    def __rmul__(self, other):
        return FunctionMerge(operator.mul, (other, self))

    @binary_operation
    def __truediv__(self, other):
        return FunctionMerge(operator.truediv, (self, other))

    @binary_operation
    def __rtruediv__(self, other):
        return FunctionMerge(operator.truediv, (other, self))

    @binary_operation
    def __matmul__(self, other):
        return BroadcastChain.combine(self, other)

    @binary_operation
    def __rmatmul__(self, other):
        return BroadcastChain.combine(other, self)

class FunctionChain(MetaFunction):
    _function_join_str = '|'
    def __init__(self, *functions):
        '''A FunctionChain is a metafunction that calls its functions in sequence, passing the
        results of the first function subsequent functions.
        '''
        super().__init__()
        self._functions = functions

    @inject_call_state
    def __call__(self, *args, **kwargs):
        f_iter = iter(self._functions)
        result = next(f_iter)(*args, **kwargs)
        for f in f_iter:
            result = f(result, **kwargs)
        return result

    def __repr__(self):
        return f'{self.__class__.__name__}{self.functions}'

    @classmethod
    def combine(cls, *funcs):
        '''Merge chains; i.e., combine all FunctionChains in `funcs` into a single FunctionChain.
        '''
        new_funcs = []
        for f in funcs:
            if type(f) is cls:
                new_funcs.extend(f.functions)
            else:
                new_funcs.append(f)
        return cls(*new_funcs)

class BroadcastChain(FunctionChain):
    _function_join_str = '@'

    @inject_call_state
    def __call__(self, *args, **kwargs):
        f_iter = iter(self._functions)
        result = next(f_iter)(*args, **kwargs)
        for f in f_iter:
            result = f(*result, **kwargs)
        return result

class FunctionMerge(MetaFunction):
    _character_to_operator = {
        '+': operator.add,
        '-': operator.sub,
        '*': operator.mul,
        '/': operator.truediv,
        '&': concat,
    }
    _operator_to_character = {v: k for k, v in _character_to_operator.items()}

    def __init__(self, merge_func:tp.Callable, functions:tuple, function_join_str=''):
        '''
        A FunctionMerge merges its functions by executing all of them and passing their results to
        `merge_func`

        Args:

            function_join_str: If you're using a `merge_func` that is not one of the standard
            operator functions, use this argument to provide a custom character to use in string
            formatting. If not provided, we default to using str(merge_func).
        '''
        super().__init__()
        self._merge_func = merge_func
        self._functions = functions
        self._function_join_str = function_join_str or self._operator_to_character.get(
                merge_func, str(merge_func))

    @inject_call_state
    def __call__(self, *args, **kwargs):
        results = (f(*args, **kwargs) for f in self.functions)
        return self._merge_func(*results)

    def __repr__(self):
        return f'{self.__class__.__name__}({self._merge_func}, {self.functions})'

    @classmethod
    def combine(cls, merge_func: tp.Callable, *funcs, function_join_str=None):
        '''Combine FunctionMerges. If consecutive FunctionMerges have the same merge_funcs, combine
        them into a single FunctionMerge.

        NOTE: combine does not check to make sure the merge_func can accept the new number of
        arguments, or that combining is appropriate for the operator. (e.g., it is inappropriate to
        combine FunctionMerges where order of operations matter. 5 / 2 / 3 != 5 / (2 / 3))
        '''
        new_funcs = []
        for f in funcs:
            if isinstance(f, cls) and f._merge_func == merge_func:
                new_funcs.extend(f.functions)
            else:
                new_funcs.append(f)
        return cls(merge_func, tuple(new_funcs), function_join_str=function_join_str)

class SimpleFunction(MetaFunction):
    def __init__(self, function, print_location_in_traceback=True):
        '''A MetaFunction-aware wrapper around a single function
        The `bind` parameter causes us to pass a meta object as the first argument to our inherited function, but it is only respected if the wrapped function is not another metafunction.
        '''
        # An interesting side effect of wraps: it causes simplefunctions to collapse into each
        # other. Because calling wraps on a function copies all that function's attributes to the
        # new function, we copy _function, etc from the wrapped function. Essentially
        # absorbing it. I'm not sure if that's good or bad.
        functools.wraps(function)(self)

        super().__init__()
        self._function = function
        self.add_location_to_traceback = print_location_in_traceback

    @inject_call_state
    def __call__(self, *args, call_state, **kwargs):
        call_state._called_functions.append(self)
        if getattr(self._function, '_receives_call_state', False):
            kwargs['call_state'] = call_state
        try:
            return self._function(*args, **kwargs)
        except Exception as e:
            self._handle_exception(call_state, e)

    def __repr__(self):
        return f'{self.__class__.__name__}({self.functions[0]!r})'

    def __str__(self):
        try:
            return self.__name__
        except AttributeError:
            # We're usually wrapping a function, but it's possible we're wrapping another
            # metafunction
            return str(self.functions[0])

    @property
    def functions(self):
        return (self._function, )

    def _handle_exception(self, call_state, e):
        if self.add_location_to_traceback:
            from metafunctions.util import highlight_current_function
            detailed_message = str(e)
            if call_state:
                detailed_message = f"{str(e)} \n\nOccured in the following function: {highlight_current_function(call_state)}"
            raise type(e)(detailed_message).with_traceback(e.__traceback__)
        raise

class DeferredValue(SimpleFunction):
    def __init__(self, value):
        '''A simple Deferred Value. Returns `value` when called. Equivalent to lambda x: x.
        '''
        self._value = value
        self.__name__ = repr(value)

    def __call__(self, *args, **kwargs):
        return self._value

    def __repr__(self):
        return f'{self.__class__.__name__}({self._value!r})'

    @property
    def functions(self):
        return (self, )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#
# License: MIT
# Copyright: Patrick Ng - 2012
#

from .retry_decorator import *

__title__ = 'retry_decorator'
__version__ = "1.1.0"
<commit_msg>Bump major version number<commit_after># -*- coding: utf-8 -*-

#
# License: MIT
# Copyright: Patrick Ng - 2012
#

from .retry_decorator import *

__title__ = 'retry_decorator'
__version__ = "2.0.0"
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Tests for requests_unixsocket"""

import logging
import multiprocessing
import os
import uuid

import pytest
import requests
import waitress

from requests_unixsocket import UnixAdapter

logger = logging.getLogger(__name__)

def wsgiapp():
    def _wsgiapp(environ, start_response):
        start_response(
            '200 OK',
            [('X-Transport', 'unix domain socket'),
             ('X-Socket-Path', environ['SERVER_PORT']),
             ('X-Requested-Path', environ['PATH_INFO'])])
        return [b'Hello world!']

    return _wsgiapp

class UnixSocketServerProcess(multiprocessing.Process):
    def __init__(self, *args, **kwargs):
        super(UnixSocketServerProcess, self).__init__(*args, **kwargs)
        self.usock = self.get_tempfile_name()

    def get_tempfile_name(self):
        # I'd rather use tempfile.NamedTemporaryFile but IDNA limits
        # the hostname to 63 characters and we'll get a "InvalidURL:
        # URL has an invalid label" error if we exceed that.
        args = (os.stat(__file__).st_ino, os.getpid(), uuid.uuid4().hex[-8:])
        return '/tmp/test_requests.%s_%s_%s' % args

    def run(self):
        logger.debug('Call waitress.serve in %r (pid %d) ...', self, self.pid)
        waitress.serve(wsgiapp(), unix_socket=self.usock)

    def __enter__(self):
        logger.debug('Starting %r ...' % self)
        self.start()
        logger.debug('Started %r (pid %d)...', self, self.pid)
        return self

    def __exit__(self, *args):
        logger.debug('Terminating %r (pid %d) ...', self, self.pid)
        self.terminate()
        logger.debug('Terminated %r (pid %d) ...', self, self.pid)

def test_unix_domain_adapter_ok():
    with UnixSocketServerProcess() as usock_process:
        session = requests.Session()
        session.mount('http+unix://', UnixAdapter())
        urlencoded_usock = requests.compat.quote_plus(usock_process.usock)
        url = 'http+unix://%s/path/to/page' % urlencoded_usock
        logger.debug('Calling session.get(%r) ...', url)
        r = session.get(url)
        logger.debug(
            'Received response: %r with text: %r and headers: %r',
            r, r.text, r.headers)
        assert r.status_code == 200
        assert r.headers['server'] == 'waitress'
        assert r.headers['X-Transport'] == 'unix domain socket'
        assert r.headers['X-Requested-Path'] == '/path/to/page'
        assert r.headers['X-Socket-Path'] == usock_process.usock
        assert isinstance(r.connection, UnixAdapter)
        assert r.url == url
        assert r.text == 'Hello world!'

def test_unix_domain_adapter_connection_error():
    session = requests.Session()
    session.mount('http+unix://', UnixAdapter())

    with pytest.raises(requests.ConnectionError):
        session.get('http+unix://socket_does_not_exist/path/to/page')
<commit_msg>Add test for proxies error<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Tests for requests_unixsocket"""

import logging
import multiprocessing
import os
import uuid

import pytest
import requests
import waitress

from requests_unixsocket import UnixAdapter

logger = logging.getLogger(__name__)

def wsgiapp():
    def _wsgiapp(environ, start_response):
        start_response(
            '200 OK',
            [('X-Transport', 'unix domain socket'),
             ('X-Socket-Path', environ['SERVER_PORT']),
             ('X-Requested-Path', environ['PATH_INFO'])])
        return [b'Hello world!']

    return _wsgiapp

class UnixSocketServerProcess(multiprocessing.Process):
    def __init__(self, *args, **kwargs):
        super(UnixSocketServerProcess, self).__init__(*args, **kwargs)
        self.usock = self.get_tempfile_name()

    def get_tempfile_name(self):
        # I'd rather use tempfile.NamedTemporaryFile but IDNA limits
        # the hostname to 63 characters and we'll get a "InvalidURL:
        # URL has an invalid label" error if we exceed that.
        args = (os.stat(__file__).st_ino, os.getpid(), uuid.uuid4().hex[-8:])
        return '/tmp/test_requests.%s_%s_%s' % args

    def run(self):
        logger.debug('Call waitress.serve in %r (pid %d) ...', self, self.pid)
        waitress.serve(wsgiapp(), unix_socket=self.usock)

    def __enter__(self):
        logger.debug('Starting %r ...' % self)
        self.start()
        logger.debug('Started %r (pid %d)...', self, self.pid)
        return self

    def __exit__(self, *args):
        logger.debug('Terminating %r (pid %d) ...', self, self.pid)
        self.terminate()
        logger.debug('Terminated %r (pid %d) ...', self, self.pid)

def test_unix_domain_adapter_ok():
    with UnixSocketServerProcess() as usock_process:
        session = requests.Session()
        session.mount('http+unix://', UnixAdapter())
        urlencoded_usock = requests.compat.quote_plus(usock_process.usock)
        url = 'http+unix://%s/path/to/page' % urlencoded_usock
        logger.debug('Calling session.get(%r) ...', url)
        r = session.get(url)
        logger.debug(
            'Received response: %r with text: %r and headers: %r',
            r, r.text, r.headers)
        assert r.status_code == 200
        assert r.headers['server'] == 'waitress'
        assert r.headers['X-Transport'] == 'unix domain socket'
        assert r.headers['X-Requested-Path'] == '/path/to/page'
        assert r.headers['X-Socket-Path'] == usock_process.usock
        assert isinstance(r.connection, UnixAdapter)
        assert r.url == url
        assert r.text == 'Hello world!'

def test_unix_domain_adapter_connection_error():
    session = requests.Session()
    session.mount('http+unix://', UnixAdapter())

    with pytest.raises(requests.ConnectionError):
        session.get('http+unix://socket_does_not_exist/path/to/page')

def test_unix_domain_adapter_connection_proxies_error():
    session = requests.Session()
    session.mount('http+unix://', UnixAdapter())

    with pytest.raises(ValueError) as excinfo:
        session.get('http+unix://socket_does_not_exist/path/to/page',
                    proxies={"http": "http://10.10.1.10:1080"})
    assert ('UnixAdapter does not support specifying proxies'
            in str(excinfo.value))
<|endoftext|>


<commit_before>from numpy.linalg import inv,cholesky

class BaseMetricLearner(object):
  def __init__(self):
    raise NotImplementedError('BaseMetricLearner should not be instantiated')

  def metric(self):
    """Computes the Mahalanobis matrix from the transformation matrix.

    .. math:: M = L^{\\top} L

    Returns
    -------
    M : (d x d) matrix
    """
    L = self.transformer()
    return L.T.dot(L)

  def transformer(self):
    """Computes the transformation matrix from the Mahalanobis matrix.

    L = inv(cholesky(M))

    Returns
    -------
    L : (d x d) matrix
    """
    return inv(cholesky(self.metric()))

  def transform(self, X=None):
    """Applies the metric transformation.

    Parameters
    ----------
    X : (n x d) matrix, optional
        Data to transform. If not supplied, the training data will be used.

    Returns
    -------
    transformed : (n x d) matrix
        Input data transformed to the metric space by :math:`XL^{\\top}`
    """
    if X is None:
      X = self.X
    L = self.transformer()
    return X.dot(L.T)

  def get_params(self, deep=False):
    """Get parameters for this estimator.

    Parameters
    ----------
    deep: boolean, optional
        @WARNING doesn't do anything, only exists because scikit-learn has this on BaseEstimator.

    Returns
    -------
    params : mapping of string to any
        Parameter names mapped to their values.
    """
    return self.params

  def set_params(self, **kwarg):
    """Set the parameters of this estimator.

    Overwrites any default parameters or parameters specified in constructor.

    Returns
    -------
    self
    """
    self.params = {**self.params, **kwarg}
<commit_msg>Comment typo<commit_after>from numpy.linalg import inv,cholesky

class BaseMetricLearner(object):
  def __init__(self):
    raise NotImplementedError('BaseMetricLearner should not be instantiated')

  def metric(self):
    """Computes the Mahalanobis matrix from the transformation matrix.

    .. math:: M = L^{\\top} L

    Returns
    -------
    M : (d x d) matrix
    """
    L = self.transformer()
    return L.T.dot(L)

  def transformer(self):
    """Computes the transformation matrix from the Mahalanobis matrix.

    L = inv(cholesky(M))

    Returns
    -------
    L : (d x d) matrix
    """
    return inv(cholesky(self.metric()))

  def transform(self, X=None):
    """Applies the metric transformation.

    Parameters
    ----------
    X : (n x d) matrix, optional
        Data to transform. If not supplied, the training data will be used.

    Returns
    -------
    transformed : (n x d) matrix
        Input data transformed to the metric space by :math:`XL^{\\top}`
    """
    if X is None:
      X = self.X
    L = self.transformer()
    return X.dot(L.T)

  def get_params(self, deep=False):
    """Get parameters for this metric learner.

    Parameters
    ----------
    deep: boolean, optional
        @WARNING doesn't do anything, only exists because scikit-learn has this on BaseEstimator.

    Returns
    -------
    params : mapping of string to any
        Parameter names mapped to their values.
    """
    return self.params

  def set_params(self, **kwarg):
    """Set the parameters of this metric learner.

    Overwrites any default parameters or parameters specified in constructor.

    Returns
    -------
    self
    """
    self.params = {**self.params, **kwarg}
<|endoftext|>