<commit_before>import collections
from collections import defaultdict
import numpy as np
import pandas as pd

__author__ = 'palmer'

def get_adducts(json_filename):
    import json
    config = json.loads(open(json_filename).read())
    target_adducts = [a['adduct'] for a in config['fdr']['pl_adducts']]
    decoy_adducts = list(set([a['adduct'] for a in config['isotope_generation']['adducts']])-set(target_adducts))
    return target_adducts, decoy_adducts

def calc_fdr_df(target_df, decoy_df, col='mult', ascending=False):
    # possible backward-incompatibility: calc_fdr_df previously returned the last argument as a series, not as an array
    return calc_fdr_arr(target_df[col], decoy_df[col], ascending=ascending)

def calc_fdr_arr(target_arr, decoy_arr, ascending=False):
    """
    Calculate the FDR curve for arrays of target scores and decoy scores.

    :param target_arr: sequence of scores from sum formulae with target adducts
    :param decoy_arr: sequence of scores from sum formulae with decoy adducts
    :param ascending: True if smaller scores are better, False if higher scores are better
    :raise TypeError: if the two sequences do not have the same length
    :return:
     fdr_curve: 1d array, twice as long as each of the input sequences, where the i-th element is the number of
     decoy hits divided by the number of target hits among the i highest scores (combined).

     target_hits: 1d array, twice as long as each of the input sequences, where the i-th element is the number of
     target hits among the i highest scores (combined)

     sorted_values: 1d array, twice as long as each of the input sequences, containing the sorted values from both
     input sequences
    """
    n, m = len(target_arr), len(decoy_arr)
    if n != m:
        raise TypeError('target should be same length as decoy {} {}'.format(n, m))
    ordering = 1 if ascending else -1  # reversed sorting if score is not ascending
    combined = np.concatenate((target_arr, decoy_arr))
    combined.sort()
    # count how often each value occurs
    target_bag, decoy_bag = _count(target_arr), _count(decoy_arr)
    unique_sorted = np.unique(combined)[::ordering]
    target_hits, decoy_hits = zip(*_iter_hits(target_bag, decoy_bag, unique_sorted))
    target_hits, decoy_hits = np.cumsum(target_hits), np.cumsum(decoy_hits)
    fdr_curve = decoy_hits / target_hits
    fdr_curve[target_hits == 0] = 0
    return fdr_curve, target_hits, combined[::ordering]

def _count(it):
    bag = defaultdict(int)
    for x in it:
        bag[x] += 1
    return bag

def _iter_hits(bag1, bag2, unique_sorted):
    # For each unique value, determine how often the value occurs in the target array and decoy array respectively
    # and yield the fractions (into the hits arrays).
    # Example: if there are two target scores and one decoy score with the same value, append 2/3, 2/3, 2/3 to the
    # target_hits array and 1/3, 1/3, 1/3 to the decoy_hits array.
    # This is to avoid the previously randomized approach of randomly assigning the value to one of the arrays.
    for x in unique_sorted:
        tc, dc = bag1[x], bag2[x]
        tc_part, dc_part = float(tc) / (tc + dc), float(dc) / (tc + dc)
        for _ in range(int(tc + dc)):
            yield (tc_part, dc_part)

def get_decoy_df(decoy_adducts, results_data, sf):
    import numpy as np
    import bisect
    import pandas as pd
    # decoy selection
    nDecoy = len(sf)
    decoy_adducts = np.random.choice(decoy_adducts, size=nDecoy, replace=True)
    dc_df = []
    for s, da in zip(sf, decoy_adducts):
        dc_df.append(results_data[da].iloc[bisect.bisect_left(sf, s)])
        dc_df = pd.DataFrame.from_records(dc_df)
    return dc_df

def select_passes(score_data, t_holds):
    import numpy as np
    moc_t, spec_t, spat_t = t_holds
    pass_tf = np.all([score_data['moc'] > moc_t, score_data['spat'] > spat_t, score_data['spec'] > spec_t], axis=0)
    pass_data = score_data[pass_tf]
    return pass_data

def select_passes_linear(score_data, t_hold):
    import numpy as np
    pass_tf = np.asarray(score_data['moc'] + score_data['spat'] + score_data['spec']) > t_hold
    pass_data = score_data[pass_tf]
    return pass_data

def select_passes_mult(score_data, t_hold):
    import numpy as np
    pass_tf = np.asarray(score_data['moc'] * score_data['spat'] * score_data['spec']) > t_hold
    pass_data = score_data[pass_tf]
    return pass_data

def select_passes_l2(score_data, t_hold):
    import numpy as np
    pass_tf = np.sqrt(np.asarray(score_data['moc'] ** 2 + score_data['spat'] ** 2 + score_data['spec'])) > t_hold
    pass_data = score_data[pass_tf]
    return pass_data

def count_adducts(pass_data, adducts):
    ## number of false hits by adduct
    fd_count = pass_data['adduct'].value_counts()  # count adduct occurances
    set_diff = set(adducts) - set(fd_count.keys())
    for s in set_diff:  # add zero occurance back in
        fd_count[s] = 0
    return fd_count.loc[[f in adducts for f in fd_count.keys()]]

def calc_fdr_adducts(fd_count, adducts, plausible_adducts, average='mean'):
    import numpy as np
    implausible_adducts = [a for a in adducts if a not in plausible_adducts]
    implausible_list = fd_count[implausible_adducts]
    implausible_hits = sum(implausible_list)
    if average == "mean":
        mean_implausible = float(implausible_hits) / len(implausible_list)
    elif average == 'median':
        mean_implausible = np.percentile(implausible_list, 50)
    else:
        raise ValueError('average type not recognised: {}'.format(average))
    plausible_hits = np.sum(fd_count[list(plausible_adducts)])
    if plausible_hits == 0:
        return 1, plausible_hits, implausible_hits
    fdr = (len(plausible_adducts) * mean_implausible) / plausible_hits
    return fdr, plausible_hits, implausible_hits

def is_fdr_curve(fdr_curve):
    """
    A FDR curve has these properties:

    - It starts at 0 and ends at 1
    - All points are greater than or equal to zero

    :param fdr_curve: the sequence to be tested
    :return: True if fdr_curve satisfies the conditions above, False otherwise
    """
    if len(fdr_curve) < 2:
        raise ValueError("FDR length < 2")
    if fdr_curve[0] != 0:
        raise ValueError("FDR start not 0")
    if not np.allclose(fdr_curve[-1], 1.0):
        raise ValueError("FDR end not 1")
    if any(fdr_curve[1:-1] < 0):
        raise ValueError("FDR values not all > 0")
    return True

def find_crossing(fdr_curve, fdr_target):
    """
    Find the index of the point before the rightmost crossing point between an FDR curve and a FDR target value.

    Formally speaking, given an array fdr_curve and a number fdr_target, find the smallest index i such that
    fdr_curve[j] >= fdr_target for all j > i

    :param fdr_curve: the FDR curve as a 1d array
    :param fdr_target: the target FDR
    :type fdr_target: float between 0 and 1
    :return: the index i or -1 if no crossing was found
    :raise ValueError: if fdr_curve is not a valid FDR curve
    """
    #if not is_fdr_curve(fdr_curve):
    #    raise ValueError("Not a valid FDR curve") #ADP - need to review is_fdr_curve criteria +noise means can start above 0
    if not 0 < fdr_target < 1:
        return -1

    less_zero_indices = np.where(fdr_curve <= fdr_target)[0]
    if len(less_zero_indices) == 0:
        return len(fdr_curve)-1
    i = less_zero_indices[-1]
    return i

def score_msm(score_data_df):
    return score_data_df['moc'] * score_data_df['spat'] * score_data_df['spec']

class decoy_adducts():
    #### Some class for dealing with FDR results ####
    def __init__(self, fname, target_adducts, decoy_adducts, shuf=None):
        self.target_adducts = target_adducts
        self.decoy_adducts = decoy_adducts
        self.shuf = shuf or _shuffle_rows
        # read in raw score file and calculate metabolite signal match
        with open(fname) as f_in:
            self.score_data_df = pd.read_csv(f_in, quotechar='"').fillna(0)
        self.score_data_df["msm"] = score_msm(self.score_data_df)
        self.score_data_df.sort_values(by="sf", inplace=True)
        self._count_formulas_per_adduct()

    def _count_formulas_per_adduct(self):
        # store some data info
        self.sf_l = {}
        self.n_sf = {}
        for a in self.target_adducts:
            self.sf_l[a] = np.unique(self.score_data_df.ix[self.score_data_df['adduct'] == a]["sf"])
            self.n_sf[a] = len(self.sf_l[a])

    def decoy_adducts_get_pass_list(self, fdr_target, n_reps, col='msm', return_decoy=False):
        # Get MSM threshold @ target fdr
        # Return molecules with higher MSM value
        msm_vals = self.get_msm_threshold_per_adduct(fdr_target, n_reps)
        pass_list = {}
        for a in self.target_adducts:
            target_df = self.score_data_df.ix[self.score_data_df["adduct"] == a]
            pass_list[a] = target_df.ix[target_df[col] > msm_vals[a]]['sf'].values
        if not return_decoy:
            return pass_list
        else:
            pass_list_decoy = {}
            for a in self.target_adducts:
                decoy_df = self.score_data_df.ix[~self.score_data_df["adduct"].isin(self.target_adducts)]
                pass_list_decoy[a] = decoy_df.ix[decoy_df[col] > msm_vals[a]][['sf', 'adduct']].values
            return pass_list, pass_list_decoy

    def get_msm_thresholds(self, adduct, fdr_target, n_reps=10, col='msm'):
        """
            Calculate the MSM crossing point at a given target fdr
        """
        fdr_curves, _, score_vects = self.get_fdr_curve(adduct, n_reps, col)
        msm_vals = []
        for fdr_curve, score_vect in zip(fdr_curves, score_vects):
            crossing_idx = find_crossing(fdr_curve, fdr_target)
            if crossing_idx > -1:
                msm_vals.append(score_vect[crossing_idx])
            else:
                msm_vals.append(0)
        return msm_vals

    def get_msm_threshold_per_adduct(self, fdr_target, n_reps=10, col='msm'):
        # Repeatedly calcualte FDR curves
        #   Find target crossing point -> find correspdoning msm score
        # return average score per adduct
        msm_vals = {}
        for a in self.target_adducts:
            msm_vals[a] = self.get_msm_thresholds(a, fdr_target, n_reps=n_reps, col=col)
            # calculate average
            msm_vals[a] = np.median(msm_vals[a])
        return msm_vals

    def get_fdr_curve(self, adduct, n_reps=10, col='msm'):
        # for a particular adduct, calcualte n_reps fdr curves
        target_df = self.score_data_df.ix[self.score_data_df["adduct"] == adduct]
        col_vector_decoy = self.score_data_df.ix[self.score_data_df['adduct'].isin(self.decoy_adducts) &
                                                 self.score_data_df['sf'].isin(self.sf_l[adduct])][col].values
        data_reps = len(col_vector_decoy) / len(self.sf_l[adduct])
        try:
            col_vector_decoy = col_vector_decoy.reshape((self.n_sf[adduct], data_reps))
        except ValueError as e:
            print len(col_vector_decoy) , len(self.sf_l[adduct]), np.shape(self.score_data_df), adduct, self.n_sf[adduct], data_reps
            raise
        self.shuf(col_vector_decoy)
        fdr_curves = []
        target_hits = []
        score_vects = []
        for col_vector in col_vector_decoy.T[:n_reps]:
            fdr_curve, target_hit, score_vect = calc_fdr_arr(target_df[col].values, col_vector, ascending=False)
            fdr_curves.append(fdr_curve)
            target_hits.append(target_hit)
            score_vects.append(score_vect)
        return fdr_curves, target_hits, score_vects

def _shuffle_rows(scores):
    # consume the iterator without using the values by "inserting" into an empty immutable queue;
    # see https://docs.python.org/2/library/itertools.html#recipes
    collections.deque((np.random.shuffle(row) for row in scores), maxlen=0)
<commit_msg>allow to pass dataframes to decoy_results constructor<commit_after>import collections
from collections import defaultdict
import numpy as np
import pandas as pd

__author__ = 'palmer'

def get_adducts(json_filename):
    import json
    config = json.loads(open(json_filename).read())
    target_adducts = [a['adduct'] for a in config['fdr']['pl_adducts']]
    decoy_adducts = list(set([a['adduct'] for a in config['isotope_generation']['adducts']])-set(target_adducts))
    return target_adducts, decoy_adducts

def calc_fdr_df(target_df, decoy_df, col='mult', ascending=False):
    # possible backward-incompatibility: calc_fdr_df previously returned the last argument as a series, not as an array
    return calc_fdr_arr(target_df[col], decoy_df[col], ascending=ascending)

def calc_fdr_arr(target_arr, decoy_arr, ascending=False):
    """
    Calculate the FDR curve for arrays of target scores and decoy scores.

    :param target_arr: sequence of scores from sum formulae with target adducts
    :param decoy_arr: sequence of scores from sum formulae with decoy adducts
    :param ascending: True if smaller scores are better, False if higher scores are better
    :raise TypeError: if the two sequences do not have the same length
    :return:
     fdr_curve: 1d array, twice as long as each of the input sequences, where the i-th element is the number of
     decoy hits divided by the number of target hits among the i highest scores (combined).

     target_hits: 1d array, twice as long as each of the input sequences, where the i-th element is the number of
     target hits among the i highest scores (combined)

     sorted_values: 1d array, twice as long as each of the input sequences, containing the sorted values from both
     input sequences
    """
    n, m = len(target_arr), len(decoy_arr)
    if n != m:
        raise TypeError('target should be same length as decoy {} {}'.format(n, m))
    ordering = 1 if ascending else -1  # reversed sorting if score is not ascending
    combined = np.concatenate((target_arr, decoy_arr))
    combined.sort()
    # count how often each value occurs
    target_bag, decoy_bag = _count(target_arr), _count(decoy_arr)
    unique_sorted = np.unique(combined)[::ordering]
    target_hits, decoy_hits = zip(*_iter_hits(target_bag, decoy_bag, unique_sorted))
    target_hits, decoy_hits = np.cumsum(target_hits), np.cumsum(decoy_hits)
    fdr_curve = decoy_hits / target_hits
    fdr_curve[target_hits == 0] = 0
    return fdr_curve, target_hits, combined[::ordering]

def _count(it):
    bag = defaultdict(int)
    for x in it:
        bag[x] += 1
    return bag

def _iter_hits(bag1, bag2, unique_sorted):
    # For each unique value, determine how often the value occurs in the target array and decoy array respectively
    # and yield the fractions (into the hits arrays).
    # Example: if there are two target scores and one decoy score with the same value, append 2/3, 2/3, 2/3 to the
    # target_hits array and 1/3, 1/3, 1/3 to the decoy_hits array.
    # This is to avoid the previously randomized approach of randomly assigning the value to one of the arrays.
    for x in unique_sorted:
        tc, dc = bag1[x], bag2[x]
        tc_part, dc_part = float(tc) / (tc + dc), float(dc) / (tc + dc)
        for _ in range(int(tc + dc)):
            yield (tc_part, dc_part)

def get_decoy_df(decoy_adducts, results_data, sf):
    import numpy as np
    import bisect
    import pandas as pd
    # decoy selection
    nDecoy = len(sf)
    decoy_adducts = np.random.choice(decoy_adducts, size=nDecoy, replace=True)
    dc_df = []
    for s, da in zip(sf, decoy_adducts):
        dc_df.append(results_data[da].iloc[bisect.bisect_left(sf, s)])
        dc_df = pd.DataFrame.from_records(dc_df)
    return dc_df

def select_passes(score_data, t_holds):
    import numpy as np
    moc_t, spec_t, spat_t = t_holds
    pass_tf = np.all([score_data['moc'] > moc_t, score_data['spat'] > spat_t, score_data['spec'] > spec_t], axis=0)
    pass_data = score_data[pass_tf]
    return pass_data

def select_passes_linear(score_data, t_hold):
    import numpy as np
    pass_tf = np.asarray(score_data['moc'] + score_data['spat'] + score_data['spec']) > t_hold
    pass_data = score_data[pass_tf]
    return pass_data

def select_passes_mult(score_data, t_hold):
    import numpy as np
    pass_tf = np.asarray(score_data['moc'] * score_data['spat'] * score_data['spec']) > t_hold
    pass_data = score_data[pass_tf]
    return pass_data

def select_passes_l2(score_data, t_hold):
    import numpy as np
    pass_tf = np.sqrt(np.asarray(score_data['moc'] ** 2 + score_data['spat'] ** 2 + score_data['spec'])) > t_hold
    pass_data = score_data[pass_tf]
    return pass_data

def count_adducts(pass_data, adducts):
    ## number of false hits by adduct
    fd_count = pass_data['adduct'].value_counts()  # count adduct occurances
    set_diff = set(adducts) - set(fd_count.keys())
    for s in set_diff:  # add zero occurance back in
        fd_count[s] = 0
    return fd_count.loc[[f in adducts for f in fd_count.keys()]]

def calc_fdr_adducts(fd_count, adducts, plausible_adducts, average='mean'):
    import numpy as np
    implausible_adducts = [a for a in adducts if a not in plausible_adducts]
    implausible_list = fd_count[implausible_adducts]
    implausible_hits = sum(implausible_list)
    if average == "mean":
        mean_implausible = float(implausible_hits) / len(implausible_list)
    elif average == 'median':
        mean_implausible = np.percentile(implausible_list, 50)
    else:
        raise ValueError('average type not recognised: {}'.format(average))
    plausible_hits = np.sum(fd_count[list(plausible_adducts)])
    if plausible_hits == 0:
        return 1, plausible_hits, implausible_hits
    fdr = (len(plausible_adducts) * mean_implausible) / plausible_hits
    return fdr, plausible_hits, implausible_hits

def is_fdr_curve(fdr_curve):
    """
    A FDR curve has these properties:

    - It starts at 0 and ends at 1
    - All points are greater than or equal to zero

    :param fdr_curve: the sequence to be tested
    :return: True if fdr_curve satisfies the conditions above, False otherwise
    """
    if len(fdr_curve) < 2:
        raise ValueError("FDR length < 2")
    if fdr_curve[0] != 0:
        raise ValueError("FDR start not 0")
    if not np.allclose(fdr_curve[-1], 1.0):
        raise ValueError("FDR end not 1")
    if any(fdr_curve[1:-1] < 0):
        raise ValueError("FDR values not all > 0")
    return True

def find_crossing(fdr_curve, fdr_target):
    """
    Find the index of the point before the rightmost crossing point between an FDR curve and a FDR target value.

    Formally speaking, given an array fdr_curve and a number fdr_target, find the smallest index i such that
    fdr_curve[j] >= fdr_target for all j > i

    :param fdr_curve: the FDR curve as a 1d array
    :param fdr_target: the target FDR
    :type fdr_target: float between 0 and 1
    :return: the index i or -1 if no crossing was found
    :raise ValueError: if fdr_curve is not a valid FDR curve
    """
    #if not is_fdr_curve(fdr_curve):
    #    raise ValueError("Not a valid FDR curve") #ADP - need to review is_fdr_curve criteria +noise means can start above 0
    if not 0 < fdr_target < 1:
        return -1

    less_zero_indices = np.where(fdr_curve <= fdr_target)[0]
    if len(less_zero_indices) == 0:
        return len(fdr_curve)-1
    i = less_zero_indices[-1]
    return i

def score_msm(score_data_df):
    return score_data_df['moc'] * score_data_df['spat'] * score_data_df['spec']

class decoy_adducts():
    #### Some class for dealing with FDR results ####
    def __init__(self, fname, target_adducts, decoy_adducts, shuf=None):
        self.target_adducts = target_adducts
        self.decoy_adducts = decoy_adducts
        self.shuf = shuf or _shuffle_rows
        if type(fname) == str:
            # read in raw score file and calculate metabolite signal match
            with open(fname) as f_in:
                self.score_data_df = pd.read_csv(f_in, quotechar='"').fillna(0)
        else:
            # accept ready-to-use dataframes as well as filenames
            self.score_data_df = fname
        self.score_data_df["msm"] = score_msm(self.score_data_df)
        self.score_data_df.sort_values(by="sf", inplace=True)
        self._count_formulas_per_adduct()

    def _count_formulas_per_adduct(self):
        # store some data info
        self.sf_l = {}
        self.n_sf = {}
        for a in self.target_adducts:
            self.sf_l[a] = np.unique(self.score_data_df.ix[self.score_data_df['adduct'] == a]["sf"])
            self.n_sf[a] = len(self.sf_l[a])

    def decoy_adducts_get_pass_list(self, fdr_target, n_reps, col='msm', return_decoy=False):
        # Get MSM threshold @ target fdr
        # Return molecules with higher MSM value
        msm_vals = self.get_msm_threshold_per_adduct(fdr_target, n_reps)
        pass_list = {}
        for a in self.target_adducts:
            target_df = self.score_data_df.ix[self.score_data_df["adduct"] == a]
            pass_list[a] = target_df.ix[target_df[col] > msm_vals[a]]['sf'].values
        if not return_decoy:
            return pass_list
        else:
            pass_list_decoy = {}
            for a in self.target_adducts:
                decoy_df = self.score_data_df.ix[~self.score_data_df["adduct"].isin(self.target_adducts)]
                pass_list_decoy[a] = decoy_df.ix[decoy_df[col] > msm_vals[a]][['sf', 'adduct']].values
            return pass_list, pass_list_decoy

    def get_msm_thresholds(self, adduct, fdr_target, n_reps=10, col='msm'):
        """
            Calculate the MSM crossing point at a given target fdr
        """
        fdr_curves, _, score_vects = self.get_fdr_curve(adduct, n_reps, col)
        msm_vals = []
        for fdr_curve, score_vect in zip(fdr_curves, score_vects):
            crossing_idx = find_crossing(fdr_curve, fdr_target)
            if crossing_idx > -1:
                msm_vals.append(score_vect[crossing_idx])
            else:
                msm_vals.append(0)
        return msm_vals

    def get_msm_threshold_per_adduct(self, fdr_target, n_reps=10, col='msm'):
        # Repeatedly calcualte FDR curves
        #   Find target crossing point -> find correspdoning msm score
        # return average score per adduct
        msm_vals = {}
        for a in self.target_adducts:
            msm_vals[a] = self.get_msm_thresholds(a, fdr_target, n_reps=n_reps, col=col)
            # calculate average
            msm_vals[a] = np.median(msm_vals[a])
        return msm_vals

    def get_fdr_curve(self, adduct, n_reps=10, col='msm'):
        # for a particular adduct, calcualte n_reps fdr curves
        target_df = self.score_data_df.ix[self.score_data_df["adduct"] == adduct]
        col_vector_decoy = self.score_data_df.ix[self.score_data_df['adduct'].isin(self.decoy_adducts) &
                                                 self.score_data_df['sf'].isin(self.sf_l[adduct])][col].values
        data_reps = len(col_vector_decoy) / len(self.sf_l[adduct])
        try:
            col_vector_decoy = col_vector_decoy.reshape((self.n_sf[adduct], data_reps))
        except ValueError as e:
            print len(col_vector_decoy) , len(self.sf_l[adduct]), np.shape(self.score_data_df), adduct, self.n_sf[adduct], data_reps
            raise
        self.shuf(col_vector_decoy)
        fdr_curves = []
        target_hits = []
        score_vects = []
        for col_vector in col_vector_decoy.T[:n_reps]:
            fdr_curve, target_hit, score_vect = calc_fdr_arr(target_df[col].values, col_vector, ascending=False)
            fdr_curves.append(fdr_curve)
            target_hits.append(target_hit)
            score_vects.append(score_vect)
        return fdr_curves, target_hits, score_vects

def _shuffle_rows(scores):
    # consume the iterator without using the values by "inserting" into an empty immutable queue;
    # see https://docs.python.org/2/library/itertools.html#recipes
    collections.deque((np.random.shuffle(row) for row in scores), maxlen=0)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__version__ = '0.3'

from _uniout import Uniout, runs_in_ipython
import sys

if runs_in_ipython():
    from IPython.utils import io
    io.stdout = Uniout(sys.stdout)
    io.stderr = Uniout(sys.stderr)
else:
    sys.stdout = Uniout(sys.stdout)
    sys.stderr = Uniout(sys.stderr)
<commit_msg>sys should be preceding<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__version__ = '0.3'

import sys
from _uniout import Uniout, runs_in_ipython

if runs_in_ipython():
    from IPython.utils import io
    io.stdout = Uniout(sys.stdout)
    io.stderr = Uniout(sys.stderr)
else:
    sys.stdout = Uniout(sys.stdout)
    sys.stderr = Uniout(sys.stderr)
<|endoftext|>


<commit_before>"""Provide a base HTML template variable for population with appropriate
statistics in the report.py module.
"""

base_template = \
"""

<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css">
<link rel="stylesheet" href="http://uwadataanalysis.cloudapp.net/static/report/main.css">

<title>Analysis Report</title>

          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
            Toggle navigation

          </button>
          Contents

Invalid/Empty

Numerical

String

Boolean

Enum

Identifier

Date

Time

Day

Email

Character

Hyperlink

Currency

&nbsp

Analysis Report

Invalid Rows ({len_invalid_rows})

These rows contain either too many or too few columns.

            {invalid_rows}

Empty Columns ({len_empty_columns})

These columns contain >= 90% empty values.

            {empty_columns}

Error Cells ({len_error_columns})

These cells contain invalid values.

            {error_columns}

Delimiter

This file contains the delimiter type:

{delimiter_type}

Column Analysis (Based on {len_columns} rows)

Charts

Click 'Show Data' at the end of the table to view a chart.

Note: If numerical/currency data contains more than 10,000 values only the top 10,000 will be displayed in the chart.

Showing chart for column:

datda here

Numerical

                    Column
                    Min
                    Max
                    Mode
                    Mean
                    Median Low
                    Median
                    Median High
                    Normally Distributed
                    Standard Deviation
                    Outliers
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {numerical_analysis}

String

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {string_analysis}

Boolean

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    Total "True"
                    Total "False"
                    Total "Yes"
                    Total "No"
                    Total Boolean Values
                    View Chart

                {boolean_analysis}

Enumerated

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {enum_analysis}

Identifier

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {identifier_analysis}

Date

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {date_analysis}

Time

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {time_analysis}

Day

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {day_analysis}

Email

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {email_analysis}

Character

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {char_analysis}

Hyperlink

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {hyper_analysis}

Currency

                    Column
                    Min
                    Max
                    Mode
                    Mean
                    Median Low
                    Median
                    Median High
                    Normally Distributed
                    Standard Deviation
                    Outliers
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {currency_analysis}

"""
<commit_msg>Fixed typo<commit_after>"""Provide a base HTML template variable for population with appropriate
statistics in the report.py module.
"""

base_template = \
"""

<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css">
<link rel="stylesheet" href="http://uwadataanalysis.cloudapp.net/static/report/main.css">

<title>Analysis Report</title>

          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
            Toggle navigation

          </button>
          Contents

Invalid/Empty

Numerical

String

Boolean

Enum

Identifier

Date

Time

Day

Email

Character

Hyperlink

Currency

&nbsp

Analysis Report

Invalid Rows ({len_invalid_rows})

These rows contain either too many or too few columns.

            {invalid_rows}

Empty Columns ({len_empty_columns})

These columns contain >= 90% empty values.

            {empty_columns}

Error Cells ({len_error_columns})

These cells contain invalid values.

            {error_columns}

Delimiter

This file contains the delimiter type:

{delimiter_type}

Column Analysis (Based on {len_columns} rows)

Charts

Click 'Show Data' at the end of the table to view a chart.

Note: If numerical/currency data contains more than 10,000 values only the top 10,000 will be displayed in the chart.

Showing chart for column:

datda here

Numerical

                    Column
                    Min
                    Max
                    Mode
                    Mean
                    Median Low
                    Median
                    Median High
                    Normally Distributed
                    Standard Deviation
                    Outliers
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {numerical_analysis}

String

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {string_analysis}

Boolean

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    Total "True"
                    Total "False"
                    Total "Yes"
                    Total "No"
                    Total Boolean Values
                    View Chart

                {boolean_analysis}

Enumerated

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {enum_analysis}

Identifier

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {identifier_analysis}

Date

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {date_analysis}

Time

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {time_analysis}

Day

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {day_analysis}

Email

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {email_analysis}

Character

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {char_analysis}

Hyperlink

                    Column
                    Mode
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {hyper_analysis}

Currency

                    Column
                    Min
                    Max
                    Mode
                    Mean
                    Median Low
                    Median
                    Median High
                    Normally Distributed
                    Standard Deviation
                    Outliers
                    Most Common (Top 5)
                    Least Common (Top 5)
                    Unique Items
                    View Chart

                {currency_analysis}

"""
<|endoftext|>


<commit_before>import tensorflow as tf
import os
from src.val import build_graph
from src.train import evaluate_metric
from src import input_data
import numpy as np
import argparse
from src.utils import parse_argstring, enum_dict, process_cli_params, get_cli_params
from skopt import gp_minimize, dump
import sys
from tqdm import tqdm

class Hyperopt(object):
    def __init__(self):
        # Parse command line and default parameters
        self.params = process_cli_params(get_cli_params())

        # for k in sorted(self.params_dict.keys()):
        #     print(k, self.params_dict[k])

    def objective(self, x):

        print("----------------------------------------")
        print("----------------------------------------")

        p = self.convert_dims_to_params(x)

        tf.reset_default_graph()

        # -----------------------------
        # Set GPU device to use
        os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
        os.environ["CUDA_VISIBLE_DEVICES"] = str(p.which_gpu)

        # Set seeds
        np.random.seed(p.seed)
        tf.set_random_seed(p.seed)

        # Load data
        print("===  Loading Data ===")
        mnist = input_data.read_data_sets("MNIST_data",
                                          n_labeled=p.num_labeled,
                                          validation_size=p.validation,
                                          one_hot=True,
                                          disjoint=False)
        num_examples = mnist.train.num_examples
        if p.validation > 0:
            mnist.test = mnist.validation
        iter_per_epoch = (num_examples // p.batch_size)
        num_iter = iter_per_epoch * p.end_epoch

        # Build graph

        g, m, trainable_parameters = build_graph(p)

        print("=== Starting Session ===")
        with tf.Session() as sess:
            init = tf.global_variables_initializer()
            sess.run(init)
            print("=== Training ===")
            for i in tqdm(range(num_iter)):

                images, labels = mnist.train.next_batch(p.batch_size,
                                                        p.ul_batch_size)
                _ = sess.run(
                    [g['train_step']],
                    feed_dict={g['images']: images,
                               g['labels']: labels,
                               g['train_flag']: True})

            print("=== Evaluating ===")
            error = tf.constant(100.0) - m['acc']
            val_err = evaluate_metric(mnist.validation, sess, error, graph=g,
                                      params=p)

        return val_err

    def get_dims(self):

        dims = [
            # rc_weights
            (500., 5000, 'log-uniform'),  # 0 rc_0
            (5.00, 50., 'log-uniform'),  # 1 rc_1
            (0.01, 1.0, 'log-uniform'),  # 2 rc_2:6
            (0.01, 1.0),                # 3 corrupt_sd
            (0.01, 1.0),                # 4 vadv_sd
            (1, 2, 3, 4),               # 5 num_power_iters
            (0.01, 10.0, 'log-uniform')  # 6: eps_0
        ]
        x0 = [1000, 10, 0.1, 0.3, 0.5, 3, 1.0]

        if self.params.model == 'clw' or self.params.model == 'nlw':
            dims += [
                (1e-3, 0.5, 'log-uniform'),  # 7 eps_1
                (1e-5, 0.1, 'log-uniform'),  # 8 eps_2
            ]
            x0 += [0.1, 0.001]

        return dims, x0

    def convert_dims_to_params(self, x):

        # -------------------------
        # Optimize
        self.params.rc_weights = {0: x[0], 1: x[1],
                                  2: x[2], 3: x[2],
                                  4: x[2], 5: x[2],
                                  6: x[2]
                                  }
        self.params.corrupt_sd = x[3]
        self.params.vadv_sd = x[4]
        self.params.num_power_iters = x[5]

        if self.params.model == 'clw' or self.params.model == 'nlw':
            self.params.epsilon = {0: x[6], 1: x[7],
                                   2: x[8], 3: x[8],
                                   4: x[8], 5: x[8],
                                   6: x[8]
                                   }
        else:
            self.params.epsilon = {0: x[6]}

        return self.params

# class HyperoptPowerIters(Hyperopt):
#     def convert_dims_to_params(self, x):
#         self.params.num_power_iters = x
#         return self.params
#
#     def get_default_params(self):
#         super(HyperoptPowerIters, self).get_default_params()
#
#         if self.params.model == "c":
#             self.params.rc_weights = enum_dict([898.44421, 20.73306, 0.17875, 0.31394, 0.02214, 0.39981, 0.04065])
#             self.params.epsilon = enum_dict([0.03723])
#         elif self.params.model == "clw":
#             self.params.rc_weights = enum_dict([898.44421, 8.81609, 0.61101, 0.11661, 0.13746, 0.50335, 0.63461])
#             self.params.epsilon = enum_dict([0.11002, 0.0093, 0.00508, 1e-05, 0.00073, 0.00113, 0.00019])
#         elif self.params.model == "vat":
#             self.params.epsilon = enum_dict([5.0])
#             self.params.encoder_layers = parse_argstring(
#                 "784-1200-600-300-150-10", dtype=int)
#             self.params.beta1_during_decay = 0.5
#             self.params.decay_start = 0.5
#             self.params.decay_start_epoch = self.params.decay_start * \
#                                             self.params.end_epoch
#
#
#
# def tune_single_parameter():
#     os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  # filter out info, warnings
#     hyperopt = HyperoptPowerIters()
#     dims = [1, 2, 3, 4, 5]
#     res = {}
#     for x in dims:
#         res[x] = hyperopt.objective(x)
#         print(x, res[x])
#
#     print("----------------------------------------")
#     print("----------------------------------------")
#     print("=== Complete ===")
#     for k, v in res.items():
#         print(k, v)
#

def main():

    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  # filter out info, warnings

    hyperopt = Hyperopt()
    dims, x0 = hyperopt.get_dims()

    print("=== Beginning Search ===")
    res = gp_minimize(hyperopt.objective, dims, n_calls=11, x0=x0, verbose=True)
    print(res.fun, ":", *res.x)

    dump(res, hyperopt.params.logdir + hyperopt.params.id + '/hyperopt_res.gz')

if __name__ == '__main__':
    # tune_single_parameter()
    main()

<commit_msg>Hyperopt: use minimum validation error as the objective<commit_after>import tensorflow as tf
import os
from src.val import build_graph
from src.train import evaluate_metric
from src import input_data
import numpy as np
import argparse
from src.utils import parse_argstring, enum_dict, process_cli_params, get_cli_params
from skopt import gp_minimize, dump
import sys
from tqdm import tqdm

class Hyperopt(object):
    def __init__(self):
        # Parse command line and default parameters
        self.params = process_cli_params(get_cli_params())

        # for k in sorted(self.params_dict.keys()):
        #     print(k, self.params_dict[k])

    def objective(self, x):

        print("----------------------------------------")
        print("----------------------------------------")

        p = self.convert_dims_to_params(x)

        tf.reset_default_graph()

        # -----------------------------
        # Set GPU device to use
        os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
        os.environ["CUDA_VISIBLE_DEVICES"] = str(p.which_gpu)

        # Set seeds
        np.random.seed(p.seed)
        tf.set_random_seed(p.seed)

        # Load data
        print("===  Loading Data ===")
        mnist = input_data.read_data_sets("MNIST_data",
                                          n_labeled=p.num_labeled,
                                          validation_size=p.validation,
                                          one_hot=True,
                                          disjoint=False)
        num_examples = mnist.train.num_examples
        if p.validation > 0:
            mnist.test = mnist.validation
        iter_per_epoch = (num_examples // p.batch_size)
        num_iter = iter_per_epoch * p.end_epoch

        # Build graph

        g, m, trainable_parameters = build_graph(p)
        val_errs = []
        error = tf.constant(100.0) - m['acc']

        print("=== Starting Session ===")
        with tf.Session() as sess:
            init = tf.global_variables_initializer()
            sess.run(init)
            print("=== Training ===")
            for i in tqdm(range(num_iter)):

                images, labels = mnist.train.next_batch(p.batch_size,
                                                        p.ul_batch_size)
                _ = sess.run(
                    [g['train_step']],
                    feed_dict={g['images']: images,
                               g['labels']: labels,
                               g['train_flag']: True})

                if (i > 1) and ((i + 1) % int(p.test_frequency_in_epochs *
                                                  iter_per_epoch) == 0):
            # print("=== Evaluating ===")

                    val_errs.append(evaluate_metric(mnist.validation, sess, error, graph=g, params=p))

        val_err = min(val_errs)

        return val_err

    def get_dims(self):

        dims = [
            # rc_weights
            (500., 5000, 'log-uniform'),  # 0 rc_0
            (5.00, 50., 'log-uniform'),  # 1 rc_1
            (0.01, 1.0, 'log-uniform'),  # 2 rc_2:6
            (0.01, 1.0),                # 3 corrupt_sd
            (0.01, 1.0),                # 4 vadv_sd
            (1, 2, 3, 4),               # 5 num_power_iters
            (0.01, 10.0, 'log-uniform')  # 6: eps_0
        ]
        x0 = [1000, 10, 0.1, 0.3, 0.5, 3, 1.0]

        if self.params.model == 'clw' or self.params.model == 'nlw':
            dims += [
                (1e-3, 0.5, 'log-uniform'),  # 7 eps_1
                (1e-5, 0.1, 'log-uniform'),  # 8 eps_2
            ]
            x0 += [0.1, 0.001]

        return dims, x0

    def convert_dims_to_params(self, x):

        # -------------------------
        # Optimize
        self.params.rc_weights = {0: x[0], 1: x[1],
                                  2: x[2], 3: x[2],
                                  4: x[2], 5: x[2],
                                  6: x[2]
                                  }
        self.params.corrupt_sd = x[3]
        self.params.vadv_sd = x[4]
        self.params.num_power_iters = x[5]

        if self.params.model == 'clw' or self.params.model == 'nlw':
            self.params.epsilon = {0: x[6], 1: x[7],
                                   2: x[8], 3: x[8],
                                   4: x[8], 5: x[8],
                                   6: x[8]
                                   }
        else:
            self.params.epsilon = {0: x[6]}

        return self.params

# class HyperoptPowerIters(Hyperopt):
#     def convert_dims_to_params(self, x):
#         self.params.num_power_iters = x
#         return self.params
#
#     def get_default_params(self):
#         super(HyperoptPowerIters, self).get_default_params()
#
#         if self.params.model == "c":
#             self.params.rc_weights = enum_dict([898.44421, 20.73306, 0.17875, 0.31394, 0.02214, 0.39981, 0.04065])
#             self.params.epsilon = enum_dict([0.03723])
#         elif self.params.model == "clw":
#             self.params.rc_weights = enum_dict([898.44421, 8.81609, 0.61101, 0.11661, 0.13746, 0.50335, 0.63461])
#             self.params.epsilon = enum_dict([0.11002, 0.0093, 0.00508, 1e-05, 0.00073, 0.00113, 0.00019])
#         elif self.params.model == "vat":
#             self.params.epsilon = enum_dict([5.0])
#             self.params.encoder_layers = parse_argstring(
#                 "784-1200-600-300-150-10", dtype=int)
#             self.params.beta1_during_decay = 0.5
#             self.params.decay_start = 0.5
#             self.params.decay_start_epoch = self.params.decay_start * \
#                                             self.params.end_epoch
#
#
#
# def tune_single_parameter():
#     os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  # filter out info, warnings
#     hyperopt = HyperoptPowerIters()
#     dims = [1, 2, 3, 4, 5]
#     res = {}
#     for x in dims:
#         res[x] = hyperopt.objective(x)
#         print(x, res[x])
#
#     print("----------------------------------------")
#     print("----------------------------------------")
#     print("=== Complete ===")
#     for k, v in res.items():
#         print(k, v)
#

def main():

    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  # filter out info, warnings

    hyperopt = Hyperopt()
    dims, x0 = hyperopt.get_dims()

    print("=== Beginning Search ===")
    res = gp_minimize(hyperopt.objective, dims, n_calls=11, x0=x0, verbose=True)
    print(res.fun, ":", *res.x)

    dump(res, hyperopt.params.logdir + hyperopt.params.id + '/hyperopt_res.gz')

if __name__ == '__main__':
    # tune_single_parameter()
    main()

<|endoftext|>


<commit_before>"""
Responsible for reading in configuration files, validating the proper
format and providing sane defaults for parts that don't have any.

"""
import logging
from os import path
import platform
import sys
import yaml

(major, minor, rev) = platform.python_version_tuple()
if float('%s.%s' % (major, minor)) < 2.7:
    import logutils.dictconfig as logging_config
else:
    from logging import config as logging_config

from helper import NullHandler

LOGGER = logging.getLogger(__name__)

class Config(object):
    """The Config object holds the current state of the configuration for an
    application. If no configuration file is provided, it will used a set of
    defaults with very basic behavior for logging and daemonization.

    """
    APPLICATION = {'wake_interval': 60}
    DAEMON = {'user': None,
              'group': None,
              'pidfile': None,
              'prevent_core': True}
    LOGGING_FORMAT = ('%(levelname) -10s %(asctime)s %(process)-6d '
                      '%(processName) -20s %(threadName)-12s %(name) -30s '
                      '%(funcName) -25s L%(lineno)-6d: %(message)s')
    LOGGING = {'disable_existing_loggers': True,
               'filters': dict(),
               'formatters': {'verbose': {'datefmt': '%Y-%m-%d %H:%M:%S',
                                          'format': LOGGING_FORMAT}},
               'handlers': {'console': {'class': 'logging.StreamHandler',
                                        'debug_only': True,
                                        'formatter': 'verbose'}},
               'incremental': False,
               'loggers': {'helper': {'handlers': ['console'],
                                      'level': 'INFO',
                                      'propagate': True}},
               'root': {'handlers': [],
                        'level': logging.CRITICAL,
                        'propagate': True},
               'version': 1}

    def __init__(self, file_path=None):
        """Create a new instance of the configuration object, passing in the
        path to the configuration file.

        :param str file_path:

        """
        self.application = Data(self.APPLICATION)
        self.daemon = Data(self.DAEMON)
        self._file_path = None
        self._values = Data()
        if file_path:
            self._file_path = self._validate(file_path)
            self._values = self._load_config_file()
        self._assign_values(self.application, self._values.get('Application'))
        self._assign_values(self.daemon, self._values.get('Daemon'))

    def _assign_values(self, obj, values):
        """Assign values to the object passed in from the dictionary of values.

        :param Data obj: Data object to assign values to
        :param dict values: Values to assign

        """
        for key in values or dict():
            setattr(obj, key, values[key])

    def get(self, name, default=None):
        """Return the value for key if key is in the configuration, else default.

        :param str name: The key name to return
        :param mixed default: The default value for the key
        :return: mixed

        """
        return self._values.get(name, default)

    @property
    def logging(self):
        """Return the logging configuration in the form of a dictionary.

        :rtype: dict

        """
        config = self.LOGGING
        config_in = self._values.get('Logging', dict())
        for section in ['formatters', 'handlers', 'loggers', 'filters', 'root']:
            if section in config_in:
                for key in config_in[section]:
                    config[section][key] = config_in[section][key]
        LOGGER.debug(config)
        return config

    def reload(self):
        """Reload the configuration from disk returning True if the
        configuration has changed from the previous values.

        """
        if self._file_path:

            # Try and reload the configuration file from disk
            try:
                values = Data(self._load_config_file())
            except ValueError as error:
                LOGGER.error('Could not reload configuration: %s', error)
                return False

            # Only update the configuration if the values differ
            if hash(values) != hash(self._values):
                self._values = values
                self._assign_defaults()
                return True

        return False

    def _load_config_file(self):
        """Load the configuration file into memory, returning the content.

        """
        LOGGER.info('Loading configuration from %s', self._file_path)
        try:
            config = open(self._file_path).read()
        except OSError as error:
            raise ValueError('Could not read configuration file: %s' % error)
        try:
            return yaml.safe_load(config)
        except yaml.YAMLError as error:
            message = '\n'.join(['    > %s' % line
                                 for line in str(error).split('\n')])
            sys.stderr.write("\n\n  Error in the configuration file:\n\n"
                             "%s\n\n" % message)
            sys.stderr.write("  Configuration should be a valid YAML file.\n\n")
            sys.stderr.write("  YAML format validation available at "
                             "http://yamllint.com\n")
            raise ValueError(error)

    def _validate(self, file_path):
        """Normalize the path provided and ensure the file path, raising a
        ValueError if the file does not exist.

        :param str file_path:
        :return: str
        :raises: ValueError

        """
        file_path = path.abspath(file_path)
        if not path.exists(file_path):
            raise ValueError('Configuration file not found: %s' % file_path)
        return file_path

class LoggingConfig(object):
    """The Logging class is used for abstracting away dictConfig logging
    semantics and can be used by sub-processes to ensure consistent logging
    rule application.

    """
    DEBUG_ONLY = 'debug_only'
    HANDLERS = 'handlers'
    LOGGERS = 'loggers'

    def __init__(self, configuration, debug=None):
        """Create a new instance of the Logging object passing in the
        DictConfig syntax logging configuration and a debug flag.

        :param dict configuration: The logging configuration
        :param bool debug: Toggles use of debug_only loggers

        """
        # Force a NullLogger for some libraries that require it
        root_logger = logging.getLogger()
        root_logger.addHandler(NullHandler())

        self.config = configuration
        self.debug = debug
        self.configure()

    def update(self, configuration, debug=None):
        """Update the internal configuration values, removing debug_only
        handlers if debug is False. Returns True if the configuration has
        changed from previous configuration values.

        :param dict configuration: The logging configuration
        :param bool debug: Toggles use of debug_only loggers
        :rtype: bool

        """

        if hash(self.config) != hash(configuration) and debug != self.debug:
            self.config = configuration
            self.debug = debug
            self.configure()
            return True
        return False

    def configure(self):
        """Configure the Python stdlib logger"""
        if self.debug is not None and not self.debug:
            self._remove_debug_handlers()
        self._remove_debug_only()
        logging_config.dictConfig(self.config)
        try:
            logging.captureWarnings(True)
        except AttributeError:
            pass

    def _remove_debug_only(self):
        """Iterate through each handler removing the invalid dictConfig key of
        debug_only.

        """
        LOGGER.debug('Removing debug only from handlers')
        for handler in self.config[self.HANDLERS]:
            if self.DEBUG_ONLY in self.config[self.HANDLERS][handler]:
                del self.config[self.HANDLERS][handler][self.DEBUG_ONLY]

    def _remove_debug_handlers(self):
        """Remove any handlers with an attribute of debug_only that is True and
        remove the references to said handlers from any loggers that are
        referencing them.

        """
        remove = list()
        for handler in self.config[self.HANDLERS]:
            if self.config[self.HANDLERS][handler].get('debug_only'):
                remove.append(handler)
        for handler in remove:
            del self.config[self.HANDLERS][handler]
            for logger in self.config[self.LOGGERS].keys():
                logger = self.config[self.LOGGERS][logger]
                if handler in logger[self.HANDLERS]:
                    logger[self.HANDLERS].remove(handler)
        self._remove_debug_only()

class Data(object):
    """Data object configuration is wrapped in, can be used as a object with
    attributes or as a dict.

    """
    def __init__(self, value=None):
        super(Data, self).__init__()
        if value and isinstance(value, dict):
            for name in value.keys():
                if isinstance(value[name], dict):
                    object.__setattr__(self, name, Data(value[name]))
                else:
                    object.__setattr__(self, name, value[name])

    def __contains__(self, name):
        return name in self.__dict__.keys()

    def __delattr__(self, name):
        object.__delattr__(self, name)

    def __delitem__(self, name):
        if not name in self.__dict__:
            raise KeyError(name)
        object.__delattr__(self, name)

    def __getattribute__(self, name):
        return object.__getattribute__(self, name)

    def __getitem__(self, name):
        return object.__getattribute__(self, name)

    def __setitem__(self, name, value):
        if isinstance(value, dict) and name != '__dict__':
            value = Data(value)
        object.__setattr__(self, name, value)

    def __setattr__(self, name, value):
        if isinstance(value, dict) and name != '__dict__':
            value = Data(value)
        object.__setattr__(self, name, value)

    def __repr__(self):
        return repr(self.__dict__)

    def __len__(self):
        return len(self.__dict__.keys())

    def __iter__(self):
        for name in self.__dict__.keys():
            yield name

    def str(self):
        """Return a string representation of the data object.

        :rtype: str

        """
        return str(self.__dict__)

    def dict(self):
        """Return the data object as a dictionary, recursively casting children
        that are Data objects as well.

        :rtype: dict

        """
        output = dict()
        for key, value in self.iteritems():
            if isinstance(value, Data):
                output[key] = value.dict()
            else:
                output[key] = value
        return output

    def get(self, name, default=None):
        """Return the value for key if key is in the dictionary, else default.
        If default is not given, it defaults to None, so that this method
        never raises a KeyError.

        :param str name: The key name to return
        :param mixed default: The default value for the key
        :return: mixed

        """
        return self.__dict__.get(name, default)

    def has_key(self, name):
        """Test for the presence of key in the data object. has_key() is
        deprecated in favor of key in d.

        :param name:
        :return: bool

        """
        return name in self.__dict__

    def items(self):
        """Return a copy of the dictionary's list of (key, value) pairs.

        :rtype: list

        """
        return self.__dict__.items()

    def iteritems(self):
        """Return an iterator over the data keys. See the note for
        Data.items().

        Using itervalues() while adding or deleting entries in the data object
        may raise a RuntimeError or fail to iterate over all entries.

        :rtype: iterator
        :raises: RuntimeError

        """
        return self.__dict__.iteritems()

    def itervalues(self):
        """Return an iterator over the data values. See the note for
        Data.items().

        Using itervalues() while adding or deleting entries in the data object
        may raise a RuntimeError or fail to iterate over all entries.

        :rtype: iterator
        :raises: RuntimeError

        """
        return self.__dict__.itervalues()

    def keys(self):
        """Return a copy of the dictionary's list of keys. See the note for
        Data.items()

        :rtype: list

        """
        return self.__dict__.keys()

    def pop(self, name, default=None):
        """If key is in the dictionary, remove it and return its value, else
        return default. If default is not given and key is not in the
        dictionary, a KeyError is raised.

        :param str name: The key name
        :param mixed default: The default value
        :raises: KeyError

        """
        return self.__dict__.pop(name, default)

    def setdefault(self, name, default=None):
        """If key is in the dictionary, return its value. If not, insert key
        with a value of default and return default. default defaults to None.

        :param str name: The key
        :param mixed default: The value
        :return: mixed

        """
        self.__dict__.setdefault(name, default)

    def update(self, other=None, **kwargs):
        """Update the dictionary with the key/value pairs from other,
        overwriting existing keys. update() accepts either another dictionary
        object or an iterable of key/value pairs (as tuples or other iterables
        of length two). If keyword arguments are specified, the dictionary is
        then updated with those key/value pairs: d.update(red=1, blue=2).

        :param dict other: Dict or other iterable
        :param dict **kwargs: Key/value pairs to update
        :rtype: None

        """
        self.__dict__.update(other, **kwargs)

    def values(self):
        """Return the configuration values

        :rtype: list

        """
        return self.__dict__.values()
<commit_msg>Remove deprecated iteritems, add json support<commit_after>"""
Responsible for reading in configuration files, validating the proper
format and providing sane defaults for parts that don't have any.

"""
import json
import logging
from os import path
import platform
import sys
import yaml

from helper import NullHandler

(major, minor, rev) = platform.python_version_tuple()
if float('%s.%s' % (major, minor)) < 2.7:
    import logutils.dictconfig as logging_config
else:
    from logging import config as logging_config

LOGGER = logging.getLogger(__name__)

class Config(object):
    """The Config object holds the current state of the configuration for an
    application. If no configuration file is provided, it will used a set of
    defaults with very basic behavior for logging and daemonization.

    """
    APPLICATION = {'wake_interval': 60}
    DAEMON = {'user': None,
              'group': None,
              'pidfile': None,
              'prevent_core': True}
    LOGGING_FORMAT = ('%(levelname) -10s %(asctime)s %(process)-6d '
                      '%(processName) -20s %(threadName)-12s %(name) -30s '
                      '%(funcName) -25s L%(lineno)-6d: %(message)s')
    LOGGING = {'disable_existing_loggers': True,
               'filters': dict(),
               'formatters': {'verbose': {'datefmt': '%Y-%m-%d %H:%M:%S',
                                          'format': LOGGING_FORMAT}},
               'handlers': {'console': {'class': 'logging.StreamHandler',
                                        'debug_only': True,
                                        'formatter': 'verbose'}},
               'incremental': False,
               'loggers': {'helper': {'handlers': ['console'],
                                      'level': 'INFO',
                                      'propagate': True}},
               'root': {'handlers': [],
                        'level': logging.CRITICAL,
                        'propagate': True},
               'version': 1}

    def __init__(self, file_path=None):
        """Create a new instance of the configuration object, passing in the
        path to the configuration file.

        :param str file_path:

        """
        self.application = Data(self.APPLICATION)
        self.daemon = Data(self.DAEMON)
        self._file_path = None
        self._values = Data()
        if file_path:
            self._file_path = self._validate(file_path)
            self._values = self._load_config_file()

        app = self._values.get('Application', self._values.get('application'))
        self._assign_values(self.application, app)
        daemon = self._values.get('Daemon', self._values.get('daemon'))
        self._assign_values(self.daemon, daemon)

    @staticmethod
    def _assign_values(obj, values):
        """Assign values to the object passed in from the dictionary of values.

        :param Data obj: Data object to assign values to
        :param dict values: Values to assign

        """
        for key in values or dict():
            setattr(obj, key, values[key])

    def get(self, name, default=None):
        """Return the value for key if key is in the configuration, else default.

        :param str name: The key name to return
        :param mixed default: The default value for the key
        :return: mixed

        """
        return self._values.get(name, default)

    @property
    def logging(self):
        """Return the logging configuration in the form of a dictionary.

        :rtype: dict

        """
        config = self.LOGGING
        config_in = self._values.get('Logging',
                                     self._values.get('logging', {}))
        for section in ['formatters', 'handlers', 'loggers',
                        'filters', 'root']:
            if section in config_in:
                for key in config_in[section]:
                    config[section][key] = config_in[section][key]
        LOGGER.debug(config)
        return config

    def reload(self):
        """Reload the configuration from disk returning True if the
        configuration has changed from the previous values.

        """
        if self._file_path:

            # Try and reload the configuration file from disk
            try:
                values = Data(self._load_config_file())
            except ValueError as error:
                LOGGER.error('Could not reload configuration: %s', error)
                return False

            # Only update the configuration if the values differ
            if hash(values) != hash(self._values):
                self._values = values
                return True

        return False

    def _load_config_file(self):
        """Load the configuration file into memory, returning the content.

        """
        LOGGER.info('Loading configuration from %s', self._file_path)
        if self._file_path.endswith('json'):
            return self._load_json_config()
        # Keep the behavior as it was with regard to file extensions for YAML
        return self._load_yaml_config()

    def _load_json_config(self):
        """Load the configuration file in JSON format

        :rtype: dict

        """
        try:
            with open(self._file_path, 'r') as handle:
                return json.load(handle)
        except (OSError, ValueError) as error:
            raise ValueError('Could not read configuration file: %s' % error)

    def _load_yaml_config(self):
        """Loads the configuration file from a .yaml or .yml file

        :type: dict

        """
        try:
            config = open(self._file_path).read()
        except OSError as error:
            raise ValueError('Could not read configuration file: %s' % error)
        try:
            return yaml.safe_load(config)
        except yaml.YAMLError as error:
            message = '\n'.join(['    > %s' % line
                                 for line in str(error).split('\n')])
            sys.stderr.write("\n\n  Error in the configuration file:\n\n"
                             "%s\n\n" % message)
            sys.stderr.write("  Configuration should be a valid YAML file.\n")
            sys.stderr.write("  YAML format validation available at "
                             "http://yamllint.com\n")
            raise ValueError(error)

    @staticmethod
    def _validate(file_path):
        """Normalize the path provided and ensure the file path, raising a
        ValueError if the file does not exist.

        :param str file_path:
        :return: str
        :raises: ValueError

        """
        file_path = path.abspath(file_path)
        if not path.exists(file_path):
            raise ValueError('Configuration file not found: %s' % file_path)
        return file_path

class LoggingConfig(object):
    """The Logging class is used for abstracting away dictConfig logging
    semantics and can be used by sub-processes to ensure consistent logging
    rule application.

    """
    DEBUG_ONLY = 'debug_only'
    HANDLERS = 'handlers'
    LOGGERS = 'loggers'

    def __init__(self, configuration, debug=None):
        """Create a new instance of the Logging object passing in the
        DictConfig syntax logging configuration and a debug flag.

        :param dict configuration: The logging configuration
        :param bool debug: Toggles use of debug_only loggers

        """
        # Force a NullLogger for some libraries that require it
        root_logger = logging.getLogger()
        root_logger.addHandler(NullHandler())

        self.config = configuration
        self.debug = debug
        self.configure()

    def update(self, configuration, debug=None):
        """Update the internal configuration values, removing debug_only
        handlers if debug is False. Returns True if the configuration has
        changed from previous configuration values.

        :param dict configuration: The logging configuration
        :param bool debug: Toggles use of debug_only loggers
        :rtype: bool

        """

        if hash(self.config) != hash(configuration) and debug != self.debug:
            self.config = configuration
            self.debug = debug
            self.configure()
            return True
        return False

    def configure(self):
        """Configure the Python stdlib logger"""
        if self.debug is not None and not self.debug:
            self._remove_debug_handlers()
        self._remove_debug_only()
        logging_config.dictConfig(self.config)
        try:
            logging.captureWarnings(True)
        except AttributeError:
            pass

    def _remove_debug_only(self):
        """Iterate through each handler removing the invalid dictConfig key of
        debug_only.

        """
        LOGGER.debug('Removing debug only from handlers')
        for handler in self.config[self.HANDLERS]:
            if self.DEBUG_ONLY in self.config[self.HANDLERS][handler]:
                del self.config[self.HANDLERS][handler][self.DEBUG_ONLY]

    def _remove_debug_handlers(self):
        """Remove any handlers with an attribute of debug_only that is True and
        remove the references to said handlers from any loggers that are
        referencing them.

        """
        remove = list()
        for handler in self.config[self.HANDLERS]:
            if self.config[self.HANDLERS][handler].get('debug_only'):
                remove.append(handler)
        for handler in remove:
            del self.config[self.HANDLERS][handler]
            for logger in self.config[self.LOGGERS].keys():
                logger = self.config[self.LOGGERS][logger]
                if handler in logger[self.HANDLERS]:
                    logger[self.HANDLERS].remove(handler)
        self._remove_debug_only()

class Data(object):
    """Data object configuration is wrapped in, can be used as a object with
    attributes or as a dict.

    """
    def __init__(self, value=None):
        super(Data, self).__init__()
        if value and isinstance(value, dict):
            for name in value.keys():
                if isinstance(value[name], dict):
                    object.__setattr__(self, name, Data(value[name]))
                else:
                    object.__setattr__(self, name, value[name])

    def __contains__(self, name):
        return name in self.__dict__.keys()

    def __delattr__(self, name):
        object.__delattr__(self, name)

    def __delitem__(self, name):
        if name not in self.__dict__:
            raise KeyError(name)
        object.__delattr__(self, name)

    def __getattribute__(self, name):
        return object.__getattribute__(self, name)

    def __getitem__(self, name):
        return object.__getattribute__(self, name)

    def __setitem__(self, name, value):
        if isinstance(value, dict) and name != '__dict__':
            value = Data(value)
        object.__setattr__(self, name, value)

    def __setattr__(self, name, value):
        if isinstance(value, dict) and name != '__dict__':
            value = Data(value)
        object.__setattr__(self, name, value)

    def __repr__(self):
        return repr(self.__dict__)

    def __len__(self):
        return len(self.__dict__)

    def __iter__(self):
        for name in self.__dict__.keys():
            yield name

    def str(self):
        """Return a string representation of the data object.

        :rtype: str

        """
        return str(self.__dict__)

    def dict(self):
        """Return the data object as a dictionary, recursively casting children
        that are Data objects as well.

        :rtype: dict

        """
        output = dict()
        for key, value in self.items():
            if isinstance(value, Data):
                output[key] = value.dict()
            else:
                output[key] = value
        return output

    def get(self, name, default=None):
        """Return the value for key if key is in the dictionary, else default.
        If default is not given, it defaults to None, so that this method
        never raises a KeyError.

        :param str name: The key name to return
        :param mixed default: The default value for the key
        :return: mixed

        """
        return self.__dict__.get(name, default)

    def has_key(self, name):
        """Test for the presence of key in the data object. has_key() is
        deprecated in favor of key in d.

        :param name:
        :return: bool

        """
        return name in self.__dict__

    def items(self):
        """Return a copy of the dictionary's list of (key, value) pairs.

        :rtype: list

        """
        return self.__dict__.items()

    def itervalues(self):
        """Return an iterator over the data values. See the note for
        Data.items().

        Using itervalues() while adding or deleting entries in the data object
        may raise a RuntimeError or fail to iterate over all entries.

        :rtype: iterator
        :raises: RuntimeError

        """
        return self.__dict__.itervalues()

    def keys(self):
        """Return a copy of the dictionary's list of keys. See the note for
        Data.items()

        :rtype: list

        """
        return self.__dict__.keys()

    def pop(self, name, default=None):
        """If key is in the dictionary, remove it and return its value, else
        return default. If default is not given and key is not in the
        dictionary, a KeyError is raised.

        :param str name: The key name
        :param mixed default: The default value
        :raises: KeyError

        """
        return self.__dict__.pop(name, default)

    def setdefault(self, name, default=None):
        """If key is in the dictionary, return its value. If not, insert key
        with a value of default and return default. default defaults to None.

        :param str name: The key
        :param mixed default: The value
        :return: mixed

        """
        self.__dict__.setdefault(name, default)

    def update(self, other=None, **kwargs):
        """Update the dictionary with the key/value pairs from other,
        overwriting existing keys. update() accepts either another dictionary
        object or an iterable of key/value pairs (as tuples or other iterables
        of length two). If keyword arguments are specified, the dictionary is
        then updated with those key/value pairs: d.update(red=1, blue=2).

        :param dict other: Dict or other iterable
        :param dict kwargs: Key/value pairs to update
        :rtype: None

        """
        self.__dict__.update(other, **kwargs)

    def values(self):
        """Return the configuration values

        :rtype: list

        """
        return self.__dict__.values()
<|endoftext|>