<commit_before>import logging
from collections import defaultdict

import joblib
import numpy as np
from fri.model.base_cvxproblem import Relevance_CVXProblem
from fri.model.base_initmodel import InitModel
from fri.model.base_type import ProblemType
from fri.utils import permutate_feature_in_data
from scipy import stats

MIN_N_PROBE_FEATURES = 20  # Lower bound of probe features

def _start_solver_worker(bound: Relevance_CVXProblem):
    """
    Worker thread method for parallel computation
    """
    return bound.solve()

class RelevanceBoundsIntervals(object):
    def __init__(self, data, problem_type: ProblemType, best_init_model: InitModel, random_state, n_resampling, n_jobs,
                 verbose):
        self.data = data
        self.problem_type = problem_type
        self.verbose = verbose
        self.n_jobs = n_jobs
        self.n_resampling = n_resampling
        self.random_state = random_state
        self.best_init_model = best_init_model
        self.best_hyperparameters = best_init_model.hyperparam

        # Relax constraints to improve stability
        relaxed_constraints = problem_type.get_relaxed_constraints(best_init_model.constraints)
        self.init_constraints = relaxed_constraints

    def get_normalized_lupi_intervals(self, lupi_features, presetModel=None):

        # We define a list of all the features we want to compute relevance bounds for
        X, _ = self.data  # TODO: handle other data formats
        all_d = X.shape[1]
        normal_d = all_d - lupi_features

        # Compute relevance bounds and probes for normal features and LUPI
        with joblib.Parallel(n_jobs=self.n_jobs, verbose=self.verbose) as parallel:
            d_n = _get_necessary_dimensions(normal_d, presetModel)
            rb = self.compute_relevance_bounds(d_n, parallel=parallel)
            probe_upper = self.compute_probe_values(d_n, True, parallel=parallel)
            probe_lower = self.compute_probe_values(d_n, False, parallel=parallel)

            d_l = _get_necessary_dimensions(all_d, presetModel, start=normal_d)
            rb_l = self.compute_relevance_bounds(d_l, parallel=parallel)
            probe_priv_upper = self.compute_probe_values(d_l, True, parallel=parallel)
            probe_priv_lower = self.compute_probe_values(d_l, True, parallel=parallel)
        probes = [probe_lower, probe_upper, probe_priv_lower, probe_priv_upper]
        #
        # Postprocess
        #

        # Get Scaling Parameters
        l1 = self.init_constraints["w_l1"]
        l1_priv = self.init_constraints["w_priv_l1"]

        # Normalize Normal and Lupi features
        rb_norm = _postprocessing(l1, rb)
        rb_l_norm = _postprocessing(l1_priv, rb_l)
        interval_ = np.concatenate([rb_norm, rb_l_norm])

        # Normalize Probes
        for probe in probes:
            probe = _postprocessing(l1_priv, probe)
        # pr_norm = _postprocessing(l1, pr)
        # pr_l_norm = _postprocessing(l1_priv, pr_l)

        #
        #
        # Classify features
        fc = feature_classification(probe_lower, probe_upper, rb_norm, verbose=self.verbose)
        fc_l = feature_classification(probe_priv_lower, probe_priv_upper, rb_l_norm, verbose=self.verbose)
        fc_both = np.concatenate([fc, fc_l])

        return interval_, fc_both

    def get_normalized_intervals(self, presetModel=None):
        # We define a list of all the features we want to compute relevance bounds for
        X, _ = self.data  # TODO: handle other data formats
        d = X.shape[1]
        # Depending on the preset model, we dont need to compute all bounds
        # e.g. in the case of fixed features we skip those
        dims = _get_necessary_dimensions(d, presetModel)

        with joblib.Parallel(n_jobs=self.n_jobs, verbose=self.verbose) as parallel:
            relevance_bounds = self.compute_relevance_bounds(dims, parallel=parallel, presetModel=presetModel)
            probe_values_upper = self.compute_probe_values(dims, isUpper=True, parallel=parallel,
                                                           presetModel=presetModel)
            probe_values_lower = self.compute_probe_values(dims, isUpper=False, parallel=parallel,
                                                           presetModel=presetModel)

        # Postprocess bounds
        norm_bounds = _postprocessing(self.best_init_model.L1_factor, relevance_bounds)
        norm_probe_values_upper = _postprocessing(self.best_init_model.L1_factor, probe_values_upper)
        norm_probe_values_lower = _postprocessing(self.best_init_model.L1_factor, probe_values_lower)
        feature_classes = feature_classification(norm_probe_values_lower, norm_probe_values_upper, norm_bounds,
                                                 verbose=self.verbose)

        return norm_bounds, feature_classes

    def compute_relevance_bounds(self, dims, parallel=None, presetModel=None, solverargs=None):
        init_model_state = self.best_init_model.model_state

        work_queue = self._generate_relevance_bounds_tasks(dims, self.data, presetModel,
                                                           init_model_state)

        # Solve relevance bounds in parallel (when available)
        if parallel is None:
            parallel = joblib.Parallel(n_jobs=self.n_jobs, verbose=self.verbose)
        bound_results = parallel(map(joblib.delayed(_start_solver_worker), work_queue))

        # Retrieve results and aggregate values in dict
        solved_bounds = defaultdict(list)
        for finished_bound in bound_results:

            # Only add bounds with feasible solutions
            if finished_bound.is_solved:
                solved_bounds[finished_bound.current_feature].append(finished_bound)

        # Initalize array for pair of bounds(= intervals)
        length = len(dims)
        intervals = np.zeros((length, 2))
        for abs_index, rel_index in zip(dims, range(length)):
            # Return interval for feature i (can be a fixed value when set beforehand)
            interval_i = self._create_interval(abs_index, solved_bounds, presetModel)
            intervals[rel_index] = interval_i

        return intervals  # TODO: add model model_state (omega, bias) to return value

    def compute_probe_values(self, dims, isUpper=True, parallel=None, presetModel=None, max_loops=0):
        # Get model parameters
        init_model_state = self.best_init_model.model_state

        # Prepare parallel framework
        if parallel is None:
            parallel = joblib.Parallel(n_jobs=self.n_jobs, verbose=self.verbose)

        # Run loop until we got enough samples
        enough_samples = False
        i = 0
        probe_values = []
        while not enough_samples:
            # Generate
            probe_queue = self._generate_probe_value_tasks(dims, self.data,
                                                           self.n_resampling,
                                                           self.random_state, presetModel, init_model_state)
            # Compute solution
            probe_results = parallel(map(joblib.delayed(_start_solver_worker), probe_queue))
            # probe_values.extend([probe.objective.value for probe in probe_results if probe.is_solved])

            candidates = defaultdict(list)
            for candidate in probe_results:
                # Only add bounds with feasible solutions
                if candidate.is_solved:
                    candidates[candidate.probeID].append(candidate)
            for j in candidates.values():
                if isUpper:
                    probe_values.append(self.problem_type.get_cvxproblem_template.aggregate_max_candidates(j))
                else:
                    probe_values.append(self.problem_type.get_cvxproblem_template.aggregate_min_candidates(j))

            n_probes = len(probe_values)
            if self.n_resampling > MIN_N_PROBE_FEATURES > n_probes:
                print(f"isUpper:{isUpper}: Only {n_probes} probe features were feasible.")
            else:
                enough_samples = True
            if i >= max_loops:
                enough_samples = True
            i += 1

        return np.array(probe_values)

    def _generate_relevance_bounds_tasks(self, dims, data, preset_model=None,
                                         best_model_state=None):
        # Do not compute bounds for fixed features
        if preset_model is not None:
            dims = [di for di in dims if di not in preset_model]

        # Instantiate objects for computation later
        for di in dims:
            # Add Lower Bound problem(s) to work list
            yield from self.problem_type.get_cvxproblem_template.generate_lower_bound_problem(self.best_hyperparameters,
                                                                                self.init_constraints,
                                                                                best_model_state, data, di, preset_model)

            # Add problem(s) for Upper bound
            yield from self.problem_type.get_cvxproblem_template.generate_upper_bound_problem(self.best_hyperparameters,
                                                                                self.init_constraints,
                                                                                best_model_state, data, di, preset_model)

    def _generate_probe_value_tasks(self, dims, data, n_resampling, random_state,
                                    preset_model=None, best_model_state=None):
        # Random sample n_resampling shadow features by permuting real features and computing upper bound
        random_choice = random_state.choice(a=dims, size=n_resampling)

        # Instantiate objects
        for i, di in enumerate(random_choice):
            data_perm = permutate_feature_in_data(data, di, random_state)

            # We only use upper bounds as probe features
            yield from self.problem_type.get_cvxproblem_template.generate_upper_bound_problem(self.best_hyperparameters,
                                                                                self.init_constraints,
                                                                                best_model_state, data_perm, di, preset_model,
                                                                                probeID=i)

    def _create_interval(self, feature: int, solved_bounds: dict, presetModel: dict = None):
        # Return preset values for fixed features
        if presetModel is not None:
            if feature in presetModel:
                return presetModel[feature]

        all_bounds = solved_bounds[feature]
        min_problems_candidates = [p for p in all_bounds if p.isLowerBound]
        max_problems_candidates = [p for p in all_bounds if not p.isLowerBound]
        if len(all_bounds) < 2:
            logging.error(f"(Some) relevance bounds for feature {feature} were not solved.")
            raise Exception("Infeasible bound(s).")
        lower_bound = self.problem_type.get_cvxproblem_template.aggregate_min_candidates(min_problems_candidates)
        upper_bound = self.problem_type.get_cvxproblem_template.aggregate_max_candidates(max_problems_candidates)
        return lower_bound, upper_bound

    def compute_single_preset_relevance_bounds(self, i: int, signed_preset_i: [float, float]):
        """
        Method to run method once for one restricted feature
        Parameters
        ----------
        i:
            restricted feature
        signed_preset_i:
            restricted range of feature i (set before optimization = preset)

        """
        preset = {i: signed_preset_i}

        rangevector = self.compute_multi_preset_relevance_bounds(preset)

        return rangevector

    def compute_multi_preset_relevance_bounds(self, preset, normalized=True, lupi_features=0):
        """
        Method to run method with preset values

        Parameters
        ----------
        lupi_features
        """
        X, y = self.data

        # The user is working with normalized values while we compute them unscaled
        if normalized:
            for k, v in preset.items():
                preset[k] = np.asarray(v) * self.best_init_model.L1_factor

        # Add sign to presets
        preset = self._add_sign_to_preset(preset)

        # Calculate all bounds with feature i set to min_i
        if lupi_features > 0:
            rangevector, f_classes = self.get_normalized_lupi_intervals(lupi_features, presetModel=preset)
        else:
            rangevector, f_classes = self.get_normalized_intervals(presetModel=preset)

        return rangevector

    def _add_sign_to_preset(self, unsigned_presets):
        """
        We need signed presets for our convex problem definition later.
        We reuse the coefficients of the optimal model for this

        Parameters
        ----------
        unsigned_presets : dict

        Returns
        -------
        dict
        """

        signed_presets = {}
        # Obtain optimal model parameters
        w = self.best_init_model.model_state["w"]
        sum = 0
        for i, preset in unsigned_presets.items():
            unsigned_preset_i = np.sign(w[i]) * preset
            # accumulate maximal feature  contribution
            sum += unsigned_preset_i[1]
            signed_presets[i] = unsigned_preset_i

        # Check if unsigned_presets makes sense
        l1 = self.init_constraints["w_l1"]
        if sum > l1:
            print("maximum L1 norm of presets: ", sum)
            print("L1 allowed:", l1)
            print("Presets are not feasible. Try lowering values.")
            return

        return signed_presets

def _get_necessary_dimensions(d: int, presetModel: dict = None, start=0):
    dims = np.arange(start, d)

    # if presetModel is not None:
    #    # Exclude fixed (preset) dimensions from being redundantly computed
    #    dims = [di for di in dims if di not in presetModel.keys()]
    # TODO: check the removal of this block
    return dims

def _postprocessing(L1, rangevector, normalize=False, round_to_zero=True):
    if normalize:
        assert L1 > 0
        rangevector = rangevector.copy() / L1

    if round_to_zero:
        rangevector[rangevector <= 1e-11] = 0
    return rangevector

def feature_classification(probes_low, probes_up, relevance_bounds, fpr=1e-4, verbose=0):
    print("**** Feature Selection ****")

    lower_stat = create_probe_statistic(probes_low, fpr, verbose)
    upper_stat = create_probe_statistic(probes_up, fpr, verbose)

    weakly = np.logical_not(
        np.logical_and(upper_stat[0] <= relevance_bounds[:, 1], relevance_bounds[:, 1] <= upper_stat[1]))
    strongly = np.logical_not(
        np.logical_and(lower_stat[0] <= relevance_bounds[:, 0], relevance_bounds[:, 0] <= lower_stat[1]))
    both = np.logical_and(weakly, strongly)
    prediction = np.zeros(relevance_bounds.shape[0], dtype=np.int)
    prediction[weakly] = 1
    prediction[both] = 2

    return prediction

def create_probe_statistic(probe_values, fpr, verbose=0):
    # Create prediction interval statistics based on randomly permutated probe features (based on real features)
    n = len(probe_values)
    if n == 0:
        if verbose > 0:
            print("All probes were infeasible. All features considered relevant.")
        #    # If all probes were infeasible we expect an empty list
        #    # If they are infeasible it also means that only strongly relevant features were in the data
        #    # As such we just set the prediction without considering the statistics
        mean = 0
    else:
        probe_values = np.asarray(probe_values)
        mean = probe_values.mean()

    if mean == 0:
        lower_threshold, upper_threshold = mean, mean
    else:
        s = probe_values.std()
        lower_threshold = mean + stats.t(df=n - 1).ppf(fpr) * s * np.sqrt(1 + (1 / n))
        upper_threshold = mean - stats.t(df=n - 1).ppf(fpr) * s * np.sqrt(1 + (1 / n))

    if verbose > 0:
        print(f"Using {n} probe features")
        print(f"FS threshold: {lower_threshold}-{upper_threshold}, Mean:{mean}")

    return lower_threshold, upper_threshold
<commit_msg>PATCH ... adding real lower probe features<commit_after>import logging
from collections import defaultdict

import joblib
import numpy as np
from fri.model.base_cvxproblem import Relevance_CVXProblem
from fri.model.base_initmodel import InitModel
from fri.model.base_type import ProblemType
from fri.utils import permutate_feature_in_data
from scipy import stats

MIN_N_PROBE_FEATURES = 20  # Lower bound of probe features

def _start_solver_worker(bound: Relevance_CVXProblem):
    """
    Worker thread method for parallel computation
    """
    return bound.solve()

class RelevanceBoundsIntervals(object):
    def __init__(self, data, problem_type: ProblemType, best_init_model: InitModel, random_state, n_resampling, n_jobs,
                 verbose):
        self.data = data
        self.problem_type = problem_type
        self.verbose = verbose
        self.n_jobs = n_jobs
        self.n_resampling = n_resampling
        self.random_state = random_state
        self.best_init_model = best_init_model
        self.best_hyperparameters = best_init_model.hyperparam

        # Relax constraints to improve stability
        relaxed_constraints = problem_type.get_relaxed_constraints(best_init_model.constraints)
        self.init_constraints = relaxed_constraints

    def get_normalized_lupi_intervals(self, lupi_features, presetModel=None):

        # We define a list of all the features we want to compute relevance bounds for
        X, _ = self.data  # TODO: handle other data formats
        all_d = X.shape[1]
        normal_d = all_d - lupi_features

        # Compute relevance bounds and probes for normal features and LUPI
        with joblib.Parallel(n_jobs=self.n_jobs, verbose=self.verbose) as parallel:
            d_n = _get_necessary_dimensions(normal_d, presetModel)
            rb = self.compute_relevance_bounds(d_n, parallel=parallel)
            probe_upper = self.compute_probe_values(d_n, True, parallel=parallel)
            probe_lower = self.compute_probe_values(d_n, False, parallel=parallel)

            d_l = _get_necessary_dimensions(all_d, presetModel, start=normal_d)
            rb_l = self.compute_relevance_bounds(d_l, parallel=parallel)
            probe_priv_upper = self.compute_probe_values(d_l, True, parallel=parallel)
            probe_priv_lower = self.compute_probe_values(d_l, True, parallel=parallel)
        probes = [probe_lower, probe_upper, probe_priv_lower, probe_priv_upper]
        #
        # Postprocess
        #

        # Get Scaling Parameters
        l1 = self.init_constraints["w_l1"]
        l1_priv = self.init_constraints["w_priv_l1"]

        # Normalize Normal and Lupi features
        rb_norm = _postprocessing(l1, rb)
        rb_l_norm = _postprocessing(l1_priv, rb_l)
        interval_ = np.concatenate([rb_norm, rb_l_norm])

        # Normalize Probes
        for probe in probes:
            probe = _postprocessing(l1_priv, probe)
        # pr_norm = _postprocessing(l1, pr)
        # pr_l_norm = _postprocessing(l1_priv, pr_l)

        #
        #
        # Classify features
        fc = feature_classification(probe_lower, probe_upper, rb_norm, verbose=self.verbose)
        fc_l = feature_classification(probe_priv_lower, probe_priv_upper, rb_l_norm, verbose=self.verbose)
        fc_both = np.concatenate([fc, fc_l])

        return interval_, fc_both

    def get_normalized_intervals(self, presetModel=None):
        # We define a list of all the features we want to compute relevance bounds for
        X, _ = self.data  # TODO: handle other data formats
        d = X.shape[1]
        # Depending on the preset model, we dont need to compute all bounds
        # e.g. in the case of fixed features we skip those
        dims = _get_necessary_dimensions(d, presetModel)

        with joblib.Parallel(n_jobs=self.n_jobs, verbose=self.verbose) as parallel:
            relevance_bounds = self.compute_relevance_bounds(dims, parallel=parallel, presetModel=presetModel)
            probe_values_upper = self.compute_probe_values(dims, isUpper=True, parallel=parallel,
                                                           presetModel=presetModel)
            probe_values_lower = self.compute_probe_values(dims, isUpper=False, parallel=parallel,
                                                           presetModel=presetModel)

        # Postprocess bounds
        norm_bounds = _postprocessing(self.best_init_model.L1_factor, relevance_bounds)
        norm_probe_values_upper = _postprocessing(self.best_init_model.L1_factor, probe_values_upper)
        norm_probe_values_lower = _postprocessing(self.best_init_model.L1_factor, probe_values_lower)
        feature_classes = feature_classification(norm_probe_values_lower, norm_probe_values_upper, norm_bounds,
                                                 verbose=self.verbose)

        return norm_bounds, feature_classes

    def compute_relevance_bounds(self, dims, parallel=None, presetModel=None, solverargs=None):
        init_model_state = self.best_init_model.model_state

        work_queue = self._generate_relevance_bounds_tasks(dims, self.data, presetModel,
                                                           init_model_state)

        # Solve relevance bounds in parallel (when available)
        if parallel is None:
            parallel = joblib.Parallel(n_jobs=self.n_jobs, verbose=self.verbose)
        bound_results = parallel(map(joblib.delayed(_start_solver_worker), work_queue))

        # Retrieve results and aggregate values in dict
        solved_bounds = defaultdict(list)
        for finished_bound in bound_results:

            # Only add bounds with feasible solutions
            if finished_bound.is_solved:
                solved_bounds[finished_bound.current_feature].append(finished_bound)

        # Initalize array for pair of bounds(= intervals)
        length = len(dims)
        intervals = np.zeros((length, 2))
        for abs_index, rel_index in zip(dims, range(length)):
            # Return interval for feature i (can be a fixed value when set beforehand)
            interval_i = self._create_interval(abs_index, solved_bounds, presetModel)
            intervals[rel_index] = interval_i

        return intervals  # TODO: add model model_state (omega, bias) to return value

    def compute_probe_values(self, dims, isUpper=True, parallel=None, presetModel=None, max_loops=0):
        # Get model parameters
        init_model_state = self.best_init_model.model_state

        # Prepare parallel framework
        if parallel is None:
            parallel = joblib.Parallel(n_jobs=self.n_jobs, verbose=self.verbose)

        # Run loop until we got enough samples
        enough_samples = False
        i = 0
        probe_values = []
        while not enough_samples:
            # Generate
            probe_queue = self._generate_probe_value_tasks(self.data, dims, True, self.n_resampling, self.random_state,
                                                           presetModel, init_model_state)
            # Compute solution
            probe_results = parallel(map(joblib.delayed(_start_solver_worker), probe_queue))
            # probe_values.extend([probe.objective.value for probe in probe_results if probe.is_solved])

            candidates = defaultdict(list)
            for candidate in probe_results:
                # Only add bounds with feasible solutions
                if candidate.is_solved:
                    candidates[candidate.probeID].append(candidate)
            for j in candidates.values():
                if isUpper:
                    probe_values.append(self.problem_type.get_cvxproblem_template.aggregate_max_candidates(j))
                else:
                    probe_values.append(self.problem_type.get_cvxproblem_template.aggregate_min_candidates(j))

            n_probes = len(probe_values)
            if self.n_resampling > MIN_N_PROBE_FEATURES > n_probes:
                print(f"isUpper:{isUpper}: Only {n_probes} probe features were feasible.")
            else:
                enough_samples = True
            if i >= max_loops:
                enough_samples = True
            i += 1

        return np.array(probe_values)

    def _generate_relevance_bounds_tasks(self, dims, data, preset_model=None,
                                         best_model_state=None):
        # Do not compute bounds for fixed features
        if preset_model is not None:
            dims = [di for di in dims if di not in preset_model]

        # Instantiate objects for computation later
        for di in dims:
            # Add Lower Bound problem(s) to work list
            yield from self.problem_type.get_cvxproblem_template.generate_lower_bound_problem(self.best_hyperparameters,
                                                                                self.init_constraints,
                                                                                best_model_state, data, di, preset_model)

            # Add problem(s) for Upper bound
            yield from self.problem_type.get_cvxproblem_template.generate_upper_bound_problem(self.best_hyperparameters,
                                                                                self.init_constraints,
                                                                                best_model_state, data, di, preset_model)

    def _generate_probe_value_tasks(self, data, dims, isUpper, n_resampling, random_state, preset_model=None,
                                    best_model_state=None):
        if isUpper:
            factory = self.problem_type.get_cvxproblem_template.generate_upper_bound_problem
        else:
            factory = self.problem_type.get_cvxproblem_template.generate_lower_bound_problem

        # Random sample n_resampling shadow features by permuting real features and computing upper bound
        random_choice = random_state.choice(a=dims, size=n_resampling)

        # Instantiate objects
        for i, di in enumerate(random_choice):
            data_perm = permutate_feature_in_data(data, di, random_state)

            # We only use upper bounds as probe features
            yield from factory(self.best_hyperparameters, self.init_constraints,
                               best_model_state, data_perm, di, preset_model,
                               probeID=i)

    def _create_interval(self, feature: int, solved_bounds: dict, presetModel: dict = None):
        # Return preset values for fixed features
        if presetModel is not None:
            if feature in presetModel:
                return presetModel[feature]

        all_bounds = solved_bounds[feature]
        min_problems_candidates = [p for p in all_bounds if p.isLowerBound]
        max_problems_candidates = [p for p in all_bounds if not p.isLowerBound]
        if len(all_bounds) < 2:
            logging.error(f"(Some) relevance bounds for feature {feature} were not solved.")
            raise Exception("Infeasible bound(s).")
        lower_bound = self.problem_type.get_cvxproblem_template.aggregate_min_candidates(min_problems_candidates)
        upper_bound = self.problem_type.get_cvxproblem_template.aggregate_max_candidates(max_problems_candidates)
        return lower_bound, upper_bound

    def compute_single_preset_relevance_bounds(self, i: int, signed_preset_i: [float, float]):
        """
        Method to run method once for one restricted feature
        Parameters
        ----------
        i:
            restricted feature
        signed_preset_i:
            restricted range of feature i (set before optimization = preset)

        """
        preset = {i: signed_preset_i}

        rangevector = self.compute_multi_preset_relevance_bounds(preset)

        return rangevector

    def compute_multi_preset_relevance_bounds(self, preset, normalized=True, lupi_features=0):
        """
        Method to run method with preset values

        Parameters
        ----------
        lupi_features
        """
        X, y = self.data

        # The user is working with normalized values while we compute them unscaled
        if normalized:
            for k, v in preset.items():
                preset[k] = np.asarray(v) * self.best_init_model.L1_factor

        # Add sign to presets
        preset = self._add_sign_to_preset(preset)

        # Calculate all bounds with feature i set to min_i
        if lupi_features > 0:
            rangevector, f_classes = self.get_normalized_lupi_intervals(lupi_features, presetModel=preset)
        else:
            rangevector, f_classes = self.get_normalized_intervals(presetModel=preset)

        return rangevector

    def _add_sign_to_preset(self, unsigned_presets):
        """
        We need signed presets for our convex problem definition later.
        We reuse the coefficients of the optimal model for this

        Parameters
        ----------
        unsigned_presets : dict

        Returns
        -------
        dict
        """

        signed_presets = {}
        # Obtain optimal model parameters
        w = self.best_init_model.model_state["w"]
        sum = 0
        for i, preset in unsigned_presets.items():
            unsigned_preset_i = np.sign(w[i]) * preset
            # accumulate maximal feature  contribution
            sum += unsigned_preset_i[1]
            signed_presets[i] = unsigned_preset_i

        # Check if unsigned_presets makes sense
        l1 = self.init_constraints["w_l1"]
        if sum > l1:
            print("maximum L1 norm of presets: ", sum)
            print("L1 allowed:", l1)
            print("Presets are not feasible. Try lowering values.")
            return

        return signed_presets

def _get_necessary_dimensions(d: int, presetModel: dict = None, start=0):
    dims = np.arange(start, d)

    # if presetModel is not None:
    #    # Exclude fixed (preset) dimensions from being redundantly computed
    #    dims = [di for di in dims if di not in presetModel.keys()]
    # TODO: check the removal of this block
    return dims

def _postprocessing(L1, rangevector, normalize=False, round_to_zero=True):
    if normalize:
        assert L1 > 0
        rangevector = rangevector.copy() / L1

    if round_to_zero:
        rangevector[rangevector <= 1e-11] = 0
    return rangevector

def feature_classification(probes_low, probes_up, relevance_bounds, fpr=1e-4, verbose=0):
    print("**** Feature Selection ****")

    lower_stat = create_probe_statistic(probes_low, fpr, verbose)
    upper_stat = create_probe_statistic(probes_up, fpr, verbose)

    weakly = np.logical_not(
        np.logical_and(upper_stat[0] <= relevance_bounds[:, 1], relevance_bounds[:, 1] <= upper_stat[1]))
    strongly = np.logical_not(
        np.logical_and(lower_stat[0] <= relevance_bounds[:, 0], relevance_bounds[:, 0] <= lower_stat[1]))
    both = np.logical_and(weakly, strongly)
    prediction = np.zeros(relevance_bounds.shape[0], dtype=np.int)
    prediction[weakly] = 1
    prediction[both] = 2

    return prediction

def create_probe_statistic(probe_values, fpr, verbose=0):
    # Create prediction interval statistics based on randomly permutated probe features (based on real features)
    n = len(probe_values)
    if n == 0:
        if verbose > 0:
            print("All probes were infeasible. All features considered relevant.")
        #    # If all probes were infeasible we expect an empty list
        #    # If they are infeasible it also means that only strongly relevant features were in the data
        #    # As such we just set the prediction without considering the statistics
        mean = 0
    else:
        probe_values = np.asarray(probe_values)
        mean = probe_values.mean()

    if mean == 0:
        lower_threshold, upper_threshold = mean, mean
    else:
        s = probe_values.std()
        lower_threshold = mean + stats.t(df=n - 1).ppf(fpr) * s * np.sqrt(1 + (1 / n))
        upper_threshold = mean - stats.t(df=n - 1).ppf(fpr) * s * np.sqrt(1 + (1 / n))

    if verbose > 0:
        print(f"Using {n} probe features")
        print(f"FS threshold: {lower_threshold}-{upper_threshold}, Mean:{mean}")

    return lower_threshold, upper_threshold
<|endoftext|>


<commit_before>#! /usr/bin/env python3
"""Minimal macro processor.  Used for generating VC++ makefiles.

The available template commands are:

    Expand a template section for each file in a list of file patterns::
        ###MAKTEMPLATE:FOREACH my/path*/*.cxx,other*.cxx
        ...
        ###MAKTEMPLATE:ENDFOREACH

    In the template section, you can use `###BASENAME###` to get the base name
    of the file being processed (e.g. "base" for "../base.cxx"), and you can
    use `###FILENAME###` to get the full filename.

Copyright (c) 2006-2021, Bart Samwel and Jeroen T. Vermeulen.
"""

from __future__ import (
    absolute_import,
    print_function,
    unicode_literals,
    )

from argparse import (
    ArgumentError,
    ArgumentParser,
    RawDescriptionHelpFormatter,
    )
from contextlib import contextmanager
from glob import glob
import os
from sys import (
    argv,
    stdin,
    stderr,
    stdout,
    )
import sys
from textwrap import dedent

def expand_foreach_file(path, block, outfile):
    """Expand a "foreach" block for a single file path.

    Write the results to outfile.
    """
    basepath, _ = os.path.splitext(os.path.basename(path))
    for line in block:
        line = line.replace("###FILENAME###", path)
        line = line.replace("###BASENAME###", basepath)
        outfile.write(line)

def match_globs(globs):
    """List all files matching any item in globs.

    Eliminates duplicates.
    """
    return sorted({
        path
        for pattern in globs
        for path in glob(pattern)
        })

def expand_foreach(globs, block, outfile):
    """Expand a foreach block for each file matching one of globs.

    Write the results to outfile.
    """
    # We'll be iterating over block a variable number of times.  Turn it
    # from a generic iterable into an immutable array.
    block = tuple(block)
    for path in match_globs(globs):
        expand_foreach_file(path, block, outfile)

# Header to be prefixed to the generated file.
OUTPUT_HEADER = dedent("""\
    # AUTOMATICALLY GENERATED FILE -- DO NOT EDIT.
    #
    # This file is generated automatically by libpqxx's {script} script, and
    # will be rewritten from time to time.
    #
    # If you modify this file, chances are your modifications will be lost.
    #
    # The {script} script should be available in the tools directory of the
    # libpqxx source archive.
    """)

foreach_marker = r"###MAKTEMPLATE:FOREACH "
end_foreach_marker = r"###MAKTEMPLATE:ENDFOREACH"

def parse_foreach(line):
    """Parse FOREACH directive, if line contains one.

    :param line: One line of template input.
    :return: A list of FOREACH globs, or None if this was not a FOREACH line.
    """
    line = line.strip()
    if line.startswith(foreach_marker):
        return line[len(foreach_marker):].split(',')
    else:
        return None

def read_foreach_block(infile):
    """Read a FOREACH block from infile (not including the FOREACH directive).

    Assumes that the FOREACH directive was in the preceding line.  Consumes
    the line with the ENDFOREACH directive, but does not yield it.

    :return: Iterable of lines.
    """
    for line in infile:
        if line.strip().startswith(end_foreach_marker):
            return
        yield line

def expand_template(infile, outfile):
    """Expand the template in infile, and write the results to outfile."""
    for line in infile:
        globs = parse_foreach(line)
        if globs is None:
            # Not a FOREACH line.  Copy to output.
            outfile.write(line)
        else:
            block = read_foreach_block(infile)
            expand_foreach(globs, block, outfile)

@contextmanager
def open_stream(path=None, default=None, mode='r'):
    """Open file at given path, or yield default.  Close as appropriate.

    The default should be a stream, not a path; closing the context will not
    close it.
    """
    if path is None:
        yield default
    else:
        with open(path, mode) as stream:
            yield stream

def parse_args():
    """Parse command-line arguments.

    :return: Tuple of: input path (or None for stdin), output path (or None
        for stdout).
    """
    parser = ArgumentParser(
        description=__doc__, formatter_class=RawDescriptionHelpFormatter)

    parser.add_argument(
        'template', nargs='?',
        help="Input template.  Defaults to standard input.")
    parser.add_argument(
        'output', nargs='?',
        help="Output file.  Defaults to standard output.")

    args = parser.parse_args()
    return args.template, args.output

def write_header(stream, template_path=None):
    """Write header to stream."""
    hr = ('# ' + '#' * 78) + "\n"
    script = os.path.basename(argv[0])

    outstream.write(hr)
    outstream.write(OUTPUT_HEADER.format(script=script))
    if template_path is not None:
        outstream.write("#\n")
        outstream.write("# Generated from template '%s'.\n" % template_path)
    outstream.write(hr)

if __name__ == '__main__':
    try:
        template_path, output_path = parse_args()
    except ArgumentError as error:
        stderr.write('%s\n' % error)
        sys.exit(2)

    input_stream = open_stream(template_path, stdin, 'r')
    output_stream = open_stream(output_path, stdout, 'w')
    with input_stream as instream, output_stream as outstream:
        write_header(outstream, template_path)
        expand_template(instream, outstream)
<commit_msg>Copyright year is global.<commit_after>#! /usr/bin/env python3
"""Minimal macro processor.  Used for generating VC++ makefiles.

The available template commands are:

    Expand a template section for each file in a list of file patterns::
        ###MAKTEMPLATE:FOREACH my/path*/*.cxx,other*.cxx
        ...
        ###MAKTEMPLATE:ENDFOREACH

    In the template section, you can use `###BASENAME###` to get the base name
    of the file being processed (e.g. "base" for "../base.cxx"), and you can
    use `###FILENAME###` to get the full filename.

Copyright (c) 2000-2021, Bart Samwel and Jeroen T. Vermeulen.
"""

from __future__ import (
    absolute_import,
    print_function,
    unicode_literals,
    )

from argparse import (
    ArgumentError,
    ArgumentParser,
    RawDescriptionHelpFormatter,
    )
from contextlib import contextmanager
from glob import glob
import os
from sys import (
    argv,
    stdin,
    stderr,
    stdout,
    )
import sys
from textwrap import dedent

def expand_foreach_file(path, block, outfile):
    """Expand a "foreach" block for a single file path.

    Write the results to outfile.
    """
    basepath, _ = os.path.splitext(os.path.basename(path))
    for line in block:
        line = line.replace("###FILENAME###", path)
        line = line.replace("###BASENAME###", basepath)
        outfile.write(line)

def match_globs(globs):
    """List all files matching any item in globs.

    Eliminates duplicates.
    """
    return sorted({
        path
        for pattern in globs
        for path in glob(pattern)
        })

def expand_foreach(globs, block, outfile):
    """Expand a foreach block for each file matching one of globs.

    Write the results to outfile.
    """
    # We'll be iterating over block a variable number of times.  Turn it
    # from a generic iterable into an immutable array.
    block = tuple(block)
    for path in match_globs(globs):
        expand_foreach_file(path, block, outfile)

# Header to be prefixed to the generated file.
OUTPUT_HEADER = dedent("""\
    # AUTOMATICALLY GENERATED FILE -- DO NOT EDIT.
    #
    # This file is generated automatically by libpqxx's {script} script, and
    # will be rewritten from time to time.
    #
    # If you modify this file, chances are your modifications will be lost.
    #
    # The {script} script should be available in the tools directory of the
    # libpqxx source archive.
    """)

foreach_marker = r"###MAKTEMPLATE:FOREACH "
end_foreach_marker = r"###MAKTEMPLATE:ENDFOREACH"

def parse_foreach(line):
    """Parse FOREACH directive, if line contains one.

    :param line: One line of template input.
    :return: A list of FOREACH globs, or None if this was not a FOREACH line.
    """
    line = line.strip()
    if line.startswith(foreach_marker):
        return line[len(foreach_marker):].split(',')
    else:
        return None

def read_foreach_block(infile):
    """Read a FOREACH block from infile (not including the FOREACH directive).

    Assumes that the FOREACH directive was in the preceding line.  Consumes
    the line with the ENDFOREACH directive, but does not yield it.

    :return: Iterable of lines.
    """
    for line in infile:
        if line.strip().startswith(end_foreach_marker):
            return
        yield line

def expand_template(infile, outfile):
    """Expand the template in infile, and write the results to outfile."""
    for line in infile:
        globs = parse_foreach(line)
        if globs is None:
            # Not a FOREACH line.  Copy to output.
            outfile.write(line)
        else:
            block = read_foreach_block(infile)
            expand_foreach(globs, block, outfile)

@contextmanager
def open_stream(path=None, default=None, mode='r'):
    """Open file at given path, or yield default.  Close as appropriate.

    The default should be a stream, not a path; closing the context will not
    close it.
    """
    if path is None:
        yield default
    else:
        with open(path, mode) as stream:
            yield stream

def parse_args():
    """Parse command-line arguments.

    :return: Tuple of: input path (or None for stdin), output path (or None
        for stdout).
    """
    parser = ArgumentParser(
        description=__doc__, formatter_class=RawDescriptionHelpFormatter)

    parser.add_argument(
        'template', nargs='?',
        help="Input template.  Defaults to standard input.")
    parser.add_argument(
        'output', nargs='?',
        help="Output file.  Defaults to standard output.")

    args = parser.parse_args()
    return args.template, args.output

def write_header(stream, template_path=None):
    """Write header to stream."""
    hr = ('# ' + '#' * 78) + "\n"
    script = os.path.basename(argv[0])

    outstream.write(hr)
    outstream.write(OUTPUT_HEADER.format(script=script))
    if template_path is not None:
        outstream.write("#\n")
        outstream.write("# Generated from template '%s'.\n" % template_path)
    outstream.write(hr)

if __name__ == '__main__':
    try:
        template_path, output_path = parse_args()
    except ArgumentError as error:
        stderr.write('%s\n' % error)
        sys.exit(2)

    input_stream = open_stream(template_path, stdin, 'r')
    output_stream = open_stream(output_path, stdout, 'w')
    with input_stream as instream, output_stream as outstream:
        write_header(outstream, template_path)
        expand_template(instream, outstream)
<|endoftext|>


<commit_before>import json
from ..models import (User, UserSchema, Project, Like, ProjectSchema,
                      Comment, CommentSchema, Category, CategorySchema,
                      Group, GroupSchema, LikeSchema, LogSchema)
from flask import Blueprint, jsonify, request, abort, url_for
from ..extensions import db
from .toolshed import require_login, current_user
from datetime import datetime
from ..importer import create_project
from ..updater import update_projects

api = Blueprint('api', __name__)

# Schemas

all_users_schema = UserSchema(many=True)
single_user_schema = UserSchema()
all_projects_schema = ProjectSchema(many=True, exclude=("logs",))
all_projects_with_logs = ProjectSchema(many=True)
single_project_schema = ProjectSchema()
single_comment_schema = CommentSchema()
all_comments_schema = CommentSchema(many=True)
single_category_schema = CategorySchema()
all_categories_schema = CategorySchema(many=True)
single_group_schema = GroupSchema()
all_groups_schema = GroupSchema(many=True)
single_like_schema = LikeSchema()
all_likes_schema = LikeSchema(many=True)
all_logs_schema = LogSchema(many=True)
single_log_schema = LogSchema()

# response functions

def success_response(schema, data):
    results = schema.dump(data)
    return jsonify({"status": "success", "data": results.data})

def failure_response(reason, code):
    return jsonify({"status": "fail", "data": {"title": reason}}), code

# User routes

@api.route('/user')
def get_user():
    name = current_user()
    user = User.query.filter_by(github_name=name).first()
    if user:
        return success_response(single_user_schema, user)
    else:
        return failure_response("User not logged in", 401)

@api.route("/users")
def users():
    users = User.query.all()
    if users:
        return success_response(all_users_schema, users)
    else:
        return failure_response("There are no users.", 404)

@api.route("/users/<int:id>")
def user(id):
    user = User.query.get(id)
    if user:
        return success_response(single_user_schema, user)
    else:
        return failure_response("There was no such user.", 404)

@api.route("/users/<int:id>/pending_submissions")
def get_pending_submissions(id):
    user = User.query.get(id)
    if user.submissions:
        pending = Project.query.filter_by(submitted_by_id=user.id).filter_by(status=False).all()
        return success_response(all_projects_schema, pending)
    else:
        return failure_response("No pending submissions.", 404)

@api.route("/users/<int:id>/submissions")
def get_submissions(id):
    user = User.query.get(id)
    if user.submissions:
        submissions = Project.query.filter_by(submitted_by_id=user.id).filter_by(status=True).all()
        return success_response(all_projects_schema, submissions)
    else:
        return failure_response("No submissions.", 404)

# project routes

@api.route("/projects")
def projects():
    projects = Project.query.order_by(Project.name)
    if projects:
        return success_response(all_projects_schema, projects)
    else:
        return failure_response("There are no projects.", 404)

@api.route("/projects/newest")
def newest_projects():
    projects = Project.query.order_by(Project.date_added)
    if projects:
        return success_response(all_projects_schema, projects)
    else:
        return failure_response("There are no projects.", 404)

@api.route("/projects/popular")
def popular_projects():
    projects = Project.query.order_by(Project.score)
    if projects:
        return success_response(all_projects_schema, projects)

@api.route("/projects/<int:id>")
def project(id):
    project = Project.query.get(id)
    if project:
        return success_response(single_project_schema, project)
    else:
        return failure_response("There was no such project.", 404)

@api.route("/projects", methods=["POST"])
def make_project():
    urls = request.get_json()
    project = create_project(**urls)
    if not project:
        return failure_response("This project already exsists.", 409)
    user_name = current_user()
    user = User.query.filter_by(github_name=user_name).first()
    project.submitted_by_id = user.id
    user.submissions.append(project)
    db.session.add(project)
    db.session.commit()
    return success_response(single_project_schema, project)

# Logs routes

@api.route("/projects/logs")
def projects_logs():
    projects = Project.query.all()
    if projects:
        return success_response(all_projects_with_logs, projects)
    else:
        return failure_response("There are no projects", 404)

@api.route("/projects/<int:id>/logs")
def project_logs(id):
    project = Project.query.get(id)
    if project:
        return success_response(single_project_schema, project)
    else:
        return failure_response("There was no such project.", 404)

# Category routes

@api.route("/groups")
def all_groups():
    groups = Group.query.all()
    if groups:
        return success_response(all_groups_schema, groups)
    else:
        return failure_response("There are no groups.", 404)

@api.route("/groups/<int:id>/projects")
def group_projects(id):
    group = Group.query.get(id)
    if group.projects:
        return success_response(single_group_schema, group.projects)
    else:
        return failure_response("There is no such group.", 404)

# Group routes

@api.route("/categories")
def all_categories():
    categories = Category.query.all()
    if categories:
        return success_response(all_categories_schema, categories)
    else:
        return failure_response("There are no categories.", 404)

@api.route("/categories/<int:id>/groups")
def group_categories(id):
    category = Category.query.get(id)
    if category.groups:
        return success_response(single_category_schema, category.groups)
    else:
        return failure_response("There is no such category.", 404)

# Comment routes

@api.route("/users/<int:id>/comments")
def user_comments(id):
    user = User.query.get_or_404(id)
    if user.comments:
        return success_response(all_comments_schema, user.comments)
    else:
        return failure_response("This user has no comments.", 404)

@api.route("/projects/<int:id>/comments")
def project_comments(id):
    project = Project.query.get(id)
    if project.comments:
        return success_response(all_comments_schema, project.comments)
    else:
        return failure_response("This project has no comments.", 404)

@api.route("/projects/<int:id>/comments", methods=["POST"])
@require_login
def add_project_comment(id):
    user_name = current_user()
    user = User.query.filter_by(github_name=user_name).first()
    comment_data = request.get_json()
    project = Project.query.get(id)
    if project:
        comment = Comment(text=comment_data['text'],
                          created=datetime.utcnow(),
                          project_id=project.id,
                          user_id=user.id)

        user.comments.append(comment)
        db.session.add(comment)
        db.session.commit()
        return success_response(single_comment_schema, comment)
    else:
        return failure_response("There was no such project", 404)

@api.route("/comments/<int:id>", methods=["PUT"])
def edit_comment(id):
    user_name = current_user()
    user = User.query.filter_by(github_name=user_name).first()
    comment = Comment.query.get_or_404(id)
    comment_data = request.get_json()
    if comment.user_id != user.id:
        return failure_response("You are not authorized to edit this comment.")
    comment.text = comment_data['text']
    db.session.commit()
    return success_response(single_comment_schema, comment)

@api.route("/comments/<int:id>", methods=["DELETE"])
def delete_comment(id):
    comment = Comment.query.get_or_404(id)
    db.session.delete(comment)
    db.session.commit()
    return success_response(single_comment_schema, comment)

# Like Routes

@api.route("/likes/projects/<int:id>", methods=["POST"])
def like_project(id):
    project = Project.query.get_or_404(id)
    user_name = current_user()
    user = User.query.filter_by(github_name=user_name).first()
    new_like = Like(user_id=user.id,
                     project_id=project.id)
    user.like.append(new_like)
    db.session.add(new_like)
    db.session.commit()
    return success_response(single_like_schema, new_like)

@api.route("/likes/<int:id>", methods=["DELETE"])
def unlike_project(id):
    like = Like.query.get_or_404(id)
    db.session.delete(like)
    db.session.commit()
    return success_response(single_like_schema, like)

@api.route("/users/<int:id>/likes")
def get_user_likes(id):
    user = User.query.get_or_404(id)
    if user.likes:
        return success_response(all_likes_schema, user.likes)
    else:
        return failure_response("User has no likes", 404)

@api.route("/projects/<int:id>/likes")
def get_project_likes(id):
    project = Project.query.get_or_404(id)
    if project.user_likes:
        return success_response(all_likes_schema, project.user_likes)
    else:
        return failure_response("Project has no likes.", 404)

@api.route("/projects/update")
def update_call():
    projects = Project.query.all()
    update_projects(projects)
    return success_response()

@api.route("/projects/dump", methods=["POST"])
def projects_seed():
    urls_json = request.get_json()
<commit_msg>fixed api endpoints for categories and groups<commit_after>import json
from ..models import (User, UserSchema, Project, Like, ProjectSchema,
                      Comment, CommentSchema, Category, CategorySchema,
                      Group, GroupSchema, LikeSchema, LogSchema)
from flask import Blueprint, jsonify, request, abort, url_for
from ..extensions import db
from .toolshed import require_login, current_user
from datetime import datetime
from ..importer import create_project
from ..updater import update_projects

api = Blueprint('api', __name__)

# Schemas

all_users_schema = UserSchema(many=True)
single_user_schema = UserSchema()
all_projects_schema = ProjectSchema(many=True, exclude=("logs",))
all_projects_with_logs = ProjectSchema(many=True)
single_project_schema = ProjectSchema()
single_comment_schema = CommentSchema()
all_comments_schema = CommentSchema(many=True)
single_category_schema = CategorySchema()
all_categories_schema = CategorySchema(many=True)
single_group_schema = GroupSchema()
all_groups_schema = GroupSchema(many=True)
single_like_schema = LikeSchema()
all_likes_schema = LikeSchema(many=True)
all_logs_schema = LogSchema(many=True)
single_log_schema = LogSchema()

# response functions

def success_response(schema, data):
    results = schema.dump(data)
    return jsonify({"status": "success", "data": results.data})

def failure_response(reason, code):
    return jsonify({"status": "fail", "data": {"title": reason}}), code

# User routes

@api.route('/user')
def get_user():
    name = current_user()
    user = User.query.filter_by(github_name=name).first()
    if user:
        return success_response(single_user_schema, user)
    else:
        return failure_response("User not logged in", 401)

@api.route("/users")
def users():
    users = User.query.all()
    if users:
        return success_response(all_users_schema, users)
    else:
        return failure_response("There are no users.", 404)

@api.route("/users/<int:id>")
def user(id):
    user = User.query.get(id)
    if user:
        return success_response(single_user_schema, user)
    else:
        return failure_response("There was no such user.", 404)

@api.route("/users/<int:id>/pending_submissions")
def get_pending_submissions(id):
    user = User.query.get(id)
    if user.submissions:
        pending = Project.query.filter_by(submitted_by_id=user.id).filter_by(status=False).all()
        return success_response(all_projects_schema, pending)
    else:
        return failure_response("No pending submissions.", 404)

@api.route("/users/<int:id>/submissions")
def get_submissions(id):
    user = User.query.get(id)
    if user.submissions:
        submissions = Project.query.filter_by(submitted_by_id=user.id).filter_by(status=True).all()
        return success_response(all_projects_schema, submissions)
    else:
        return failure_response("No submissions.", 404)

# project routes

@api.route("/projects")
def projects():
    projects = Project.query.order_by(Project.name)
    if projects:
        return success_response(all_projects_schema, projects)
    else:
        return failure_response("There are no projects.", 404)

@api.route("/projects/newest")
def newest_projects():
    projects = Project.query.order_by(Project.date_added)
    if projects:
        return success_response(all_projects_schema, projects)
    else:
        return failure_response("There are no projects.", 404)

@api.route("/projects/popular")
def popular_projects():
    projects = Project.query.order_by(Project.score)
    if projects:
        return success_response(all_projects_schema, projects)

@api.route("/projects/<int:id>")
def project(id):
    project = Project.query.get(id)
    if project:
        return success_response(single_project_schema, project)
    else:
        return failure_response("There was no such project.", 404)

@api.route("/projects", methods=["POST"])
def make_project():
    urls = request.get_json()
    project = create_project(**urls)
    if not project:
        return failure_response("This project already exsists.", 409)
    user_name = current_user()
    user = User.query.filter_by(github_name=user_name).first()
    project.submitted_by_id = user.id
    user.submissions.append(project)
    db.session.add(project)
    db.session.commit()
    return success_response(single_project_schema, project)

# Logs routes

@api.route("/projects/logs")
def projects_logs():
    projects = Project.query.all()
    if projects:
        return success_response(all_projects_with_logs, projects)
    else:
        return failure_response("There are no projects", 404)

@api.route("/projects/<int:id>/logs")
def project_logs(id):
    project = Project.query.get(id)
    if project:
        return success_response(single_project_schema, project)
    else:
        return failure_response("There was no such project.", 404)

# Group routes

@api.route("/groups")
def all_groups():
    groups = Group.query.all()
    if groups:
        return success_response(all_groups_schema, groups)
    else:
        return failure_response("There are no groups.", 404)

@api.route("/groups/<int:id>")
def group_projects(id):
    group = Group.query.get(id)
    if group.projects:
        return success_response(single_group_schema, group)
    else:
        return failure_response("There is no such group.", 404)

# Category routes

@api.route("/categories")
def all_categories():
    categories = Category.query.all()
    if categories:
        return success_response(all_categories_schema, categories)
    else:
        return failure_response("There are no categories.", 404)

@api.route("/categories/<int:id>")
def group_categories(id):
    category = Category.query.get(id)
    if category:
        return success_response(single_category_schema, category)
    else:
        return failure_response("There is no such category.", 404)

# Comment routes

@api.route("/users/<int:id>/comments")
def user_comments(id):
    user = User.query.get_or_404(id)
    if user.comments:
        return success_response(all_comments_schema, user.comments)
    else:
        return failure_response("This user has no comments.", 404)

@api.route("/projects/<int:id>/comments")
def project_comments(id):
    project = Project.query.get(id)
    if project.comments:
        return success_response(all_comments_schema, project.comments)
    else:
        return failure_response("This project has no comments.", 404)

@api.route("/projects/<int:id>/comments", methods=["POST"])
@require_login
def add_project_comment(id):
    user_name = current_user()
    user = User.query.filter_by(github_name=user_name).first()
    comment_data = request.get_json()
    project = Project.query.get(id)
    if project:
        comment = Comment(text=comment_data['text'],
                          created=datetime.utcnow(),
                          project_id=project.id,
                          user_id=user.id)

        user.comments.append(comment)
        db.session.add(comment)
        db.session.commit()
        return success_response(single_comment_schema, comment)
    else:
        return failure_response("There was no such project", 404)

@api.route("/comments/<int:id>", methods=["PUT"])
def edit_comment(id):
    user_name = current_user()
    user = User.query.filter_by(github_name=user_name).first()
    comment = Comment.query.get_or_404(id)
    comment_data = request.get_json()
    if comment.user_id != user.id:
        return failure_response("You are not authorized to edit this comment.")
    comment.text = comment_data['text']
    db.session.commit()
    return success_response(single_comment_schema, comment)

@api.route("/comments/<int:id>", methods=["DELETE"])
def delete_comment(id):
    comment = Comment.query.get_or_404(id)
    db.session.delete(comment)
    db.session.commit()
    return success_response(single_comment_schema, comment)

# Like Routes

@api.route("/likes/projects/<int:id>", methods=["POST"])
def like_project(id):
    project = Project.query.get_or_404(id)
    user_name = current_user()
    user = User.query.filter_by(github_name=user_name).first()
    new_like = Like(user_id=user.id,
                     project_id=project.id)
    user.like.append(new_like)
    db.session.add(new_like)
    db.session.commit()
    return success_response(single_like_schema, new_like)

@api.route("/likes/<int:id>", methods=["DELETE"])
def unlike_project(id):
    like = Like.query.get_or_404(id)
    db.session.delete(like)
    db.session.commit()
    return success_response(single_like_schema, like)

@api.route("/users/<int:id>/likes")
def get_user_likes(id):
    user = User.query.get_or_404(id)
    if user.likes:
        return success_response(all_likes_schema, user.likes)
    else:
        return failure_response("User has no likes", 404)

@api.route("/projects/<int:id>/likes")
def get_project_likes(id):
    project = Project.query.get_or_404(id)
    if project.user_likes:
        return success_response(all_likes_schema, project.user_likes)
    else:
        return failure_response("Project has no likes.", 404)

@api.route("/projects/update")
def update_call():
    projects = Project.query.all()
    update_projects(projects)
    return success_response()

@api.route("/projects/dump", methods=["POST"])
def projects_seed():
    urls_json = request.get_json()
<|endoftext|>


<commit_before>import json
import hashlib
import logging
import mailbox
import os
import random
import string
import tarfile
import time
from datetime import datetime
from email.message import Message
from shutil import rmtree

from celery import task, chain, group, chord
from pytz import utc

from django.conf import settings
from django.db import transaction

from inboxen.models import Domain, Email, Inbox, PartList, Tag, User

log = logging.getLogger(__name__)

for setting_name in ('LIBERATION_BODY', 'LIBERATION_SUBJECT', 'LIBERATION_PATH'):
    assert hasattr(settings, setting_name), "%s has not been set" % setting_name

TAR_TYPES = {
    'tar.gz': {'writer': 'w:gz', 'mime-type': 'application/x-gzip'},
    'tar.bz2': {'writer': 'w:bz2', 'mime-type': 'application/x-bzip2'},
    'tar': {'writer': 'w:', 'mime-type': 'application/x-tar'}
    }

@task(rate_limit='4/h')
def liberate(user, options=None):
    """ Get set for liberation, expects User object """

    if options == None:
        options = {}

    options['user'] = user.id

    rstr = ""
    for i in range(7):
        rstr += string.ascii_letters[random.randint(0, 50)]
    mail_path = "%s/%s_%s_%s_%s" % (settings.LIBERATION_PATH, time.time(), os.getpid(), rstr, hashlib.sha256(user.username + rstr).hexdigest()[:50])

    # make maildir
    mailbox.Maildir(mail_path)

    tasks = chord(
                [liberate_inbox.s(mail_path, inbox.id) for inbox in Inbox.objects.filter(user=user, deleted=False).only('id')],
                liberate_collect_emails.s(mail_path, options)
                )
    tasks.apply_async()

@task(rate_limit='10/m')
def liberate_inbox(mail_path, inbox_id):
    """ Gather email IDs """

    inbox = Inbox.objects.get(id=inbox_id, deleted=False)
    maildir = mailbox.Maildir(mail_path)
    maildir.add_folder(str(inbox))

    return {
            'folder': str(inbox),
            'ids': [email.id for email in Email.objects.filter(inbox=inbox, deleted=False).only('id')]
            }

@task()
def liberate_collect_emails(results, mail_path, options):
    """ Send off data mining tasks """

    msg_tasks = []
    for result in results:
        inbox = [liberate_message.s(mail_path, result['folder'], email_id) for email_id in result['ids']]
        msg_tasks.extend(inbox)
    if len(msg_tasks) > 0:
        msg_tasks = chain(
                        group(msg_tasks),
                        liberate_convert_box.s(mail_path, options),
                        liberate_tarball.s(mail_path, options),
                        liberation_finish.s(options)
                        )
    else:
        options["noEmails"] = True
        msg_tasks = liberation_finish.s(options)

    msg_tasks.apply_async()

def make_message(message):
    """ Make a Python  email.message.Message from our models """
    parents = {}
    part_list = message.parts.all()
    first = None

    for part in part_list:
        msg = Message()

        header_set = part.header_set.order_by("ordinal").select_related("name__name", "data__data")
        for header in header_set:
            msg[header.name.name] = header.data.data

        if msg.is_multipart():
            parents[part.id] = msg
        else:
            msg.set_payload(part.body.data)

        if first is None:
            first = msg
        else:
            parents[part.parent_id].attach(msg)

    return first

@task(rate_limit='1000/m')
@transaction.atomic()
def liberate_message(mail_path, inbox, email_id):
    """ Take email from database and put on filesystem """
    maildir = mailbox.Maildir(mail_path).get_folder(inbox)

    try:
        msg = Email.objects.get(id=email_id, deleted=False)
        msg = make_message(msg)
    except Exception, exc:
        msg_id = hex(int(email_id))[2:]
        log.debug("Exception processing %s", msg_id, exc_info=exc)
        raise Exception(msg_id)

    maildir.add(msg)

@task()
def liberate_convert_box(result, mail_path, options):
    """ Convert maildir to mbox if needed """
    if options['storage_type'] == 'maildir':
        pass

    elif options['storage_type'] == 'mailbox':
        maildir = mailbox.Maildir(mail_path)
        mbox = mailbox.mbox(mail_path + '.mbox')
        mbox.lock()

        for inbox in maildir.list_folders():
            folder = maildir.get_folder(inbox)

            for key in folder.iterkeys():
                msg = str(folder.pop(key))
                mbox.add(msg)
            maildir.remove_folder(inbox)

        rmtree(mail_path)
        mbox.close()

    return result

@task(default_retry_delay=600)
def liberate_tarball(result, mail_path, options):
    """ Tar up and delete the maildir """

    tar_type = TAR_TYPES[options.get('compression_type', 'tar.gz')]
    tar_name = "%s.%s" % (mail_path, options.get('compression_type', 'tar.gz'))

    try:
        tar = tarfile.open(tar_name, tar_type['writer'])
    except (IOError, OSError), error:
        log.debug("Couldn't open tarfile at %s", tar_name)
        raise liberate_tarball.retry(exc=error)

    date = str(datetime.now(utc).date())
    dir_name = "inboxen-%s" % date

    if options['storage_type'] == 'maildir':
        try:
            tar.add("%s/" % mail_path, dir_name) # directories are added recursively by default
        finally:
            tar.close()
        rmtree(mail_path)

    elif options['storage_type'] == 'mailbox':
        try:
            tar.add("%s.mbox" % mail_path, dir_name)
        finally:
            tar.close()
        os.remove("%s.mbox" % mail_path)

    return {'path': tar_name, 'mime-type': tar_type['mime-type'], 'date': date, 'results': result}

@task()
@transaction.atomic()
def liberation_finish(result, options):
    """ Create email to send to user """
    tags = liberate_inbox_tags(options['user'], result['date'])
    profile = liberate_user_profile(options['user'], result['results'], result['date'])

    inbox_tags = ["Inboxen", "data", "liberation"]
    inbox = Inbox.objects
    for tag in inbox_tags:
        inbox = inbox.filter(tag__tag=tag)

    try:
        inbox = inbox.get(user__id=options["user"])
    except Inbox.MultipleObjectsReturned:
        inbox = inbox.filter(user__id=options["user"])[0]
    except Inbox.DoesNotExist:
        domain = random.choice(Domain.objects.all())
        inbox = Inbox.objects.create(domain=domain, user_id=options["user"])
        for tag in inbox_tags:
            tag = Tag(tag=tag, inbox=inbox)
            tag.save()

    email = Email(inbox=inbox, reveived_date=datetime.now())
    main_body = Body.objects.get_or_create(data="")
    main_part = PartList(body=main_body, email=email)
    main_part.save()

    main_headers = main_part.header_set
    main_headers.get_or_create(name="From", data="support@inboxen.org", ordinal=0)
    main_headers.get_or_create(name="Subject", data=settings.LIBERATION_SUBJECT, ordinal=1)
    main_headers.get_or_create(name="Content-Type", data="multipart/mixed; boundary=\"InboxenIsTheBest\"")

    msg_body = Body.objects.get_or_create(data=settings.LIBERATION_BODY)
    msg_part = PartList(body=msg_body, email=email, parent=main_part)
    msg_part.save()
    msg_part.headers_set.get_or_create(name="Content-Type", data="text/plain")

    if not options.get("noEmail", False):
        archive_body = Body.objects.get_or_create(path=result["path"])[0]
        archive_part = PartList(body=archive_body, email=email, parent=main_part)
        archive_part.save()
        archive_headers = archive.header_set
        cont_dispos = "attachment; filename=\"emails-{0}.{1}\"".format(result['date'], options.get('compressType', 'tar.gz'))
        archive_headers.get_or_create(name="Content-Type", data=result['mime-type'], ordinal=0)
        archive_headers.get_or_create(name="Content-Disposition", data=cont_dispos, ordinal=1)

    profile_body = Body.objects.get_or_create(data=profile['data'])[0]
    profile_part = PartList(body=profile_body, email=email, parent=main_part)
    profile_part.save()
    profile_headers = profile_part.header_set
    cont_dispos = "attachment; filename=\"{0}\"".format(profile['name'])
    profile_headers.get_or_create(name="Content-Type", data=profile['type'], ordinal=0)
    profile_headers.get_or_create(name="Content-Disposition", data=cont_dispos, ordinal=1)

    tags_body = Body.objects.get_or_create(data=tags['data'])[0]
    tags_part = PartList(body=tags_body, email=email, partent=main_part)
    tags_part.save()
    tags_headers = tags_part.header_set
    cont_dispos = "attachment; filename=\"{0}\"".format(tags['name'])
    tags_headers.get_or_create(name="Content-Type", data=tags['type'], ordinal=0)
    tags_headers.get_or_create(name="Content-Disposition", data=cont_dispos, ordinal=1)

    log.debug("Finished liberation for %s", user.username)

def liberate_user_profile(user_id, email_results, date):
    """ User profile data """
    data = {
        'preferences':{}
    }
    user = User.objects.get(id=user_id)

    # user's preferences
    profile = user.userprofile
    if profile.html_preference == 1:
        data['preferences']['html_preference'] = 'Prefer plain-text'
    else:
        data['preferences']['html_preference'] = 'Prefer HTML'

    data['preferences']['pool_amount'] = profile.pool_amount

    # user data
    data["id"] = user.id
    data['username'] = user.username
    data['is_staff'] = user.is_staff
    data['is_superuser'] = user.is_superuser
    data["is_active"] = user.is_active
    data['last_login'] = user.last_login.isoformat()
    data['join_date'] = user.date_joined.isoformat()
    data['groups'] = [str(groups) for groups in user.groups.all()]

    data['errors'] = []
    for result in email_results:
        if result != None:
            data['errors'].append(str(result))

    data = json.dumps(data)

    return {
        'data':data,
        'type':'application/json',
        'name':"user-%s.json" % date
    }

def liberate_inbox_tags(user_id, date):
    """ Grab tags from inboxes """
    data = {}

    inboxes = Inbox.objects.filter(user__id=user_id)
    for inbox in inboxes:
        email = "%s@%s" % (inbox.inbox, inbox.domain)
        tags = [tag.tag for tag in Tag.objects.filter(inbox=inbox)]
        data[email] = {
            "created":inbox.created.isoformat(),
            "deleted":inbox.deleted,
            "tags":tags,
        }

    data = json.dumps(data)

    return {
        "data":data,
        "type":"application/json",
        "name":"inboxes-%s.json" % date
    }

<commit_msg>Use Email flags now<commit_after>import json
import hashlib
import logging
import mailbox
import os
import random
import string
import tarfile
import time
from datetime import datetime
from email.message import Message
from shutil import rmtree

from celery import task, chain, group, chord
from pytz import utc

from django.conf import settings
from django.db import transaction

from inboxen.models import Domain, Email, Inbox, PartList, Tag, User

log = logging.getLogger(__name__)

for setting_name in ('LIBERATION_BODY', 'LIBERATION_SUBJECT', 'LIBERATION_PATH'):
    assert hasattr(settings, setting_name), "%s has not been set" % setting_name

TAR_TYPES = {
    'tar.gz': {'writer': 'w:gz', 'mime-type': 'application/x-gzip'},
    'tar.bz2': {'writer': 'w:bz2', 'mime-type': 'application/x-bzip2'},
    'tar': {'writer': 'w:', 'mime-type': 'application/x-tar'}
    }

@task(rate_limit='4/h')
def liberate(user, options=None):
    """ Get set for liberation, expects User object """

    if options == None:
        options = {}

    options['user'] = user.id

    rstr = ""
    for i in range(7):
        rstr += string.ascii_letters[random.randint(0, 50)]
    mail_path = "%s/%s_%s_%s_%s" % (settings.LIBERATION_PATH, time.time(), os.getpid(), rstr, hashlib.sha256(user.username + rstr).hexdigest()[:50])

    # make maildir
    mailbox.Maildir(mail_path)

    tasks = chord(
                [liberate_inbox.s(mail_path, inbox.id) for inbox in Inbox.objects.filter(user=user, deleted=False).only('id')],
                liberate_collect_emails.s(mail_path, options)
                )
    tasks.apply_async()

@task(rate_limit='10/m')
def liberate_inbox(mail_path, inbox_id):
    """ Gather email IDs """

    inbox = Inbox.objects.get(id=inbox_id, deleted=False)
    maildir = mailbox.Maildir(mail_path)
    maildir.add_folder(str(inbox))

    return {
            'folder': str(inbox),
            'ids': [email.id for email in Email.objects.filter(inbox=inbox, flags=~Email.flags.deleted).only('id')]
            }

@task()
def liberate_collect_emails(results, mail_path, options):
    """ Send off data mining tasks """

    msg_tasks = []
    for result in results:
        inbox = [liberate_message.s(mail_path, result['folder'], email_id) for email_id in result['ids']]
        msg_tasks.extend(inbox)
    if len(msg_tasks) > 0:
        msg_tasks = chain(
                        group(msg_tasks),
                        liberate_convert_box.s(mail_path, options),
                        liberate_tarball.s(mail_path, options),
                        liberation_finish.s(options)
                        )
    else:
        options["noEmails"] = True
        msg_tasks = liberation_finish.s(options)

    msg_tasks.apply_async()

def make_message(message):
    """ Make a Python  email.message.Message from our models """
    parents = {}
    part_list = message.parts.all()
    first = None

    for part in part_list:
        msg = Message()

        header_set = part.header_set.order_by("ordinal").select_related("name__name", "data__data")
        for header in header_set:
            msg[header.name.name] = header.data.data

        if msg.is_multipart():
            parents[part.id] = msg
        else:
            msg.set_payload(part.body.data)

        if first is None:
            first = msg
        else:
            parents[part.parent_id].attach(msg)

    return first

@task(rate_limit='1000/m')
@transaction.atomic()
def liberate_message(mail_path, inbox, email_id):
    """ Take email from database and put on filesystem """
    maildir = mailbox.Maildir(mail_path).get_folder(inbox)

    try:
        msg = Email.objects.get(id=email_id, flags=~Email.flags.deleted)
        msg = make_message(msg)
    except Exception, exc:
        msg_id = hex(int(email_id))[2:]
        log.debug("Exception processing %s", msg_id, exc_info=exc)
        raise Exception(msg_id)

    maildir.add(msg)

@task()
def liberate_convert_box(result, mail_path, options):
    """ Convert maildir to mbox if needed """
    if options['storage_type'] == 'maildir':
        pass

    elif options['storage_type'] == 'mailbox':
        maildir = mailbox.Maildir(mail_path)
        mbox = mailbox.mbox(mail_path + '.mbox')
        mbox.lock()

        for inbox in maildir.list_folders():
            folder = maildir.get_folder(inbox)

            for key in folder.iterkeys():
                msg = str(folder.pop(key))
                mbox.add(msg)
            maildir.remove_folder(inbox)

        rmtree(mail_path)
        mbox.close()

    return result

@task(default_retry_delay=600)
def liberate_tarball(result, mail_path, options):
    """ Tar up and delete the maildir """

    tar_type = TAR_TYPES[options.get('compression_type', 'tar.gz')]
    tar_name = "%s.%s" % (mail_path, options.get('compression_type', 'tar.gz'))

    try:
        tar = tarfile.open(tar_name, tar_type['writer'])
    except (IOError, OSError), error:
        log.debug("Couldn't open tarfile at %s", tar_name)
        raise liberate_tarball.retry(exc=error)

    date = str(datetime.now(utc).date())
    dir_name = "inboxen-%s" % date

    if options['storage_type'] == 'maildir':
        try:
            tar.add("%s/" % mail_path, dir_name) # directories are added recursively by default
        finally:
            tar.close()
        rmtree(mail_path)

    elif options['storage_type'] == 'mailbox':
        try:
            tar.add("%s.mbox" % mail_path, dir_name)
        finally:
            tar.close()
        os.remove("%s.mbox" % mail_path)

    return {'path': tar_name, 'mime-type': tar_type['mime-type'], 'date': date, 'results': result}

@task()
@transaction.atomic()
def liberation_finish(result, options):
    """ Create email to send to user """
    tags = liberate_inbox_tags(options['user'], result['date'])
    profile = liberate_user_profile(options['user'], result['results'], result['date'])

    inbox_tags = ["Inboxen", "data", "liberation"]
    inbox = Inbox.objects
    for tag in inbox_tags:
        inbox = inbox.filter(tag__tag=tag)

    try:
        inbox = inbox.get(user__id=options["user"])
    except Inbox.MultipleObjectsReturned:
        inbox = inbox.filter(user__id=options["user"])[0]
    except Inbox.DoesNotExist:
        domain = random.choice(Domain.objects.all())
        inbox = Inbox.objects.create(domain=domain, user_id=options["user"])
        for tag in inbox_tags:
            tag = Tag(tag=tag, inbox=inbox)
            tag.save()

    email = Email(inbox=inbox, reveived_date=datetime.now())
    main_body = Body.objects.get_or_create(data="")
    main_part = PartList(body=main_body, email=email)
    main_part.save()

    main_headers = main_part.header_set
    main_headers.get_or_create(name="From", data="support@inboxen.org", ordinal=0)
    main_headers.get_or_create(name="Subject", data=settings.LIBERATION_SUBJECT, ordinal=1)
    main_headers.get_or_create(name="Content-Type", data="multipart/mixed; boundary=\"InboxenIsTheBest\"")

    msg_body = Body.objects.get_or_create(data=settings.LIBERATION_BODY)
    msg_part = PartList(body=msg_body, email=email, parent=main_part)
    msg_part.save()
    msg_part.headers_set.get_or_create(name="Content-Type", data="text/plain")

    if not options.get("noEmail", False):
        archive_body = Body.objects.get_or_create(path=result["path"])[0]
        archive_part = PartList(body=archive_body, email=email, parent=main_part)
        archive_part.save()
        archive_headers = archive.header_set
        cont_dispos = "attachment; filename=\"emails-{0}.{1}\"".format(result['date'], options.get('compressType', 'tar.gz'))
        archive_headers.get_or_create(name="Content-Type", data=result['mime-type'], ordinal=0)
        archive_headers.get_or_create(name="Content-Disposition", data=cont_dispos, ordinal=1)

    profile_body = Body.objects.get_or_create(data=profile['data'])[0]
    profile_part = PartList(body=profile_body, email=email, parent=main_part)
    profile_part.save()
    profile_headers = profile_part.header_set
    cont_dispos = "attachment; filename=\"{0}\"".format(profile['name'])
    profile_headers.get_or_create(name="Content-Type", data=profile['type'], ordinal=0)
    profile_headers.get_or_create(name="Content-Disposition", data=cont_dispos, ordinal=1)

    tags_body = Body.objects.get_or_create(data=tags['data'])[0]
    tags_part = PartList(body=tags_body, email=email, partent=main_part)
    tags_part.save()
    tags_headers = tags_part.header_set
    cont_dispos = "attachment; filename=\"{0}\"".format(tags['name'])
    tags_headers.get_or_create(name="Content-Type", data=tags['type'], ordinal=0)
    tags_headers.get_or_create(name="Content-Disposition", data=cont_dispos, ordinal=1)

    log.debug("Finished liberation for %s", user.username)

def liberate_user_profile(user_id, email_results, date):
    """ User profile data """
    data = {
        'preferences':{}
    }
    user = User.objects.get(id=user_id)

    # user's preferences
    profile = user.userprofile
    if profile.html_preference == 1:
        data['preferences']['html_preference'] = 'Prefer plain-text'
    else:
        data['preferences']['html_preference'] = 'Prefer HTML'

    data['preferences']['pool_amount'] = profile.pool_amount

    # user data
    data["id"] = user.id
    data['username'] = user.username
    data['is_staff'] = user.is_staff
    data['is_superuser'] = user.is_superuser
    data["is_active"] = user.is_active
    data['last_login'] = user.last_login.isoformat()
    data['join_date'] = user.date_joined.isoformat()
    data['groups'] = [str(groups) for groups in user.groups.all()]

    data['errors'] = []
    for result in email_results:
        if result is not None:
            data['errors'].append(str(result))

    data = json.dumps(data)

    return {
        'data':data,
        'type':'application/json',
        'name':"user-%s.json" % date
    }

def liberate_inbox_tags(user_id, date):
    """ Grab tags from inboxes """
    data = {}

    inboxes = Inbox.objects.filter(user__id=user_id)
    for inbox in inboxes:
        email = "%s@%s" % (inbox.inbox, inbox.domain)
        tags = [tag.tag for tag in Tag.objects.filter(inbox=inbox)]
        data[email] = {
            "created":inbox.created.isoformat(),
            "deleted":inbox.deleted,
            "tags":tags,
        }

    data = json.dumps(data)

    return {
        "data":data,
        "type":"application/json",
        "name":"inboxes-%s.json" % date
    }

<|endoftext|>


<commit_before>#!/usr/bin/python
"""
  ROS wrapper for Husky
  usage:
     ./husky.py [<metalog file> [F]|<IP node> <IP master>]
"""

from node import NodeROS, setIPs
import sys
import math

def normalizeAnglePIPI( angle ):
    while angle < -math.pi:
        angle += 2*math.pi
    while angle > math.pi:
        angle -= 2*math.pi
    return angle

class HuskyROS:
    def __init__( self, metalog=None, assertWrite=True ):
        self.node = NodeROS(
            subscribe=['/imu/data', '/imu/mag', '/imu/rpy', '/imu/temperature',
                '/husky/data/encoders',
                '/husky/data/power_status',
                '/husky/data/safety_status',
                '/joy',
                '/husky/data/system_status'],
            publish=['/husky/cmd_vel'],
            heartbeat='/husky/data/encoders',
            metalog=metalog, assertWrite=assertWrite )
        self.speed = 0.0
        self.angularSpeed = 0.0
        self.enc = None # unknown
        self.time = None
        self.imu = None
        self.mag = None
        self.azimuth = None
        self.greenPressed = None
        self.redPressed = None
        self.joyAxis = (0,0, 0,0, 0,0)
        self.update()

    def update( self ):
        self.node.cmdList.append( ('/husky/cmd_vel', (self.speed,self.angularSpeed)) )
        status = self.node.update()
        for topic,data in status:
            if topic == '/joy':
                self.joyAxis, buttons = data
                self.greenPressed = (buttons[1]==1)
                self.redPressed = (buttons[2]==1)
            if topic == '/imu/data':
                q0,q1,q2,q3 = data # quaternion
#                print "[%.2f %.2f %.2f]" % (1-2*(q2*q2+q3*q3), 2*(q1*q2-q0*q3), 2*(q0*q2+q1*q3)),
#                print "[%.2f %.2f %.2f]" % (2*(q1*q2+q0*q3), 1-2*(q1*q1+q3*q3), 2*(q2*q3-q0*q1)),
#                print "[%.2f %.2f %.2f]" % (2*(q1*q3-q0*q2), 2*(q0*q1+q2*q3), 1-2*(q1*q1+q2*q2))
#                a2 = math.acos(q0)   # q0 = cos(alpha/2)
#                sinA2 = math.sin(a2)
#                bx = math.acos(q1/sinA2) # q1 = sin(alpha/2)*cos(bx)
#                by = math.acos(q2/sinA2)
#                bz = math.acos(q3/sinA2)
#                self.imu = data[:]
#                print bx, by, bz #math.degrees(2*a2)
                self.imu =  math.atan2(2*(q0*q1+q2*q3), 1-2*(q1*q1+q2*q2)) # along X axis
#                self.imu =  math.degrees( math.asin(2*(q0*q2-q3*q1)) ) # along Y axis ... max 5deg
#                self.imu =  math.degrees( math.atan2(2*(q0*q3+q1*q2), 1-2*(q2*q2+q3*q3)) ) # along Z axis ~80 deg
            if topic == '/imu/mag':
                self.mag = data[:]
                self.azimuth = math.atan2( data[2], data[0] )

            if topic == '/husky/data/encoders':
                self.time = data[0]
                self.enc = data[1][0], data[2][0] # only distance traveled

    def setSpeedPxPa( self, speed, angularSpeed ):
        self.speed = speed
        self.angularSpeed = angularSpeed

    def goStraight( self, dist, maxSpeed=0.1 ):
        "go forward for given distance"
        print "GO!", dist, maxSpeed
        assert self.enc != None
        prevEncL, prevEncR = self.enc
        startTime = self.time
        while ((self.enc[0]-prevEncL) + (self.enc[1]-prevEncR))/2.0 < dist:
            self.setSpeedPxPa( maxSpeed, 0 )
            self.update()
        self.setSpeedPxPa( 0, 0 )
        self.update()
        print "TIME", self.time-startTime

    def turn( self, angle ):
        "go forward for given distance"
        WIDTH = 0.55
        assert self.enc != None
        prevEncL, prevEncR = self.enc
        startTime = self.time
        prevImu = self.imu
#        while abs((self.enc[0]-prevEncL) - (self.enc[1]-prevEncR)) < WIDTH*angle:  # unusable
        while abs( normalizeAnglePIPI(self.imu - prevImu) ) < angle:
            self.setSpeedPxPa( 0.0, 0.3 )
            self.update()
        self.setSpeedPxPa( 0, 0 )
        self.update()
        print "TIME", self.time-startTime, math.degrees( normalizeAnglePIPI(self.imu - prevImu) )

    def turnToAzimuth( self, azimuth ):
        print "AZI", math.degrees( self.azimuth ), math.degrees( azimuth )
        while abs( normalizeAnglePIPI(self.azimuth - azimuth) ) > math.radians(2):
            self.setSpeedPxPa( 0.0, 0.3 )
            self.update()
        self.setSpeedPxPa( 0, 0 )
        self.update()

    def wait( self, seconds ):
        startTime = self.time
        while startTime + seconds > self.time:
            self.setSpeedPxPa( 0.0, 0.0 )
            self.update()

def test( robot ):
    robot.speed = 0.0
    while True:
        robot.update()
        if robot.greenPressed:
            print "GREEN"
            robot.speed = 0.1
        else:
            print "PAUSE"
            robot.speed = 0.0
        if robot.redPressed:
            print "RED"
            robot.speed = 0.0
            break
    robot.speed = 0.0
    robot.update()

def test2( robot ):
    VEL_SCALE = 0.5
    ROT_SCALE = 1.0
    prevPressed = False
    prevEnc = robot.enc[:]
    while True:
        robot.update()
        if not prevPressed and robot.greenPressed:
            print robot.azimuth, prevEnc[0] - robot.enc[0], prevEnc[1] - robot.enc[1]
            prevEnc = robot.enc[:]
        if robot.greenPressed:
            robot.speed = VEL_SCALE*robot.joyAxis[1]
            robot.angularSpeed = ROT_SCALE*robot.joyAxis[0]
        else:
            robot.speed,robot.angularSpeed = 0,0
        prevPressed = robot.greenPressed
        if robot.redPressed:
            print "RED"
            break
    robot.speed = 0.0
    robot.update()

def test3( robot ):
    print math.degrees(robot.imu), robot.mag, math.degrees(robot.azimuth)
    robot.goStraight( 1.0 )
    print math.degrees(robot.imu), robot.mag, math.degrees(robot.azimuth)
    robot.wait( 3 )
    print math.degrees(robot.imu), robot.mag, math.degrees(robot.azimuth)
    robot.turn( math.radians(90) )
    print math.degrees(robot.imu), robot.mag
    robot.wait( 3 )
    print math.degrees(robot.imu), robot.mag

def test4( robot ):
    robot.wait( 3 )
    for i in xrange(20):
        robot.wait( 2 )
        print math.degrees(robot.azimuth)
        robot.turn( math.radians(20) )
    robot.wait( 3 )

def test5( robot ):
    for i in xrange(5):
        robot.turnToAzimuth( math.radians(0.0) )
        robot.goStraight( 1.0 )
        robot.turnToAzimuth( math.radians(180.0) )
        robot.goStraight( 1.0 )

def test6( robot ):
    for azi,dist in [ (-1.13311392218, 11.091),
            (0.175614628559, 3.584),
            (1.62315635666, 11.842),
            (-2.77704236902, 4.407),
            (-1.14243399246, 1.0)]: # unknown
        robot.turnToAzimuth( azi )
        robot.goStraight( dist, maxSpeed=0.3 )

if __name__ == "__main__":
    if len(sys.argv) < 2 or (len(sys.argv)==2 and "meta" not in sys.argv[1]):
        print __doc__
        sys.exit(1)

    metalog = None
    assertWrite = True
    if "meta" in sys.argv[1]:
        metalog = sys.argv[1]
        if len(sys.argv) > 2 and sys.argv[2] == 'F':
            assertWrite = False
    else:
        setIPs( sys.argv[1], 'http://'+sys.argv[2]+':11311' )
    robot = HuskyROS( metalog=metalog, assertWrite=assertWrite )
    test6( robot )

#-------------------------------------------------------------------
# vim: expandtab sw=4 ts=4

<commit_msg>fixed executable on Husky<commit_after>#!/usr/bin/python
"""
  ROS wrapper for Husky
  usage:
     ./husky.py [<metalog file> [F]|<IP node> <IP master>]
"""

from node import NodeROS, setIPs
import sys
import math

def normalizeAnglePIPI( angle ):
    while angle < -math.pi:
        angle += 2*math.pi
    while angle > math.pi:
        angle -= 2*math.pi
    return angle

class HuskyROS:
    def __init__( self, metalog=None, assertWrite=True ):
        self.node = NodeROS(
            subscribe=['/imu/data', '/imu/mag', '/imu/rpy', '/imu/temperature',
                '/husky/data/encoders',
                '/husky/data/power_status',
                '/husky/data/safety_status',
                '/joy',
                '/husky/data/system_status'],
            publish=['/husky/cmd_vel'],
            heartbeat='/husky/data/encoders',
            metalog=metalog, assertWrite=assertWrite )
        self.speed = 0.0
        self.angularSpeed = 0.0
        self.enc = None # unknown
        self.time = None
        self.imu = None
        self.mag = None
        self.azimuth = None
        self.greenPressed = None
        self.redPressed = None
        self.joyAxis = (0,0, 0,0, 0,0)
        self.update()

    def update( self ):
        self.node.cmdList.append( ('/husky/cmd_vel', (self.speed,self.angularSpeed)) )
        status = self.node.update()
        for topic,data in status:
            if topic == '/joy':
                self.joyAxis, buttons = data
                self.greenPressed = (buttons[1]==1)
                self.redPressed = (buttons[2]==1)
            if topic == '/imu/data':
                q0,q1,q2,q3 = data # quaternion
#                print "[%.2f %.2f %.2f]" % (1-2*(q2*q2+q3*q3), 2*(q1*q2-q0*q3), 2*(q0*q2+q1*q3)),
#                print "[%.2f %.2f %.2f]" % (2*(q1*q2+q0*q3), 1-2*(q1*q1+q3*q3), 2*(q2*q3-q0*q1)),
#                print "[%.2f %.2f %.2f]" % (2*(q1*q3-q0*q2), 2*(q0*q1+q2*q3), 1-2*(q1*q1+q2*q2))
#                a2 = math.acos(q0)   # q0 = cos(alpha/2)
#                sinA2 = math.sin(a2)
#                bx = math.acos(q1/sinA2) # q1 = sin(alpha/2)*cos(bx)
#                by = math.acos(q2/sinA2)
#                bz = math.acos(q3/sinA2)
#                self.imu = data[:]
#                print bx, by, bz #math.degrees(2*a2)
                self.imu =  math.atan2(2*(q0*q1+q2*q3), 1-2*(q1*q1+q2*q2)) # along X axis
#                self.imu =  math.degrees( math.asin(2*(q0*q2-q3*q1)) ) # along Y axis ... max 5deg
#                self.imu =  math.degrees( math.atan2(2*(q0*q3+q1*q2), 1-2*(q2*q2+q3*q3)) ) # along Z axis ~80 deg
            if topic == '/imu/mag':
                self.mag = data[:]
                self.azimuth = math.atan2( data[2], data[0] )

            if topic == '/husky/data/encoders':
                self.time = data[0]
                self.enc = data[1][0], data[2][0] # only distance traveled

    def setSpeedPxPa( self, speed, angularSpeed ):
        self.speed = speed
        self.angularSpeed = angularSpeed

    def goStraight( self, dist, maxSpeed=0.1 ):
        "go forward for given distance"
        print "GO!", dist, maxSpeed
        assert self.enc != None
        prevEncL, prevEncR = self.enc
        startTime = self.time
        while ((self.enc[0]-prevEncL) + (self.enc[1]-prevEncR))/2.0 < dist:
            self.setSpeedPxPa( maxSpeed, 0 )
            self.update()
        self.setSpeedPxPa( 0, 0 )
        self.update()
        print "TIME", self.time-startTime

    def turn( self, angle ):
        "go forward for given distance"
        WIDTH = 0.55
        assert self.enc != None
        prevEncL, prevEncR = self.enc
        startTime = self.time
        prevImu = self.imu
#        while abs((self.enc[0]-prevEncL) - (self.enc[1]-prevEncR)) < WIDTH*angle:  # unusable
        while abs( normalizeAnglePIPI(self.imu - prevImu) ) < angle:
            self.setSpeedPxPa( 0.0, 0.3 )
            self.update()
        self.setSpeedPxPa( 0, 0 )
        self.update()
        print "TIME", self.time-startTime, math.degrees( normalizeAnglePIPI(self.imu - prevImu) )

    def turnToAzimuth( self, azimuth ):
        print "AZI", math.degrees( self.azimuth ), math.degrees( azimuth )
        while abs( normalizeAnglePIPI(self.azimuth - azimuth) ) > math.radians(2):
            self.setSpeedPxPa( 0.0, 0.3 )
            self.update()
        self.setSpeedPxPa( 0, 0 )
        self.update()

    def wait( self, seconds ):
        startTime = self.time
        while startTime + seconds > self.time:
            self.setSpeedPxPa( 0.0, 0.0 )
            self.update()

def test( robot ):
    robot.speed = 0.0
    while True:
        robot.update()
        if robot.greenPressed:
            print "GREEN"
            robot.speed = 0.1
        else:
            print "PAUSE"
            robot.speed = 0.0
        if robot.redPressed:
            print "RED"
            robot.speed = 0.0
            break
    robot.speed = 0.0
    robot.update()

def test2( robot ):
    VEL_SCALE = 0.5
    ROT_SCALE = 1.0
    prevPressed = False
    prevEnc = robot.enc[:]
    while True:
        robot.update()
        if not prevPressed and robot.greenPressed:
            print robot.azimuth, prevEnc[0] - robot.enc[0], prevEnc[1] - robot.enc[1]
            prevEnc = robot.enc[:]
        if robot.greenPressed:
            robot.speed = VEL_SCALE*robot.joyAxis[1]
            robot.angularSpeed = ROT_SCALE*robot.joyAxis[0]
        else:
            robot.speed,robot.angularSpeed = 0,0
        prevPressed = robot.greenPressed
        if robot.redPressed:
            print "RED"
            break
    robot.speed = 0.0
    robot.update()

def test3( robot ):
    print math.degrees(robot.imu), robot.mag, math.degrees(robot.azimuth)
    robot.goStraight( 1.0 )
    print math.degrees(robot.imu), robot.mag, math.degrees(robot.azimuth)
    robot.wait( 3 )
    print math.degrees(robot.imu), robot.mag, math.degrees(robot.azimuth)
    robot.turn( math.radians(90) )
    print math.degrees(robot.imu), robot.mag
    robot.wait( 3 )
    print math.degrees(robot.imu), robot.mag

def test4( robot ):
    robot.wait( 3 )
    for i in xrange(20):
        robot.wait( 2 )
        print math.degrees(robot.azimuth)
        robot.turn( math.radians(20) )
    robot.wait( 3 )

def test5( robot ):
    for i in xrange(5):
        robot.turnToAzimuth( math.radians(0.0) )
        robot.goStraight( 1.0 )
        robot.turnToAzimuth( math.radians(180.0) )
        robot.goStraight( 1.0 )

def test6( robot ):
    for azi,dist in [ (-1.13311392218, 11.091),
            (0.175614628559, 3.584),
            (1.62315635666, 11.842),
            (-2.77704236902, 4.407),
            (-1.14243399246, 1.0)]: # unknown
        robot.turnToAzimuth( azi )
        robot.goStraight( dist, maxSpeed=0.3 )

if __name__ == "__main__":
    if len(sys.argv) < 2 or (len(sys.argv)==2 and "meta" not in sys.argv[1]):
        print __doc__
        sys.exit(1)

    metalog = None
    assertWrite = True
    if "meta" in sys.argv[1]:
        metalog = sys.argv[1]
        if len(sys.argv) > 2 and sys.argv[2] == 'F':
            assertWrite = False
    else:
        setIPs( sys.argv[1], 'http://'+sys.argv[2]+':11311' )
    robot = HuskyROS( metalog=metalog, assertWrite=assertWrite )
    test6( robot )

#-------------------------------------------------------------------
# vim: expandtab sw=4 ts=4

<|endoftext|>