<commit_before>import numpy as np
from opt.utils_logarithmic_barrier import logarithmic_barrier, \
                                           constraint_inequalities_funcs_eval, \
                                           plot_inner_iterations, \
                                           plot_central_path, \
                                           line_search_for_log_barrier_by_backtracking
from opt.utils import compute_error, print_iterations

def primal_dual_method(f, constraints_ineq,
                       x_0, tol, tol_backtracking,
                       t,
                       plot=True,
                       x_ast=None, p_ast=None,
                       max_inner_iter=30,
                       mu=None,
                       gf_B=None,
                       Hf_B=None,
                       tol_outer_iter=1e-6,
                       max_total_iterations=30):
    """
    Outer iterations for logarithmic barrier method
    to numerically approximate solution of PL with only inequalities.
    Args:
        f_B (fun): definition of logarithmic barrier function
                   as lambda expression or function definition.
        constraints_ineq: dictionary of inequalities constraints
                          in "<= 0" form.
        x_0 (numpy ndarray): initial point for Newton's method.
        tol (float): tolerance that will halt method.
                     Controls stopping criteria.
        tol_backtracking (float): tolerance that will halt method.
                                  Controls value of line search
                                  by backtracking.
        t (float): parameter barrier.
        plot (bool): if true make plots of Err vs iterations and
                     plot of inner iterations.
        x_ast (numpy ndarray): solution of min f_o subject to Ax <= b.
                               It's required that user knows the solution...
        p_ast (float): value of f_B(x_ast).
        max_inner_iter (int): maximum number of inner iterations.
        mu (float): parameter that will multiply barrier parameter.
        gf_B (fun): definition of gradient of f_B as lambda
                     expressionr or function definition.
        Hf_B (fun): definition of Hessian of f_B as lambda
                     expressionr or function definition.
        tol_outer_iter (float): tolerance that will halt method.
                                Controls stopping criteria.
        max_total_iter (int): maximum number of total iterations.
    Returns:
        x (numpy ndarray): numpy array, approximation of x_ast.
        total_iter (int): number of iterations regarding outer and
                          inner iterations.
        t (float): updated barrier parameter.
        x_plot_total_iter (numpy ndarray): numpy array that containts
                                           in columns vector of
                                           approximations of all inner
                                           iterations. Last column contains
                                           x, approximation of solution.
                                           Useful for plotting.
    """
    x = x_0
    n = x.size
    log_barrier_eval = logarithmic_barrier(f,x,t,constraints_ineq)
    m = len(constraints_ineq.keys())
    stopping_criteria = m/t
    outer_iter = 0
    total_iter = 0
    const_funcs_eval = constraint_inequalities_funcs_eval(x,constraints_ineq)
    print("Outer iterations of path following method")
    print('{} {:0.2e}'.format("Mu value:", mu))
    columns = ["Outer iter", "logbarrier", "t_log_barrier",
               "duality gap"]
    list_values = [outer_iter, log_barrier_eval, t,
                   stopping_criteria]
    data = {"row" + str(outer_iter): list_values}
    print_iterations(data, columns)
    print("-"*65 + "\n" + "-"*65)
    error_between_iterations = 1

    while(stopping_criteria > tol_outer_iter \
          and error_between_iterations > tol_outer_iter \
          and total_iter < max_total_iterations):
        x_aux = x
        [x,
         inner_iterations,
         Err_plot,
         x_plot] = logarithmic_barrier_newton_method(f,
                                                     constraints_ineq,
                                                     t, x, tol,
                                                     tol_backtracking,
                                                     x_ast, p_ast,
                                                     max_inner_iter,
                                                     gf_B,
                                                     Hf_B)
        if(outer_iter == 0):
            x_plot_total_iter = x_plot
        else:
            x_plot_total_iter = np.column_stack((x_plot_total_iter,
                                                 x_plot))
        error_between_iterations = compute_error(x, x_aux)
        if(error_between_iterations > tol_outer_iter):
            if plot:
                plot_inner_iterations(Err_plot)
                plot_central_path(x_plot)
            print("Inner iterations")
            print(x)
            const_ineq_funcs_eval = constraint_inequalities_funcs_eval(x,
                                                                       constraints_ineq)
            t=mu*t
            log_barrier_eval = logarithmic_barrier(f,x,t,
                                                   constraints_ineq)
            if(sum(const_ineq_funcs_eval < -np.nextafter(0,1)) <=-1):
                print("Some constraint inequalities evaluated in x were nonpositive, check approximations")
            outer_iter+=1
            stopping_criteria = m/t
            total_iter+= total_iter + inner_iterations
            #print
            print("-"*65 + "\n" + "-"*65)
            print("Outer iterations of path following method")
            print('{} {:0.2e}'.format("Mu value:", mu))
            list_values = [outer_iter, log_barrier_eval, t,
                           stopping_criteria]
            data = {"row" + str(outer_iter): list_values}
            print_iterations(data, columns)
            print("-"*65 + "\n" + "-"*65)
    return [x,total_iter,t,x_plot_total_iter]

def logarithmic_barrier_newton_method(f_B, constraints_ineq,
                                      t_B, x_0, tol,
                                      tol_backtracking, x_ast=None,
                                      p_ast=None, maxiter=30,
                                      gf_B=None,
                                      Hf_B=None):
    """
    Logarithmic barrier primal-dual method using Newton's method
    to numerically approximate solution of PL with only inequalities.
    Args:
        f_B (fun): definition of logarithmic barrier function
                   as lambda expression or function definition.
        constraints_ineq: dictionary of inequalities constraints
                          in "<= 0" form.
        t_path: barrier parameter.
        x_0 (numpy ndarray): initial point for Newton's method.
        tol (float): tolerance that will halt method.
                     Controls stopping criteria.
        tol_backtracking (float): tolerance that will halt method.
                                  Controls value of line search
                                  by backtracking.
        x_ast (numpy ndarray): solution of min f_o subject to Ax <= b.
                               It's required that user knows the solution...
        p_ast (float): value of f_o(x_ast).
        maxiter (int): maximum number of iterations
        gf_B (fun): definition of gradient of f_B as lambda
                     expressionr or function definition.
        Hf_B (fun): definition of Hessian of f_B as lambda
                     expressionr or function definition.
    Returns:
        x (numpy ndarray): numpy array, approximation solution of
                           inner iterations problem.
        iteration (int): number of iterations.
        Err_plot (numpy ndarray): numpy array of absolute error
                                  between p_ast and f(x) with x
                                  approximation of x_ast. Useful for
                                  plotting.
        x_plot (numpy ndarray): numpy array that containts in columns
                                vector of approximations. Last column
                                contains x, approximation of solution.
                                Useful for plotting.
    """
    iteration = 0

    x = x_0
    n = x.size

    f_B_eval = f_B(x)
    gf_B_eval = gf_B(x, t_B)
    Hf_B_eval = Hf_B(x, t_B)

    normgf = np.linalg.norm(gf_B_eval)
    condHf= np.linalg.cond(Hf_B_eval)
    Err_plot_aux = np.zeros(maxiter)
    Err_plot_aux[iteration]=compute_error(p_ast,f_B_eval)

    Err = compute_error(x_ast,x)
    system_matrix = Hf_B_eval
    rhs = -gf_B_eval

    x_plot = np.zeros((n,maxiter))
    x_plot[:,iteration] = x

    #Newton's direction and Newton's decrement
    dir_Newton = np.linalg.solve(system_matrix, rhs)
    dec_Newton = rhs.dot(dir_Newton)
    columns = ["Iter", "Normgf", "Newtons decrement",
               "Err x ast", "Err p ast",
               "line search", "CondHf"]
    list_values = [iteration, normgf, dec_Newton,
                   Err, Err_plot_aux[iteration], "---", condHf]
    data = {"row" + str(iteration): list_values}
    print_iterations(data, columns)
    stopping_criteria = dec_Newton/2
    iteration+=1

    while(stopping_criteria>tol and iteration < maxiter):
        der_direct = -dec_Newton
        t = line_search_for_log_barrier_by_backtracking(f_B,dir_Newton,
                                                        x,t_B,
                                                        constraints_ineq,
                                                        der_direct)
        x = x + t*dir_Newton

        #Eval
        f_B_eval = f_B(x)
        gf_B_eval = gf_B(x, t_B)
        Hf_B_eval = Hf_B(x, t_B)

        system_matrix = Hf_B_eval
        rhs = -gf_B_eval

        #Update
        dir_Newton = np.linalg.solve(system_matrix, rhs)
        dec_Newton = dec_Newton = rhs.dot(dir_Newton)
        Err_plot_aux[iteration]=compute_error(p_ast,f_B_eval)
        x_plot[:,iteration] = x
        Err = compute_error(x_ast,x)
        stopping_criteria = dec_Newton/2
        condHf= np.linalg.cond(Hf_B_eval)
        normgf = np.linalg.norm(gf_B_eval)

        #print
        list_values = [iteration, normgf, dec_Newton,
                       Err, Err_plot_aux[iteration], t, condHf]
        data = {"row" + str(iteration): list_values}
        print_iterations(data, columns)

        if t<tol_backtracking: #if t is less than tol_backtracking
                               #then we need to check the reason
            iter_salida=iteration
            iteration = maxiter - 1
        iteration+=1
    print('{} {:0.2e}'.format("Error of x with respect to x_ast:",Err))
    print('{} {}'.format("Approximate solution:", x))
    idx_columns_for_plot = Err_plot_aux > np.finfo(float).eps*10**(-2)
    Err_plot = Err_plot_aux[idx_columns_for_plot]

    if iteration == maxiter and t < tol_backtracking:
        print("Backtracking value less than tol_backtracking, check approximation")
        iteration=iter_salida
    else:
        if iteration == maxiter:
            print("Reached maximum of iterations, check approximation")
    x_plot = x_plot[:,:iteration]
    return [x,iteration,Err_plot,x_plot]
<commit_msg>renaming path following method as primal-dual LB method<commit_after>import numpy as np
from opt.utils_logarithmic_barrier import logarithmic_barrier, \
                                           constraint_inequalities_funcs_eval, \
                                           plot_inner_iterations, \
                                           plot_central_path, \
                                           line_search_for_log_barrier_by_backtracking
from opt.utils import compute_error, print_iterations

def primal_dual_method(f, constraints_ineq,
                       x_0, tol, tol_backtracking,
                       t,
                       plot=True,
                       x_ast=None, p_ast=None,
                       max_inner_iter=30,
                       mu=None,
                       gf_B=None,
                       Hf_B=None,
                       tol_outer_iter=1e-6,
                       max_total_iterations=30):
    """
    Outer iterations for logarithmic barrier method
    to numerically approximate solution of PL with only inequalities.
    Args:
        f_B (fun): definition of logarithmic barrier function
                   as lambda expression or function definition.
        constraints_ineq: dictionary of inequalities constraints
                          in "<= 0" form.
        x_0 (numpy ndarray): initial point for Newton's method.
        tol (float): tolerance that will halt method.
                     Controls stopping criteria.
        tol_backtracking (float): tolerance that will halt method.
                                  Controls value of line search
                                  by backtracking.
        t (float): parameter barrier.
        plot (bool): if true make plots of Err vs iterations and
                     plot of inner iterations.
        x_ast (numpy ndarray): solution of min f_o subject to Ax <= b.
                               It's required that user knows the solution...
        p_ast (float): value of f_B(x_ast).
        max_inner_iter (int): maximum number of inner iterations.
        mu (float): parameter that will multiply barrier parameter.
        gf_B (fun): definition of gradient of f_B as lambda
                     expressionr or function definition.
        Hf_B (fun): definition of Hessian of f_B as lambda
                     expressionr or function definition.
        tol_outer_iter (float): tolerance that will halt method.
                                Controls stopping criteria.
        max_total_iter (int): maximum number of total iterations.
    Returns:
        x (numpy ndarray): numpy array, approximation of x_ast.
        total_iter (int): number of iterations regarding outer and
                          inner iterations.
        t (float): updated barrier parameter.
        x_plot_total_iter (numpy ndarray): numpy array that containts
                                           in columns vector of
                                           approximations of all inner
                                           iterations. Last column contains
                                           x, approximation of solution.
                                           Useful for plotting.
    """
    x = x_0
    n = x.size
    log_barrier_eval = logarithmic_barrier(f,x,t,constraints_ineq)
    m = len(constraints_ineq.keys())
    stopping_criteria = m/t
    outer_iter = 0
    total_iter = 0
    const_funcs_eval = constraint_inequalities_funcs_eval(x,constraints_ineq)
    print("Outer iterations of path following method")
    print('{} {:0.2e}'.format("Mu value:", mu))
    columns = ["Outer iter", "logbarrier", "t_log_barrier",
               "duality gap"]
    list_values = [outer_iter, log_barrier_eval, t,
                   stopping_criteria]
    data = {"row" + str(outer_iter): list_values}
    print_iterations(data, columns)
    print("-"*65 + "\n" + "-"*65)
    error_between_iterations = 1

    while(stopping_criteria > tol_outer_iter \
          and error_between_iterations > tol_outer_iter \
          and total_iter < max_total_iterations):
        x_aux = x
        [x,
         inner_iterations,
         Err_plot,
         x_plot] = logarithmic_barrier_newton_method(f,
                                                     constraints_ineq,
                                                     t, x, tol,
                                                     tol_backtracking,
                                                     x_ast, p_ast,
                                                     max_inner_iter,
                                                     gf_B,
                                                     Hf_B)
        if(outer_iter == 0):
            x_plot_total_iter = x_plot
        else:
            x_plot_total_iter = np.column_stack((x_plot_total_iter,
                                                 x_plot))
        error_between_iterations = compute_error(x, x_aux)
        if(error_between_iterations > tol_outer_iter):
            if plot:
                plot_inner_iterations(Err_plot)
                plot_central_path(x_plot)
            print("Inner iterations")
            print(x)
            const_ineq_funcs_eval = constraint_inequalities_funcs_eval(x,
                                                                       constraints_ineq)
            t=mu*t
            log_barrier_eval = logarithmic_barrier(f,x,t,
                                                   constraints_ineq)
            if(sum(const_ineq_funcs_eval < -np.nextafter(0,1)) <=-1):
                print("Some constraint inequalities evaluated in x were nonpositive, check approximations")
            outer_iter+=1
            stopping_criteria = m/t
            total_iter+= total_iter + inner_iterations
            #print
            print("-"*65 + "\n" + "-"*65)
            print("Outer iterations of primal-dual LB method")
            print('{} {:0.2e}'.format("Mu value:", mu))
            list_values = [outer_iter, log_barrier_eval, t,
                           stopping_criteria]
            data = {"row" + str(outer_iter): list_values}
            print_iterations(data, columns)
            print("-"*65 + "\n" + "-"*65)
    return [x,total_iter,t,x_plot_total_iter]

def logarithmic_barrier_newton_method(f_B, constraints_ineq,
                                      t_B, x_0, tol,
                                      tol_backtracking, x_ast=None,
                                      p_ast=None, maxiter=30,
                                      gf_B=None,
                                      Hf_B=None):
    """
    Logarithmic barrier primal-dual method using Newton's method
    to numerically approximate solution of PL with only inequalities.
    Args:
        f_B (fun): definition of logarithmic barrier function
                   as lambda expression or function definition.
        constraints_ineq: dictionary of inequalities constraints
                          in "<= 0" form.
        t_path: barrier parameter.
        x_0 (numpy ndarray): initial point for Newton's method.
        tol (float): tolerance that will halt method.
                     Controls stopping criteria.
        tol_backtracking (float): tolerance that will halt method.
                                  Controls value of line search
                                  by backtracking.
        x_ast (numpy ndarray): solution of min f_o subject to Ax <= b.
                               It's required that user knows the solution...
        p_ast (float): value of f_o(x_ast).
        maxiter (int): maximum number of iterations
        gf_B (fun): definition of gradient of f_B as lambda
                     expressionr or function definition.
        Hf_B (fun): definition of Hessian of f_B as lambda
                     expressionr or function definition.
    Returns:
        x (numpy ndarray): numpy array, approximation solution of
                           inner iterations problem.
        iteration (int): number of iterations.
        Err_plot (numpy ndarray): numpy array of absolute error
                                  between p_ast and f(x) with x
                                  approximation of x_ast. Useful for
                                  plotting.
        x_plot (numpy ndarray): numpy array that containts in columns
                                vector of approximations. Last column
                                contains x, approximation of solution.
                                Useful for plotting.
    """
    iteration = 0

    x = x_0
    n = x.size

    #Eval
    f_B_eval = f_B(x)
    gf_B_eval = gf_B(x, t_B)
    Hf_B_eval = Hf_B(x, t_B)

    normgf = np.linalg.norm(gf_B_eval)
    condHf= np.linalg.cond(Hf_B_eval)
    Err_plot_aux = np.zeros(maxiter)
    Err_plot_aux[iteration]=compute_error(p_ast,f_B_eval)

    Err = compute_error(x_ast,x)

    system_matrix = Hf_B_eval
    rhs = -gf_B_eval

    x_plot = np.zeros((n,maxiter))
    x_plot[:,iteration] = x

    #Newton's direction and Newton's decrement
    dir_Newton = np.linalg.solve(system_matrix, rhs)
    dec_Newton = rhs.dot(dir_Newton)
    columns = ["Iter", "Normgf", "Newtons decrement",
               "Err x ast", "Err p ast",
               "line search", "CondHf"]
    list_values = [iteration, normgf, dec_Newton,
                   Err, Err_plot_aux[iteration], "---", condHf]
    data = {"row" + str(iteration): list_values}

    #print
    print_iterations(data, columns)

    #stopping criteria
    stopping_criteria = dec_Newton/2
    iteration+=1

    while(stopping_criteria>tol and iteration < maxiter):
        der_direct = -dec_Newton
        t = line_search_for_log_barrier_by_backtracking(f_B,dir_Newton,
                                                        x,t_B,
                                                        constraints_ineq,
                                                        der_direct)
        x = x + t*dir_Newton

        #Eval
        f_B_eval = f_B(x)
        gf_B_eval = gf_B(x, t_B)
        Hf_B_eval = Hf_B(x, t_B)

        system_matrix = Hf_B_eval
        rhs = -gf_B_eval

        #Update
        dir_Newton = np.linalg.solve(system_matrix, rhs)
        dec_Newton = dec_Newton = rhs.dot(dir_Newton)
        Err_plot_aux[iteration]=compute_error(p_ast,f_B_eval)
        x_plot[:,iteration] = x
        Err = compute_error(x_ast,x)
        stopping_criteria = dec_Newton/2
        condHf= np.linalg.cond(Hf_B_eval)
        normgf = np.linalg.norm(gf_B_eval)

        #print
        list_values = [iteration, normgf, dec_Newton,
                       Err, Err_plot_aux[iteration], t, condHf]
        data = {"row" + str(iteration): list_values}
        print_iterations(data, columns)

        if t<tol_backtracking: #if t is less than tol_backtracking
                               #then we need to check the reason
            iter_salida=iteration
            iteration = maxiter - 1
        iteration+=1
    print('{} {:0.2e}'.format("Error of x with respect to x_ast:",Err))
    print('{} {}'.format("Approximate solution:", x))
    idx_columns_for_plot = Err_plot_aux > np.finfo(float).eps*10**(-2)
    Err_plot = Err_plot_aux[idx_columns_for_plot]

    if iteration == maxiter and t < tol_backtracking:
        print("Backtracking value less than tol_backtracking, check approximation")
        iteration=iter_salida
    else:
        if iteration == maxiter:
            print("Reached maximum of iterations, check approximation")
    x_plot = x_plot[:,:iteration]
    return [x,iteration,Err_plot,x_plot]
<|endoftext|>


<commit_before>from os import environ
from unittest import TestCase

environ['DJANGO_SETTINGS_MODULE'] = 'test_settings'
from incuna_auth.middleware import LoginRequiredMiddleware

class AuthenticatedUser(object):
    def is_authenticated(self):
        return True

class AnonymousUser(object):
    def is_authenticated(self):
        return False

class Request(object):

    def __init__(self, path_info, method='GET'):
        self.path_info = path_info
        self.method = method

class TestLoginRequiredMiddleware(TestCase):
    def setUp(self):
        self.middleware = LoginRequiredMiddleware()

    def test_skip_middleware_if_url_is_exempt(self):
        self.request = Request('exempt-and-protected-url/')
        self.request.user = AnonymousUser()
        response = self.middleware.process_request(self.request)
        self.assertEqual(response, None)

    def test_skip_middleware_if_url_is_not_protected(self):
        self.request = Request('non-protected-url/')
        self.request.user = AnonymousUser()
        response = self.middleware.process_request(self.request)
        self.assertEqual(response, None)

    def test_skip_middleware_if_user_is_authenticated(self):
        self.request = Request('protected-url/')
        self.request.user = AuthenticatedUser()
        response = self.middleware.process_request(self.request)
        self.assertEqual(response, None)
<commit_msg>Add test for non-GET requests<commit_after>from os import environ
from unittest import TestCase

environ['DJANGO_SETTINGS_MODULE'] = 'test_settings'
from incuna_auth.middleware import LoginRequiredMiddleware

class AuthenticatedUser(object):
    def is_authenticated(self):
        return True

class AnonymousUser(object):
    def is_authenticated(self):
        return False

class Request(object):

    def __init__(self, path_info, method='GET'):
        self.path_info = path_info
        self.method = method

class TestLoginRequiredMiddleware(TestCase):
    def setUp(self):
        self.middleware = LoginRequiredMiddleware()

    def test_skip_middleware_if_url_is_exempt(self):
        self.request = Request('exempt-and-protected-url/')
        self.request.user = AnonymousUser()
        response = self.middleware.process_request(self.request)
        self.assertEqual(response, None)

    def test_skip_middleware_if_url_is_not_protected(self):
        self.request = Request('non-protected-url/')
        self.request.user = AnonymousUser()
        response = self.middleware.process_request(self.request)
        self.assertEqual(response, None)

    def test_skip_middleware_if_user_is_authenticated(self):
        self.request = Request('protected-url/')
        self.request.user = AuthenticatedUser()
        response = self.middleware.process_request(self.request)
        self.assertEqual(response, None)

    def test_403_result_if_non_get_request(self):
        self.request = Request('protected-url/', 'POST')
        self.request.user = AnonymousUser()
        response = self.middleware.process_request(self.request)
        self.assertEqual(response.status_code, 403)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
A convenience system to manage jobs, both active and already run
'''

from __future__ import print_function

from __future__ import absolute_import

# Import python libs
import fnmatch
import os

# Import salt libs
import salt.client
import salt.payload
import salt.utils
import salt.minion

from salt.ext.six import string_types

import logging
log = logging.getLogger(__name__)

def active(outputter=None, display_progress=False):
    '''
    Return a report on all actively running jobs from a job id centric
    perspective

    CLI Example:

    .. code-block:: bash

        salt-run jobs.active
    '''
    ret = {}
    client = salt.client.get_local_client(__opts__['conf_file'])
    active_ = client.cmd('*', 'saltutil.running', timeout=__opts__['timeout'])
    if display_progress:
        __progress__('Attempting to contact minions: {0}'.format(list(active_.keys())))
    for minion, data in active_.items():
        if display_progress:
            __progress__('Received reply from minion {0}'.format(minion))
        if not isinstance(data, list):
            continue
        for job in data:
            if not job['jid'] in ret:
                ret[job['jid']] = _format_job_instance(job)
                ret[job['jid']].update({'Running': [{minion: job.get('pid', None)}], 'Returned': []})
            else:
                ret[job['jid']]['Running'].append({minion: job['pid']})

    mminion = salt.minion.MasterMinion(__opts__)
    for jid in ret:
        returner = _get_returner((__opts__['ext_job_cache'], __opts__['master_job_cache']))
        data = mminion.returners['{0}.get_jid'.format(returner)](jid)
        for minion in data:
            if minion not in ret[jid]['Returned']:
                ret[jid]['Returned'].append(minion)

    if outputter:
        return {'outputter': outputter, 'data': ret}
    else:
        return ret

def lookup_jid(jid,
               ext_source=None,
               missing=False,
               outputter=None,
               display_progress=False):
    '''
    Return the printout from a previously executed job

    CLI Example:

    .. code-block:: bash

        salt-run jobs.lookup_jid 20130916125524463507
        salt-run jobs.lookup_jid 20130916125524463507 outputter=highstate
    '''
    ret = {}
    mminion = salt.minion.MasterMinion(__opts__)
    returner = _get_returner((__opts__['ext_job_cache'], ext_source, __opts__['master_job_cache']))
    if display_progress:
        __progress__('Querying returner: {0}'.format(returner))

    try:
        data = mminion.returners['{0}.get_jid'.format(returner)](jid)
    except TypeError:
        return 'Requested returner could not be loaded. No JIDs could be retrieved.'
    for minion in data:
        if display_progress:
            __progress__(minion)
        if u'return' in data[minion]:
            ret[minion] = data[minion].get(u'return')
        else:
            ret[minion] = data[minion].get('return')
    if missing:
        ckminions = salt.utils.minions.CkMinions(__opts__)
        exp = ckminions.check_minions(data['tgt'], data['tgt_type'])
        for minion_id in exp:
            if minion_id not in data:
                ret[minion_id] = 'Minion did not return'
    if outputter:
        return {'outputter': outputter, 'data': ret}
    else:
        return ret

def list_job(jid, ext_source=None, outputter=None):
    '''
    List a specific job given by its jid

    CLI Example:

    .. code-block:: bash

        salt-run jobs.list_job 20130916125524463507
    '''
    ret = {'jid': jid}
    mminion = salt.minion.MasterMinion(__opts__)
    returner = _get_returner((__opts__['ext_job_cache'], ext_source, __opts__['master_job_cache']))

    job = mminion.returners['{0}.get_load'.format(returner)](jid)
    ret.update(_format_jid_instance(jid, job))
    ret['Result'] = mminion.returners['{0}.get_jid'.format(returner)](jid)
    if outputter:
        return {'outputter': outputter, 'data': ret}
    else:
        return ret

def list_jobs(ext_source=None,
              outputter=None,
              search_metadata=None,
              search_function=None,
              search_target=None,
              display_progress=False):
    '''
    List all detectable jobs and associated functions

    CLI Example:

    .. code-block:: bash

        salt-run jobs.list_jobs
    '''
    returner = _get_returner((__opts__['ext_job_cache'], ext_source, __opts__['master_job_cache']))
    if display_progress:
        __progress__('Querying returner {0} for jobs.'.format(returner))
    mminion = salt.minion.MasterMinion(__opts__)

    try:
        ret = mminion.returners['{0}.get_jids'.format(returner)]()
    except TypeError:
        return 'Error: Requested returner could not be loaded. No jobs could be retrieved.'

    if search_metadata:
        mret = {}
        for item in ret:
            if 'Metadata' in ret[item]:
                if isinstance(search_metadata, dict):
                    for key in search_metadata:
                        if key in ret[item]['Metadata']:
                            if ret[item]['Metadata'][key] == search_metadata[key]:
                                mret[item] = ret[item]
                else:
                    log.info('The search_metadata parameter must be specified'
                             ' as a dictionary.  Ignoring.')
    else:
        mret = ret.copy()

    if search_target:
        _mret = {}
        for item in mret:
            if 'Target' in ret[item]:
                if isinstance(search_target, list):
                    for key in search_target:
                        if fnmatch.fnmatch(ret[item]['Target'], key):
                            _mret[item] = ret[item]
                elif isinstance(search_target, string_types):
                    if fnmatch.fnmatch(ret[item]['Target'], search_target):
                        _mret[item] = ret[item]
        mret = _mret.copy()

    if search_function:
        _mret = {}
        for item in mret:
            if 'Function' in ret[item]:
                if isinstance(search_function, list):
                    for key in search_function:
                        if fnmatch.fnmatch(ret[item]['Function'], key):
                            _mret[item] = ret[item]
                elif isinstance(search_function, string_types):
                    if fnmatch.fnmatch(ret[item]['Function'], search_function):
                        _mret[item] = ret[item]
        mret = _mret.copy()

    if outputter:
        return {'outputter': outputter, 'data': mret}
    else:
        return mret

def print_job(jid, ext_source=None, outputter=None):
    '''
    Print a specific job's detail given by it's jid, including the return data.

    CLI Example:

    .. code-block:: bash

        salt-run jobs.print_job 20130916125524463507
    '''
    ret = {}

    returner = _get_returner((__opts__['ext_job_cache'], ext_source, __opts__['master_job_cache']))
    mminion = salt.minion.MasterMinion(__opts__)

    try:
        job = mminion.returners['{0}.get_load'.format(returner)](jid)
        ret[jid] = _format_jid_instance(jid, job)
    except TypeError:
        ret[jid]['Result'] = ('Requested returner {0} is not available. Jobs cannot be retrieved. '
            'Check master log for details.'.format(returner))
        return ret
    ret[jid]['Result'] = mminion.returners['{0}.get_jid'.format(returner)](jid)
    if outputter:
        return {'outputter': outputter, 'data': ret}
    else:
        return ret

def _get_returner(returner_types):
    '''
    Helper to iterate over returner_types and pick the first one
    '''
    for returner in returner_types:
        if returner and returner is not None:
            return returner

def _format_job_instance(job):
    '''
    Helper to format a job instance
    '''
    ret = {'Function': job.get('fun', 'unknown-function'),
           'Arguments': list(job.get('arg', [])),
           # unlikely but safeguard from invalid returns
           'Target': job.get('tgt', 'unknown-target'),
           'Target-type': job.get('tgt_type', []),
           'User': job.get('user', 'root')}

    if 'metadata' in job:
        ret['Metadata'] = job.get('metadata', {})
    else:
        if 'kwargs' in job:
            if 'metadata' in job['kwargs']:
                ret['Metadata'] = job['kwargs'].get('metadata', {})

    if 'Minions' in job:
        ret['Minions'] = job['Minions']
    return ret

def _format_jid_instance(jid, job):
    '''
    Helper to format jid instance
    '''
    ret = _format_job_instance(job)
    ret.update({'StartTime': salt.utils.jid_to_time(jid)})
    return ret

def _walk_through(job_dir, display_progress=False):
    '''
    Walk through the job dir and return jobs
    '''
    serial = salt.payload.Serial(__opts__)

    for top in os.listdir(job_dir):
        t_path = os.path.join(job_dir, top)

        for final in os.listdir(t_path):
            load_path = os.path.join(t_path, final, '.load.p')
            job = serial.load(salt.utils.fopen(load_path, 'rb'))

            if not os.path.isfile(load_path):
                continue

            job = serial.load(salt.utils.fopen(load_path, 'rb'))
            jid = job['jid']
            if display_progress:
                __progress__('Found JID {0}'.format(jid))
            yield jid, job, t_path, final
<commit_msg>Add docs to runner.jobs.lookup_jid and versionadded tags<commit_after># -*- coding: utf-8 -*-
'''
A convenience system to manage jobs, both active and already run
'''

from __future__ import print_function

from __future__ import absolute_import

# Import python libs
import fnmatch
import os

# Import salt libs
import salt.client
import salt.payload
import salt.utils
import salt.minion

from salt.ext.six import string_types

import logging
log = logging.getLogger(__name__)

def active(outputter=None, display_progress=False):
    '''
    Return a report on all actively running jobs from a job id centric
    perspective

    CLI Example:

    .. code-block:: bash

        salt-run jobs.active
    '''
    ret = {}
    client = salt.client.get_local_client(__opts__['conf_file'])
    active_ = client.cmd('*', 'saltutil.running', timeout=__opts__['timeout'])
    if display_progress:
        __progress__('Attempting to contact minions: {0}'.format(list(active_.keys())))
    for minion, data in active_.items():
        if display_progress:
            __progress__('Received reply from minion {0}'.format(minion))
        if not isinstance(data, list):
            continue
        for job in data:
            if not job['jid'] in ret:
                ret[job['jid']] = _format_job_instance(job)
                ret[job['jid']].update({'Running': [{minion: job.get('pid', None)}], 'Returned': []})
            else:
                ret[job['jid']]['Running'].append({minion: job['pid']})

    mminion = salt.minion.MasterMinion(__opts__)
    for jid in ret:
        returner = _get_returner((__opts__['ext_job_cache'], __opts__['master_job_cache']))
        data = mminion.returners['{0}.get_jid'.format(returner)](jid)
        for minion in data:
            if minion not in ret[jid]['Returned']:
                ret[jid]['Returned'].append(minion)

    if outputter:
        return {'outputter': outputter, 'data': ret}
    else:
        return ret

def lookup_jid(jid,
               ext_source=None,
               missing=False,
               outputter=None,
               display_progress=False):
    '''
    Return the printout from a previously executed job

    jid
        The jid to look up.

    ext_source
        The external job cache to use. Default: `None`.

    missing
        When set to `True`, adds the minions that did not return from the command.
        Default: `False`.

    outputter
        The outputter to use. Default: `None`.

        .. versionadded:: Lithium

    display_progress
        Displays progress events when set to `True`. Default: `False`.

        .. versionadded:: Lithium

    CLI Example:

    .. code-block:: bash

        salt-run jobs.lookup_jid 20130916125524463507
        salt-run jobs.lookup_jid 20130916125524463507 outputter=highstate
    '''
    ret = {}
    mminion = salt.minion.MasterMinion(__opts__)
    returner = _get_returner((__opts__['ext_job_cache'], ext_source, __opts__['master_job_cache']))
    if display_progress:
        __progress__('Querying returner: {0}'.format(returner))

    try:
        data = mminion.returners['{0}.get_jid'.format(returner)](jid)
    except TypeError:
        return 'Requested returner could not be loaded. No JIDs could be retrieved.'
    for minion in data:
        if display_progress:
            __progress__(minion)
        if u'return' in data[minion]:
            ret[minion] = data[minion].get(u'return')
        else:
            ret[minion] = data[minion].get('return')
    if missing:
        ckminions = salt.utils.minions.CkMinions(__opts__)
        exp = ckminions.check_minions(data['tgt'], data['tgt_type'])
        for minion_id in exp:
            if minion_id not in data:
                ret[minion_id] = 'Minion did not return'
    if outputter:
        return {'outputter': outputter, 'data': ret}
    else:
        return ret

def list_job(jid, ext_source=None, outputter=None):
    '''
    List a specific job given by its jid

    CLI Example:

    .. code-block:: bash

        salt-run jobs.list_job 20130916125524463507
    '''
    ret = {'jid': jid}
    mminion = salt.minion.MasterMinion(__opts__)
    returner = _get_returner((__opts__['ext_job_cache'], ext_source, __opts__['master_job_cache']))

    job = mminion.returners['{0}.get_load'.format(returner)](jid)
    ret.update(_format_jid_instance(jid, job))
    ret['Result'] = mminion.returners['{0}.get_jid'.format(returner)](jid)
    if outputter:
        return {'outputter': outputter, 'data': ret}
    else:
        return ret

def list_jobs(ext_source=None,
              outputter=None,
              search_metadata=None,
              search_function=None,
              search_target=None,
              display_progress=False):
    '''
    List all detectable jobs and associated functions

    CLI Example:

    .. code-block:: bash

        salt-run jobs.list_jobs
    '''
    returner = _get_returner((__opts__['ext_job_cache'], ext_source, __opts__['master_job_cache']))
    if display_progress:
        __progress__('Querying returner {0} for jobs.'.format(returner))
    mminion = salt.minion.MasterMinion(__opts__)

    try:
        ret = mminion.returners['{0}.get_jids'.format(returner)]()
    except TypeError:
        return 'Error: Requested returner could not be loaded. No jobs could be retrieved.'

    if search_metadata:
        mret = {}
        for item in ret:
            if 'Metadata' in ret[item]:
                if isinstance(search_metadata, dict):
                    for key in search_metadata:
                        if key in ret[item]['Metadata']:
                            if ret[item]['Metadata'][key] == search_metadata[key]:
                                mret[item] = ret[item]
                else:
                    log.info('The search_metadata parameter must be specified'
                             ' as a dictionary.  Ignoring.')
    else:
        mret = ret.copy()

    if search_target:
        _mret = {}
        for item in mret:
            if 'Target' in ret[item]:
                if isinstance(search_target, list):
                    for key in search_target:
                        if fnmatch.fnmatch(ret[item]['Target'], key):
                            _mret[item] = ret[item]
                elif isinstance(search_target, string_types):
                    if fnmatch.fnmatch(ret[item]['Target'], search_target):
                        _mret[item] = ret[item]
        mret = _mret.copy()

    if search_function:
        _mret = {}
        for item in mret:
            if 'Function' in ret[item]:
                if isinstance(search_function, list):
                    for key in search_function:
                        if fnmatch.fnmatch(ret[item]['Function'], key):
                            _mret[item] = ret[item]
                elif isinstance(search_function, string_types):
                    if fnmatch.fnmatch(ret[item]['Function'], search_function):
                        _mret[item] = ret[item]
        mret = _mret.copy()

    if outputter:
        return {'outputter': outputter, 'data': mret}
    else:
        return mret

def print_job(jid, ext_source=None, outputter=None):
    '''
    Print a specific job's detail given by it's jid, including the return data.

    CLI Example:

    .. code-block:: bash

        salt-run jobs.print_job 20130916125524463507
    '''
    ret = {}

    returner = _get_returner((__opts__['ext_job_cache'], ext_source, __opts__['master_job_cache']))
    mminion = salt.minion.MasterMinion(__opts__)

    try:
        job = mminion.returners['{0}.get_load'.format(returner)](jid)
        ret[jid] = _format_jid_instance(jid, job)
    except TypeError:
        ret[jid]['Result'] = ('Requested returner {0} is not available. Jobs cannot be retrieved. '
            'Check master log for details.'.format(returner))
        return ret
    ret[jid]['Result'] = mminion.returners['{0}.get_jid'.format(returner)](jid)
    if outputter:
        return {'outputter': outputter, 'data': ret}
    else:
        return ret

def _get_returner(returner_types):
    '''
    Helper to iterate over returner_types and pick the first one
    '''
    for returner in returner_types:
        if returner and returner is not None:
            return returner

def _format_job_instance(job):
    '''
    Helper to format a job instance
    '''
    ret = {'Function': job.get('fun', 'unknown-function'),
           'Arguments': list(job.get('arg', [])),
           # unlikely but safeguard from invalid returns
           'Target': job.get('tgt', 'unknown-target'),
           'Target-type': job.get('tgt_type', []),
           'User': job.get('user', 'root')}

    if 'metadata' in job:
        ret['Metadata'] = job.get('metadata', {})
    else:
        if 'kwargs' in job:
            if 'metadata' in job['kwargs']:
                ret['Metadata'] = job['kwargs'].get('metadata', {})

    if 'Minions' in job:
        ret['Minions'] = job['Minions']
    return ret

def _format_jid_instance(jid, job):
    '''
    Helper to format jid instance
    '''
    ret = _format_job_instance(job)
    ret.update({'StartTime': salt.utils.jid_to_time(jid)})
    return ret

def _walk_through(job_dir, display_progress=False):
    '''
    Walk through the job dir and return jobs
    '''
    serial = salt.payload.Serial(__opts__)

    for top in os.listdir(job_dir):
        t_path = os.path.join(job_dir, top)

        for final in os.listdir(t_path):
            load_path = os.path.join(t_path, final, '.load.p')
            job = serial.load(salt.utils.fopen(load_path, 'rb'))

            if not os.path.isfile(load_path):
                continue

            job = serial.load(salt.utils.fopen(load_path, 'rb'))
            jid = job['jid']
            if display_progress:
                __progress__('Found JID {0}'.format(jid))
            yield jid, job, t_path, final
<|endoftext|>


<commit_before>#!/usr/bin/env python
# git-state.py
# GusE 2014.12.16 V0.1
"""
Git utility to show the state of all subfolder git repositories
"""

import getopt
import sys
import os
import subprocess
import traceback
import logging
import logging.handlers

__app__ = os.path.basename(__file__)
__author__ = "Gus Esquivel"
__copyright__ = "Copyright 2014"
__credits__ = ["Gus Esquivel"]
__license__ = "GPL"
__version__ = "0.1"
__maintainer__ = "Gus Esquivel"
__email__ = "gesquive@gmail.com"
__status__ = "Beta"

#--------------------------------------
# Configurable Constants
LOG_FILE = '/var/log/' + os.path.splitext(__app__)[0] + '.log'
LOG_SIZE = 1024*1024*200

verbose = False
debug = False

logger = logging.getLogger(__app__)

def usage():
    usage = \
"""Usage: %s [options] forced_arg
    Git utility to show the state of all subfolder git repositories
Options and arguments:
  -h --help                         Prints this message.
  -v --verbose                      Writes all messages to console.

    v%s
""" % (__app__, __version__)

    print usage

def main():
    global verbose, debug

    try:
        opts, args = getopt.getopt(sys.argv[1:], "hv", \
        ["help", "verbose", "debug"])
    except getopt.GetoptError, err:
        print str(err)
        print usage()
        sys.exit(2)

    verbose = False
    debug = False
    forced_arg = None

    # Save forced arg
    if len(args) > 0:
        forced_arg = args[0]
    elif len(args) > 1:
        usage()
        sys.exit(2)

    for o, a in opts:
        if o in ("-h", "--help"):
            # Print out help and exit
            usage()
            sys.exit()
        elif o in ("-dd", "--debug"):
            debug = True
        elif o in ("-v", "--verbose"):
            verbose = True

    log_file = LOG_FILE
    if not os.access(log_file, os.W_OK):
        print "Cannot write to '%(log_file)s'.\nExiting." % locals()
        sys.exit(2)
    file_handler = logging.handlers.RotatingFileHandler(log_file,
                                            maxBytes=LOG_SIZE, backupCount=9)
    file_formater = logging.Formatter('%(asctime)s,%(levelname)s,%(thread)d,%(message)s')
    file_handler.setFormatter(file_formater)
    logger.addHandler(file_handler)

    if verbose:
        console_handler = logging.StreamHandler(sys.stdout)
        console_formatter = logging.Formatter("[%(asctime)s] %(levelname)-5.5s: %(message)s")
        console_handler.setFormatter(console_formatter)
        logger.addHandler(console_handler)

    logger.setLevel(logging.DEBUG)

    try:
        # DO SOMETHING
        do_something()
    except (KeyboardInterrupt, SystemExit):
        pass
    except Exception, e:
        print traceback.format_exc()

def do_something():
    print "Hello World!"

if __name__ == '__main__':
    main()
<commit_msg>Herping the derp<commit_after>#!/usr/bin/env python
# git-state.py
# GusE 2014.12.16 V0.1
"""
Git utility to show the state of all subfolder git repositories
"""

import getopt
import sys
import os
import subprocess
import traceback
import logging
import logging.handlers
import argparse

__app__ = os.path.basename(__file__)
__author__ = "Gus Esquivel"
__copyright__ = "Copyright 2014"
__credits__ = ["Gus Esquivel"]
__license__ = "GPL"
__version__ = "0.1"
__maintainer__ = "Gus Esquivel"
__email__ = "gesquive@gmail"
__status__ = "Beta"

#--------------------------------------
# Configurable Constants
LOG_FILE = '/var/log/' + os.path.splitext(__app__)[0] + '.log'
LOG_SIZE = 1024*1024*200

verbose = False
debug = False

logger = logging.getLogger(__app__)

def main():
    global verbose, debug

    parser = argparse.ArgumentParser(add_help=False,
        description="Git utility to show the state of all subfolder git repositories",
        epilog="%(__app__)s v%(__version__)s\n" % globals())

    group = parser.add_argument_group("Options")
    group.add_argument("-d", "--dir", default=".",
        help="The parent directory to the git repositories.")
    group.add_argument("-h", "--help", action="help",
        help="Show this help message and exit.")
    group.add_argument("-v", "--verbose", action="store_true", dest="verbose",
        help="Writes all messages to console.")
    group.add_argument("-D", "--debug", action="store_true", dest="debug",
        help=argparse.SUPPRESS)
    group.add_argument("-V", "--version", action="version",
                    version="%(__app__)s v%(__version__)s" % globals())

    args = parser.parse_args()
    verbose = args.verbose
    debug = args.debug

    console_handler = logging.StreamHandler(sys.stdout)
    console_formatter = logging.Formatter("[%(asctime)s] %(levelname)-5.5s: %(message)s")
    console_handler.setFormatter(console_formatter)
    if verbose:
        logger.addHandler(console_handler)

    logger.setLevel(logging.DEBUG)

    try:
        # DO SOMETHING
        do_something()
    except (KeyboardInterrupt, SystemExit):
        pass
    except Exception, e:
        print traceback.format_exc()

def do_something():
    print "Hello World!"

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import pdb
import sys
import traceback
import subprocess
import os
import time
import logging
import yaml
from conda_build.metadata import MetaData
import signal
from pprint import pformat
# create the anaconda cli
import binstar_client
from argparse import Namespace

current_subprocs = set()
shutdown = False

def handle_signal(signum, frame):
    # send signal recieved to subprocesses
    global shutdown
    shutdown = True
    for proc in current_subprocs:
        if proc.poll() is None:
            proc.send_signal(signum)
    print("Killing build script due to receiving signum={}"
          "".format(signum))
    sys.exit(1)

signal.signal(signal.SIGINT, handle_signal)
signal.signal(signal.SIGTERM, handle_signal)

def get_file_names_on_anaconda_channel(username, anaconda_cli,
                                       channel='main'):
    """Get the names of **all** the files on anaconda.org/username

    Parameters
    ----------
    username : str
    anaconda_cli : return value from binstar_client.utils.get_binstar()
    channel : str, optional
        The channel on anaconda.org/username to upload to.
        Defaults to 'main'

    Returns
    -------
    set
        The file names of all files on an anaconda channel.
        Something like 'linux-64/album-0.0.2.post0-0_g6b05c00_py27.tar.bz2'
    """
    return set([f['basename']
                for f in anaconda_cli.show_channel(channel, username)['files']])

def Popen(cmd):
    """Returns stdout and stderr

    Parameters
    ----------
    cmd : list
        List of strings to be sent to subprocess.Popen

    Returns
    -------
    stdout : """
    # capture the output with subprocess.Popen
    try:
        proc = subprocess.Popen(cmd, stderr=subprocess.PIPE)
        current_subprocs.add(proc)
    except subprocess.CalledProcessError as cpe:
        print(cpe)
        # pdb.set_trace()
    stdout, stderr = proc.communicate()
    current_subprocs.remove(proc)
    return stdout, stderr, proc.returncode

def check_output(cmd):
    try:
        ret = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as cpe:
        print(cmd)
        print(cpe.output.decode())
        raise RuntimeError("{} raised with check_output comand {}".format(
            cpe.output.decode(), cmd))
    else:
        name = ret.decode().strip().split('\n')
        return name

def determine_build_name(path_to_recipe, *conda_build_args):
    """Figure out what conda says the output built package name is going to be
    Parameters
    ----------
    path_to_recipe : str
        The location of the recipe on disk
    *conda_build_args : list
        List of extra arguments to be appeneded to the conda build command.
        For example, this might include ['--python', '3.4'], to tell
        conda-build to build a python 3.4 package

    Returns
    -------
    package_name : str
        Something like:
        /home/edill/mc/conda-bld/linux-64/pims-0.3.3.post0-0_g1bea480_py27.tar.bz2
    """
    conda_build_args = [] if conda_build_args is None else list(conda_build_args)
    cmd = ['conda', 'build', path_to_recipe, '--output'] + conda_build_args
    ret = check_output(cmd)
    if len(ret) > 1:
        # Then this is the first time we are getting the build name and conda
        # has to check out the source. Call it a second time and you get the4
        # full path to the file that will be spit out by conda-build
        return determine_build_name(path_to_recipe, *conda_build_args)
    # we want to keep track of the exact command so we can run it later.
    # Obviously drop the `--output` flag so that conda-build actually builds
    # the package.
    cmd.remove('--output')
    return ret[0], cmd

def run_build(recipes_path, anaconda_cli, username, pyver,
              token=None, npver=None):
    """Build and upload packages that do not already exist at {{ username }}

    Parameters
    ----------
    recipes_path : str
        Folder where the recipes are located
    anaconda_cli : return value from binstar_client.utils.get_binstar()
    username : str
        The anaconda user to upload all the built packages to
    pyver : list
        The python versions that need to be build. Should be a list of python
        versions to be passed to conda-build /path --python pyver.
        ['2.7', '3.4', '3.5']
    token : str, optional
        The binstar token that should be used to upload packages to
        anaconda.org/username. If no token is provided, no uploading will occur
    npver : str, optional
    """
    if npver is None:
        npver = os.environ.get("CONDA_NPY", "1.11")
    if token is None:
        print("No binstar token available. There will be no uploading."
              "Consider setting the BINSTAR_TOKEN environmental "
              "variable or passing one in via the --token command "
              "line argument")
    # get all file names that are in the channel I am interested in
    packages = get_file_names_on_anaconda_channel(username, anaconda_cli)

    dont_build = []
    to_build = []
    print("Determining package build names...")
    print('{: <8} | {}'.format('to build', 'built package name'))
    logging.info("Build Plan")
    for folder in sorted(os.listdir(recipes_path)):
        recipe_dir = os.path.join(recipes_path, folder)
        for py in pyver:
            try:
                path_to_built_package, build_cmd = determine_build_name(
                    recipe_dir, '--python', py, '--numpy', '1.10')
            except RuntimeError as re:
                logging.error(re)
                continue
            name_on_anaconda = os.sep.join(
                path_to_built_package.split(os.sep)[-2:])
            # pdb.set_trace()
            meta = MetaData(recipe_dir)
            on_anaconda_channel = name_on_anaconda in packages
            if on_anaconda_channel:
                dont_build.append((path_to_built_package, name_on_anaconda,
                                   build_cmd, meta))
            else:
                to_build.append((path_to_built_package, name_on_anaconda,
                                 build_cmd, meta))

            logging.info('{:<8} | {}'.format(
                         str(not bool(on_anaconda_channel)), name_on_anaconda))
    # build all the packages that need to be built
    UPLOAD_CMD = ['anaconda', '-t', token, 'upload', '-u',
                  username]
    # pdb.set_trace()
    no_token = []
    uploaded = []
    upload_failed = []
    build_or_test_failed = []
    # for each package
    for full_path, pkg_name, cmd, metadata in to_build:
        # output the package build name
        print("Building: %s"% pkg_name)
        # output the build command
        print("Build cmd: %s" % ' '.join(cmd))
        stdout, stderr, returncode = Popen(cmd)
        if returncode != 0:
            build_or_test_failed.append(pkg_name)
            print('stdout\n', stdout)
            print('stderr\n', stderr)
            continue
        if token:
            print("UPLOAD START")
            stdout, stderr, returncode = Popen(UPLOAD_CMD + [full_path])
            if returncode != 0:
                print('stdout\n', stdout)
                print('stderr\n', stderr)
                upload_failed.append(pkg_name)
                continue
            uploaded.append(pkg_name)
            continue

        no_token.append(pkg_name)

    return {
        'alreadybuilt': sorted([dont[1] for dont in dont_build]),
        'uploaded': sorted(uploaded),
        'no_token': sorted(no_token),
        'upload_failed': sorted(upload_failed),
        'build_or_test_failed': sorted(build_or_test_failed),
    }

def set_binstar_upload(on=False):
    """Turn on or off binstar uploading

    Parameters
    ----------
    on : bool, optional
        True: turn on binstar uploading
        False: turn off binstar uploading
        Defaults to False

    Raises exception if disabling binstar upload fails
    """
    rcpath = os.path.join(os.path.expanduser('~'), '.condarc')
    with open(rcpath, 'r') as f:
        rc = yaml.load(f.read())
    binstar_upload = rc.get('binstar_upload', False)
    if binstar_upload != on:
        rc['binstar_upload'] = on
        with open(rcpath, 'w') as f:
            # write the new yaml in `rcpath`
            f.write(yaml.dump(rc))

def cli():
    from argparse import ArgumentParser
    p = ArgumentParser(
        description="""
Tool for building a folder of conda recipes where only the ones that don't
already exist are built.
""",
    )
    p.add_argument(
        'recipes_path',
        nargs='?',
        help="path to recipes that should be built"
    )
    p.add_argument(
        '-p', "--pyver",
        action='store',
        nargs='*',
        help="Directory to write recipes to (default: %(default)s).",
        default=["2.7", "3.4", "3.5"],
    )
    p.add_argument(
        '-t', '--token', action='store',
        nargs='?',
        help='Binstar token to use to upload build packages',
        default=None
    )
    p.add_argument(
        '-l', '--log',
        nargs='?',
        help='Name of the log file to write'
    )
    p.add_argument(
        "-s", "--site",
        nargs='?',
        help='Anaconda upload api (defaults to %(default)s',
        default='https://api.anaconda.org'
    )
    p.add_argument(
        "-u", "--username",
        action="store",
        nargs='?',
        help=("Username to upload package to")
    )
    p.add_argument(
        '--npver', action='store', nargs='*',
        help=('List the numpy versions to build packages for. Defaults to '
              '%(default)s'),
        default=['1.11']
    )
    args = p.parse_args()
    if args.token is None and 'BINSTAR_TOKEN' in os.environ:
        args.token = os.environ.get('BINSTAR_TOKEN')
    # pdb.set_trace()
    args.recipes_path = os.path.abspath(args.recipes_path)
    print(args)
    # pdb.set_trace()
    run(**dict(args._get_kwargs()))

def run(recipes_path, pyver, log, site, username, token, npver):
    # set up logging
    if not log:
        log_dirname = os.path.join(os.path.expanduser('~'),
                                      'auto-build-logs', 'dev')
        os.makedirs(log_dirname, exist_ok=True)
        log_filename = time.strftime("%m.%d-%H.%M")
        log = os.path.join(log_dirname, log_filename)
    FORMAT = "%(levelname)s | %(asctime)-15s | %(message)s"
    stream_handler = logging.StreamHandler()
    file_handler = logging.FileHandler(log)
    logging.basicConfig(level=logging.DEBUG, format=FORMAT,
                        handlers=[stream_handler, file_handler])
    # build all python versions by default
    if not pyver:
        pyver = ['2.7', '3.4', '3.5']
    # check to make sure that the recipes_path exists
    if not os.path.exists(recipes_path):
        logging.Error("The recipes_path: '%s' does not exist." % recipes_path)
        sys.exit(1)
    print('token={}'.format(token))
    # just disable binstar uploading whenever this script is running.
    print('Disabling binstar upload. If you want to turn it back on, '
          'execute: `conda config --set binstar_upload true`')
    set_binstar_upload(False)
    # set up logging
    full_recipes_path = os.path.abspath(recipes_path)
    print('Logging summary to %s' % log)

    # site = 'https://pergamon.cs.nsls2.local:8443/api'
    # os.environ['REQUESTS_CA_BUNDLE'] = '/etc/certificates/ca_cs_nsls2_local.crt'

    anaconda_cli = binstar_client.utils.get_binstar(Namespace(token=token,
                                                              site=site))
    try:
        results = run_build(full_recipes_path, anaconda_cli, username,
                            pyver, token=token, npver=npver)
    except Exception as e:
        logging.error("Major error encountered in attempt to build {}"
                      "".format(full_recipes_path))
        logging.error("Error in run_build!")
        logging.error(e)
        logging.error("Full stack trace")
        logging.error(traceback.format_exc())
    else:
        logging.info("Build summary")
        logging.info('Expected {} packages'.format(
            len(pyver) * len(os.listdir(full_recipes_path))))
        num_builds = {k: len(v) for k, v in results.items()}
        logging.info('Got {} packages.'.format(
            sum([n for n in num_builds.values()])))
        logging.info('Breakdown is as follows')
        for k, v in num_builds.items():
            logging.info('section: {:<25}. number build: {}'.format(k, v))
        if results['build_or_test_failed']:
            logging.error("Some packages failed to build")
            logging.error(pformat(results['build_or_test_failed']))
        if results['no_token']:
            logging.error("No anaconda token. Cannot upload these packages")
            logging.error(pformat(results['no_token']))
        if results['upload_failed']:
            logging.error('Upload failed for these packages')
            logging.error(pformat(results['upload_failed']))
        if results['alreadybuilt']:
            logging.info('Packages that already exist in {}'.format(username))
            logging.info(pformat(results['alreadybuilt']))
        if results['uploaded']:
            logging.info('Packages that were built and uploaded')
            logging.info(pformat(results['uploaded']))
    finally:
        # close the file handler
        file_handler.close()

if __name__ == "__main__":
    run('../staged-recipes-dev/pyall/xray-vision', ['2.7'], None, None,
        'ericdill', None)
<commit_msg>Decode the Popen output. pprint instead of print<commit_after>import pdb
import sys
import traceback
import subprocess
import os
import time
import logging
import yaml
from conda_build.metadata import MetaData
import signal
from pprint import pformat, pprint
# create the anaconda cli
import binstar_client
from argparse import Namespace

current_subprocs = set()
shutdown = False

def handle_signal(signum, frame):
    # send signal recieved to subprocesses
    global shutdown
    shutdown = True
    for proc in current_subprocs:
        if proc.poll() is None:
            proc.send_signal(signum)
    print("Killing build script due to receiving signum={}"
          "".format(signum))
    sys.exit(1)

signal.signal(signal.SIGINT, handle_signal)
signal.signal(signal.SIGTERM, handle_signal)

def get_file_names_on_anaconda_channel(username, anaconda_cli,
                                       channel='main'):
    """Get the names of **all** the files on anaconda.org/username

    Parameters
    ----------
    username : str
    anaconda_cli : return value from binstar_client.utils.get_binstar()
    channel : str, optional
        The channel on anaconda.org/username to upload to.
        Defaults to 'main'

    Returns
    -------
    set
        The file names of all files on an anaconda channel.
        Something like 'linux-64/album-0.0.2.post0-0_g6b05c00_py27.tar.bz2'
    """
    return set([f['basename']
                for f in anaconda_cli.show_channel(channel, username)['files']])

def Popen(cmd):
    """Returns stdout and stderr

    Parameters
    ----------
    cmd : list
        List of strings to be sent to subprocess.Popen

    Returns
    -------
    stdout : """
    # capture the output with subprocess.Popen
    try:
        proc = subprocess.Popen(cmd, stderr=subprocess.PIPE)
        current_subprocs.add(proc)
    except subprocess.CalledProcessError as cpe:
        print(cpe)
        # pdb.set_trace()
    stdout, stderr = proc.communicate()
    current_subprocs.remove(proc)
    return stdout.decode(), stderr.decode(), proc.returncode

def check_output(cmd):
    try:
        ret = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as cpe:
        print(cmd)
        print(cpe.output.decode())
        raise RuntimeError("{} raised with check_output comand {}".format(
            cpe.output.decode(), cmd))
    else:
        name = ret.decode().strip().split('\n')
        return name

def determine_build_name(path_to_recipe, *conda_build_args):
    """Figure out what conda says the output built package name is going to be
    Parameters
    ----------
    path_to_recipe : str
        The location of the recipe on disk
    *conda_build_args : list
        List of extra arguments to be appeneded to the conda build command.
        For example, this might include ['--python', '3.4'], to tell
        conda-build to build a python 3.4 package

    Returns
    -------
    package_name : str
        Something like:
        /home/edill/mc/conda-bld/linux-64/pims-0.3.3.post0-0_g1bea480_py27.tar.bz2
    """
    conda_build_args = [] if conda_build_args is None else list(conda_build_args)
    cmd = ['conda', 'build', path_to_recipe, '--output'] + conda_build_args
    ret = check_output(cmd)
    if len(ret) > 1:
        # Then this is the first time we are getting the build name and conda
        # has to check out the source. Call it a second time and you get the4
        # full path to the file that will be spit out by conda-build
        return determine_build_name(path_to_recipe, *conda_build_args)
    # we want to keep track of the exact command so we can run it later.
    # Obviously drop the `--output` flag so that conda-build actually builds
    # the package.
    cmd.remove('--output')
    return ret[0], cmd

def run_build(recipes_path, anaconda_cli, username, pyver,
              token=None, npver=None):
    """Build and upload packages that do not already exist at {{ username }}

    Parameters
    ----------
    recipes_path : str
        Folder where the recipes are located
    anaconda_cli : return value from binstar_client.utils.get_binstar()
    username : str
        The anaconda user to upload all the built packages to
    pyver : list
        The python versions that need to be build. Should be a list of python
        versions to be passed to conda-build /path --python pyver.
        ['2.7', '3.4', '3.5']
    token : str, optional
        The binstar token that should be used to upload packages to
        anaconda.org/username. If no token is provided, no uploading will occur
    npver : str, optional
    """
    if npver is None:
        npver = os.environ.get("CONDA_NPY", "1.11")
    if token is None:
        print("No binstar token available. There will be no uploading."
              "Consider setting the BINSTAR_TOKEN environmental "
              "variable or passing one in via the --token command "
              "line argument")
    # get all file names that are in the channel I am interested in
    packages = get_file_names_on_anaconda_channel(username, anaconda_cli)

    dont_build = []
    to_build = []
    print("Determining package build names...")
    print('{: <8} | {}'.format('to build', 'built package name'))
    logging.info("Build Plan")
    for folder in sorted(os.listdir(recipes_path)):
        recipe_dir = os.path.join(recipes_path, folder)
        for py in pyver:
            try:
                path_to_built_package, build_cmd = determine_build_name(
                    recipe_dir, '--python', py, '--numpy', '1.10')
            except RuntimeError as re:
                logging.error(re)
                continue
            name_on_anaconda = os.sep.join(
                path_to_built_package.split(os.sep)[-2:])
            # pdb.set_trace()
            meta = MetaData(recipe_dir)
            on_anaconda_channel = name_on_anaconda in packages
            if on_anaconda_channel:
                dont_build.append((path_to_built_package, name_on_anaconda,
                                   build_cmd, meta))
            else:
                to_build.append((path_to_built_package, name_on_anaconda,
                                 build_cmd, meta))

            logging.info('{:<8} | {}'.format(
                         str(not bool(on_anaconda_channel)), name_on_anaconda))
    # build all the packages that need to be built
    UPLOAD_CMD = ['anaconda', '-t', token, 'upload', '-u',
                  username]
    # pdb.set_trace()
    no_token = []
    uploaded = []
    upload_failed = []
    build_or_test_failed = []
    # for each package
    for full_path, pkg_name, cmd, metadata in to_build:
        # output the package build name
        print("Building: %s"% pkg_name)
        # output the build command
        print("Build cmd: %s" % ' '.join(cmd))
        stdout, stderr, returncode = Popen(cmd)
        if returncode != 0:
            build_or_test_failed.append(pkg_name)
            pprint('stdout\n', stdout)
            pprint('stderr\n', stderr)
            continue
        if token:
            print("UPLOAD START")
            stdout, stderr, returncode = Popen(UPLOAD_CMD + [full_path])
            if returncode != 0:
                pprint('stdout\n', stdout)
                pprint('stderr\n', stderr)
                upload_failed.append(pkg_name)
                continue
            uploaded.append(pkg_name)
            continue

        no_token.append(pkg_name)

    return {
        'alreadybuilt': sorted([dont[1] for dont in dont_build]),
        'uploaded': sorted(uploaded),
        'no_token': sorted(no_token),
        'upload_failed': sorted(upload_failed),
        'build_or_test_failed': sorted(build_or_test_failed),
    }

def set_binstar_upload(on=False):
    """Turn on or off binstar uploading

    Parameters
    ----------
    on : bool, optional
        True: turn on binstar uploading
        False: turn off binstar uploading
        Defaults to False

    Raises exception if disabling binstar upload fails
    """
    rcpath = os.path.join(os.path.expanduser('~'), '.condarc')
    with open(rcpath, 'r') as f:
        rc = yaml.load(f.read())
    binstar_upload = rc.get('binstar_upload', False)
    if binstar_upload != on:
        rc['binstar_upload'] = on
        with open(rcpath, 'w') as f:
            # write the new yaml in `rcpath`
            f.write(yaml.dump(rc))

def cli():
    from argparse import ArgumentParser
    p = ArgumentParser(
        description="""
Tool for building a folder of conda recipes where only the ones that don't
already exist are built.
""",
    )
    p.add_argument(
        'recipes_path',
        nargs='?',
        help="path to recipes that should be built"
    )
    p.add_argument(
        '-p', "--pyver",
        action='store',
        nargs='*',
        help="Directory to write recipes to (default: %(default)s).",
        default=["2.7", "3.4", "3.5"],
    )
    p.add_argument(
        '-t', '--token', action='store',
        nargs='?',
        help='Binstar token to use to upload build packages',
        default=None
    )
    p.add_argument(
        '-l', '--log',
        nargs='?',
        help='Name of the log file to write'
    )
    p.add_argument(
        "-s", "--site",
        nargs='?',
        help='Anaconda upload api (defaults to %(default)s',
        default='https://api.anaconda.org'
    )
    p.add_argument(
        "-u", "--username",
        action="store",
        nargs='?',
        help=("Username to upload package to")
    )
    p.add_argument(
        '--npver', action='store', nargs='*',
        help=('List the numpy versions to build packages for. Defaults to '
              '%(default)s'),
        default=['1.11']
    )
    args = p.parse_args()
    if args.token is None and 'BINSTAR_TOKEN' in os.environ:
        args.token = os.environ.get('BINSTAR_TOKEN')
    # pdb.set_trace()
    args.recipes_path = os.path.abspath(args.recipes_path)
    print(args)
    # pdb.set_trace()
    run(**dict(args._get_kwargs()))

def run(recipes_path, pyver, log, site, username, token, npver):
    # set up logging
    if not log:
        log_dirname = os.path.join(os.path.expanduser('~'),
                                      'auto-build-logs', 'dev')
        os.makedirs(log_dirname, exist_ok=True)
        log_filename = time.strftime("%m.%d-%H.%M")
        log = os.path.join(log_dirname, log_filename)
    FORMAT = "%(levelname)s | %(asctime)-15s | %(message)s"
    stream_handler = logging.StreamHandler()
    file_handler = logging.FileHandler(log)
    logging.basicConfig(level=logging.DEBUG, format=FORMAT,
                        handlers=[stream_handler, file_handler])
    # build all python versions by default
    if not pyver:
        pyver = ['2.7', '3.4', '3.5']
    # check to make sure that the recipes_path exists
    if not os.path.exists(recipes_path):
        logging.Error("The recipes_path: '%s' does not exist." % recipes_path)
        sys.exit(1)
    print('token={}'.format(token))
    # just disable binstar uploading whenever this script is running.
    print('Disabling binstar upload. If you want to turn it back on, '
          'execute: `conda config --set binstar_upload true`')
    set_binstar_upload(False)
    # set up logging
    full_recipes_path = os.path.abspath(recipes_path)
    print('Logging summary to %s' % log)

    # site = 'https://pergamon.cs.nsls2.local:8443/api'
    # os.environ['REQUESTS_CA_BUNDLE'] = '/etc/certificates/ca_cs_nsls2_local.crt'

    anaconda_cli = binstar_client.utils.get_binstar(Namespace(token=token,
                                                              site=site))
    try:
        results = run_build(full_recipes_path, anaconda_cli, username,
                            pyver, token=token, npver=npver)
    except Exception as e:
        logging.error("Major error encountered in attempt to build {}"
                      "".format(full_recipes_path))
        logging.error("Error in run_build!")
        logging.error(e)
        logging.error("Full stack trace")
        logging.error(traceback.format_exc())
    else:
        logging.info("Build summary")
        logging.info('Expected {} packages'.format(
            len(pyver) * len(os.listdir(full_recipes_path))))
        num_builds = {k: len(v) for k, v in results.items()}
        logging.info('Got {} packages.'.format(
            sum([n for n in num_builds.values()])))
        logging.info('Breakdown is as follows')
        for k, v in num_builds.items():
            logging.info('section: {:<25}. number build: {}'.format(k, v))
        if results['build_or_test_failed']:
            logging.error("Some packages failed to build")
            logging.error(pformat(results['build_or_test_failed']))
        if results['no_token']:
            logging.error("No anaconda token. Cannot upload these packages")
            logging.error(pformat(results['no_token']))
        if results['upload_failed']:
            logging.error('Upload failed for these packages')
            logging.error(pformat(results['upload_failed']))
        if results['alreadybuilt']:
            logging.info('Packages that already exist in {}'.format(username))
            logging.info(pformat(results['alreadybuilt']))
        if results['uploaded']:
            logging.info('Packages that were built and uploaded')
            logging.info(pformat(results['uploaded']))
    finally:
        # close the file handler
        file_handler.close()

if __name__ == "__main__":
    run('../staged-recipes-dev/pyall/xray-vision', ['2.7'], None, None,
        'ericdill', None)
<|endoftext|>


<commit_before>import unittest

import numpy as np

class RegionTestCase(unittest.TestCase):

    def test_region(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 1, 1],
                               [0, 0, 1],
                               [0, 0, 0]])

        region = Region(test_array)

        bitmap = region.bitmap

        self.assertFalse(bitmap[0, 0])
        self.assertTrue(bitmap[0, 1])
        self.assertEqual(bitmap.shape, (3, 3))

    def test_region_select_from_array(self):
        from jicimagelib.region import Region

        id_array = np.array([[0, 0, 0],
                             [1, 1, 1],
                             [2, 2, 2]])

        region_1 = Region.select_from_array(id_array, 1)

        self.assertFalse(region_1.bitmap[0, 0])
        self.assertTrue(region_1.bitmap[1, 0])
        self.assertFalse(region_1.bitmap[2, 0])

        self.assertEqual(region_1.area, 3)

    def test_region_area(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 1, 1],
                               [0, 0, 1],
                               [0, 0, 0]])

        region = Region(test_array)

        self.assertEqual(region.area, 3)

    def test_region_perimeter(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 0, 0, 0, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 0, 0, 0, 0]])

        region = Region(test_array)

        self.assertEqual(region.perimeter, 8)

    def test_region_border(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 0, 0, 0, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 0, 0, 0, 0]])

        region = Region(test_array)

        border_array = np.array([[0, 0, 0, 0, 0],
                                 [0, 1, 1, 1, 0],
                                 [0, 1, 0, 1, 0],
                                 [0, 1, 1, 1, 0],
                                 [0, 0, 0, 0, 0]])

        border_region = Region(border_array)

        self.assertTrue( np.array_equal(region.border.bitmap,
            border_region.bitmap))

    def test_region_inner(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 0, 0, 0, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 0, 0, 0, 0]])

        region = Region(test_array)

        inner_array = np.array([[0, 0, 0, 0, 0],
                                [0, 0, 0, 0, 0],
                                [0, 0, 1, 0, 0],
                                [0, 0, 0, 0, 0],
                                [0, 0, 0, 0, 0]])

        inner_region = Region(inner_array)

        self.assertTrue(np.array_equal(region.inner.bitmap,
            inner_region.bitmap))

    def test_region_constructor(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 1, 2],
                               [0, 0, 1],
                               [0, 0, 0]])

        with self.assertRaises(ValueError):
            Region(test_array)

    def test_region_convex_hull(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 0, 0, 0, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 0, 0, 0],
                               [0, 1, 0, 0, 0],
                               [0, 0, 0, 0, 0]])

        region = Region(test_array)

        convex_hull_array = np.array([[0, 0, 0, 0, 0],
                                      [0, 1, 1, 1, 0],
                                      [0, 1, 1, 0, 0],
                                      [0, 1, 0, 0, 0],
                                      [0, 0, 0, 0, 0]], dtype=bool)

        self.assertTrue(np.array_equal(region.convex_hull.bitmap,
            convex_hull_array))

    def test_index_arrays(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 1, 1],
                               [0, 0, 1],
                               [0, 0, 0]])

        region = Region(test_array)
        x_array, y_array =  region.index_arrays
        self.assertTrue(np.array_equal(x_array, np.array([0, 0, 1])))
        self.assertTrue(np.array_equal(y_array, np.array([1, 2, 2])))

    def test_points(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 1, 1],
                               [0, 0, 1],
                               [0, 0, 0]])

        region = Region(test_array)
        x_array, y_array =  region.index_arrays
        self.assertEqual(region.points,
            [(0,1), (0,2), (1,2)])

    def test_dilate(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 0, 0, 0, 0],
                               [0, 0, 1, 1, 0],
                               [0, 1, 1, 0, 0],
                               [0, 1, 0, 0, 0],
                               [0, 0, 0, 0, 0]])

        region = Region(test_array)

        dilate_array = np.array([[0, 0, 1, 1, 0],
                                 [0, 1, 1, 1, 1],
                                 [1, 1, 1, 1, 0],
                                 [1, 1, 1, 0, 0],
                                 [0, 1, 0, 0, 0]], dtype=bool)

        print region.dilate()
        self.assertTrue(np.array_equal(region.dilate().bitmap,
            dilate_array))
<commit_msg>Added Region tests for str and repr.<commit_after>import unittest

import numpy as np

class RegionTestCase(unittest.TestCase):

    def test_region(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 1, 1],
                               [0, 0, 1],
                               [0, 0, 0]])

        region = Region(test_array)

        bitmap = region.bitmap

        self.assertFalse(bitmap[0, 0])
        self.assertTrue(bitmap[0, 1])
        self.assertEqual(bitmap.shape, (3, 3))

    def test_region_select_from_array(self):
        from jicimagelib.region import Region

        id_array = np.array([[0, 0, 0],
                             [1, 1, 1],
                             [2, 2, 2]])

        region_1 = Region.select_from_array(id_array, 1)

        self.assertFalse(region_1.bitmap[0, 0])
        self.assertTrue(region_1.bitmap[1, 0])
        self.assertFalse(region_1.bitmap[2, 0])

        self.assertEqual(region_1.area, 3)

    def test_region_area(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 1, 1],
                               [0, 0, 1],
                               [0, 0, 0]])

        region = Region(test_array)

        self.assertEqual(region.area, 3)

    def test_region_perimeter(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 0, 0, 0, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 0, 0, 0, 0]])

        region = Region(test_array)

        self.assertEqual(region.perimeter, 8)

    def test_region_border(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 0, 0, 0, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 0, 0, 0, 0]])

        region = Region(test_array)

        border_array = np.array([[0, 0, 0, 0, 0],
                                 [0, 1, 1, 1, 0],
                                 [0, 1, 0, 1, 0],
                                 [0, 1, 1, 1, 0],
                                 [0, 0, 0, 0, 0]])

        border_region = Region(border_array)

        self.assertTrue( np.array_equal(region.border.bitmap,
            border_region.bitmap))

    def test_region_inner(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 0, 0, 0, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 1, 1, 0],
                               [0, 0, 0, 0, 0]])

        region = Region(test_array)

        inner_array = np.array([[0, 0, 0, 0, 0],
                                [0, 0, 0, 0, 0],
                                [0, 0, 1, 0, 0],
                                [0, 0, 0, 0, 0],
                                [0, 0, 0, 0, 0]])

        inner_region = Region(inner_array)

        self.assertTrue(np.array_equal(region.inner.bitmap,
            inner_region.bitmap))

    def test_region_constructor(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 1, 2],
                               [0, 0, 1],
                               [0, 0, 0]])

        with self.assertRaises(ValueError):
            Region(test_array)

    def test_region_convex_hull(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 0, 0, 0, 0],
                               [0, 1, 1, 1, 0],
                               [0, 1, 0, 0, 0],
                               [0, 1, 0, 0, 0],
                               [0, 0, 0, 0, 0]])

        region = Region(test_array)

        convex_hull_array = np.array([[0, 0, 0, 0, 0],
                                      [0, 1, 1, 1, 0],
                                      [0, 1, 1, 0, 0],
                                      [0, 1, 0, 0, 0],
                                      [0, 0, 0, 0, 0]], dtype=bool)

        self.assertTrue(np.array_equal(region.convex_hull.bitmap,
            convex_hull_array))

    def test_index_arrays(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 1, 1],
                               [0, 0, 1],
                               [0, 0, 0]])

        region = Region(test_array)
        x_array, y_array =  region.index_arrays
        self.assertTrue(np.array_equal(x_array, np.array([0, 0, 1])))
        self.assertTrue(np.array_equal(y_array, np.array([1, 2, 2])))

    def test_points(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 1, 1],
                               [0, 0, 1],
                               [0, 0, 0]])

        region = Region(test_array)
        x_array, y_array =  region.index_arrays
        self.assertEqual(region.points,
            [(0,1), (0,2), (1,2)])

    def test_dilate(self):
        from jicimagelib.region import Region

        test_array = np.array([[0, 0, 0, 0, 0],
                               [0, 0, 1, 1, 0],
                               [0, 1, 1, 0, 0],
                               [0, 1, 0, 0, 0],
                               [0, 0, 0, 0, 0]])

        region = Region(test_array)

        dilate_array = np.array([[0, 0, 1, 1, 0],
                                 [0, 1, 1, 1, 1],
                                 [1, 1, 1, 1, 0],
                                 [1, 1, 1, 0, 0],
                                 [0, 1, 0, 0, 0]], dtype=bool)

        print region.dilate()
        self.assertTrue(np.array_equal(region.dilate().bitmap,
            dilate_array))

    def test_repr(self):
        from jicimagelib.region import Region
        test_array = np.array([[0, 1, 1],
                               [0, 0, 1],
                               [0, 0, 0]])
        region = Region(test_array)
        self.assertEqual(repr(region), repr(region.bitmap))

    def test_str(self):
        from jicimagelib.region import Region
        test_array = np.array([[0, 1, 1],
                               [0, 0, 1],
                               [0, 0, 0]])
        region = Region(test_array)
        self.assertEqual(str(region), str(region.bitmap))
<|endoftext|>