<commit_before>"""Approximate bilateral rank filter for local (custom kernel) mean.

The local histogram is computed using a sliding window similar to the method
described in [1]_.

The pixel neighborhood is defined by:

* the given structuring element
* an interval [g-s0, g+s1] in greylevel around g the processed pixel greylevel

The kernel is flat (i.e. each pixel belonging to the neighborhood contributes
equally).

Result image is 8-/16-bit or double with respect to the input image and the
rank filter operation.

References
----------

.. [1] Huang, T. ,Yang, G. ;  Tang, G.. "A fast two-dimensional
       median filtering algorithm", IEEE Transactions on Acoustics, Speech and
       Signal Processing, Feb 1979. Volume: 27 , Issue: 1, Page(s): 13 - 18.

"""

import numpy as np
from skimage import img_as_ubyte

from . import bilateral_cy
from .generic import _handle_input

__all__ = ['mean_bilateral', 'pop_bilateral', 'sum_bilateral']

def _apply(func, image, selem, out, mask, shift_x, shift_y, s0, s1,
           out_dtype=None):

    image, selem, out, mask, max_bin = _handle_input(image, selem, out, mask,
                                                     out_dtype)

    func(image, selem, shift_x=shift_x, shift_y=shift_y, mask=mask,
         out=out, max_bin=max_bin, s0=s0, s1=s1)

    return out

def mean_bilateral(image, selem, out=None, mask=None, shift_x=False,
                   shift_y=False, s0=10, s1=10):
    """Apply a flat kernel bilateral filter.

    This is an edge-preserving and noise reducing denoising filter. It averages
    pixels based on their spatial closeness and radiometric similarity.

    Spatial closeness is measured by considering only the local pixel
    neighborhood given by a structuring element (selem).

    Radiometric similarity is defined by the greylevel interval [g-s0, g+s1]
    where g is the current pixel greylevel. Only pixels belonging to the
    structuring element AND having a greylevel inside this interval are
    averaged. Return greyscale local bilateral_mean of an image.

    Parameters
    ----------
    image : ndarray (uint8, uint16)
        Image array.
    selem : ndarray
        The neighborhood expressed as a 2-D array of 1's and 0's.
    out : ndarray (same dtype as input)
        If None, a new array will be allocated.
    mask : ndarray
        Mask array that defines (>0) area of the image included in the local
        neighborhood. If None, the complete image is used (default).
    shift_x, shift_y : int
        Offset added to the structuring element center point. Shift is bounded
        to the structuring element sizes (center must be inside the given
        structuring element).
    s0, s1 : int
        Define the [s0, s1] interval around the greyvalue of the center pixel
        to be considered for computing the value.

    Returns
    -------
    out : ndarray (same dtype as input image)
        Output image.

    See also
    --------
    skimage.filter.denoise_bilateral for a gaussian bilateral filter.

    Examples
    --------
    >>> from skimage import data
    >>> from skimage.morphology import disk
    >>> from skimage.filter.rank import bilateral_mean
    >>> # Load test image / cast to uint16
    >>> ima = data.camera().astype(np.uint16)
    >>> # bilateral filtering of cameraman image using a flat kernel
    >>> bilat_ima = bilateral_mean(ima, disk(20), s0=10,s1=10)

    """

    return _apply(bilateral_cy._mean, image, selem, out=out,
                  mask=mask, shift_x=shift_x, shift_y=shift_y, s0=s0, s1=s1)

def pop_bilateral(image, selem, out=None, mask=None, shift_x=False,
                  shift_y=False, s0=10, s1=10):
    """Return the number (population) of pixels actually inside the bilateral
    neighborhood, i.e. being inside the structuring element AND having a gray
    level inside the interval [g-s0, g+s1].

    Parameters
    ----------
    image : ndarray (uint8, uint16)
        Image array.
    selem : ndarray
        The neighborhood expressed as a 2-D array of 1's and 0's.
    out : ndarray (same dtype as input)
        If None, a new array will be allocated.
    mask : ndarray
        Mask array that defines (>0) area of the image included in the local
        neighborhood. If None, the complete image is used (default).
    shift_x, shift_y : int
        Offset added to the structuring element center point. Shift is bounded
        to the structuring element sizes (center must be inside the given
        structuring element).
    s0, s1 : int
        Define the [s0, s1] interval around the greyvalue of the center pixel
        to be considered for computing the value.

    Returns
    -------
    out : ndarray (same dtype as input image)
        Output image.

    Examples
    --------
    >>> # Local mean
    >>> from skimage.morphology import square
    >>> import skimage.filter.rank as rank
    >>> ima16 = 255 * np.array([[0, 0, 0, 0, 0],
    ...                         [0, 1, 1, 1, 0],
    ...                         [0, 1, 1, 1, 0],
    ...                         [0, 1, 1, 1, 0],
    ...                         [0, 0, 0, 0, 0]], dtype=np.uint16)
    >>> rank.bilateral_pop(ima16, square(3), s0=10,s1=10)
    array([[3, 4, 3, 4, 3],
           [4, 4, 6, 4, 4],
           [3, 6, 9, 6, 3],
           [4, 4, 6, 4, 4],
           [3, 4, 3, 4, 3]], dtype=uint16)

    """

    return _apply(bilateral_cy._pop, image, selem, out=out,
                  mask=mask, shift_x=shift_x, shift_y=shift_y, s0=s0, s1=s1)

def sum_bilateral(image, selem, out=None, mask=None, shift_x=False,
                   shift_y=False, s0=10, s1=10):
    """Apply a flat kernel bilateral filter.

    This is an edge-preserving and noise reducing denoising filter. It averages
    pixels based on their spatial closeness and radiometric similarity.

    Spatial closeness is measured by considering only the local pixel
    neighborhood given by a structuring element (selem).

    Radiometric similarity is defined by the greylevel interval [g-s0, g+s1]
    where g is the current pixel greylevel. Only pixels belonging to the
    structuring element AND having a greylevel inside this interval are
    summed. Return greyscale local bilateral sum of an image.
    Result is truncated (8bit or 16bit).

    Parameters
    ----------
    image : ndarray (uint8, uint16)
        Image array.
    selem : ndarray
        The neighborhood expressed as a 2-D array of 1's and 0's.
    out : ndarray (same dtype as input)
        If None, a new array will be allocated.
    mask : ndarray
        Mask array that defines (>0) area of the image included in the local
        neighborhood. If None, the complete image is used (default).
    shift_x, shift_y : int
        Offset added to the structuring element center point. Shift is bounded
        to the structuring element sizes (center must be inside the given
        structuring element).
    s0, s1 : int
        Define the [s0, s1] interval around the greyvalue of the center pixel
        to be considered for computing the value.

    Returns
    -------
    out : ndarray (same dtype as input image)
        Output image.

    See also
    --------
    skimage.filter.denoise_bilateral for a gaussian bilateral filter.

    Examples
    --------
    >>> from skimage import data
    >>> from skimage.morphology import disk
    >>> from skimage.filter.rank import sum_bilateral
    >>> # Load test image / cast to uint16
    >>> ima = data.camera().astype(np.uint16)
    >>> # bilateral filtering of cameraman image using a flat kernel
    >>> bilat_ima = sum_bilateral(ima, disk(20), s0=10,s1=10)

    """

    return _apply(bilateral_cy._sum, image, selem, out=out,
                  mask=mask, shift_x=shift_x, shift_y=shift_y, s0=s0, s1=s1)
<commit_msg>Improve doc strings of bilateral rank filters<commit_after>"""Approximate bilateral rank filter for local (custom kernel) mean.

The local histogram is computed using a sliding window similar to the method
described in [1]_.

The pixel neighborhood is defined by:

* the given structuring element
* an interval [g-s0, g+s1] in greylevel around g the processed pixel greylevel

The kernel is flat (i.e. each pixel belonging to the neighborhood contributes
equally).

Result image is 8-/16-bit or double with respect to the input image and the
rank filter operation.

References
----------

.. [1] Huang, T. ,Yang, G. ;  Tang, G.. "A fast two-dimensional
       median filtering algorithm", IEEE Transactions on Acoustics, Speech and
       Signal Processing, Feb 1979. Volume: 27 , Issue: 1, Page(s): 13 - 18.

"""

import numpy as np
from skimage import img_as_ubyte

from . import bilateral_cy
from .generic import _handle_input

__all__ = ['mean_bilateral', 'pop_bilateral', 'sum_bilateral']

def _apply(func, image, selem, out, mask, shift_x, shift_y, s0, s1,
           out_dtype=None):

    image, selem, out, mask, max_bin = _handle_input(image, selem, out, mask,
                                                     out_dtype)

    func(image, selem, shift_x=shift_x, shift_y=shift_y, mask=mask,
         out=out, max_bin=max_bin, s0=s0, s1=s1)

    return out

def mean_bilateral(image, selem, out=None, mask=None, shift_x=False,
                   shift_y=False, s0=10, s1=10):
    """Apply a flat kernel bilateral filter.

    This is an edge-preserving and noise reducing denoising filter. It averages
    pixels based on their spatial closeness and radiometric similarity.

    Spatial closeness is measured by considering only the local pixel
    neighborhood given by a structuring element.

    Radiometric similarity is defined by the greylevel interval [g-s0, g+s1]
    where g is the current pixel greylevel.

    Only pixels belonging to the structuring element and having a greylevel
    inside this interval are averaged.

    Parameters
    ----------
    image : 2-D array (uint8, uint16)
        Input image.
    selem : 2-D array
        The neighborhood expressed as a 2-D array of 1's and 0's.
    out : 2-D array (same dtype as input)
        If None, a new array is allocated.
    mask : ndarray
        Mask array that defines (>0) area of the image included in the local
        neighborhood. If None, the complete image is used (default).
    shift_x, shift_y : int
        Offset added to the structuring element center point. Shift is bounded
        to the structuring element sizes (center must be inside the given
        structuring element).
    s0, s1 : int
        Define the [s0, s1] interval around the greyvalue of the center pixel
        to be considered for computing the value.

    Returns
    -------
    out : 2-D array (same dtype as input image)
        Output image.

    See also
    --------
    skimage.filter.denoise_bilateral for a Gaussian bilateral filter.

    Examples
    --------
    >>> from skimage import data
    >>> from skimage.morphology import disk
    >>> from skimage.filter.rank import mean_bilateral
    >>> img = data.camera().astype(np.uint16)
    >>> bilat_img = mean_bilateral(img, disk(20), s0=10,s1=10)

    """

    return _apply(bilateral_cy._mean, image, selem, out=out,
                  mask=mask, shift_x=shift_x, shift_y=shift_y, s0=s0, s1=s1)

def pop_bilateral(image, selem, out=None, mask=None, shift_x=False,
                  shift_y=False, s0=10, s1=10):
    """Return the local number (population) of pixels.

    The number of pixels is defined as the number of pixels which are included
    in the structuring element and the mask. Additionally the must have a
    greylevel inside the interval [g-s0, g+s1] where g is the greyvalue of the
    center pixel.

    Parameters
    ----------
    image : 2-D array (uint8, uint16)
        Input image.
    selem : 2-D array
        The neighborhood expressed as a 2-D array of 1's and 0's.
    out : 2-D array (same dtype as input)
        If None, a new array is allocated.
    mask : ndarray
        Mask array that defines (>0) area of the image included in the local
        neighborhood. If None, the complete image is used (default).
    shift_x, shift_y : int
        Offset added to the structuring element center point. Shift is bounded
        to the structuring element sizes (center must be inside the given
        structuring element).
    s0, s1 : int
        Define the [s0, s1] interval around the greyvalue of the center pixel
        to be considered for computing the value.

    Returns
    -------
    out : 2-D array (same dtype as input image)
        Output image.

    Examples
    --------
    >>> from skimage.morphology import square
    >>> import skimage.filter.rank as rank
    >>> img = 255 * np.array([[0, 0, 0, 0, 0],
    ...                       [0, 1, 1, 1, 0],
    ...                       [0, 1, 1, 1, 0],
    ...                       [0, 1, 1, 1, 0],
    ...                       [0, 0, 0, 0, 0]], dtype=np.uint16)
    >>> rank.pop_bilateral(imgsquare(3), s0=10, s1=10)
    array([[3, 4, 3, 4, 3],
           [4, 4, 6, 4, 4],
           [3, 6, 9, 6, 3],
           [4, 4, 6, 4, 4],
           [3, 4, 3, 4, 3]], dtype=uint16)

    """

    return _apply(bilateral_cy._pop, image, selem, out=out,
                  mask=mask, shift_x=shift_x, shift_y=shift_y, s0=s0, s1=s1)

def sum_bilateral(image, selem, out=None, mask=None, shift_x=False,
                   shift_y=False, s0=10, s1=10):
    """Apply a flat kernel bilateral filter.

    This is an edge-preserving and noise reducing denoising filter. It averages
    pixels based on their spatial closeness and radiometric similarity.

    Spatial closeness is measured by considering only the local pixel
    neighborhood given by a structuring element (selem).

    Radiometric similarity is defined by the greylevel interval [g-s0, g+s1]
    where g is the current pixel greylevel.

    Only pixels belonging to the structuring element AND having a greylevel
    inside this interval are summed.

    Parameters
    ----------
    image : 2-D array (uint8, uint16)
        Input image.
    selem : 2-D array
        The neighborhood expressed as a 2-D array of 1's and 0's.
    out : 2-D array (same dtype as input)
        If None, a new array is allocated.
    mask : ndarray
        Mask array that defines (>0) area of the image included in the local
        neighborhood. If None, the complete image is used (default).
    shift_x, shift_y : int
        Offset added to the structuring element center point. Shift is bounded
        to the structuring element sizes (center must be inside the given
        structuring element).
    s0, s1 : int
        Define the [s0, s1] interval around the greyvalue of the center pixel
        to be considered for computing the value.

    Returns
    -------
    out : 2-D array (same dtype as input image)
        Output image.

    See also
    --------
    skimage.filter.denoise_bilateral for a Gaussian bilateral filter.

    Examples
    --------
    >>> from skimage import data
    >>> from skimage.morphology import disk
    >>> from skimage.filter.rank import sum_bilateral
    >>> img = data.camera().astype(np.uint16)
    >>> bilat_img = sum_bilateral(img, disk(10), s0=10, s1=10)

    """

    return _apply(bilateral_cy._sum, image, selem, out=out,
                  mask=mask, shift_x=shift_x, shift_y=shift_y, s0=s0, s1=s1)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Defines components for holding properties of rocks or samples or whatevers.

:copyright: 2015 Agile Geoscience
:license: Apache 2.0
"""
import re

class ComponentError(Exception):
    """
    Generic error class.
    """
    pass

class Component(object):
    """
    Initialize with a dictionary of properties. You can use any
    properties you want e.g.:

        - lithology: a simple one-word rock type
        - colour, e.g. 'grey'
        - grainsize or range, e.g. 'vf-f'
        - modifier, e.g. 'rippled'
        - quantity, e.g. '35%', or 'stringers'
        - description, e.g. from cuttings

    You can include as many other things as you want, e.g.

        - porosity
        - cementation
        - lithology code
    """

    def __init__(self, properties):
        for k, v in properties.items():
            if k and v:
                setattr(self, k.lower(), v.lower())

    def __repr__(self):
        s = str(self)
        return "Component({0})".format(s)

    def __str__(self):
        s = []
        for key in self.__dict__:
            t = '"{key}":"{value}"'
            s.append(t.format(key=key, value=self.__dict__[key]))
        return ', '.join(s)

    def __bool__(self):
        if not self.__dict__.keys():
            return False
        else:
            return True

    # For Python 2
    __nonzero__ = __bool__

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return False

        # Weed out empty elements
        s = {k: v for k, v in self.__dict__.items() if v}
        o = {k: v for k, v in other.__dict__.items() if v}

        # Compare
        if s == o:
            return True
        else:
            return False

    def __ne__(self, other):
        return not self.__eq__(other)

    # If we define __eq__ we also need __hash__ otherwise the object
    # becomes unhashable. All this does is hash the frozenset of the
    # keys. (You can only hash immutables.)
    def __hash__(self):
        return hash(frozenset(self.__dict__.keys()))

    @classmethod
    def from_text(cls, text, lexicon, required=None, first_only=True):
        """
        Generate a Component from a text string, using a Lexicon.

        Args:
            text (str): The text string to parse.
            lexicon (Lexicon): The dictionary to use for the
                categories and lexemes.
            first_only (bool): Whether to only take the first
                match of a lexeme against the text string.

        Returns:
            Component: A Component object, or None if there was no
                must-have field.
        """
        component = lexicon.get_component(text, first_only=first_only)
        if required and (required not in component):
            return None
        else:
            return cls(component)

    def summary(self, fmt=None, initial=True, default=''):
        """
        Given a format string, return a summary description of a component.

        Args:
            component (dict): A component dictionary.
            fmt (str): Describes the format with a string. Use '%'
                to signal a field in the component, which is analogous
                to a dictionary. If no format is given, you will
                just get a list of attributes.
            initial (bool): Whether to capitialize the first letter.
            default (str): What to give if there's no component defined.

        Returns:
            str: A summary string.

        Example:

            r = Component({'colour': 'Red',
                           'grainsize': 'VF-F',
                           'lithology': 'Sandstone'})

            r.summary()  -->  'Red, vf-f, sandstone'
        """
        if default and not self.__dict__:
            return default

        if not fmt:
            string, flist = '', []
            for item in self.__dict__:
                string += '{}, '
                flist.append(item)
            string = string.strip(', ')
        else:
            fmt = re.sub(r'  ', '_dblspc_', fmt)
            string = re.sub(r'\{(\w+)\}', '{}', fmt)
            flist = re.findall(r'\{(\w+)\}', fmt)

        words = []
        for key in flist:
            word = self.__dict__.get(key.lower())
            if word and key[0].isupper():
                word = word.capitalize()
            if word and key.isupper():
                word = word.upper()
            if not word:
                word = ''
            words.append(word)

        try:
            summary = string.format(*words)
        except KeyError as e:
            raise ComponentError("No such attribute, "+str(e))

        if initial and summary:
            summary = summary[0].upper() + summary[1:]

        # Tidy up double spaces
        summary = re.sub(r'  ', ' ', summary)
        summary = re.sub(r'_dblspc_', '  ', summary)

        return summary
<commit_msg>first go at #29<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Defines components for holding properties of rocks or samples or whatevers.

:copyright: 2015 Agile Geoscience
:license: Apache 2.0
"""
import re

class ComponentError(Exception):
    """
    Generic error class.
    """
    pass

class Component(object):
    """
    Initialize with a dictionary of properties. You can use any
    properties you want e.g.:

        - lithology: a simple one-word rock type
        - colour, e.g. 'grey'
        - grainsize or range, e.g. 'vf-f'
        - modifier, e.g. 'rippled'
        - quantity, e.g. '35%', or 'stringers'
        - description, e.g. from cuttings

    You can include as many other things as you want, e.g.

        - porosity
        - cementation
        - lithology code
    """

    def __init__(self, properties):
        for k, v in properties.items():
            if k and v:
                setattr(self, k.lower(), v.lower())

    def __repr__(self):
        s = str(self)
        return "Component({0})".format(s)

    def __str__(self):
        s = []
        for key in self.__dict__:
            t = '"{key}":"{value}"'
            s.append(t.format(key=key, value=self.__dict__[key]))
        return ', '.join(s)

    def __bool__(self):
        if not self.__dict__.keys():
            return False
        else:
            return True

    # For Python 2
    __nonzero__ = __bool__

    def __eq__(self, other):
        if not isinstance(other, self.__class__):
            return False

        # Weed out empty elements
        s = {k: v for k, v in self.__dict__.items() if v}
        o = {k: v for k, v in other.__dict__.items() if v}

        # Compare
        if s == o:
            return True
        else:
            return False

    def __ne__(self, other):
        return not self.__eq__(other)

    # If we define __eq__ we also need __hash__ otherwise the object
    # becomes unhashable. All this does is hash the frozenset of the
    # keys. (You can only hash immutables.)
    def __hash__(self):
        return hash(frozenset(self.__dict__.keys()))

    def _repr_html_(self):
        """
        IPython Notebook magic repr function.
        """
        rows = ''
        s = '
{k}	{v}
'
        for k, v in self.__dict__.items():
            rows += s.format(k=k, v=v)
        html = '
{}
'.format(rows)
        return html

    @classmethod
    def from_text(cls, text, lexicon, required=None, first_only=True):
        """
        Generate a Component from a text string, using a Lexicon.

        Args:
            text (str): The text string to parse.
            lexicon (Lexicon): The dictionary to use for the
                categories and lexemes.
            first_only (bool): Whether to only take the first
                match of a lexeme against the text string.

        Returns:
            Component: A Component object, or None if there was no
                must-have field.
        """
        component = lexicon.get_component(text, first_only=first_only)
        if required and (required not in component):
            return None
        else:
            return cls(component)

    def summary(self, fmt=None, initial=True, default=''):
        """
        Given a format string, return a summary description of a component.

        Args:
            component (dict): A component dictionary.
            fmt (str): Describes the format with a string. Use '%'
                to signal a field in the component, which is analogous
                to a dictionary. If no format is given, you will
                just get a list of attributes.
            initial (bool): Whether to capitialize the first letter.
            default (str): What to give if there's no component defined.

        Returns:
            str: A summary string.

        Example:

            r = Component({'colour': 'Red',
                           'grainsize': 'VF-F',
                           'lithology': 'Sandstone'})

            r.summary()  -->  'Red, vf-f, sandstone'
        """
        if default and not self.__dict__:
            return default

        if not fmt:
            string, flist = '', []
            for item in self.__dict__:
                string += '{}, '
                flist.append(item)
            string = string.strip(', ')
        else:
            fmt = re.sub(r'  ', '_dblspc_', fmt)
            string = re.sub(r'\{(\w+)\}', '{}', fmt)
            flist = re.findall(r'\{(\w+)\}', fmt)

        words = []
        for key in flist:
            word = self.__dict__.get(key.lower())
            if word and key[0].isupper():
                word = word.capitalize()
            if word and key.isupper():
                word = word.upper()
            if not word:
                word = ''
            words.append(word)

        try:
            summary = string.format(*words)
        except KeyError as e:
            raise ComponentError("No such attribute, "+str(e))

        if initial and summary:
            summary = summary[0].upper() + summary[1:]

        # Tidy up double spaces
        summary = re.sub(r'  ', ' ', summary)
        summary = re.sub(r'_dblspc_', '  ', summary)

        return summary
<|endoftext|>


<commit_before>import argparse
import tabulate

from typing import List
from operator import itemgetter
from repairbox.manager import RepairBox

def list_sources(rbox: 'RepairBox') -> None:
    """
    Produces a list of all the sources known to RepairBox.
    """
    hdrs = ['Source', 'URL', 'Version']
    tbl = []
    for src in rbox.sources:
        tbl.append([src.name, src.url, src.version])

    # transform into a pretty table
    tbl = tabulate.tabulate(tbl, headers=hdrs, tablefmt='simple')
    print('')
    print(tbl)

def add_source(rbox: 'RepairBox', src: str) -> None:
    rbox.sources.add(src)
    print('added dataset: {}'.format(src))

def remove_source(rbox: 'RepairBox', src: str) -> None:
    rbox.sources.remove(ds)
    print('removed source: {}'.format(src))

def update_sources(rbox: 'RepairBox', ) -> None:
    print('updating sources...')
    rbox.sources.update()

def install_artefact(rbox: 'RepairBox', name: str, update: bool) -> None:
    print('installing artefact: {}'.format(name))
    artefact = rbox.artefacts[name]
    artefact.install(upgrade=update)

def build_artefact(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('building artefact: {}'.format(name))
    artefact = rbox.artefacts[name]
    artefact.build(force=force)

def download_artefact(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('downloading artefact: {}'.format(name))
    artefact = rbox.artefacts[name]
    artefact.download(force=force)

def upload_artefact(rbox: 'RepairBox', name: str) -> None:
    print('uploading artefact: {}'.format(name))
    artefact = rbox.artefacts[name]
    artefact.upload()

# TODO: wrong name!
def uninstall_bug(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('uninstalling artefact: {}'.format(name))
    artefact = rbox.artefacts[name]
    artefact.uninstall(force=force)

###############################################################################
# [tool] group
###############################################################################

def install_tool(rbox: 'RepairBox', name: str, update: bool) -> None:
    print('installing tool: {}'.format(name))
    t = rbox.tools[name]
    t.install(upgrade=update)

def uninstall_tool(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('uninstalling tool: {}'.format(name))
    rbox.tools[name].uninstall(force=force)

def build_tool(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('building tool: {}'.format(name))
    rbox.tools[name].build(force=force)

def download_tool(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('downloading tool: {}'.format(name))
    rbox.tools[name].download(force=force)

def upload_tool(rbox: 'RepairBox', name: str) -> None:
    print('uploading tool: {}'.format(name))
    rbox.tools[name].upload()

# TODO: tidy up copypasta
def list_tools(rbox: 'RepairBox', show_installed=None) -> None:
    tbl = []
    hdrs = ['Tool', 'Source', 'Installed?']
    for tool in rbox.tools:

        # apply filtering based on installation status
        if show_installed is not None:
            if show_installed != tool.installed:
                continue

        installed = 'Yes' if tool.installed else 'No'
        row = [tool.name, tool.url, installed]
        tbl.append(row)

    # sort by dataset then by artefact
    tbl = sorted(tbl, key=itemgetter(1,2))

    # transform into a pretty table
    tbl = tabulate.tabulate(tbl, headers=hdrs, tablefmt='simple')
    print('')
    print(tbl)

###############################################################################
# [container] group
###############################################################################

def launch(rbox: 'RepairBox', name: str, tools: List[str] = []) -> None:
    artefact = rbox.artefacts[name]
    artefact.install()
    tools = [rbox.tools[t] for t in tools]
    try:
        c = None
        c = artefact.provision(tty=True, tools=tools)
        c.interact()
    finally:
        if c: # ensure the container is destroyed
            c.destroy()

def list_artefacts(rbox: 'RepairBox', show_installed=None) -> None:
    tbl = []
    hdrs = ['Artefact', 'Source', 'Installed?']
    for artefact in rbox.artefacts:

        # apply filtering based on installation status
        if show_installed is not None:
            if show_installed != artefact.installed:
                continue

        installed = 'Yes' if artefact.installed else 'No'
        row = [artefact.identifier, artefact.dataset.name, installed]
        tbl.append(row)

    # sort by dataset then by artefact
    tbl = sorted(tbl, key=itemgetter(1,2))

    # transform into a pretty table
    tbl = tabulate.tabulate(tbl, headers=hdrs, tablefmt='simple')
    print('')
    print(tbl)

def main():
    #with open(os.path.join(os.path.dirname(__file__), "banner.txt"), "r") as f:
    #    desc = f.read()
    rbox = RepairBox()

    desc = ':-)'
    parser = argparse.ArgumentParser(description=desc,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    subparsers = parser.add_subparsers()
    parser.add_argument('--version', action='version', version='2.0.0')

    ###########################################################################
    # [source] group
    ###########################################################################
    g_src = subparsers.add_parser('source')
    g_subparsers = g_src.add_subparsers()

    # [source list]
    cmd = g_subparsers.add_parser('list')
    cmd.set_defaults(func=lambda args: list_sources(rbox))

    # [source add :dataset]
    cmd = g_subparsers.add_parser('add')
    cmd.add_argument('source')
    cmd.set_defaults(func=lambda args: add_source(rbox, args.source))

    # [source remove :dataset]
    cmd = g_subparsers.add_parser('remove')
    cmd.add_argument('source')
    cmd.set_defaults(func=lambda args: remove_source(rbox, args.source))

    # [source update]
    cmd = g_subparsers.add_parser('update')
    cmd.set_defaults(func=lambda args: update_sources(rbox))

    ###########################################################################
    # [tool] group
    ###########################################################################
    g_tool = subparsers.add_parser('tool')
    g_subparsers = g_tool.add_subparsers()

    # [tool install (--update) :tool]
    cmd = g_subparsers.add_parser('install')
    cmd.add_argument('tool')
    cmd.add_argument('--update',
                     action='store_true')
    cmd.set_defaults(func=lambda args: install_tool(rbox, args.tool, args.update))

    # [tool uninstall (--force) :tool]
    cmd = g_subparsers.add_parser('uninstall')
    cmd.add_argument('tool')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: uninstall_tool(rbox, args.tool, force=args.force))

    # [tool build (--update) :artefact]
    cmd = g_subparsers.add_parser('build')
    cmd.add_argument('tool')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: build_tool(rbox, args.tool, args.force))

    # [tool download (--force) :artefact]
    cmd = g_subparsers.add_parser('download')
    cmd.add_argument('tool')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: download_tool(rbox, args.tool, args.force))

    # [tool upload :artefact]
    cmd = g_subparsers.add_parser('upload')
    cmd.add_argument('tool')
    cmd.set_defaults(func=lambda args: upload_tool(rbox, args.tool))

    # [tool list]
    cmd = g_subparsers.add_parser('list')
    cmd.add_argument('--installed',
                     dest='installed',
                     action='store_true')
    cmd.add_argument('--uninstalled',
                     dest='installed',
                     action='store_false')
    cmd.set_defaults(installed=None)
    cmd.set_defaults(func=lambda args: list_tools(rbox, args.installed))

    ###########################################################################
    # [container] group
    ###########################################################################
    g_container = subparsers.add_parser('container')
    g_subparsers = g_container.add_subparsers()

    # [container launch :artefact]
    cmd = g_subparsers.add_parser('launch')
    cmd.add_argument('artefact')
    # TODO: add volumes
    cmd.add_argument('--with',
                     help='name of a tool',
                     dest='tools',
                     action='append',
                     default=[])
    cmd.set_defaults(func=lambda args: launch(rbox, args.artefact, args.tools))

    # [container run :artefact]

    # [container connect :artefact]

    ###########################################################################
    # [artefact] group
    ###########################################################################
    g_artefact = subparsers.add_parser('artefact')
    g_subparsers = g_artefact.add_subparsers()

    # [artefact install (--update) :artefact]
    cmd = g_subparsers.add_parser('install')
    cmd.add_argument('artefact')
    cmd.add_argument('--update',
                     action='store_true')
    cmd.set_defaults(func=lambda args: install_artefact(rbox, args.artefact, args.update))

    # [artefact uninstall (--force) :artefact]
    cmd = g_subparsers.add_parser('uninstall')
    cmd.add_argument('artefact')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: uninstall_artefact(rbox, args.artefact, force=args.force))

    # [artefact build (--update) :artefact]
    cmd = g_subparsers.add_parser('build')
    cmd.add_argument('artefact')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: build_artefact(rbox, args.artefact, args.force))

    # [artefact download (--force) :artefact]
    cmd = g_subparsers.add_parser('download')
    cmd.add_argument('artefact')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: download_artefact(rbox, args.artefact, args.force))

    # [artefact upload :artefact]
    cmd = g_subparsers.add_parser('upload')
    cmd.add_argument('artefact')
    cmd.set_defaults(func=lambda args: upload_artefact(rbox, args.artefact))

    # [artefact list]
    cmd = g_subparsers.add_parser('list')
    cmd.add_argument('--installed',
                     dest='installed',
                     action='store_true')
    cmd.add_argument('--uninstalled',
                     dest='installed',
                     action='store_false')
    cmd.set_defaults(installed=None)
    cmd.set_defaults(func=lambda args: list_artefacts(rbox, args.installed))

    # parse and process arguments
    args = parser.parse_args()
    if 'func' in vars(args):
        args.func(args)

if __name__ == "__main__":
    main()
<commit_msg>bug fix: typo in artefact uninstall command<commit_after>import argparse
import tabulate

from typing import List
from operator import itemgetter
from repairbox.manager import RepairBox

def list_sources(rbox: 'RepairBox') -> None:
    """
    Produces a list of all the sources known to RepairBox.
    """
    hdrs = ['Source', 'URL', 'Version']
    tbl = []
    for src in rbox.sources:
        tbl.append([src.name, src.url, src.version])

    # transform into a pretty table
    tbl = tabulate.tabulate(tbl, headers=hdrs, tablefmt='simple')
    print('')
    print(tbl)

def add_source(rbox: 'RepairBox', src: str) -> None:
    rbox.sources.add(src)
    print('added dataset: {}'.format(src))

def remove_source(rbox: 'RepairBox', src: str) -> None:
    rbox.sources.remove(ds)
    print('removed source: {}'.format(src))

def update_sources(rbox: 'RepairBox', ) -> None:
    print('updating sources...')
    rbox.sources.update()

def install_artefact(rbox: 'RepairBox', name: str, update: bool) -> None:
    print('installing artefact: {}'.format(name))
    artefact = rbox.artefacts[name]
    artefact.install(upgrade=update)

def build_artefact(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('building artefact: {}'.format(name))
    artefact = rbox.artefacts[name]
    artefact.build(force=force)

def download_artefact(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('downloading artefact: {}'.format(name))
    artefact = rbox.artefacts[name]
    artefact.download(force=force)

def upload_artefact(rbox: 'RepairBox', name: str) -> None:
    print('uploading artefact: {}'.format(name))
    artefact = rbox.artefacts[name]
    artefact.upload()

# TODO: wrong name!
def uninstall_artefact(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('uninstalling artefact: {}'.format(name))
    artefact = rbox.artefacts[name]
    artefact.uninstall(force=force)

###############################################################################
# [tool] group
###############################################################################

def install_tool(rbox: 'RepairBox', name: str, update: bool) -> None:
    print('installing tool: {}'.format(name))
    t = rbox.tools[name]
    t.install(upgrade=update)

def uninstall_tool(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('uninstalling tool: {}'.format(name))
    rbox.tools[name].uninstall(force=force)

def build_tool(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('building tool: {}'.format(name))
    rbox.tools[name].build(force=force)

def download_tool(rbox: 'RepairBox', name: str, force: bool) -> None:
    print('downloading tool: {}'.format(name))
    rbox.tools[name].download(force=force)

def upload_tool(rbox: 'RepairBox', name: str) -> None:
    print('uploading tool: {}'.format(name))
    rbox.tools[name].upload()

# TODO: tidy up copypasta
def list_tools(rbox: 'RepairBox', show_installed=None) -> None:
    tbl = []
    hdrs = ['Tool', 'Source', 'Installed?']
    for tool in rbox.tools:

        # apply filtering based on installation status
        if show_installed is not None:
            if show_installed != tool.installed:
                continue

        installed = 'Yes' if tool.installed else 'No'
        row = [tool.name, tool.url, installed]
        tbl.append(row)

    # sort by dataset then by artefact
    tbl = sorted(tbl, key=itemgetter(1,2))

    # transform into a pretty table
    tbl = tabulate.tabulate(tbl, headers=hdrs, tablefmt='simple')
    print('')
    print(tbl)

###############################################################################
# [container] group
###############################################################################

def launch(rbox: 'RepairBox', name: str, tools: List[str] = []) -> None:
    artefact = rbox.artefacts[name]
    artefact.install()
    tools = [rbox.tools[t] for t in tools]
    try:
        c = None
        c = artefact.provision(tty=True, tools=tools)
        c.interact()
    finally:
        if c: # ensure the container is destroyed
            c.destroy()

def list_artefacts(rbox: 'RepairBox', show_installed=None) -> None:
    tbl = []
    hdrs = ['Artefact', 'Source', 'Installed?']
    for artefact in rbox.artefacts:

        # apply filtering based on installation status
        if show_installed is not None:
            if show_installed != artefact.installed:
                continue

        installed = 'Yes' if artefact.installed else 'No'
        row = [artefact.identifier, artefact.dataset.name, installed]
        tbl.append(row)

    # sort by dataset then by artefact
    tbl = sorted(tbl, key=itemgetter(1,2))

    # transform into a pretty table
    tbl = tabulate.tabulate(tbl, headers=hdrs, tablefmt='simple')
    print('')
    print(tbl)

def main():
    #with open(os.path.join(os.path.dirname(__file__), "banner.txt"), "r") as f:
    #    desc = f.read()
    rbox = RepairBox()

    desc = ':-)'
    parser = argparse.ArgumentParser(description=desc,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    subparsers = parser.add_subparsers()
    parser.add_argument('--version', action='version', version='2.0.0')

    ###########################################################################
    # [source] group
    ###########################################################################
    g_src = subparsers.add_parser('source')
    g_subparsers = g_src.add_subparsers()

    # [source list]
    cmd = g_subparsers.add_parser('list')
    cmd.set_defaults(func=lambda args: list_sources(rbox))

    # [source add :dataset]
    cmd = g_subparsers.add_parser('add')
    cmd.add_argument('source')
    cmd.set_defaults(func=lambda args: add_source(rbox, args.source))

    # [source remove :dataset]
    cmd = g_subparsers.add_parser('remove')
    cmd.add_argument('source')
    cmd.set_defaults(func=lambda args: remove_source(rbox, args.source))

    # [source update]
    cmd = g_subparsers.add_parser('update')
    cmd.set_defaults(func=lambda args: update_sources(rbox))

    ###########################################################################
    # [tool] group
    ###########################################################################
    g_tool = subparsers.add_parser('tool')
    g_subparsers = g_tool.add_subparsers()

    # [tool install (--update) :tool]
    cmd = g_subparsers.add_parser('install')
    cmd.add_argument('tool')
    cmd.add_argument('--update',
                     action='store_true')
    cmd.set_defaults(func=lambda args: install_tool(rbox, args.tool, args.update))

    # [tool uninstall (--force) :tool]
    cmd = g_subparsers.add_parser('uninstall')
    cmd.add_argument('tool')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: uninstall_tool(rbox, args.tool, force=args.force))

    # [tool build (--update) :artefact]
    cmd = g_subparsers.add_parser('build')
    cmd.add_argument('tool')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: build_tool(rbox, args.tool, args.force))

    # [tool download (--force) :artefact]
    cmd = g_subparsers.add_parser('download')
    cmd.add_argument('tool')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: download_tool(rbox, args.tool, args.force))

    # [tool upload :artefact]
    cmd = g_subparsers.add_parser('upload')
    cmd.add_argument('tool')
    cmd.set_defaults(func=lambda args: upload_tool(rbox, args.tool))

    # [tool list]
    cmd = g_subparsers.add_parser('list')
    cmd.add_argument('--installed',
                     dest='installed',
                     action='store_true')
    cmd.add_argument('--uninstalled',
                     dest='installed',
                     action='store_false')
    cmd.set_defaults(installed=None)
    cmd.set_defaults(func=lambda args: list_tools(rbox, args.installed))

    ###########################################################################
    # [container] group
    ###########################################################################
    g_container = subparsers.add_parser('container')
    g_subparsers = g_container.add_subparsers()

    # [container launch :artefact]
    cmd = g_subparsers.add_parser('launch')
    cmd.add_argument('artefact')
    # TODO: add volumes
    cmd.add_argument('--with',
                     help='name of a tool',
                     dest='tools',
                     action='append',
                     default=[])
    cmd.set_defaults(func=lambda args: launch(rbox, args.artefact, args.tools))

    # [container run :artefact]

    # [container connect :artefact]

    ###########################################################################
    # [artefact] group
    ###########################################################################
    g_artefact = subparsers.add_parser('artefact')
    g_subparsers = g_artefact.add_subparsers()

    # [artefact install (--update) :artefact]
    cmd = g_subparsers.add_parser('install')
    cmd.add_argument('artefact')
    cmd.add_argument('--update',
                     action='store_true')
    cmd.set_defaults(func=lambda args: install_artefact(rbox, args.artefact, args.update))

    # [artefact uninstall (--force) :artefact]
    cmd = g_subparsers.add_parser('uninstall')
    cmd.add_argument('artefact')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: uninstall_artefact(rbox, args.artefact, force=args.force))

    # [artefact build (--update) :artefact]
    cmd = g_subparsers.add_parser('build')
    cmd.add_argument('artefact')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: build_artefact(rbox, args.artefact, args.force))

    # [artefact download (--force) :artefact]
    cmd = g_subparsers.add_parser('download')
    cmd.add_argument('artefact')
    cmd.add_argument('--force',
                     action='store_true')
    cmd.set_defaults(func=lambda args: download_artefact(rbox, args.artefact, args.force))

    # [artefact upload :artefact]
    cmd = g_subparsers.add_parser('upload')
    cmd.add_argument('artefact')
    cmd.set_defaults(func=lambda args: upload_artefact(rbox, args.artefact))

    # [artefact list]
    cmd = g_subparsers.add_parser('list')
    cmd.add_argument('--installed',
                     dest='installed',
                     action='store_true')
    cmd.add_argument('--uninstalled',
                     dest='installed',
                     action='store_false')
    cmd.set_defaults(installed=None)
    cmd.set_defaults(func=lambda args: list_artefacts(rbox, args.installed))

    # parse and process arguments
    args = parser.parse_args()
    if 'func' in vars(args):
        args.func(args)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Download files by observation ID, filter the event file, and separate by CCDs
"""

import argparse
import sys
import os

from chandra_suli import find_files
from chandra_suli import logging_system
from chandra_suli.run_command import CommandRunner
from chandra_suli.work_within_directory import work_within_directory

if __name__=="__main__":

    parser = argparse.ArgumentParser(description='Run following steps: download files by obsid, filter the event file, '
                                                 'and separate by CCD')

    parser.add_argument("-w", "--workdir", help="Directory for all the output (output files will be in a directory "
                                                "named after the obsid)", type=str, required=True)

    parser.add_argument("-o","--obsid",help="Observation ID Numbers", type=int, required=True)

    parser.add_argument('-r', '--region_repo', help="Path to the repository of region files",
                        type=str, required=True)

    parser.add_argument('-a',"--adj_factor",
                        help="If region files need to be adjusted, what factor to increase axes of ellipses by",
                        type=float, required=True)

    parser.add_argument("-e1","--emin",help="Minimum energy (eV)",type=int,required=True)

    parser.add_argument("-e2","--emax",help="Maximum energy (eV)",type=int,required=True)

    args = parser.parse_args()

    # Get the logger
    logger = logging_system.get_logger(os.path.basename(sys.argv[0]))

    # Get the command runner
    runner = CommandRunner(logger)

    # Sanitize the workdir
    workdir = os.path.abspath(os.path.expandvars(os.path.expanduser(args.workdir)))

    with work_within_directory(workdir):

        # Download files

        cmd_line = "download_by_obsid.py --obsid %d" %args.obsid

        runner.run(cmd_line)

        try:

            evtfile = os.path.basename(find_files.find_files(os.getcwd(),'*%s*evt3.fits' %args.obsid)[0])
            tsvfile = os.path.basename(find_files.find_files(os.getcwd(),"%s.tsv" %args.obsid)[0])
            expmap = os.path.basename(find_files.find_files(os.getcwd(),"*%s*exp3.fits*" %args.obsid)[0])

        except IndexError:

            raise RuntimeError("\n\n\nCould not find one of the downloaded files for obsid %s. Exiting..." % args.obsid)

        # Create directory named after obsid
        if not os.path.exists(str(args.obsid)):

            os.mkdir(str(args.obsid))

        # Move files in there

        os.rename(evtfile, os.path.join(str(args.obsid), evtfile))
        os.rename(tsvfile, os.path.join(str(args.obsid), tsvfile))
        os.rename(expmap, os.path.join(str(args.obsid), expmap))

        # filtered_evtfile = "%d_filtered.fits" %(args.obsid)

        # # Filter regions
        #
        # # Figure out the path for the regions files for this obsid
        #
        # region_dir = os.path.join(os.path.expandvars(os.path.expanduser(args.region_repo)), '%s' % args.obsid)
        #
        # cmd_line = "filter_event_file.py --evtfile %s --tsvfile %s --region_dir %s --outfile %s --emin %d --emax %d " \
        #            "--adj_factor %s"\
        #            %(evtfile, tsvfile, region_dir, filtered_evtfile, args.emin, args.emax, args.adj_factor)
        #
        # runner.run(cmd_line)
        #
        # # Separate CCDs
        #
        # cmd_line = "separate_CCD.py --evtfile %s" %filtered_evtfile
        #
        # runner.run(cmd_line)<commit_msg>syntax<commit_after>#!/usr/bin/env python

"""
Download files by observation ID, filter the event file, and separate by CCDs
"""

import argparse
import sys
import os

from chandra_suli import find_files
from chandra_suli import logging_system
from chandra_suli.run_command import CommandRunner
from chandra_suli.work_within_directory import work_within_directory

if __name__=="__main__":

    parser = argparse.ArgumentParser(description='Run following steps: download files by obsid, filter the event file, '
                                                 'and separate by CCD')

    parser.add_argument("-w", "--workdir", help="Directory for all the output (output files will be in a directory "
                                                "named after the obsid)", type=str, required=True)

    parser.add_argument("-o","--obsid",help="Observation ID Numbers", type=int, required=True)

    args = parser.parse_args()

    # Get the logger
    logger = logging_system.get_logger(os.path.basename(sys.argv[0]))

    # Get the command runner
    runner = CommandRunner(logger)

    # Sanitize the workdir
    workdir = os.path.abspath(os.path.expandvars(os.path.expanduser(args.workdir)))

    with work_within_directory(workdir):

        # Download files

        cmd_line = "download_by_obsid.py --obsid %d" %args.obsid

        runner.run(cmd_line)

        try:

            evtfile = os.path.basename(find_files.find_files(os.getcwd(),'*%s*evt3.fits' %args.obsid)[0])
            tsvfile = os.path.basename(find_files.find_files(os.getcwd(),"%s.tsv" %args.obsid)[0])
            expmap = os.path.basename(find_files.find_files(os.getcwd(),"*%s*exp3.fits*" %args.obsid)[0])

        except IndexError:

            raise RuntimeError("\n\n\nCould not find one of the downloaded files for obsid %s. Exiting..." % args.obsid)

        # Create directory named after obsid
        if not os.path.exists(str(args.obsid)):

            os.mkdir(str(args.obsid))

        # Move files in there

        os.rename(evtfile, os.path.join(str(args.obsid), evtfile))
        os.rename(tsvfile, os.path.join(str(args.obsid), tsvfile))
        os.rename(expmap, os.path.join(str(args.obsid), expmap))

        # filtered_evtfile = "%d_filtered.fits" %(args.obsid)

        # # Filter regions
        #
        # # Figure out the path for the regions files for this obsid
        #
        # region_dir = os.path.join(os.path.expandvars(os.path.expanduser(args.region_repo)), '%s' % args.obsid)
        #
        # cmd_line = "filter_event_file.py --evtfile %s --tsvfile %s --region_dir %s --outfile %s --emin %d --emax %d " \
        #            "--adj_factor %s"\
        #            %(evtfile, tsvfile, region_dir, filtered_evtfile, args.emin, args.emax, args.adj_factor)
        #
        # runner.run(cmd_line)
        #
        # # Separate CCDs
        #
        # cmd_line = "separate_CCD.py --evtfile %s" %filtered_evtfile
        #
        # runner.run(cmd_line)<|endoftext|>


<commit_before>"""A PLOT3D file reader.  This reader does not support a timeseries of
files.

"""
# Author: Prabhu Ramachandran <prabhu@aero.iitb.ac.in>
# Copyright (c) 2007, Enthought, Inc.
# License: BSD Style.

# Standard library imports.
from os.path import basename, isfile, exists, splitext

# Enthought library imports.
from traits.api import Trait, Instance, Str, TraitPrefixMap, Button
from traitsui.api import View, Group, Item, FileEditor
from tvtk.api import tvtk
from apptools.persistence.state_pickler import set_state
from apptools.persistence.file_path import FilePath

# Local imports.
from mayavi.core.source import Source
from mayavi.core.common import handle_children_state, error
from mayavi.core.pipeline_info import PipelineInfo

########################################################################
# `PLOT3DReader` class
########################################################################
class PLOT3DReader(Source):

    """A PLOT3D file reader.  This reader does not support a
    timeseries of files.
    """

    # The version of this class.  Used for persistence.
    __version__ = 0

    # XYZ file name
    xyz_file_name = Str('', desc='the XYZ file')

    # The (optional) Q file.
    q_file_name = Str('', desc='the Q file')

    # The active scalar name.
    scalars_name = Trait('density',
                         TraitPrefixMap({'density': 100,
                                          'pressure':110,
                                          'temperature': 120,
                                          'enthalpy': 130,
                                          'internal energy': 140,
                                          'kinetic energy': 144,
                                          'velocity magnitude': 153,
                                          'stagnation energy': 163,
                                          'entropy': 170,
                                          'swirl': 184}),
                         desc='scalar data attribute to show')
    # The active vector name.
    vectors_name = Trait('momentum',
                         TraitPrefixMap({'velocity': 200,
                                          'vorticity': 201,
                                          'momentum': 202,
                                          'pressure gradient': 210}),
                         desc='vector data attribute to show')

    # The VTK data file reader.
    reader = Instance(tvtk.MultiBlockPLOT3DReader, args=(), allow_none=False,
                      record=True)

    # Information about what this object can produce.
    output_info = PipelineInfo(datasets=['structured_grid'])

    ########################################
    # View related code.

    update_reader = Button('Update Reader')

    # Our view.
    view = View(Group(Item('xyz_file_name', editor=FileEditor()),
                      Item('q_file_name', editor=FileEditor()),
                      Item(name='scalars_name',
                           enabled_when='len(object.q_file_name) > 0'),
                      Item(name='vectors_name',
                           enabled_when='len(object.q_file_name)>0'),
                      Item(name='update_reader'),
                      label='Reader',
                      ),
                Group(Item(name='reader', style='custom',
                           resizable=True),
                      show_labels=False,
                      label='PLOT3DReader'
                      ),
                resizable=True)

    ########################################
    # Private traits.

    # The current file paths.  This is not meant to be touched by the
    # user.
    xyz_file_path = Instance(FilePath, args=(), desc='the current XYZ file path')
    q_file_path = Instance(FilePath, args=(), desc='the current Q file path')

    ######################################################################
    # `object` interface
    ######################################################################
    def __get_pure_state__(self):
        d = super(PLOT3DReader, self).__get_pure_state__()
        # These traits are dynamically created.
        for name in ('scalars_name', 'vectors_name', 'xyz_file_name',
                     'q_file_name'):
            d.pop(name, None)

        return d

    def __set_pure_state__(self, state):
        xyz_fn = state.xyz_file_path.abs_pth
        q_fn = state.q_file_path.abs_pth
        if not isfile(xyz_fn):
            msg = 'Could not find file at %s\n'%xyz_fn
            msg += 'Please move the file there and try again.'
            raise IOError, msg

        # Setup the reader state.
        set_state(self, state, first=['reader'], ignore=['*'])
        # Initialize the files.
        self.initialize(xyz_fn, q_fn, configure=False)
        # Now set the remaining state without touching the children.
        set_state(self, state, ignore=['children', 'xyz_file_path', 'q_file_path'])
        # Setup the children.
        handle_children_state(self.children, state.children)
        # Setup the children's state.
        set_state(self, state, first=['children'], ignore=['*'])

    ######################################################################
    # `FileDataSource` interface
    ######################################################################
    def initialize(self, xyz_file_name, q_file_name='', configure=True):
        """Given an xyz filename and a Q filename which may or may not
        be part of a time series, this initializes the list of files.
        This method need not be called to initialize the data.

        If configure is True, it pops up a UI to configure the
        PLOT3DReader.
        """
        if len(q_file_name) == 0:
            base = splitext(xyz_file_name)[0]
            qf = base + '.q'
            if exists(qf):
                q_file_name = qf

        if configure:
            # First set properties of the reader.  This is useful when
            # the data format has atypical defaults.  Automatic
           # detection can be disastrous sometimes due to VTK related
            # problems.
            self.reader.edit_traits(kind='livemodal')
        self.xyz_file_name = xyz_file_name
        if len(q_file_name) > 0:
            self.q_file_name = q_file_name

    def update(self):
        if len(self.xyz_file_path.get()) == 0:
            return
        self.reader.update()
        self.render()

    ######################################################################
    # Non-public interface
    ######################################################################
    def _xyz_file_name_changed(self, value):
        if len(value) == 0:
            return
        else:
            self.reader.xyz_file_name = value
            self.xyz_file_path.set(value)
            self._update_reader_output()

    def _q_file_name_changed(self, value):
        if len(value) == 0:
            return
        else:
            self.reader.q_file_name = value
            self.q_file_path.set(value)
            self._update_reader_output()

    def _update_reader_output(self):
        r = self.reader
        r.update()

        if r.error_code != 0:
            try:
                self.reader.i_blanking = True
            except AttributeError:
                pass
            else:
                r.update()

        # Try reading file.
        if r.error_code != 0:
            # No output so the file might be an ASCII file.
            try:
                # Turn off IBlanking.
                r.set(i_blanking = False, binary_file = False)
            except AttributeError:
                pass
            else:
                r.update()

        # Try again this time as ascii and with blanking.
        if r.error_code != 0:
            # No output so the file might be an ASCII file.
            try:
                # Turn on IBlanking.
                r.i_blanking = True
            except AttributeError:
                pass
            else:
                r.update()

        # If there still is an error, ask the user.
        if r.error_code != 0:
            r.edit_traits(kind='livemodal')
            r.update()

        # If there still is an error, ask the user to retry.
        if r.error_code != 0:
            msg = 'Unable to read file properly. '\
                  'Please check the settings of the reader '\
                  'on the UI and press the "Update Reader" button '\
                  'when done and try again!'
            error(msg)
            return

        # Now setup the outputs by resetting self.outputs.  Changing
        # the outputs automatically fires a pipeline_changed event.
        try:
            n = r.number_of_output_ports
        except AttributeError: # for VTK >= 4.5
            n = r.number_of_outputs
        outputs = []
        for i in range(n):
            outputs.append(r.get_output().get_block(i))

        self.outputs = outputs

        # Fire data_changed just in case the outputs are not
        # really changed.  This can happen if the dataset is of
        # the same type as before.
        self.data_changed = True

        # Change our name on the tree view
        self.name = self._get_name()

    def _scalars_name_changed(self, value):
        self.reader.scalar_function_number = self.scalars_name_
        self.reader.modified()
        self.update()
        self.data_changed = True

    def _vectors_name_changed(self, value):
        self.reader.vector_function_number = self.vectors_name_
        self.reader.modified()
        self.update()
        self.data_changed = True

    def _update_reader_fired(self):
        self.reader.modified()
        self._update_reader_output()
        self.pipeline_changed = True

    def _get_name(self):
        """ Gets the name to display on the tree view.
        """
        xyz_fname = basename(self.xyz_file_path.get())
        q_fname = basename(self.q_file_path.get())
        if len(self.q_file_name) > 0:
            ret = "PLOT3D:%s, %s"%(xyz_fname, q_fname)
        else:
            ret = "PLOT3D:%s"%(xyz_fname)
        if '[Hidden]' in self.name:
            ret += ' [Hidden]'
        return ret

<commit_msg>Fix formatting in comment<commit_after>"""A PLOT3D file reader.  This reader does not support a timeseries of
files.

"""
# Author: Prabhu Ramachandran <prabhu@aero.iitb.ac.in>
# Copyright (c) 2007, Enthought, Inc.
# License: BSD Style.

# Standard library imports.
from os.path import basename, isfile, exists, splitext

# Enthought library imports.
from traits.api import Trait, Instance, Str, TraitPrefixMap, Button
from traitsui.api import View, Group, Item, FileEditor
from tvtk.api import tvtk
from apptools.persistence.state_pickler import set_state
from apptools.persistence.file_path import FilePath

# Local imports.
from mayavi.core.source import Source
from mayavi.core.common import handle_children_state, error
from mayavi.core.pipeline_info import PipelineInfo

########################################################################
# `PLOT3DReader` class
########################################################################
class PLOT3DReader(Source):

    """A PLOT3D file reader.  This reader does not support a
    timeseries of files.
    """

    # The version of this class.  Used for persistence.
    __version__ = 0

    # XYZ file name
    xyz_file_name = Str('', desc='the XYZ file')

    # The (optional) Q file.
    q_file_name = Str('', desc='the Q file')

    # The active scalar name.
    scalars_name = Trait('density',
                         TraitPrefixMap({'density': 100,
                                          'pressure':110,
                                          'temperature': 120,
                                          'enthalpy': 130,
                                          'internal energy': 140,
                                          'kinetic energy': 144,
                                          'velocity magnitude': 153,
                                          'stagnation energy': 163,
                                          'entropy': 170,
                                          'swirl': 184}),
                         desc='scalar data attribute to show')
    # The active vector name.
    vectors_name = Trait('momentum',
                         TraitPrefixMap({'velocity': 200,
                                          'vorticity': 201,
                                          'momentum': 202,
                                          'pressure gradient': 210}),
                         desc='vector data attribute to show')

    # The VTK data file reader.
    reader = Instance(tvtk.MultiBlockPLOT3DReader, args=(), allow_none=False,
                      record=True)

    # Information about what this object can produce.
    output_info = PipelineInfo(datasets=['structured_grid'])

    ########################################
    # View related code.

    update_reader = Button('Update Reader')

    # Our view.
    view = View(Group(Item('xyz_file_name', editor=FileEditor()),
                      Item('q_file_name', editor=FileEditor()),
                      Item(name='scalars_name',
                           enabled_when='len(object.q_file_name) > 0'),
                      Item(name='vectors_name',
                           enabled_when='len(object.q_file_name)>0'),
                      Item(name='update_reader'),
                      label='Reader',
                      ),
                Group(Item(name='reader', style='custom',
                           resizable=True),
                      show_labels=False,
                      label='PLOT3DReader'
                      ),
                resizable=True)

    ########################################
    # Private traits.

    # The current file paths.  This is not meant to be touched by the
    # user.
    xyz_file_path = Instance(FilePath, args=(), desc='the current XYZ file path')
    q_file_path = Instance(FilePath, args=(), desc='the current Q file path')

    ######################################################################
    # `object` interface
    ######################################################################
    def __get_pure_state__(self):
        d = super(PLOT3DReader, self).__get_pure_state__()
        # These traits are dynamically created.
        for name in ('scalars_name', 'vectors_name', 'xyz_file_name',
                     'q_file_name'):
            d.pop(name, None)

        return d

    def __set_pure_state__(self, state):
        xyz_fn = state.xyz_file_path.abs_pth
        q_fn = state.q_file_path.abs_pth
        if not isfile(xyz_fn):
            msg = 'Could not find file at %s\n'%xyz_fn
            msg += 'Please move the file there and try again.'
            raise IOError, msg

        # Setup the reader state.
        set_state(self, state, first=['reader'], ignore=['*'])
        # Initialize the files.
        self.initialize(xyz_fn, q_fn, configure=False)
        # Now set the remaining state without touching the children.
        set_state(self, state, ignore=['children', 'xyz_file_path', 'q_file_path'])
        # Setup the children.
        handle_children_state(self.children, state.children)
        # Setup the children's state.
        set_state(self, state, first=['children'], ignore=['*'])

    ######################################################################
    # `FileDataSource` interface
    ######################################################################
    def initialize(self, xyz_file_name, q_file_name='', configure=True):
        """Given an xyz filename and a Q filename which may or may not
        be part of a time series, this initializes the list of files.
        This method need not be called to initialize the data.

        If configure is True, it pops up a UI to configure the
        PLOT3DReader.
        """
        if len(q_file_name) == 0:
            base = splitext(xyz_file_name)[0]
            qf = base + '.q'
            if exists(qf):
                q_file_name = qf

        if configure:
            # First set properties of the reader.  This is useful when
            # the data format has atypical defaults.  Automatic
            # detection can be disastrous sometimes due to VTK related
            # problems.
            self.reader.edit_traits(kind='livemodal')
        self.xyz_file_name = xyz_file_name
        if len(q_file_name) > 0:
            self.q_file_name = q_file_name

    def update(self):
        if len(self.xyz_file_path.get()) == 0:
            return
        self.reader.update()
        self.render()

    ######################################################################
    # Non-public interface
    ######################################################################
    def _xyz_file_name_changed(self, value):
        if len(value) == 0:
            return
        else:
            self.reader.xyz_file_name = value
            self.xyz_file_path.set(value)
            self._update_reader_output()

    def _q_file_name_changed(self, value):
        if len(value) == 0:
            return
        else:
            self.reader.q_file_name = value
            self.q_file_path.set(value)
            self._update_reader_output()

    def _update_reader_output(self):
        r = self.reader
        r.update()

        if r.error_code != 0:
            try:
                self.reader.i_blanking = True
            except AttributeError:
                pass
            else:
                r.update()

        # Try reading file.
        if r.error_code != 0:
            # No output so the file might be an ASCII file.
            try:
                # Turn off IBlanking.
                r.set(i_blanking = False, binary_file = False)
            except AttributeError:
                pass
            else:
                r.update()

        # Try again this time as ascii and with blanking.
        if r.error_code != 0:
            # No output so the file might be an ASCII file.
            try:
                # Turn on IBlanking.
                r.i_blanking = True
            except AttributeError:
                pass
            else:
                r.update()

        # If there still is an error, ask the user.
        if r.error_code != 0:
            r.edit_traits(kind='livemodal')
            r.update()

        # If there still is an error, ask the user to retry.
        if r.error_code != 0:
            msg = 'Unable to read file properly. '\
                  'Please check the settings of the reader '\
                  'on the UI and press the "Update Reader" button '\
                  'when done and try again!'
            error(msg)
            return

        # Now setup the outputs by resetting self.outputs.  Changing
        # the outputs automatically fires a pipeline_changed event.
        try:
            n = r.number_of_output_ports
        except AttributeError: # for VTK >= 4.5
            n = r.number_of_outputs
        outputs = []
        for i in range(n):
            outputs.append(r.get_output().get_block(i))

        self.outputs = outputs

        # Fire data_changed just in case the outputs are not
        # really changed.  This can happen if the dataset is of
        # the same type as before.
        self.data_changed = True

        # Change our name on the tree view
        self.name = self._get_name()

    def _scalars_name_changed(self, value):
        self.reader.scalar_function_number = self.scalars_name_
        self.reader.modified()
        self.update()
        self.data_changed = True

    def _vectors_name_changed(self, value):
        self.reader.vector_function_number = self.vectors_name_
        self.reader.modified()
        self.update()
        self.data_changed = True

    def _update_reader_fired(self):
        self.reader.modified()
        self._update_reader_output()
        self.pipeline_changed = True

    def _get_name(self):
        """ Gets the name to display on the tree view.
        """
        xyz_fname = basename(self.xyz_file_path.get())
        q_fname = basename(self.q_file_path.get())
        if len(self.q_file_name) > 0:
            ret = "PLOT3D:%s, %s"%(xyz_fname, q_fname)
        else:
            ret = "PLOT3D:%s"%(xyz_fname)
        if '[Hidden]' in self.name:
            ret += ' [Hidden]'
        return ret

<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-

"""Package Module

Utilities for managing Python packages.

.. moduleauthor:: Timothy Helton <timothy.j.helton@gmail.com>
"""

from collections import namedtuple
from datetime import datetime as dt
import glob
import logging
import subprocess
import os
import os.path as osp
import tarfile
from typing import Iterable

from strumenti import system

log_file = 'packages_{}.log'.format(dt.date(dt.now()))
master_log = logging.DEBUG
console_log = logging.INFO
file_log = logging.DEBUG

class Manage:
    """Class will assist with managing Python packages.

    :Attributes:

        - **log_file**: *str* name of log file
        - **outdated**: *dict* outdated packages
        - **packages**: *dict* all packages
        - **req_txt_path**: *str* path to the file containing all package \
            version requirements
        - **wheels**: *list* path to all wheels generated by installing \
            packages (this includes all dependencies)
        - **wheel_path**: *str* path to the temporary wheelhouse directory
    """
    def __init__(self, logging_file=log_file):
        self.log_file = logging_file
        self._outdated = {}
        self._packages = {}
        self.req_txt_path = 'requirements.txt'
        self._wheels = []
        self.wheel_path = 'temp_wheelhouse'

    @property
    def outdated(self):
        self.list_packages(outdated=True)
        return self._outdated

    @property
    def packages(self):
        self.list_packages()
        return self._packages

    @property
    def wheels(self):
        self.get_wheels()
        return self._wheels

    def __repr__(self):
        return 'Manage()'

    def create_tar(self):
        """Create tar file of wheels."""
        with tarfile.open('{}.tar'.format(self.wheel_path), 'w:gz') as tar:
            for name in self.wheels:
                tar.add(name)

    def create_wheels(self, packages: ([str], [tuple], [list])):
        """Create wheels for packages.

        :param packages: names of packages to upgrade
        :type: str tuple list
        """
        if not self._packages:
            self.list_packages()

        packages = system.check_list(packages)

        os.makedirs(self.wheel_path, exist_ok=True)
        for pkg in packages:
            logging.info('{:15}{}'.format('Create Wheel:', pkg))
            version = self._packages[pkg]
            proc = subprocess.Popen(['pip', 'wheel',
                                     '--wheel-dir={}'.format(self.wheel_path),
                                     '{}=={}'.format(pkg, version)],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            output, error = proc.communicate()
            logging.debug(output.decode())
            if error:
                logging.error(error.decode())
            logging.info('{:15}{}'.format('Complete:', pkg))

    def get_wheels(self):
        """Get the absolute path for all wheel files."""
        wheels = glob.glob('{}{}*'.format(self.wheel_path, os.sep))
        self._wheels = [osp.realpath(x) for x in wheels]

    def list_packages(self, outdated: bool=False):
        """Get information on installed packages.

        :param bool outdated: if True only outdated packages will be retrieved
        """
        versions = namedtuple('Versions', ['old_ver', 'new_ver'])

        cmd = ['pip', 'list']
        if outdated:
            cmd.append('-o')

        execute = subprocess.run(cmd, stdout=subprocess.PIPE)
        packages = execute.stdout.decode('utf-8').splitlines()
        packages = [x.split() for x in packages]

        if outdated:
            if not packages:
                logging.info('All installed packages are up to date.')
            else:
                self._outdated = {x[0]: versions(x[2], x[4]) for x in packages}
        else:
            self._packages = {x[0]: x[1].strip('()') for x in packages}

    @staticmethod
    def install_packages(packages: Iterable, upgrade: bool=True):
        """Install Python packages.

        .. note:: The argument packages can be a list of package names as \
            strings or tuples of package names with desired versions to be \
            installed.

            - List of Strings example:
                - ['numpy', 'scipy']
            - List of Tuples example:
                - [('numpy', '1.10.4'), ('scipy', '0.16.1')]

        :param iterable packages: packages to install
        :param bool upgrade: set to True if package is already installed \
            and the version is being upgraded or False if a new package is \
            being installed
        """
        cmd = ['pip', 'install']
        if upgrade:
            cmd.append('--upgrade')

        for pkg in iter(packages):
            if isinstance(pkg, tuple):
                pkg = '{}=={}'.format(pkg[0], pkg[1])

            logging.info('\n{:10}{}'.format('Install:', pkg))
            cmd.append(pkg)
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            output, error = proc.communicate()
            logging.debug(output.decode())
            if error:
                logging.error(error.decode())
            if not error:
                logging.info('{:10}\n'.format('Complete'))

    def update_packages(self):
        """Update to latest version of all packages."""
        update_pkgs = self.outdated.keys()
        self.install_packages(update_pkgs)
        if update_pkgs:
            self.update_requirements()

    def update_requirements(self):
        """Update the requirements.txt file.

        .. note:: The requirements file will only be updated if the path is \
            defined in the attribute req_txt_file.
        """
        try:
            execute = subprocess.run(['pip', 'freeze'], stdout=subprocess.PIPE)
            with open(self.req_txt_path, 'w') as f:
                f.write(execute.stdout.decode('utf-8'))
            logging.info(('\nUpdated Requirements file: {}'
                          '\n').format(self.req_txt_path))
        except IOError:
            logging.error('\nFile Not Found: {}\n'
                          'Requirements File Not Updated'
                          '\n').format(self.req_txt_path)

if __name__ == '__main__':
    logging = system.logger_setup(log_file=log_file,
                                  master_level=master_log,
                                  console_level=console_log,
                                  file_level=file_log)
    logging.info('Checking for package updates')
    p = Manage()
    p.update_packages()
    logging.info('Package updates complete')
<commit_msg>BUG: corrected logger call to system.logger_setup<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-

"""Package Module

Utilities for managing Python packages.

.. moduleauthor:: Timothy Helton <timothy.j.helton@gmail.com>
"""

from collections import namedtuple
from datetime import datetime as dt
import glob
import logging
import subprocess
import os
import os.path as osp
import tarfile
from typing import Iterable

from strumenti import system

log_file = 'packages_{}.log'.format(dt.date(dt.now()))
master_log = logging.DEBUG
console_log = logging.INFO
file_log = logging.DEBUG

logger = system.logger_setup(log_file=log_file,
                             master_level=master_log,
                             console_level=console_log,
                             file_level=file_log)

class Manage:
    """Class will assist with managing Python packages.

    :Attributes:

        - **log_file**: *str* name of log file
        - **outdated**: *dict* outdated packages
        - **packages**: *dict* all packages
        - **req_txt_path**: *str* path to the file containing all package \
            version requirements
        - **wheels**: *list* path to all wheels generated by installing \
            packages (this includes all dependencies)
        - **wheel_path**: *str* path to the temporary wheelhouse directory
    """
    def __init__(self, logging_file=log_file):
        self.log_file = logging_file
        self._outdated = {}
        self._packages = {}
        self.req_txt_path = 'requirements.txt'
        self._wheels = []
        self.wheel_path = 'temp_wheelhouse'

    @property
    def outdated(self):
        self.list_packages(outdated=True)
        return self._outdated

    @property
    def packages(self):
        self.list_packages()
        return self._packages

    @property
    def wheels(self):
        self.get_wheels()
        return self._wheels

    def __repr__(self):
        return 'Manage()'

    def create_tar(self):
        """Create tar file of wheels."""
        with tarfile.open('{}.tar'.format(self.wheel_path), 'w:gz') as tar:
            for name in self.wheels:
                tar.add(name)

    def create_wheels(self, packages: ([str], [tuple], [list])):
        """Create wheels for packages.

        :param packages: names of packages to upgrade
        :type: str tuple list
        """
        if not self._packages:
            self.list_packages()

        packages = system.check_list(packages)

        os.makedirs(self.wheel_path, exist_ok=True)
        for pkg in packages:
            logger.info('{:15}{}'.format('Create Wheel:', pkg))
            version = self._packages[pkg]
            proc = subprocess.Popen(['pip', 'wheel',
                                     '--wheel-dir={}'.format(self.wheel_path),
                                     '{}=={}'.format(pkg, version)],
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            output, error = proc.communicate()
            logger.debug(output.decode())
            if error:
                logger.error(error.decode())
            logger.info('{:15}{}'.format('Complete:', pkg))

    def get_wheels(self):
        """Get the absolute path for all wheel files."""
        wheels = glob.glob('{}{}*'.format(self.wheel_path, os.sep))
        self._wheels = [osp.realpath(x) for x in wheels]

    def list_packages(self, outdated: bool=False):
        """Get information on installed packages.

        :param bool outdated: if True only outdated packages will be retrieved
        """
        versions = namedtuple('Versions', ['old_ver', 'new_ver'])

        cmd = ['pip', 'list']
        if outdated:
            cmd.append('-o')

        execute = subprocess.run(cmd, stdout=subprocess.PIPE)
        packages = execute.stdout.decode('utf-8').splitlines()
        packages = [x.split() for x in packages]

        if outdated:
            if not packages:
                logger.info('All installed packages are up to date.')
            else:
                self._outdated = {x[0]: versions(x[2], x[4]) for x in packages}
        else:
            self._packages = {x[0]: x[1].strip('()') for x in packages}

    @staticmethod
    def install_packages(packages: Iterable, upgrade: bool=True):
        """Install Python packages.

        .. note:: The argument packages can be a list of package names as \
            strings or tuples of package names with desired versions to be \
            installed.

            - List of Strings example:
                - ['numpy', 'scipy']
            - List of Tuples example:
                - [('numpy', '1.10.4'), ('scipy', '0.16.1')]

        :param iterable packages: packages to install
        :param bool upgrade: set to True if package is already installed \
            and the version is being upgraded or False if a new package is \
            being installed
        """
        cmd = ['pip', 'install']
        if upgrade:
            cmd.append('--upgrade')

        for pkg in iter(packages):
            if isinstance(pkg, tuple):
                pkg = '{}=={}'.format(pkg[0], pkg[1])

            logger.info('\n{:10}{}'.format('Install:', pkg))
            cmd.append(pkg)
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            output, error = proc.communicate()
            logger.debug(output.decode())
            if error:
                logger.error(error.decode())
            if not error:
                logger.info('{:10}\n'.format('Complete'))

    def update_packages(self):
        """Update to latest version of all packages."""
        update_pkgs = self.outdated.keys()
        self.install_packages(update_pkgs)
        if update_pkgs:
            self.update_requirements()

    def update_requirements(self):
        """Update the requirements.txt file.

        .. note:: The requirements file will only be updated if the path is \
            defined in the attribute req_txt_file.
        """
        try:
            execute = subprocess.run(['pip', 'freeze'], stdout=subprocess.PIPE)
            with open(self.req_txt_path, 'w') as f:
                f.write(execute.stdout.decode('utf-8'))
            logger.info(('\nUpdated Requirements file: {}'
                         '\n').format(self.req_txt_path))
        except IOError:
            logger.error('\nFile Not Found: {}\n'
                         'Requirements File Not Updated'
                         '\n').format(self.req_txt_path)

if __name__ == '__main__':
    logger = system.logger_setup(log_file=log_file,
                                 master_level=master_log,
                                 console_level=console_log,
                                 file_level=file_log)
    logger.info('Checking for package updates')
    p = Manage()
    p.update_packages()
    logger.info('Package updates complete')
<|endoftext|>


<commit_before>"""
replace_cert.py

Simple certificate replacement for load balancers and EBS configurations.

Be careful if two certs have the same name but different paths!!!!

THIS SCRIPT DOES NOT SUPPORT CHANGING PATHS!!!!!!!!!

"""

import boto3, sys, re

def verify_certs_exist(source_cert, dest_cert):
    iam_client = boto3.client('iam')

    #Get server certificates
    certs = iam_client.list_server_certificates()
    source_exists = False
    dest_exists = False

    #Split cert name and take last element (the name of the cert), and see if it matches
    for item in certs['ServerCertificateMetadataList']:
        if item['ServerCertificateName'].split("/")[-1] == source_cert:
            source_exists = True
        if item['ServerCertificateName'].split("/")[-1] == dest_cert:
            dest_exists = True

    #Notify if they exist or error and exit if they dont
    if source_exists:
        print("Source certificate " + source_cert + " exists.")
    else:
        print("ERROR: Unable to found source certificate: " + source_cert)
        sys.exit(1)

    if dest_exists:
        print("Destination certificate " + dest_cert + " exists.")
    else:
        print("ERROR: Unable to found destination certificate: " + dest_cert)
        sys.exit(1)

def main(kwargs, args):

    #TODO: Change to false before production
    dry_run = False
    if "dry-run" in kwargs or "n" in kwargs:
        print("Dry run")
        dry_run = True

    #Assume the cert names are the only two "arguments" (argument is something that doesnt start with '-')
    try:
        source_cert = args[0]
        dest_cert = args[1]
    except IndexError:
        print("You must specify a source cert name (from) and a destination cert name (to)")
        sys.exit(1)

    #Verify certs exist
    verify_certs_exist(source_cert, dest_cert)

    if not dry_run:
        print("!!!WARNING!!!")
        print("This script could make very dangerous changes, and it's recommended to run it with a --dry-run (-n) to see what it will change before it does it.")
        response = raw_input("Please respond 'yes' if you understand the consequences: ")
        if response != "yes":
            print("Aborting changes.")
            sys.exit(1)

    #Get list of load balancers
    elb_client = boto3.client('elb')
    load_balancer_list = elb_client.describe_load_balancers()
    ebs_list = list()

    for lb in load_balancer_list["LoadBalancerDescriptions"]:

        #Describe tags takes a list with each load balancer name you want to look up
        lb_name = list()
        lb_name.append(lb["LoadBalancerName"])

        #Returns a dictionary
        response = elb_client.describe_tags(LoadBalancerNames=lb_name)

        #We need the TagDescriptions value, which is a list with one element (since we only passed in one name), and we need the Tags value
        tags = response["TagDescriptions"][0]["Tags"]
        name = None
        ebs_name = None
        stack_name = None

        for tag in tags:
            if tag["Key"] == "Name":
                name = tag["Value"]
            if tag["Key"] == "elasticbeanstalk:environment-name":
                ebs_name = tag["Value"]
            if tag["Key"] == "aws:cloudformation:stack-name":
                stack_name = tag["Value"]

        if name is None:
            if ebs_name is not None:
                name = ebs_name
            else:
                name = stack_name

        #Drill down and make sure the listener is HTTPS protocol, make sure source_cert is part of the SSLCertificateId string, and replace that part with dest_cert
        for listener in lb["ListenerDescriptions"]:
            if listener["Listener"]["Protocol"] == "HTTPS":
                if source_cert == listener["Listener"]["SSLCertificateId"].split("/")[-1]:
                    replaced_string = re.sub(source_cert, dest_cert, listener["Listener"]["SSLCertificateId"])
                    print("Replacing " + listener["Listener"]["SSLCertificateId"] + " with " + replaced_string + " in loadbalancer: " + str(name))
                    if not dry_run:
                        pass
                    #Try to get the ebs name, if there is one for this load balancer
                    if ebs_name is not None:
                        ebs_list.append(ebs_name)

    for bs in ebs_list:
        print "Replacing " + source_cert + " with " + dest_cert + " in beanstalk: " + bs

#Modified From maestro
def parse_sysargs():
    current_key = None
    kwargs = dict()
    args = list()
    try:
        for arg in sys.argv[1:]:
            if current_key is None:
                if arg.startswith('-'):
                    current_key = arg.lstrip('-')
                else:
                    args.append(arg)
                continue
            if current_key is not None and not arg.startswith("-"):
                args.append(arg)
            else:
                kwargs[current_key] = None
                current_key = arg.lstrip('-')
        if current_key is not None and current_key not in kwargs:
            kwargs[current_key] = None
            current_key = None
    except IndexError:
        args.append(sys.argv[1])

    return kwargs, args

if __name__ == "__main__":
    kwargs, args = parse_sysargs()
    main(kwargs, args)
<commit_msg>Added region selection, fixed argument handling, general fixes<commit_after>"""
replace_cert.py

Simple certificate replacement for load balancers and EBS configurations.

Be careful if two certs have the same name but different paths!!!!

THIS SCRIPT DOES NOT SUPPORT CHANGING PATHS!!!!!!!!!

"""

import boto3, sys, re

def list_certs():
    iam_client = boto3.client('iam')
    certs = iam_client.list_server_certificates()
    for item in certs['ServerCertificateMetadataList']:
        print(item["ServerCertificateName"])
        print("Expires: " + str(item["Expiration"]))

def verify_certs_exist(source_cert, dest_cert):
    iam_client = boto3.client('iam')

    #Get server certificates
    certs = iam_client.list_server_certificates()
    source_exists = False
    dest_exists = False

    #Split cert name and take last element (the path and name of the cert), and see if it matches
    for item in certs['ServerCertificateMetadataList']:
        if item['ServerCertificateName'] == source_cert:
            source_exists = True
        if item['ServerCertificateName'] == dest_cert:
            dest_exists = True

    #Notify if they exist or error and exit if they dont
    if source_exists:
        print("Source certificate " + source_cert + " exists.")
    else:
        print("ERROR: Unable to find source certificate: " + source_cert)
        sys.exit(1)

    if dest_exists:
        print("Destination certificate " + dest_cert + " exists.")
    else:
        print("ERROR: Unable to find destination certificate: " + dest_cert)
        sys.exit(1)

def main(kwargs, args):

    if "list" in kwargs or "l" in kwargs or "list" in args:
        list_certs()
        sys.exit(0)

    dry_run = False
    if "dry-run" in kwargs or "n" in kwargs:
        print("Dry run")
        dry_run = True

    region = "us-east-1"
    if "region" in kwargs:
        region = kwargs["region"]
    elif "r" in kwargs:
        region = kwargs["r"]

    #Assume the cert names are the only two "arguments" (argument is something that doesnt start with '-')
    try:
        source_cert = args[0]
        dest_cert = args[1]
    except IndexError:
        print("You must specify a source cert name (from) and a destination cert name (to)")
        sys.exit(1)

    #Verify certs exist
    verify_certs_exist(source_cert, dest_cert)

    if not dry_run:
        print("!!!WARNING!!!")
        print("This script could make very dangerous changes, and it's recommended to run it with a --dry-run (-n) to see what it will change before it does it.")
        response = raw_input("Please respond 'yes' if you understand the consequences: ")
        if response != "yes":
            print("Aborting changes.")
            sys.exit(1)

    #Get list of load balancers
    elb_client = boto3.client('elb', region_name=region)
    load_balancer_list = elb_client.describe_load_balancers()
    ebs_list = list()

    for lb in load_balancer_list["LoadBalancerDescriptions"]:

        #Describe tags takes a list with each load balancer name you want to look up
        lb_name = list()
        lb_name.append(lb["LoadBalancerName"])

        #Returns a dictionary
        response = elb_client.describe_tags(LoadBalancerNames=lb_name)

        #We need the TagDescriptions value, which is a list with one element (since we only passed in one name), and we need the Tags value
        tags = response["TagDescriptions"][0]["Tags"]
        name = None
        ebs_name = None
        stack_name = None

        for tag in tags:
            if tag["Key"] == "Name":
                name = tag["Value"]
            if tag["Key"] == "elasticbeanstalk:environment-name":
                ebs_name = tag["Value"]
            if tag["Key"] == "aws:cloudformation:stack-name":
                stack_name = tag["Value"]

        if name is None:
            if ebs_name is not None:
                name = ebs_name
            else:
                name = stack_name

        #Drill down and make sure the listener is HTTPS protocol, make sure source_cert is part of the SSLCertificateId string, and replace that part with dest_cert
        for listener in lb["ListenerDescriptions"]:
            if listener["Listener"]["Protocol"] == "HTTPS":
                if source_cert == listener["Listener"]["SSLCertificateId"].split("/")[-1]:
                    replaced_string = re.sub(source_cert, dest_cert, listener["Listener"]["SSLCertificateId"])
                    print("Replacing " + listener["Listener"]["SSLCertificateId"] + " with " + replaced_string + " in loadbalancer: " + str(name))
                    if not dry_run:
                        elb_client.set_load_balancer_listener_ssl_certificate(LoadBalancerName= lb_name[0], LoadBalancerPort= listener["Listener"]["LoadBalancerPort"], SSLCertificateId = replaced_string)
                    #Try to get the ebs name, if there is one for this load balancer
                    if ebs_name is not None:
                        ebs_list.append(ebs_name)

    for bs in ebs_list:
        print "Replacing " + source_cert + " with " + dest_cert + " in beanstalk: " + bs

#Modified From maestro
def parse_sysargs():
    current_key = None
    kwargs = dict()
    args = list()
    try:
        for arg in sys.argv[1:]:
            if current_key is None:
                if arg.startswith('-'):
                    current_key = arg.lstrip('-')
                else:
                    args.append(arg)
                continue
            if current_key is None and not arg.startswith("-"):
                args.append(arg)
            elif current_key is not None and not arg.startswith("-"):
                kwargs[current_key] = arg
                current_key = None
            else:
                kwargs[current_key] = None
                current_key = arg.lstrip('-')
        if current_key is not None and current_key not in kwargs:
            kwargs[current_key] = None
            current_key = None
    except IndexError:
        args.append(sys.argv[1])
    return kwargs, args

if __name__ == "__main__":
    kwargs, args = parse_sysargs()
    main(kwargs, args)
<|endoftext|>