<commit_before>import os
import matplotlib
from plantcv.plantcv import fatal_error
observations = {}

class Params:
    """PlantCV parameters class."""

    def __init__(self, device=0, debug=None, debug_outdir=".", line_thickness=5, dpi=100, text_size=0.55,
                 text_thickness=2, marker_size=60, color_scale="gist_rainbow", color_sequence="sequential",
                 saved_color_scale=None):
        """Initialize parameters.

        Keyword arguments/parameters:
        device            = Device number. Used to count steps in the pipeline. (default: 0)
        debug             = None, print, or plot. Print = save to file, Plot = print to screen. (default: None)
        debug_outdir      = Debug images output directory. (default: .)
        line_thickness    = Width of line drawings. (default: 5)
        dpi               = Figure plotting resolution, dots per inch. (default: 100)
        text_size         = Size of plotting text. (default: 0.55)
        text_thickness    = Thickness of plotting text. (default: 2)
        marker_size       = Size of plotting markers (default: 60)
        color_scale       = Name of plotting color scale (matplotlib colormap). (default: gist_rainbow)
        color_sequence    = Build color scales in "sequential" or "random" order. (default: sequential)
        saved_color_scale = Saved color scale that will be applied next time color_palette is called. (default: None)

        :param device: int
        :param debug: str
        :param debug_outdir: str
        :param line_thickness: numeric
        :param dpi: int
        :param text_size: float
        :param text_thickness: int
        :param marker_size: int
        :param color_scale: str
        :param color_sequence: str
        :param saved_color_scale: list
        """
        self.device = device
        self.debug = debug
        self.debug_outdir = debug_outdir
        self.line_thickness = line_thickness
        self.dpi = dpi
        self.text_size = text_size
        self.text_thickness = text_thickness
        self.marker_size = marker_size
        self.color_scale = color_scale
        self.color_sequence = color_sequence
        self.saved_color_scale = saved_color_scale

class Outputs:
    """PlantCV outputs class

    """

    def __init__(self):
        self.measurements = {}
        self.images = []
        self.observations = {}

        # Add a method to clear measurements
    def clear(self):
        self.measurements = {}
        self.images = []
        self.observations = {}

    # Method to add observation to outputs
    def add_observation(self, sample, variable, trait, method, scale, datatype, value, label):
        """
        Keyword arguments/parameters:
        sample       = Sample name. Used to distinguish between multiple samples
        variable     = A local unique identifier of a variable, e.g. a short name,
                       that is a key linking the definitions of variables with observations.
        trait        = A name of the trait mapped to an external ontology; if there is no exact mapping, an informative
                       description of the trait.
        method       = A name of the measurement method mapped to an external ontology; if there is no exact mapping, an
                       informative description of the measurement procedure
        scale        = Units of the measurement or scale in which the observations are expressed; if possible, standard
                       units and scales should be used and mapped to existing ontologies; in the case of non-standard
                       scale a full explanation should be given
        datatype     = The type of data to be stored, e.g. 'int', 'float', 'str', 'list', 'bool', etc.
        value        = The data itself
        label        = The label for each value (most useful when the data is a frequency table as in hue,
                       or other tables)

        :param sample: str
        :param variable: str
        :param trait: str
        :param method: str
        :param scale: str
        :param datatype: type
        :param value:
        :param label:
        """
        self.sample = sample
        self.variable = variable
        self.trait = trait
        self.method = method
        self.scale = scale
        self.datatype = datatype
        self.value = value
        self.label = label

        # Create an empty dictionary for the sample if it does not exist
        if sample not in self.observations:
            self.observations[sample] = {}
        supported_dtype = ["str", "list", "bool","tuple"]
        # Send an error message if datatype is not supported by json
        if not ("int" in str(type(value)) or "float" in str(type(value)) or value is None or str(type(value)) in ["<class '{}'>".format(i) for i in supported_dtype]):
            fatal_error("Data type not compatible with json! Please use only these: int, float, string, list, or bool!")
        # Save the observation for the sample and variable
        self.observations[sample][variable] = {
            "trait": trait,
            "method": method,
            "scale": scale,
            "datatype": str(datatype),
            "value": value,
            "label": label
        }

# Initialize an instance of the Params and Outputs class with default values
# params and outputs are available when plantcv is imported
params = Params()
outputs = Outputs()

class Spectral_data:
    # PlantCV Hyperspectral data class
    def __init__(self, array_data, max_wavelength, min_wavelength, max_value, min_value, d_type, wavelength_dict,
                 samples, lines, interleave, wavelength_units, array_type, pseudo_rgb, filename, default_bands):
        # The actual array/datacube
        self.array_data = array_data
        # Min/max available wavelengths (for spectral datacube)
        self.max_wavelength = max_wavelength
        self.min_wavelength = min_wavelength
        #  Min/max pixel value for single wavelength or index
        self.max_value = max_value
        self.min_value = min_value
        # Numpy data type
        self.d_type = d_type
        # Contains all available wavelengths where keys are wavelength and value are indices
        self.wavelength_dict = wavelength_dict
        # Resolution of a single band of spectral data is (samples, lines) rather than (x,y) with other arrays
        self.samples = samples
        self.lines = lines
        # Interleave type
        self.interleave = interleave
        self.wavelength_units = wavelength_units
        # The type of array data (entire datacube, specific index, first derivative, etc)
        self.array_type = array_type
        # Pseudo-RGB image if the array_type is a datacube
        self.pseudo_rgb = pseudo_rgb
        # The filename where the data originated from
        self.filename = filename
        # The default band indices needed to make an pseudo_rgb image, if not available then store None
        self.default_bands = default_bands

# Example
# spectral_array = Spectral_data(max_wavelength=1000.95, min_wavelength=379.027, d_type=numpy.float32,
#                           wavelength_dict=dictionary, samples=1600, lines=1704, interleave='bil',
#                           wavelength_units='nm', array_type="datacube", filename=fname, default_bands={159,253,520})

from plantcv.plantcv.fatal_error import fatal_error
from plantcv.plantcv.print_image import print_image
from plantcv.plantcv.plot_image import plot_image
from plantcv.plantcv.color_palette import color_palette
from plantcv.plantcv.rgb2gray import rgb2gray
from plantcv.plantcv.rgb2gray_hsv import rgb2gray_hsv
from plantcv.plantcv.rgb2gray_lab import rgb2gray_lab
from plantcv.plantcv.gaussian_blur import gaussian_blur
from plantcv.plantcv import transform
from plantcv.plantcv import hyperspectral
from plantcv.plantcv import spectral_index
from plantcv.plantcv.apply_mask import apply_mask
from plantcv.plantcv.readimage import readimage
from plantcv.plantcv.readbayer import readbayer
from plantcv.plantcv.laplace_filter import laplace_filter
from plantcv.plantcv.sobel_filter import sobel_filter
from plantcv.plantcv.scharr_filter import scharr_filter
from plantcv.plantcv.hist_equalization import hist_equalization
from plantcv.plantcv.image_add import image_add
from plantcv.plantcv.image_subtract import image_subtract
from plantcv.plantcv.erode import erode
from plantcv.plantcv.dilate import dilate
from plantcv.plantcv.watershed import watershed_segmentation
from plantcv.plantcv.rectangle_mask import rectangle_mask
from plantcv.plantcv.median_blur import median_blur
from plantcv.plantcv.fill import fill
from plantcv.plantcv.invert import invert
from plantcv.plantcv.logical_and import logical_and
from plantcv.plantcv.logical_or import logical_or
from plantcv.plantcv.logical_xor import logical_xor
from plantcv.plantcv.find_objects import find_objects
from plantcv.plantcv.roi_objects import roi_objects
from plantcv.plantcv.object_composition import object_composition
from plantcv.plantcv.within_frame import within_frame
from plantcv.plantcv.analyze_object import analyze_object
from plantcv.plantcv.analyze_bound_horizontal import analyze_bound_horizontal
from plantcv.plantcv.analyze_bound_vertical import analyze_bound_vertical
from plantcv.plantcv.analyze_color import analyze_color
from plantcv.plantcv.analyze_nir_intensity import analyze_nir_intensity
from plantcv.plantcv.print_results import print_results
from plantcv.plantcv.flip import flip
from plantcv.plantcv.crop_position_mask import crop_position_mask
from plantcv.plantcv.get_nir import get_nir
from plantcv.plantcv.report_size_marker_area import report_size_marker_area
from plantcv.plantcv.white_balance import white_balance
from plantcv.plantcv.acute_vertex import acute_vertex
from plantcv.plantcv.scale_features import scale_features
from plantcv.plantcv.landmark_reference_pt_dist import landmark_reference_pt_dist
from plantcv.plantcv.x_axis_pseudolandmarks import x_axis_pseudolandmarks
from plantcv.plantcv.y_axis_pseudolandmarks import y_axis_pseudolandmarks
from plantcv.plantcv.cluster_contours import cluster_contours
from plantcv.plantcv.cluster_contour_splitimg import cluster_contour_splitimg
from plantcv.plantcv.rotate import rotate
from plantcv.plantcv.shift_img import shift_img
from plantcv.plantcv.output_mask_ori_img import output_mask
from plantcv.plantcv.auto_crop import auto_crop
from plantcv.plantcv.background_subtraction import background_subtraction
from plantcv.plantcv.naive_bayes_classifier import naive_bayes_classifier
from plantcv.plantcv.acute import acute
from plantcv.plantcv.distance_transform import distance_transform
from plantcv.plantcv.canny_edge_detect import canny_edge_detect
from plantcv.plantcv.opening import opening
from plantcv.plantcv.closing import closing
from plantcv.plantcv import roi
from plantcv.plantcv import threshold
from plantcv.plantcv.cluster_contour_mask import cluster_contour_mask
from plantcv.plantcv.analyze_thermal_values import analyze_thermal_values
from plantcv.plantcv import visualize
from plantcv.plantcv import morphology
from plantcv.plantcv.fill_holes import fill_holes
from plantcv.plantcv.get_kernel import get_kernel
from plantcv.plantcv.crop import crop
from plantcv.plantcv.stdev_filter import stdev_filter
from plantcv.plantcv.spatial_clustering import spatial_clustering
from plantcv.plantcv import photosynthesis

# add new functions to end of lists

__all__ = ['fatal_error', 'print_image', 'plot_image', 'color_palette', 'apply_mask', 'gaussian_blur', 'transform',
           'hyperspectral', 'readimage',
           'readbayer', 'laplace_filter', 'sobel_filter', 'scharr_filter', 'hist_equalization', 'erode',
           'image_add', 'image_subtract', 'dilate', 'watershed', 'rectangle_mask', 'rgb2gray_hsv', 'rgb2gray_lab',
           'rgb2gray', 'median_blur', 'fill', 'invert', 'logical_and', 'logical_or', 'logical_xor',
           'find_objects', 'roi_objects', 'object_composition', 'analyze_object', 'morphology',
           'analyze_bound_horizontal', 'analyze_bound_vertical', 'analyze_color', 'analyze_nir_intensity',
           'print_results', 'flip', 'crop_position_mask', 'get_nir', 'report_size_marker_area',
           'white_balance', 'acute_vertex', 'scale_features', 'landmark_reference_pt_dist', 'outputs',
           'x_axis_pseudolandmarks', 'y_axis_pseudolandmarks', 'cluster_contours', 'visualize',
           'cluster_contour_splitimg', 'rotate', 'shift_img', 'output_mask', 'auto_crop', 'canny_edge_detect',
           'background_subtraction', 'naive_bayes_classifier', 'acute', 'distance_transform', 'params',
           'cluster_contour_mask', 'analyze_thermal_values', 'opening',
           'closing', 'within_frame', 'fill_holes', 'get_kernel', 'Spectral_data', 'crop', 'stdev_filter',
           'spatial_clustering', 'photosynthesis']

from ._version import get_versions
__version__ = get_versions()['version']
del get_versions
<commit_msg>Update __init__.py<commit_after>import os
import matplotlib
from plantcv.plantcv import fatal_error
observations = {}

class Params:
    """PlantCV parameters class."""

    def __init__(self, device=0, debug=None, debug_outdir=".", line_thickness=5, dpi=100, text_size=0.55,
                 text_thickness=2, marker_size=60, color_scale="gist_rainbow", color_sequence="sequential",
                 saved_color_scale=None):
        """Initialize parameters.

        Keyword arguments/parameters:
        device            = Device number. Used to count steps in the pipeline. (default: 0)
        debug             = None, print, or plot. Print = save to file, Plot = print to screen. (default: None)
        debug_outdir      = Debug images output directory. (default: .)
        line_thickness    = Width of line drawings. (default: 5)
        dpi               = Figure plotting resolution, dots per inch. (default: 100)
        text_size         = Size of plotting text. (default: 0.55)
        text_thickness    = Thickness of plotting text. (default: 2)
        marker_size       = Size of plotting markers (default: 60)
        color_scale       = Name of plotting color scale (matplotlib colormap). (default: gist_rainbow)
        color_sequence    = Build color scales in "sequential" or "random" order. (default: sequential)
        saved_color_scale = Saved color scale that will be applied next time color_palette is called. (default: None)

        :param device: int
        :param debug: str
        :param debug_outdir: str
        :param line_thickness: numeric
        :param dpi: int
        :param text_size: float
        :param text_thickness: int
        :param marker_size: int
        :param color_scale: str
        :param color_sequence: str
        :param saved_color_scale: list
        """
        self.device = device
        self.debug = debug
        self.debug_outdir = debug_outdir
        self.line_thickness = line_thickness
        self.dpi = dpi
        self.text_size = text_size
        self.text_thickness = text_thickness
        self.marker_size = marker_size
        self.color_scale = color_scale
        self.color_sequence = color_sequence
        self.saved_color_scale = saved_color_scale

class Outputs:
    """PlantCV outputs class

    """

    def __init__(self):
        self.measurements = {}
        self.images = []
        self.observations = {}

        # Add a method to clear measurements
    def clear(self):
        self.measurements = {}
        self.images = []
        self.observations = {}

    # Method to add observation to outputs
    def add_observation(self, sample, variable, trait, method, scale, datatype, value, label):
        """
        Keyword arguments/parameters:
        sample       = Sample name. Used to distinguish between multiple samples
        variable     = A local unique identifier of a variable, e.g. a short name,
                       that is a key linking the definitions of variables with observations.
        trait        = A name of the trait mapped to an external ontology; if there is no exact mapping, an informative
                       description of the trait.
        method       = A name of the measurement method mapped to an external ontology; if there is no exact mapping, an
                       informative description of the measurement procedure
        scale        = Units of the measurement or scale in which the observations are expressed; if possible, standard
                       units and scales should be used and mapped to existing ontologies; in the case of non-standard
                       scale a full explanation should be given
        datatype     = The type of data to be stored, e.g. 'int', 'float', 'str', 'list', 'bool', etc.
        value        = The data itself
        label        = The label for each value (most useful when the data is a frequency table as in hue,
                       or other tables)

        :param sample: str
        :param variable: str
        :param trait: str
        :param method: str
        :param scale: str
        :param datatype: type
        :param value:
        :param label:
        """
        self.sample = sample
        self.variable = variable
        self.trait = trait
        self.method = method
        self.scale = scale
        self.datatype = datatype
        self.value = value
        self.label = label

        # Create an empty dictionary for the sample if it does not exist
        if sample not in self.observations:
            self.observations[sample] = {}
        supported_dtype = ["str", "list", "bool","tuple", "dict"]
        # Send an error message if datatype is not supported by json
        if not ("int" in str(type(value)) or "float" in str(type(value)) or value is None or str(type(value)) in ["<class '{}'>".format(i) for i in supported_dtype]):
            fatal_error("Data type not compatible with json! Please use only these: int, float, string, list, or bool!")
        # Save the observation for the sample and variable
        self.observations[sample][variable] = {
            "trait": trait,
            "method": method,
            "scale": scale,
            "datatype": str(datatype),
            "value": value,
            "label": label
        }

# Initialize an instance of the Params and Outputs class with default values
# params and outputs are available when plantcv is imported
params = Params()
outputs = Outputs()

class Spectral_data:
    # PlantCV Hyperspectral data class
    def __init__(self, array_data, max_wavelength, min_wavelength, max_value, min_value, d_type, wavelength_dict,
                 samples, lines, interleave, wavelength_units, array_type, pseudo_rgb, filename, default_bands):
        # The actual array/datacube
        self.array_data = array_data
        # Min/max available wavelengths (for spectral datacube)
        self.max_wavelength = max_wavelength
        self.min_wavelength = min_wavelength
        #  Min/max pixel value for single wavelength or index
        self.max_value = max_value
        self.min_value = min_value
        # Numpy data type
        self.d_type = d_type
        # Contains all available wavelengths where keys are wavelength and value are indices
        self.wavelength_dict = wavelength_dict
        # Resolution of a single band of spectral data is (samples, lines) rather than (x,y) with other arrays
        self.samples = samples
        self.lines = lines
        # Interleave type
        self.interleave = interleave
        self.wavelength_units = wavelength_units
        # The type of array data (entire datacube, specific index, first derivative, etc)
        self.array_type = array_type
        # Pseudo-RGB image if the array_type is a datacube
        self.pseudo_rgb = pseudo_rgb
        # The filename where the data originated from
        self.filename = filename
        # The default band indices needed to make an pseudo_rgb image, if not available then store None
        self.default_bands = default_bands

# Example
# spectral_array = Spectral_data(max_wavelength=1000.95, min_wavelength=379.027, d_type=numpy.float32,
#                           wavelength_dict=dictionary, samples=1600, lines=1704, interleave='bil',
#                           wavelength_units='nm', array_type="datacube", filename=fname, default_bands={159,253,520})

from plantcv.plantcv.fatal_error import fatal_error
from plantcv.plantcv.print_image import print_image
from plantcv.plantcv.plot_image import plot_image
from plantcv.plantcv.color_palette import color_palette
from plantcv.plantcv.rgb2gray import rgb2gray
from plantcv.plantcv.rgb2gray_hsv import rgb2gray_hsv
from plantcv.plantcv.rgb2gray_lab import rgb2gray_lab
from plantcv.plantcv.gaussian_blur import gaussian_blur
from plantcv.plantcv import transform
from plantcv.plantcv import hyperspectral
from plantcv.plantcv import spectral_index
from plantcv.plantcv.apply_mask import apply_mask
from plantcv.plantcv.readimage import readimage
from plantcv.plantcv.readbayer import readbayer
from plantcv.plantcv.laplace_filter import laplace_filter
from plantcv.plantcv.sobel_filter import sobel_filter
from plantcv.plantcv.scharr_filter import scharr_filter
from plantcv.plantcv.hist_equalization import hist_equalization
from plantcv.plantcv.image_add import image_add
from plantcv.plantcv.image_subtract import image_subtract
from plantcv.plantcv.erode import erode
from plantcv.plantcv.dilate import dilate
from plantcv.plantcv.watershed import watershed_segmentation
from plantcv.plantcv.rectangle_mask import rectangle_mask
from plantcv.plantcv.median_blur import median_blur
from plantcv.plantcv.fill import fill
from plantcv.plantcv.invert import invert
from plantcv.plantcv.logical_and import logical_and
from plantcv.plantcv.logical_or import logical_or
from plantcv.plantcv.logical_xor import logical_xor
from plantcv.plantcv.find_objects import find_objects
from plantcv.plantcv.roi_objects import roi_objects
from plantcv.plantcv.object_composition import object_composition
from plantcv.plantcv.within_frame import within_frame
from plantcv.plantcv.analyze_object import analyze_object
from plantcv.plantcv.analyze_bound_horizontal import analyze_bound_horizontal
from plantcv.plantcv.analyze_bound_vertical import analyze_bound_vertical
from plantcv.plantcv.analyze_color import analyze_color
from plantcv.plantcv.analyze_nir_intensity import analyze_nir_intensity
from plantcv.plantcv.print_results import print_results
from plantcv.plantcv.flip import flip
from plantcv.plantcv.crop_position_mask import crop_position_mask
from plantcv.plantcv.get_nir import get_nir
from plantcv.plantcv.report_size_marker_area import report_size_marker_area
from plantcv.plantcv.white_balance import white_balance
from plantcv.plantcv.acute_vertex import acute_vertex
from plantcv.plantcv.scale_features import scale_features
from plantcv.plantcv.landmark_reference_pt_dist import landmark_reference_pt_dist
from plantcv.plantcv.x_axis_pseudolandmarks import x_axis_pseudolandmarks
from plantcv.plantcv.y_axis_pseudolandmarks import y_axis_pseudolandmarks
from plantcv.plantcv.cluster_contours import cluster_contours
from plantcv.plantcv.cluster_contour_splitimg import cluster_contour_splitimg
from plantcv.plantcv.rotate import rotate
from plantcv.plantcv.shift_img import shift_img
from plantcv.plantcv.output_mask_ori_img import output_mask
from plantcv.plantcv.auto_crop import auto_crop
from plantcv.plantcv.background_subtraction import background_subtraction
from plantcv.plantcv.naive_bayes_classifier import naive_bayes_classifier
from plantcv.plantcv.acute import acute
from plantcv.plantcv.distance_transform import distance_transform
from plantcv.plantcv.canny_edge_detect import canny_edge_detect
from plantcv.plantcv.opening import opening
from plantcv.plantcv.closing import closing
from plantcv.plantcv import roi
from plantcv.plantcv import threshold
from plantcv.plantcv.cluster_contour_mask import cluster_contour_mask
from plantcv.plantcv.analyze_thermal_values import analyze_thermal_values
from plantcv.plantcv import visualize
from plantcv.plantcv import morphology
from plantcv.plantcv.fill_holes import fill_holes
from plantcv.plantcv.get_kernel import get_kernel
from plantcv.plantcv.crop import crop
from plantcv.plantcv.stdev_filter import stdev_filter
from plantcv.plantcv.spatial_clustering import spatial_clustering
from plantcv.plantcv import photosynthesis

# add new functions to end of lists

__all__ = ['fatal_error', 'print_image', 'plot_image', 'color_palette', 'apply_mask', 'gaussian_blur', 'transform',
           'hyperspectral', 'readimage',
           'readbayer', 'laplace_filter', 'sobel_filter', 'scharr_filter', 'hist_equalization', 'erode',
           'image_add', 'image_subtract', 'dilate', 'watershed', 'rectangle_mask', 'rgb2gray_hsv', 'rgb2gray_lab',
           'rgb2gray', 'median_blur', 'fill', 'invert', 'logical_and', 'logical_or', 'logical_xor',
           'find_objects', 'roi_objects', 'object_composition', 'analyze_object', 'morphology',
           'analyze_bound_horizontal', 'analyze_bound_vertical', 'analyze_color', 'analyze_nir_intensity',
           'print_results', 'flip', 'crop_position_mask', 'get_nir', 'report_size_marker_area',
           'white_balance', 'acute_vertex', 'scale_features', 'landmark_reference_pt_dist', 'outputs',
           'x_axis_pseudolandmarks', 'y_axis_pseudolandmarks', 'cluster_contours', 'visualize',
           'cluster_contour_splitimg', 'rotate', 'shift_img', 'output_mask', 'auto_crop', 'canny_edge_detect',
           'background_subtraction', 'naive_bayes_classifier', 'acute', 'distance_transform', 'params',
           'cluster_contour_mask', 'analyze_thermal_values', 'opening',
           'closing', 'within_frame', 'fill_holes', 'get_kernel', 'Spectral_data', 'crop', 'stdev_filter',
           'spatial_clustering', 'photosynthesis']

from ._version import get_versions
__version__ = get_versions()['version']
del get_versions
<|endoftext|>


<commit_before>import re
import os

__PATH = 'dataprocessing/sql_snippets'
__KEYS = ['description', 'copyright', 'license', 'url', 'author']

SPACER = '  '

expr = r"""\/\*\s*
(?P<description>[\s\S]*)
__copyright__\s*=\s*"(?P<copyright>.*)"\s*
__license__\s*=\s*"(?P<license>.*)"\s*
__url__\s*=\s*"(?P<url>.*)"\s*
__author__\s*=\s*"(?P<author>.*)"\s*
\*\/"""

for subdir, dirs, files in os.walk('.'):
    for fi in files:
        if fi.endswith('.sql'):
            path = os.path.join(subdir, fi)

            # print('Processing %s ...'%path)
            with open(path,'r') as f:
                try:
                    lines='\n'.join(f.readlines())
                except Exception as e:
                    print('Fehler in %s - abort!'%path)
                match = re.match(expr, lines)
                if match:
                    outpath = path = os.path.join('docs', subdir, fi.replace('.sql','.rst'))
                    directory = os.path.dirname(outpath)
                    if not os.path.exists(directory):
                        os.makedirs(directory)

                    result = {key:match.group(key) for key in __KEYS}
                    with open(outpath, 'w') as out:
                        out.write('.. AUTOGENERATED - DO NOT TOUCH!\n\n')
                        for key in __KEYS:

                            if key == 'description':
                                out.write(fi+'\n'+'#'.join(['#'*len(fi)]) )
                                out.write(result[key].replace('\n\n','\n'))
                            else:
                                out.write(key+'\n')
                                out.write(SPACER+result[key].replace('\n','\n'+SPACER)+'\n')
                            out.write('\n')
                else:
                    print('No proper docstring in %s'%path)
<commit_msg>update file<commit_after>import re
import os

__PATH = 'dataprocessing/sql_snippets'
__KEYS = ['description', 'copyright', 'license', 'url', 'author']

SPACER = '  '

expr = r"""\/\*\s*
(?P<description>[\s\S]*)
__copyright__\s*=\s*"(?P<copyright>.*)"\s*
__license__\s*=\s*"(?P<license>.*)"\s*
__url__\s*=\s*"(?P<url>.*)"\s*
__author__\s*=\s*"(?P<author>.*)"\s*
\*\/"""

for subdir, dirs, files in os.walk('.'):
    for fi in files:
        if fi.endswith('.sql'):
            path = os.path.join(subdir, fi)

            # print('Processing %s ...'%path)
            with open(path,'r') as f:
                try:
                    lines='\n'.join(f.readlines())
                except Exception as e:
                    print('Fehler in %s - abort!'%path)
                match = re.match(expr, lines)
                if match:
                    outpath = path = os.path.join('docs', subdir, fi.replace('.sql','.rst'))
                    directory = os.path.dirname(outpath)
                    if not os.path.exists(directory):
                        os.makedirs(directory)

                    result = {key:match.group(key) for key in __KEYS}
                    with open(outpath, 'w') as out:
                        out.write('.. AUTOGENERATED - DO NOT TOUCH!\n\n')
                        for key in __KEYS:

                            if key == 'description':
                                out.write(fi+'\n'+'#'.join(['#'*len(fi)])+'\n\n' )
                                out.write(result[key].replace('\n\n','\n'))
                            else:
                                out.write(key+'\n')
                                out.write(SPACER+result[key].replace('\n','\n'+SPACER)+'\n')
                            out.write('\n')
                else:
                    print('No proper docstring in %s'%path)
<|endoftext|>


<commit_before>from argparse import ArgumentParser

from flows.builtin.webserver.post import run_webserver
from jobrunner.cli.parse import parse_arguments
from jobrunner.jobs import register_job

def parse_webserver_arguments(args=None):
    """
    Parse the commandline options for posting a job that
    runs a simple HTTP webserver
    :param list args: Args to pass to the arg parser.
    Will use argv if none specified.
    :return obj args: parsed arguments
    """
    parser = ArgumentParser(
        prog="jobrunner post simple_http_webserver",
        description='Post a job that runs a simple HTTP webserver'
    )
    parser.add_argument(
        '--port', '-p', type=int, default=8080,
        help="The port to use to run the webserver on. Defaults to 8080"
    )
    return parse_arguments(parser, args=args)

@register_job()
def simple_http_webserver(args=None):
    """
    Show information about running jobs
    :param list args: Args to pass to the arg parser.
    Will use argv if none specified.
    :return None:
    """
    args = parse_webserver_arguments(args=args)
    run_webserver(port=args.port)
<commit_msg>fix example http webserver docstring<commit_after>from argparse import ArgumentParser

from flows.builtin.webserver.post import run_webserver
from jobrunner.cli.parse import parse_arguments
from jobrunner.jobs import register_job

def parse_webserver_arguments(args=None):
    """
    Parse the commandline options for posting a job that
    runs a simple HTTP webserver
    :param list args: Args to pass to the arg parser.
    Will use argv if none specified.
    :return obj args: parsed arguments
    """
    parser = ArgumentParser(
        prog="jobrunner post simple_http_webserver",
        description='Post a job that runs a simple HTTP webserver'
    )
    parser.add_argument(
        '--port', '-p', type=int, default=8080,
        help="The port to use to run the webserver on. Defaults to 8080"
    )
    return parse_arguments(parser, args=args)

@register_job()
def simple_http_webserver(args=None):
    """
    Post a job that runs the simple http webserver
    :param list args: Args to pass to the arg parser.
    Will use argv if none specified.
    :return None:
    """
    args = parse_webserver_arguments(args=args)
    run_webserver(port=args.port)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os.path

import config
import logging

from datetime import datetime
from datetime import timedelta

from flask import abort
from flask import flash
from flask import redirect
from flask import request
from flask import url_for

import logic.alias
import logic.department
import logic.employee
import logic.event
import logic.love
import logic.love_link
import logic.love_count
import logic.subscription
from errors import NoSuchEmployee
from errors import TaintedLove
from google.appengine.api import taskqueue
from logic import TIMESPAN_LAST_WEEK
from logic import TIMESPAN_THIS_WEEK
from logic import to_the_future
from logic import utc_week_limits
from logic.love_link import create_love_link
from main import app
from models import AccessKey
from models import Alias
from models import Employee
from models import Subscription
from util.decorators import admin_required
from util.decorators import csrf_protect
from util.decorators import user_required
from util.recipient import sanitize_recipients
from util.render import render_template
from views import common

@app.route('/', methods=['GET'])
@user_required
def home():
    recipients = request.args.get('recipients', request.args.get('recipient'))
    message = request.args.get('message')

    return render_template(
        'home.html',
        current_time=datetime.utcnow(),
        current_user=Employee.get_current_employee(),
        recipients=recipients,
        message=message,
    )

@app.route('/me', methods=['GET'])
@user_required
def me():
    current_employee = Employee.get_current_employee()

    sent_love = logic.love.recent_sent_love(current_employee.key, limit=20)
    received_love = logic.love.recent_received_love(current_employee.key, limit=20)

    return render_template(
        'me.html',
        current_time=datetime.utcnow(),
        current_user=current_employee,
        sent_loves=sent_love.get_result(),
        received_loves=received_love.get_result()
    )

@app.route('/<regex("[a-zA-Z]{3,30}"):user>', methods=['GET'])
@user_required
def me_or_explore(user):
    current_employee = Employee.get_current_employee()
    username = logic.alias.name_for_alias(user.lower().strip())

    try:
        user_key = Employee.get_key_for_username(username)
    except NoSuchEmployee:
        abort(404)

    if current_employee.key == user_key:
        return redirect(url_for('me'))
    else:
        return redirect(url_for('explore', user=username))

@app.route('/l/<string:hash_key>', methods=['GET'])
@user_required
def love_link(hash_key):
    loveLink = logic.love_link.get_love_link(hash_key)
    logging.info(loveLink)
    return redirect(url_for('home') + '?recipient={}&message={}'.format(loveLink.recipient_list, loveLink.message))

@app.route('/explore', methods=['GET'])
@user_required
def explore():
    username = request.args.get('user', None)
    if not username:
        if username is not None:
            flash('Enter a name, lover.', 'error')
        return render_template(
            'explore.html',
            current_time=datetime.utcnow(),
            user=None
        )
    username = username.lower().strip()

    user_key = Employee.query(
        Employee.username == username,
        Employee.terminated == False,  # noqa
    ).get(keys_only=True)

    if not user_key:
        flash('Sorry, "{}" is not a valid user.'.format(username), 'error')
        return redirect(url_for('explore'))

    sent_love = logic.love.recent_sent_love(user_key, include_secret=False, limit=20)
    received_love = logic.love.recent_received_love(user_key, include_secret=False, limit=20)

    return render_template(
        'explore.html',
        current_time=datetime.utcnow(),
        sent_loves=sent_love.get_result(),
        received_loves=received_love.get_result(),
        user=user_key.get()
    )

@app.route('/leaderboard', methods=['GET'])
@user_required
def leaderboard():
    timespan = request.args.get('timespan', TIMESPAN_THIS_WEEK)
    department = request.args.get('department', None)

    # If last week, we need to subtract *before* getting the week limits to
    # avoid being off by one hour on weeks that include a DST transition
    utc_now = datetime.utcnow()
    if timespan == TIMESPAN_LAST_WEEK:
        utc_now -= timedelta(days=7)
    utc_week_start, _ = utc_week_limits(utc_now)

    top_lovers, top_lovees = logic.love_count.top_lovers_and_lovees(utc_week_start, dept=department)

    top_lover_dicts = [
        {
            'employee': employee_key.get_async(),
            'num_sent': sent_count
        }
        for employee_key, sent_count
        in top_lovers
    ]

    top_loved_dicts = [
        {
            'employee': employee_key.get_async(),
            'num_received': received_count
        }
        for employee_key, received_count
        in top_lovees
    ]

    # get results for the futures set up previously
    map(to_the_future, top_lover_dicts)
    map(to_the_future, top_loved_dicts)

    return render_template(
        'leaderboard.html',
        top_loved=top_loved_dicts,
        top_lovers=top_lover_dicts,
        departments=logic.department.META_DEPARTMENTS,
        sub_departments=logic.department.META_DEPARTMENT_MAP,
        selected_dept=department,
        selected_timespan=timespan,
        org_title=config.ORG_TITLE,
    )

@app.route('/sent', methods=['GET'])
@user_required
def sent():
    link_id = request.args.get('link_id', None)

    if not link_id:
        return redirect(url_for('home'))

    return render_template(
        'sent.html',
        current_time=datetime.utcnow(),
        current_user=Employee.get_current_employee(),
        link=url_for('l/' + link_id),
    )

@app.route('/keys', methods=['GET'])
@admin_required
def keys():
    api_keys = AccessKey.query().fetch()
    return render_template(
        'keys.html',
        keys=api_keys
    )

@app.route('/keys/create', methods=['POST'])
@csrf_protect
@admin_required
def create_key():
    description = request.form.get('description')
    new_key = AccessKey.create(description)

    flash('Your API key {} has been created. Refresh the page to see it below.'.format(new_key.access_key))
    return redirect(url_for('keys'))

@app.route('/love', methods=['POST'])
@csrf_protect
@user_required
def love():
    recipients = sanitize_recipients(request.form.get('recipients'))
    message = request.form.get('message').strip()
    secret = (request.form.get('secret') == 'true')

    if not recipients:
        flash('Enter a name, lover.', 'error')
        return redirect(url_for('home'))

    recipients_display_str = ', '.join(recipients)

    if not message:
        flash('Enter a message, lover.', 'error')
        return redirect(url_for('home', recipients=recipients_display_str))

    try:
        real_recipients, hash = logic.love.send_loves(recipients, message, secret=secret)
        # actual recipients may have the sender stripped from the list
        real_display_str = ', '.join(real_recipients)

        if secret:
            flash('Secret love sent to {}!'.format(real_display_str))
            return redirect(url_for('home'))
        else:
            logging.info(real_recipients)
            hash_key = create_love_link(','.join(map(str, real_recipients)), message)
            return redirect(url_for('sent', recipients=recipients_display_str, message=message, link_id=hash_key))

    except TaintedLove as exc:
        if exc.is_error:
            flash(exc.user_message, 'error')
        else:
            flash(exc.user_message)

@app.route('/user/autocomplete', methods=['GET'])
@user_required
def autocomplete_web():
    return common.autocomplete(request)

@app.route('/subscriptions', methods=['GET'])
@admin_required
def subscriptions():
    return render_template(
        'subscriptions.html',
        subscriptions=Subscription.query().fetch(),
        events=logic.event.EVENTS,
    )

@app.route('/subscriptions/create', methods=['POST'])
@csrf_protect
@admin_required
def create_subscription():
    subscription_dict = dict(
        request_url=request.form.get('request_url').strip(),
        event=request.form.get('event').strip(),
        active=(request.form.get('active') == 'true'),
        secret=request.form.get('secret').strip(),
    )

    try:
        Subscription.create_from_dict(subscription_dict)
        flash('Subscription created. Refresh the page to see it.')
    except ValueError:
        flash('Something went wrong. Please check your input.', 'error')

    return redirect(url_for('subscriptions'))

@app.route('/subscriptions/<int:subscription_id>/delete', methods=['POST'])
@csrf_protect
@admin_required
def delete_subscription(subscription_id):
    logic.subscription.delete_subscription(subscription_id)
    flash('Subscription deleted. Refresh the page to see it\'s gone.', 'info')
    return redirect(url_for('subscriptions'))

@app.route('/aliases', methods=['GET'])
@admin_required
def aliases():
    return render_template(
        'aliases.html',
        aliases=Alias.query().fetch(),
    )

@app.route('/aliases', methods=['POST'])
@csrf_protect
@admin_required
def create_alias():
    try:
        logic.alias.save_alias(
            request.form.get('alias').strip(),
            request.form.get('username').strip(),
        )
        flash('New alias successfully saved. Refresh the page to see it.')
    except Exception as e:
        flash('Something went wrong: {}.'.format(e.message), 'error')

    return redirect(url_for('aliases'))

@app.route('/aliases/<int:alias_id>/delete', methods=['POST'])
@csrf_protect
@admin_required
def delete_alias(alias_id):
    logic.alias.delete_alias(alias_id)
    flash('Alias successfully deleted. Refresh the page to see it\'s gone.', 'info')
    return redirect(url_for('aliases'))

@app.route('/employees', methods=['GET'])
@admin_required
def employees():
    return render_template(
        'employees.html',
        pagination_result=Employee.paginate(
            order_by=Employee.username,
            prev_cursor_str=request.args.get('prev_cursor'),
            next_cursor_str=request.args.get('next_cursor'),
        ),
    )

@app.route('/employees/import', methods=['GET'])
@admin_required
def import_employees_form():
    import_file_exists = os.path.isfile(logic.employee.csv_import_file())
    return render_template(
        'import.html',
        import_file_exists=import_file_exists,
    )

@app.route('/employees/import', methods=['POST'])
@admin_required
def import_employees():
    flash('We started importing employee data in the background. Refresh the page to see it.', 'info')
    taskqueue.add(url='/tasks/employees/load/csv')
    return redirect(url_for('employees'))
<commit_msg>small refactorings in web.py<commit_after># -*- coding: utf-8 -*-
import os.path

import config
import logging

from datetime import datetime
from datetime import timedelta

from flask import abort
from flask import flash
from flask import redirect
from flask import request
from flask import url_for

import logic.alias
import logic.department
import logic.employee
import logic.event
import logic.love
import logic.love_link
import logic.love_count
import logic.subscription
from errors import NoSuchEmployee
from errors import TaintedLove
from google.appengine.api import taskqueue
from logic import TIMESPAN_LAST_WEEK
from logic import TIMESPAN_THIS_WEEK
from logic import to_the_future
from logic import utc_week_limits
from logic.love_link import create_love_link
from main import app
from models import AccessKey
from models import Alias
from models import Employee
from models import Subscription
from util.decorators import admin_required
from util.decorators import csrf_protect
from util.decorators import user_required
from util.recipient import sanitize_recipients
from util.render import render_template
from views import common

@app.route('/', methods=['GET'])
@user_required
def home():
    recipients = request.args.get('recipients', request.args.get('recipient'))
    message = request.args.get('message')

    return render_template(
        'home.html',
        current_time=datetime.utcnow(),
        current_user=Employee.get_current_employee(),
        recipients=recipients,
        message=message,
    )

@app.route('/me', methods=['GET'])
@user_required
def me():
    current_employee = Employee.get_current_employee()

    sent_love = logic.love.recent_sent_love(current_employee.key, limit=20)
    received_love = logic.love.recent_received_love(current_employee.key, limit=20)

    return render_template(
        'me.html',
        current_time=datetime.utcnow(),
        current_user=current_employee,
        sent_loves=sent_love.get_result(),
        received_loves=received_love.get_result()
    )

@app.route('/<regex("[a-zA-Z]{3,30}"):user>', methods=['GET'])
@user_required
def me_or_explore(user):
    current_employee = Employee.get_current_employee()
    username = logic.alias.name_for_alias(user.lower().strip())

    try:
        user_key = Employee.get_key_for_username(username)
    except NoSuchEmployee:
        abort(404)

    if current_employee.key == user_key:
        return redirect(url_for('me'))
    else:
        return redirect(url_for('explore', user=username))

@app.route('/l/<string:hash_key>', methods=['GET'])
@user_required
def love_link(hash_key):
    loveLink = logic.love_link.get_love_link(hash_key)
    logging.info(loveLink)
    return redirect(url_for('home') + '?recipient={}&message={}'.format(loveLink.recipient_list, loveLink.message))

@app.route('/explore', methods=['GET'])
@user_required
def explore():
    username = request.args.get('user', None)
    if not username:
        if username is not None:
            flash('Enter a name, lover.', 'error')
        return render_template(
            'explore.html',
            current_time=datetime.utcnow(),
            user=None
        )
    username = username.lower().strip()

    user_key = Employee.query(
        Employee.username == username,
        Employee.terminated == False,  # noqa
    ).get(keys_only=True)

    if not user_key:
        flash('Sorry, "{}" is not a valid user.'.format(username), 'error')
        return redirect(url_for('explore'))

    sent_love = logic.love.recent_sent_love(user_key, include_secret=False, limit=20)
    received_love = logic.love.recent_received_love(user_key, include_secret=False, limit=20)

    return render_template(
        'explore.html',
        current_time=datetime.utcnow(),
        sent_loves=sent_love.get_result(),
        received_loves=received_love.get_result(),
        user=user_key.get()
    )

@app.route('/leaderboard', methods=['GET'])
@user_required
def leaderboard():
    timespan = request.args.get('timespan', TIMESPAN_THIS_WEEK)
    department = request.args.get('department', None)

    # If last week, we need to subtract *before* getting the week limits to
    # avoid being off by one hour on weeks that include a DST transition
    utc_now = datetime.utcnow()
    if timespan == TIMESPAN_LAST_WEEK:
        utc_now -= timedelta(days=7)
    utc_week_start, _ = utc_week_limits(utc_now)

    top_lovers, top_lovees = logic.love_count.top_lovers_and_lovees(utc_week_start, dept=department)

    top_lover_dicts = [
        {
            'employee': employee_key.get_async(),
            'num_sent': sent_count
        }
        for employee_key, sent_count
        in top_lovers
    ]

    top_loved_dicts = [
        {
            'employee': employee_key.get_async(),
            'num_received': received_count
        }
        for employee_key, received_count
        in top_lovees
    ]

    # get results for the futures set up previously
    map(to_the_future, top_lover_dicts)
    map(to_the_future, top_loved_dicts)

    return render_template(
        'leaderboard.html',
        top_loved=top_loved_dicts,
        top_lovers=top_lover_dicts,
        departments=logic.department.META_DEPARTMENTS,
        sub_departments=logic.department.META_DEPARTMENT_MAP,
        selected_dept=department,
        selected_timespan=timespan,
        org_title=config.ORG_TITLE,
    )

@app.route('/sent', methods=['GET'])
@user_required
def sent():
    link_id = request.args.get('link_id', None)

    if not link_id:
        return redirect(url_for('home'))

    return render_template(
        'sent.html',
        current_time=datetime.utcnow(),
        current_user=Employee.get_current_employee(),
        link=url_for('l/' + link_id),
    )

@app.route('/keys', methods=['GET'])
@admin_required
def keys():
    api_keys = AccessKey.query().fetch()
    return render_template(
        'keys.html',
        keys=api_keys
    )

@app.route('/keys/create', methods=['POST'])
@csrf_protect
@admin_required
def create_key():
    description = request.form.get('description')
    new_key = AccessKey.create(description)

    flash('Your API key {} has been created. Refresh the page to see it below.'.format(new_key.access_key))
    return redirect(url_for('keys'))

@app.route('/love', methods=['POST'])
@csrf_protect
@user_required
def love():
    recipients = sanitize_recipients(request.form.get('recipients'))
    message = request.form.get('message').strip()
    secret = (request.form.get('secret') == 'true')

    if not recipients:
        flash('Enter a name, lover.', 'error')
        return redirect(url_for('home'))

    recipients_display_str = ', '.join(recipients)

    if not message:
        flash('Enter a message, lover.', 'error')
        return redirect(url_for('home', recipients=recipients_display_str))

    try:
        real_recipients = logic.love.send_loves(recipients, message, secret=secret)
        # actual recipients may have the sender stripped from the list
        real_display_str = ', '.join(real_recipients)
        logging.info(real_recipients)

        if secret:
            flash('Secret love sent to {}!'.format(real_display_str))
            return redirect(url_for('home'))
        else:
            hash_key = create_love_link(real_display_str, message)
            return redirect(url_for('sent', link_id=hash_key))

    except TaintedLove as exc:
        if exc.is_error:
            flash(exc.user_message, 'error')
        else:
            flash(exc.user_message)

@app.route('/user/autocomplete', methods=['GET'])
@user_required
def autocomplete_web():
    return common.autocomplete(request)

@app.route('/subscriptions', methods=['GET'])
@admin_required
def subscriptions():
    return render_template(
        'subscriptions.html',
        subscriptions=Subscription.query().fetch(),
        events=logic.event.EVENTS,
    )

@app.route('/subscriptions/create', methods=['POST'])
@csrf_protect
@admin_required
def create_subscription():
    subscription_dict = dict(
        request_url=request.form.get('request_url').strip(),
        event=request.form.get('event').strip(),
        active=(request.form.get('active') == 'true'),
        secret=request.form.get('secret').strip(),
    )

    try:
        Subscription.create_from_dict(subscription_dict)
        flash('Subscription created. Refresh the page to see it.')
    except ValueError:
        flash('Something went wrong. Please check your input.', 'error')

    return redirect(url_for('subscriptions'))

@app.route('/subscriptions/<int:subscription_id>/delete', methods=['POST'])
@csrf_protect
@admin_required
def delete_subscription(subscription_id):
    logic.subscription.delete_subscription(subscription_id)
    flash('Subscription deleted. Refresh the page to see it\'s gone.', 'info')
    return redirect(url_for('subscriptions'))

@app.route('/aliases', methods=['GET'])
@admin_required
def aliases():
    return render_template(
        'aliases.html',
        aliases=Alias.query().fetch(),
    )

@app.route('/aliases', methods=['POST'])
@csrf_protect
@admin_required
def create_alias():
    try:
        logic.alias.save_alias(
            request.form.get('alias').strip(),
            request.form.get('username').strip(),
        )
        flash('New alias successfully saved. Refresh the page to see it.')
    except Exception as e:
        flash('Something went wrong: {}.'.format(e.message), 'error')

    return redirect(url_for('aliases'))

@app.route('/aliases/<int:alias_id>/delete', methods=['POST'])
@csrf_protect
@admin_required
def delete_alias(alias_id):
    logic.alias.delete_alias(alias_id)
    flash('Alias successfully deleted. Refresh the page to see it\'s gone.', 'info')
    return redirect(url_for('aliases'))

@app.route('/employees', methods=['GET'])
@admin_required
def employees():
    return render_template(
        'employees.html',
        pagination_result=Employee.paginate(
            order_by=Employee.username,
            prev_cursor_str=request.args.get('prev_cursor'),
            next_cursor_str=request.args.get('next_cursor'),
        ),
    )

@app.route('/employees/import', methods=['GET'])
@admin_required
def import_employees_form():
    import_file_exists = os.path.isfile(logic.employee.csv_import_file())
    return render_template(
        'import.html',
        import_file_exists=import_file_exists,
    )

@app.route('/employees/import', methods=['POST'])
@admin_required
def import_employees():
    flash('We started importing employee data in the background. Refresh the page to see it.', 'info')
    taskqueue.add(url='/tasks/employees/load/csv')
    return redirect(url_for('employees'))
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Signing script
"""
import aiohttp
import asyncio
import logging
import os
import ssl
import sys
import traceback
from urllib.parse import urlparse

import scriptworker.client
from scriptworker.context import Context
from scriptworker.exceptions import ScriptWorkerTaskException
from signingscript.task import validate_task_schema
from signingscript.worker import get_token, read_temp_creds, sign

log = logging.getLogger(__name__)

class SigningContext(Context):
    signing_servers = None

    def __init__(self):
        super(SigningContext, self).__init__()

    def write_json(self, *args):
        pass

async def async_main(context):
    loop = asyncio.get_event_loop()
    work_dir = context.config['work_dir']
    context.task = scriptworker.client.get_task(context.config)
    loop.create_task(read_temp_creds(context))
    validate_task_schema(context)
    # _ scriptworker needs to validate CoT artifact
    # await sign(context)
    # X download artifacts
    # _ _ any checks here?
    # X get token
    await get_token(context, os.path.join(work_dir, 'token'), 'nightly', 'gpg')
    # X sign bits
    # X copy bits to artifact dir
    # X periodically update temp creds from disk

# main {{{1
def main(name=None):
    if name in (None, '__main__'):
        # TODO config
        context = SigningContext()
        if os.environ.get('DOCKER_HOST'):
            # The .1 on the same subnet as the DOCKER_HOST ip
            parsed = urlparse(os.environ['DOCKER_HOST'])
            parts = parsed.hostname.split('.')
            parts[3] = "1"
            my_ip = '.'.join(parts)
        else:
            my_ip = "127.0.0.1"
        context.config = {
            # TODO genericize
            'signing_server_config': 'signing_server_config.json',
            'tools_dir': '/src/signing/tools',
            'work_dir': '/src/signing/work_dir',
            'artifact_dir': '/src/signing/artifact_dir',
            'temp_creds_refresh_seconds': 330,
            'my_ip': my_ip,
            'schema_file': os.path.join(os.getcwd(), 'signingscript', 'data', 'signing_task_schema.json'),
            'verbose': True,
        }
        if context.config.get('verbose'):
            log_level = logging.DEBUG
        else:
            log_level = logging.INFO
        logging.basicConfig(
            format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
            level=log_level
        )
        logging.getLogger("taskcluster").setLevel(logging.WARNING)
        loop = asyncio.get_event_loop()
        kwargs = {}
        # XXX can we get the signing servers' CA cert on the scriptworkers?
        if context.config.get('ssl_cert'):
            sslcontext = ssl.create_default_context(cafile='/path/to/ca-bundle.crt')
            kwargs['ssl_context'] = sslcontext
        else:
            kwargs['verify_ssl'] = False
        conn = aiohttp.TCPConnector(**kwargs)
        with aiohttp.ClientSession(connector=conn) as session:
            context.session = session
            try:
                loop.run_until_complete(async_main(context))
            except ScriptWorkerTaskException as exc:
                traceback.print_exc()
                sys.exit(exc.exit_code)

main(name=__name__)
<commit_msg>stop getting pending task errors at exit<commit_after>#!/usr/bin/env python
"""Signing script
"""
import aiohttp
import asyncio
import logging
import os
import ssl
import sys
import traceback
from urllib.parse import urlparse

import scriptworker.client
from scriptworker.context import Context
from scriptworker.exceptions import ScriptWorkerTaskException
from signingscript.task import validate_task_schema
from signingscript.worker import get_token, read_temp_creds, sign

log = logging.getLogger(__name__)

class SigningContext(Context):
    signing_servers = None

    def __init__(self):
        super(SigningContext, self).__init__()

    def write_json(self, *args):
        pass

async def async_main(context):
    loop = asyncio.get_event_loop()
    work_dir = context.config['work_dir']
    context.task = scriptworker.client.get_task(context.config)
    temp_creds_future = loop.create_task(read_temp_creds(context))
    validate_task_schema(context)
    # _ scriptworker needs to validate CoT artifact
    await get_token(context, os.path.join(work_dir, 'token'), 'nightly', 'gpg')
    # X download artifacts
    # _ _ any checks here?
    # X sign bits
    # await sign(context)
    # X copy bits to artifact dir
    temp_creds_future.cancel()

# main {{{1
def main(name=None):
    if name not in (None, '__main__'):
        return
    # TODO config
    context = SigningContext()
    if os.environ.get('DOCKER_HOST'):
        # The .1 on the same subnet as the DOCKER_HOST ip
        parsed = urlparse(os.environ['DOCKER_HOST'])
        parts = parsed.hostname.split('.')
        parts[3] = "1"
        my_ip = '.'.join(parts)
    else:
        my_ip = "127.0.0.1"
    context.config = {
        # TODO genericize
        'signing_server_config': 'signing_server_config.json',
        'tools_dir': '/src/signing/tools',
        'work_dir': '/src/signing/work_dir',
        'artifact_dir': '/src/signing/artifact_dir',
        'temp_creds_refresh_seconds': 330,
        'my_ip': my_ip,
        'schema_file': os.path.join(os.getcwd(), 'signingscript', 'data', 'signing_task_schema.json'),
        'verbose': True,
    }
    if context.config.get('verbose'):
        log_level = logging.DEBUG
    else:
        log_level = logging.INFO
    logging.basicConfig(
        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
        level=log_level
    )
    logging.getLogger("taskcluster").setLevel(logging.WARNING)
    loop = asyncio.get_event_loop()
    kwargs = {}
    # XXX can we get the signing servers' CA cert on the scriptworkers?
    # XXX if they're real certs, we can skip this
    if context.config.get('ssl_cert'):
        sslcontext = ssl.create_default_context(cafile=context.config['ssl_cert'])
        kwargs['ssl_context'] = sslcontext
    else:
        kwargs['verify_ssl'] = False
    conn = aiohttp.TCPConnector(**kwargs)
    with aiohttp.ClientSession(connector=conn) as session:
        context.session = session
        try:
            loop.run_until_complete(async_main(context))
        except ScriptWorkerTaskException as exc:
            traceback.print_exc()
            sys.exit(exc.exit_code)

main(name=__name__)
<|endoftext|>


<commit_before>import click
import os
import sys
from tabulate import tabulate
import tempfile
from shutil import rmtree
from clint.textui.progress import dots, STREAM as clint_STREAM

from floyd.exceptions import WaitTimeoutException
from floyd.client.data import DataClient
from floyd.client.resource import ResourceClient
from floyd.client.files import create_tarfile, sizeof_fmt
from floyd.client.tus_data import TusDataClient
from floyd.log import logger as floyd_logger
from floyd.manager.data_config import DataConfigManager
from floyd.model.data import DataRequest

class ResourceWaitIter(object):
    def __init__(self, resource_id):
        self.resource_id = resource_id

    def __iter__(self):
        for c in ResourceClient().wait_for_ready(self.resource_id):
            yield c

    def __len__(self):
        return ResourceClient.MAX_WAIT_RETRY

def opt_to_resume(resume_flag):
    if resume_flag:
        return True

    msg = "An unfinished upload exists. Would you like to resume it?"
    return click.confirm(msg, abort=False, default=False)

def upload_is_resumable(data_config):
    # TODO: Check to make sure server says the upload is resumable
    return ((data_config.resource_id or "")
            or (os.path.isfile(data_config.tarball_path or "")
                and data_config.data_endpoint))

def initialize_new_upload(data_config, access_token):
    data_config.increment_version()
    data_config.set_tarball_path(None)
    data_config.set_data_endpoint(None)
    data_config.set_resource_id(None)

    version = data_config.version
    data_name = "{}/{}:{}".format(access_token.username,
                                  data_config.name,
                                  version)

    # Create tarball of the data using the ID returned from the API
    temp_dir = tempfile.mkdtemp()
    tarball_path = os.path.join(temp_dir, "floydhub_data.tar.gz")

    floyd_logger.debug("Creating tarfile with contents of current directory: %s",
                       tarball_path)
    floyd_logger.info("Compressing data...")
    # TODO: purge tarball on Ctrl-C
    create_tarfile(source_dir='.', filename=tarball_path)

    # If starting a new upload fails for some reason down the line, we don't
    # want to re-tar, so save off the tarball path now
    data_config.set_tarball_path(tarball_path)
    DataConfigManager.set_config(data_config)

    # Create data object using API
    data = DataRequest(name=data_name,
                       description=version,
                       data_type='gzip',
                       version=version)
    data_id = DataClient().create(data)
    if not data_id:
        rmtree(temp_dir)
        sys.exit(1)

    data_config.set_data_predecessor(data_id)
    DataConfigManager.set_config(data_config)

    # fetch auth token for upload server
    creds = DataClient().new_tus_credentials(data_id)
    if not creds:
        # TODO: delete module from server?
        rmtree(temp_dir)
        sys.exit(1)

    data_resource_id = creds[0]
    data_endpoint = TusDataClient().initialize_upload(
        tarball_path,
        metadata={"filename": data_resource_id},
        auth=creds)
    if not data_endpoint:
        # TODO: delete module from server?
        floyd_logger.error("Failed to get upload URL from Floydhub!")
        rmtree(temp_dir)
        sys.exit(1)

    data_config.set_data_endpoint(data_endpoint)
    DataConfigManager.set_config(data_config)

def complete_upload(data_config):
    data_endpoint = data_config.data_endpoint
    data_id = data_config.data_predecessor
    tarball_path = data_config.tarball_path

    if not data_id:
        floyd_logger.error("Corrupted upload state, please start a new one.")
        sys.exit(1)

    # check for tarball upload
    if not data_config.resource_id and (tarball_path and data_endpoint):
        floyd_logger.debug("Getting fresh upload credentials")
        creds = DataClient().new_tus_credentials(data_id)
        if not creds:
            sys.exit(1)

        file_size = os.path.getsize(tarball_path)
        floyd_logger.info("Uploading compressed data. Total upload size: %s",
                        sizeof_fmt(file_size))
        tus_client = TusDataClient()
        if not tus_client.resume_upload(tarball_path, data_endpoint, auth=creds):
            floyd_logger.error("Failed to finish upload!")
            return

        try:
            floyd_logger.info("Removing compressed data...")
            rmtree(os.path.dirname(tarball_path))
        except (OSError, TypeError):
            pass

        data_id = data_config.data_predecessor
        floyd_logger.debug("Created data with id : %s", data_id)
        floyd_logger.info("Upload finished.")

        # Update data config
        data_config.set_data_predecessor(data_id)
        data_config.set_tarball_path(None)
        data_config.set_data_endpoint(None)
        data_source = DataClient().get(data_id)
        data_config.set_resource_id(data_source.resource_id)
        DataConfigManager.set_config(data_config)

    # data tarball uploaded, check for server untar
    if data_config.resource_id:
        try:
            for i in dots(ResourceWaitIter(data_config.resource_id),
                        label='Waiting for server to unpack uploaded data...'):
                pass
        except WaitTimeoutException:
            clint_STREAM.write('\n')
            clint_STREAM.flush()
            floyd_logger.info(
                "Looks like it is going to take some extra for Floydhub to unpack "
                "your data. Please check back later with the following command:"
                "\n\tfloyd data upload -r")
            sys.exit(1)
        else:
            data_config.set_resource_id(None)
            DataConfigManager.set_config(data_config)

    # Print output
    table_output = [["DATA ID", "NAME", "VERSION"],
                    [data_id, data_config.name, data_config.version]]
    floyd_logger.info(tabulate(table_output, headers="firstrow"))

def abort_previous_upload(data_config):
    if data_config.tarball_path and os.path.exists(data_config.tarball_path):
        rmtree(os.path.dirname(data_config.tarball_path))

    data_config.set_tarball_path("")
    data_config.set_data_endpoint("")
    DataConfigManager.set_config(data_config)
<commit_msg>clean up config at the end of upload<commit_after>import click
import os
import sys
from tabulate import tabulate
import tempfile
from shutil import rmtree
from clint.textui.progress import dots, STREAM as clint_STREAM

from floyd.exceptions import WaitTimeoutException
from floyd.client.data import DataClient
from floyd.client.resource import ResourceClient
from floyd.client.files import create_tarfile, sizeof_fmt
from floyd.client.tus_data import TusDataClient
from floyd.log import logger as floyd_logger
from floyd.manager.data_config import DataConfigManager
from floyd.model.data import DataRequest

class ResourceWaitIter(object):
    def __init__(self, resource_id):
        self.resource_id = resource_id

    def __iter__(self):
        for c in ResourceClient().wait_for_ready(self.resource_id):
            yield c

    def __len__(self):
        return ResourceClient.MAX_WAIT_RETRY

def opt_to_resume(resume_flag):
    if resume_flag:
        return True

    msg = "An unfinished upload exists. Would you like to resume it?"
    return click.confirm(msg, abort=False, default=False)

def upload_is_resumable(data_config):
    # TODO: Check to make sure server says the upload is resumable
    return ((data_config.resource_id or "")
            or (os.path.isfile(data_config.tarball_path or "")
                and data_config.data_endpoint))

def initialize_new_upload(data_config, access_token):
    # TODO: hit upload server to check for liveness before moving on
    data_config.increment_version()
    data_config.set_tarball_path(None)
    data_config.set_data_endpoint(None)
    data_config.set_resource_id(None)

    version = data_config.version
    data_name = "{}/{}:{}".format(access_token.username,
                                  data_config.name,
                                  version)

    # Create tarball of the data using the ID returned from the API
    temp_dir = tempfile.mkdtemp()
    tarball_path = os.path.join(temp_dir, "floydhub_data.tar.gz")

    floyd_logger.debug("Creating tarfile with contents of current directory: %s",
                       tarball_path)
    floyd_logger.info("Compressing data...")
    # TODO: purge tarball on Ctrl-C
    create_tarfile(source_dir='.', filename=tarball_path)

    # If starting a new upload fails for some reason down the line, we don't
    # want to re-tar, so save off the tarball path now
    data_config.set_tarball_path(tarball_path)
    DataConfigManager.set_config(data_config)

    # Create data object using API
    data = DataRequest(name=data_name,
                       description=version,
                       data_type='gzip',
                       version=version)
    data_id = DataClient().create(data)
    if not data_id:
        rmtree(temp_dir)
        sys.exit(1)

    data_config.set_data_predecessor(data_id)
    DataConfigManager.set_config(data_config)

    # fetch auth token for upload server
    creds = DataClient().new_tus_credentials(data_id)
    if not creds:
        # TODO: delete module from server?
        rmtree(temp_dir)
        sys.exit(1)

    data_resource_id = creds[0]
    data_endpoint = TusDataClient().initialize_upload(
        tarball_path,
        metadata={"filename": data_resource_id},
        auth=creds)
    if not data_endpoint:
        # TODO: delete module from server?
        floyd_logger.error("Failed to get upload URL from Floydhub!")
        rmtree(temp_dir)
        sys.exit(1)

    data_config.set_data_endpoint(data_endpoint)
    DataConfigManager.set_config(data_config)

def complete_upload(data_config):
    data_endpoint = data_config.data_endpoint
    data_id = data_config.data_predecessor
    tarball_path = data_config.tarball_path

    if not data_id:
        floyd_logger.error("Corrupted upload state, please start a new one.")
        sys.exit(1)

    # check for tarball upload
    if not data_config.resource_id and (tarball_path and data_endpoint):
        floyd_logger.debug("Getting fresh upload credentials")
        creds = DataClient().new_tus_credentials(data_id)
        if not creds:
            sys.exit(1)

        file_size = os.path.getsize(tarball_path)
        floyd_logger.info("Uploading compressed data. Total upload size: %s",
                        sizeof_fmt(file_size))
        tus_client = TusDataClient()
        if not tus_client.resume_upload(tarball_path, data_endpoint, auth=creds):
            floyd_logger.error("Failed to finish upload!")
            return

        try:
            floyd_logger.info("Removing compressed data...")
            rmtree(os.path.dirname(tarball_path))
        except (OSError, TypeError):
            pass

        data_id = data_config.data_predecessor
        floyd_logger.debug("Created data with id : %s", data_id)
        floyd_logger.info("Upload finished.")

        # Update data config
        data_config.set_data_predecessor(data_id)
        data_config.set_tarball_path(None)
        data_config.set_data_endpoint(None)
        data_source = DataClient().get(data_id)
        data_config.set_resource_id(data_source.resource_id)
        DataConfigManager.set_config(data_config)

    # data tarball uploaded, check for server untar
    if data_config.resource_id:
        try:
            for i in dots(ResourceWaitIter(data_config.resource_id),
                        label='Waiting for server to unpack uploaded data...'):
                pass
        except WaitTimeoutException:
            clint_STREAM.write('\n')
            clint_STREAM.flush()
            floyd_logger.info(
                "Looks like it is going to take some extra for Floydhub to unpack "
                "your data. Please check back later with the following command:"
                "\n\tfloyd data upload -r")
            sys.exit(1)
        else:
            data_config.set_resource_id(None)
            data_config.set_tarball_path(None)
            data_config.set_data_endpoint(None)
            data_config.set_resource_id(None)
            DataConfigManager.set_config(data_config)

    # Print output
    table_output = [["DATA ID", "NAME", "VERSION"],
                    [data_id, data_config.name, data_config.version]]
    floyd_logger.info(tabulate(table_output, headers="firstrow"))

def abort_previous_upload(data_config):
    if data_config.tarball_path and os.path.exists(data_config.tarball_path):
        rmtree(os.path.dirname(data_config.tarball_path))

    data_config.set_tarball_path("")
    data_config.set_data_endpoint("")
    DataConfigManager.set_config(data_config)
<|endoftext|>