<commit_before>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys
from bigdl.util.common import JavaValue
from bigdl.util.common import callBigDlFunc
from bigdl.util.common import *

if sys.version >= '3':
    long = int
    unicode = str

class FeatureTransformer(JavaValue):
    """
    FeatureTransformer is a transformer that transform ImageFeature
    """

    def __init__(self, bigdl_type="float", *args):
        self.value = callBigDlFunc(
                bigdl_type, JavaValue.jvm_class_constructor(self), *args)

    def transform(self, image_feature, bigdl_type="float"):
        """
        transform ImageFeature
        """
        callBigDlFunc(bigdl_type, "transformImageFeature", self.value, image_feature)
        return image_feature

    def __call__(self, image_frame, bigdl_type="float"):
        """
        transform ImageFrame
        """
        jframe = callBigDlFunc(bigdl_type,
                             "transformImageFrame", self.value, image_frame)
        return ImageFrame(jvalue=jframe)

class Pipeline(FeatureTransformer):
    """
    Pipeline of FeatureTransformer
    """

    def __init__(self, transformers, bigdl_type="float"):
        for transfomer in transformers:
            assert transfomer.__class__.__bases__[0].__name__ == "FeatureTransformer", "the transformer should be " \
                                                                                "subclass of FeatureTransformer"
        super(Pipeline, self).__init__(bigdl_type, transformers)

class ImageFeature(JavaValue):
    """
    Each ImageFeature keeps information about single image,
    it can include various status of an image,
    e.g. original bytes read from image file, an opencv mat,
    pixels in float array, image label, meta data and so on.
    it uses HashMap to store all these data,
    the key is string that identify the corresponding value
    """

    def __init__(self, image=None, label=None, path=None, bigdl_type="float"):
        image_tensor = JTensor.from_ndarray(image) if image is not None else None
        label_tensor = JTensor.from_ndarray(label) if label is not None else None
        self.bigdl_type = bigdl_type
        self.value = callBigDlFunc(
            bigdl_type, JavaValue.jvm_class_constructor(self), image_tensor, label_tensor, path)

    def to_sample(self, float_key="floats", to_chw=True, with_im_info=False):
        """
        ImageFeature to sample
        """
        return callBigDlFunc(self.bigdl_type, "imageFeatureToSample", self.value, float_key, to_chw, with_im_info)

    def get_image(self, float_key="floats", to_chw=True):
        """
        get image as ndarray from ImageFeature
        """
        tensor = callBigDlFunc(self.bigdl_type, "imageFeatureToImageTensor", self.value,
                               float_key, to_chw)
        return tensor.to_ndarray()

    def get_label(self):
        """
        get label as ndarray from ImageFeature
        """
        label = callBigDlFunc(self.bigdl_type, "imageFeatureToLabelTensor", self.value)
        return label.to_ndarray()

    def keys(self):
        """
        get key set from ImageFeature
        """
        return callBigDlFunc(self.bigdl_type, "imageFeatureGetKeys", self.value)

class ImageFrame(JavaValue):
    """
    ImageFrame wraps a set of ImageFeature
    """

    def __init__(self, jvalue, bigdl_type="float"):
        self.value = jvalue
        self.bigdl_type = bigdl_type
        if self.is_local():
            self.image_frame = LocalImageFrame(jvalue=self.value)
        else:
            self.image_frame = DistributedImageFrame(jvalue=self.value)

    @classmethod
    def read(cls, path, sc=None, bigdl_type="float"):
        """
        Read images as Image Frame
        if sc is defined, Read image as DistributedImageFrame from local file system or HDFS
        if sc is null, Read image as LocalImageFrame from local file system
        :param path path to read images
        if sc is defined, path can be local or HDFS. Wildcard character are supported.
        if sc is null, path is local directory/image file/image file with wildcard character
        :param sc SparkContext
        :return ImageFrame
        """
        return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "read", path, sc))

    @classmethod
    def readParquet(cls, path, sql_context, bigdl_type="float"):
        """
        Read parquet file as DistributedImageFrame
        """
        return DistributedImageFrame(jvalue=callBigDlFunc(bigdl_type, "readParquet", path, sql_context))

    def is_local(self):
        """
        whether this is a LocalImageFrame
        """
        return callBigDlFunc(self.bigdl_type, "isLocal", self.value)

    def is_distributed(self):
        """
        whether this is a DistributedImageFrame
        """
        return callBigDlFunc(self.bigdl_type, "isDistributed", self.value)

    def transform(self, transformer, bigdl_type="float"):
        """
        transformImageFrame
        """
        self.value = callBigDlFunc(bigdl_type,
                                 "transformImageFrame", transformer, self.value)
        return self

    def get_image(self, float_key="floats", to_chw=True):
        """
        get image from ImageFrame
        """
        return self.image_frame.get_image(float_key, to_chw)

    def get_label(self):
        """
        get label from ImageFrame
        """
        return self.image_frame.get_label()

    def to_sample(self, float_key="floats", to_chw=True, with_im_info=False):
        """
        ImageFrame toSample
        """
        return self.image_frame.to_sample(float_key, to_chw, with_im_info)

class LocalImageFrame(ImageFrame):
    """
    LocalImageFrame wraps a list of ImageFeature
    """
    def __init__(self, image_list=None, label_list=None, jvalue=None, bigdl_type="float"):
        assert jvalue or image_list, "jvalue and image_list cannot be None in the same time"
        if jvalue:
            self.value = jvalue
        else:
            # init from image ndarray list and label rdd(optional)
            image_tensor_list = image_list.map(lambda image: JTensor.from_ndarray(image))
            label_tensor_list = label_list.map(lambda label: JTensor.from_ndarray(label)) if label_list else None
            self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self),
                                       image_tensor_list, label_tensor_list)

        self.bigdl_type = bigdl_type

    def get_image(self, float_key="floats", to_chw=True):
        """
        get image list from ImageFrame
        """
        tensors = callBigDlFunc(self.bigdl_type,
                                   "localImageFrameToImageTensor", self.value, float_key, to_chw)
        return map(lambda tensor: tensor.to_ndarray(), tensors)

    def get_label(self):
        """
        get label list from ImageFrame
        """
        labels = callBigDlFunc(self.bigdl_type, "localImageFrameToLabelTensor", self.value)
        return map(lambda tensor: tensor.to_ndarray(), labels)

    def to_sample(self, float_key="floats", to_chw=True, with_im_info=False):
        """
        to sample list
        """
        return callBigDlFunc(self.bigdl_type,
                             "localImageFrameToSample", self.value, float_key, to_chw, with_im_info)

class DistributedImageFrame(ImageFrame):
    """
    DistributedImageFrame wraps an RDD of ImageFeature
    """

    def __init__(self, image_rdd=None, label_rdd=None, jvalue=None, bigdl_type="float"):
        assert jvalue or image_rdd, "jvalue and image_rdd cannot be None in the same time"
        if jvalue:
            self.value = jvalue
        else:
            # init from image ndarray rdd and label rdd(optional)
            image_tensor_rdd = image_rdd.map(lambda image: JTensor.from_ndarray(image))
            label_tensor_rdd = label_rdd.map(lambda label: JTensor.from_ndarray(label)) if label_rdd else None
            self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self),
                                       image_tensor_rdd, label_tensor_rdd)

        self.bigdl_type = bigdl_type

    def get_image(self, float_key="floats", to_chw=True):
        """
        get image rdd from ImageFrame
        """
        tensor_rdd = callBigDlFunc(self.bigdl_type,
                               "distributedImageFrameToImageTensorRdd", self.value, float_key, to_chw)
        return tensor_rdd.map(lambda tensor: tensor.to_ndarray())

    def get_label(self):
        """
        get label rdd from ImageFrame
        """
        tensor_rdd = callBigDlFunc(self.bigdl_type, "distributedImageFrameToLabelTensorRdd", self.value)
        return tensor_rdd.map(lambda tensor: tensor.to_ndarray())

    def to_sample(self, float_key="floats", to_chw=True, with_im_info=False):
        """
        to sample rdd
        """
        return callBigDlFunc(self.bigdl_type,
                             "distributedImageFrameToSampleRdd", self.value, float_key, to_chw, with_im_info)

class HFlip(FeatureTransformer):
    """
    Flip the image horizontally
    """
    def __init__(self, bigdl_type="float"):
            super(HFlip, self).__init__(bigdl_type)
<commit_msg>Add more image transformers (#1965)<commit_after>#
# Copyright 2016 The BigDL Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys
from bigdl.util.common import JavaValue
from bigdl.util.common import callBigDlFunc
from bigdl.util.common import *

if sys.version >= '3':
    long = int
    unicode = str

class FeatureTransformer(JavaValue):
    """
    FeatureTransformer is a transformer that transform ImageFeature
    """

    def __init__(self, bigdl_type="float", *args):
        self.value = callBigDlFunc(
                bigdl_type, JavaValue.jvm_class_constructor(self), *args)

    def transform(self, image_feature, bigdl_type="float"):
        """
        transform ImageFeature
        """
        callBigDlFunc(bigdl_type, "transformImageFeature", self.value, image_feature)
        return image_feature

    def __call__(self, image_frame, bigdl_type="float"):
        """
        transform ImageFrame
        """
        jframe = callBigDlFunc(bigdl_type,
                             "transformImageFrame", self.value, image_frame)
        return ImageFrame(jvalue=jframe)

class Pipeline(FeatureTransformer):
    """
    Pipeline of FeatureTransformer
    """

    def __init__(self, transformers, bigdl_type="float"):
        for transfomer in transformers:
            assert transfomer.__class__.__bases__[0].__name__ == "FeatureTransformer", "the transformer should be " \
                                                                                "subclass of FeatureTransformer"
        super(Pipeline, self).__init__(bigdl_type, transformers)

class ImageFeature(JavaValue):
    """
    Each ImageFeature keeps information about single image,
    it can include various status of an image,
    e.g. original bytes read from image file, an opencv mat,
    pixels in float array, image label, meta data and so on.
    it uses HashMap to store all these data,
    the key is string that identify the corresponding value
    """

    def __init__(self, image=None, label=None, path=None, bigdl_type="float"):
        image_tensor = JTensor.from_ndarray(image) if image is not None else None
        label_tensor = JTensor.from_ndarray(label) if label is not None else None
        self.bigdl_type = bigdl_type
        self.value = callBigDlFunc(
            bigdl_type, JavaValue.jvm_class_constructor(self), image_tensor, label_tensor, path)

    def to_sample(self, float_key="floats", to_chw=True, with_im_info=False):
        """
        ImageFeature to sample
        """
        return callBigDlFunc(self.bigdl_type, "imageFeatureToSample", self.value, float_key, to_chw, with_im_info)

    def get_image(self, float_key="floats", to_chw=True):
        """
        get image as ndarray from ImageFeature
        """
        tensor = callBigDlFunc(self.bigdl_type, "imageFeatureToImageTensor", self.value,
                               float_key, to_chw)
        return tensor.to_ndarray()

    def get_label(self):
        """
        get label as ndarray from ImageFeature
        """
        label = callBigDlFunc(self.bigdl_type, "imageFeatureToLabelTensor", self.value)
        return label.to_ndarray()

    def keys(self):
        """
        get key set from ImageFeature
        """
        return callBigDlFunc(self.bigdl_type, "imageFeatureGetKeys", self.value)

class ImageFrame(JavaValue):
    """
    ImageFrame wraps a set of ImageFeature
    """

    def __init__(self, jvalue, bigdl_type="float"):
        self.value = jvalue
        self.bigdl_type = bigdl_type
        if self.is_local():
            self.image_frame = LocalImageFrame(jvalue=self.value)
        else:
            self.image_frame = DistributedImageFrame(jvalue=self.value)

    @classmethod
    def read(cls, path, sc=None, bigdl_type="float"):
        """
        Read images as Image Frame
        if sc is defined, Read image as DistributedImageFrame from local file system or HDFS
        if sc is null, Read image as LocalImageFrame from local file system
        :param path path to read images
        if sc is defined, path can be local or HDFS. Wildcard character are supported.
        if sc is null, path is local directory/image file/image file with wildcard character
        :param sc SparkContext
        :return ImageFrame
        """
        return ImageFrame(jvalue=callBigDlFunc(bigdl_type, "read", path, sc))

    @classmethod
    def readParquet(cls, path, sql_context, bigdl_type="float"):
        """
        Read parquet file as DistributedImageFrame
        """
        return DistributedImageFrame(jvalue=callBigDlFunc(bigdl_type, "readParquet", path, sql_context))

    def is_local(self):
        """
        whether this is a LocalImageFrame
        """
        return callBigDlFunc(self.bigdl_type, "isLocal", self.value)

    def is_distributed(self):
        """
        whether this is a DistributedImageFrame
        """
        return callBigDlFunc(self.bigdl_type, "isDistributed", self.value)

    def transform(self, transformer, bigdl_type="float"):
        """
        transformImageFrame
        """
        self.value = callBigDlFunc(bigdl_type,
                                 "transformImageFrame", transformer, self.value)
        return self

    def get_image(self, float_key="floats", to_chw=True):
        """
        get image from ImageFrame
        """
        return self.image_frame.get_image(float_key, to_chw)

    def get_label(self):
        """
        get label from ImageFrame
        """
        return self.image_frame.get_label()

    def to_sample(self, float_key="floats", to_chw=True, with_im_info=False):
        """
        ImageFrame toSample
        """
        return self.image_frame.to_sample(float_key, to_chw, with_im_info)

class LocalImageFrame(ImageFrame):
    """
    LocalImageFrame wraps a list of ImageFeature
    """
    def __init__(self, image_list=None, label_list=None, jvalue=None, bigdl_type="float"):
        assert jvalue or image_list, "jvalue and image_list cannot be None in the same time"
        if jvalue:
            self.value = jvalue
        else:
            # init from image ndarray list and label rdd(optional)
            image_tensor_list = image_list.map(lambda image: JTensor.from_ndarray(image))
            label_tensor_list = label_list.map(lambda label: JTensor.from_ndarray(label)) if label_list else None
            self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self),
                                       image_tensor_list, label_tensor_list)

        self.bigdl_type = bigdl_type

    def get_image(self, float_key="floats", to_chw=True):
        """
        get image list from ImageFrame
        """
        tensors = callBigDlFunc(self.bigdl_type,
                                   "localImageFrameToImageTensor", self.value, float_key, to_chw)
        return map(lambda tensor: tensor.to_ndarray(), tensors)

    def get_label(self):
        """
        get label list from ImageFrame
        """
        labels = callBigDlFunc(self.bigdl_type, "localImageFrameToLabelTensor", self.value)
        return map(lambda tensor: tensor.to_ndarray(), labels)

    def to_sample(self, float_key="floats", to_chw=True, with_im_info=False):
        """
        to sample list
        """
        return callBigDlFunc(self.bigdl_type,
                             "localImageFrameToSample", self.value, float_key, to_chw, with_im_info)

class DistributedImageFrame(ImageFrame):
    """
    DistributedImageFrame wraps an RDD of ImageFeature
    """

    def __init__(self, image_rdd=None, label_rdd=None, jvalue=None, bigdl_type="float"):
        assert jvalue or image_rdd, "jvalue and image_rdd cannot be None in the same time"
        if jvalue:
            self.value = jvalue
        else:
            # init from image ndarray rdd and label rdd(optional)
            image_tensor_rdd = image_rdd.map(lambda image: JTensor.from_ndarray(image))
            label_tensor_rdd = label_rdd.map(lambda label: JTensor.from_ndarray(label)) if label_rdd else None
            self.value = callBigDlFunc(bigdl_type, JavaValue.jvm_class_constructor(self),
                                       image_tensor_rdd, label_tensor_rdd)

        self.bigdl_type = bigdl_type

    def get_image(self, float_key="floats", to_chw=True):
        """
        get image rdd from ImageFrame
        """
        tensor_rdd = callBigDlFunc(self.bigdl_type,
                               "distributedImageFrameToImageTensorRdd", self.value, float_key, to_chw)
        return tensor_rdd.map(lambda tensor: tensor.to_ndarray())

    def get_label(self):
        """
        get label rdd from ImageFrame
        """
        tensor_rdd = callBigDlFunc(self.bigdl_type, "distributedImageFrameToLabelTensorRdd", self.value)
        return tensor_rdd.map(lambda tensor: tensor.to_ndarray())

    def to_sample(self, float_key="floats", to_chw=True, with_im_info=False):
        """
        to sample rdd
        """
        return callBigDlFunc(self.bigdl_type,
                             "distributedImageFrameToSampleRdd", self.value, float_key, to_chw, with_im_info)

class HFlip(FeatureTransformer):
    """
    Flip the image horizontally
    """

    def __init__(self, bigdl_type="float"):
            super(HFlip, self).__init__(bigdl_type)

class Resize(FeatureTransformer):
    """
    Resize image
    :param resize_h height after resize
    :param resize_w width after resize
    :param resize_mode if resizeMode = -1, random select a mode from (Imgproc.INTER_LINEAR,
     Imgproc.INTER_CUBIC, Imgproc.INTER_AREA, Imgproc.INTER_NEAREST, Imgproc.INTER_LANCZOS4)
    """

    def __init__(self, resize_h, resize_w, resize_mode = 1, bigdl_type="float"):
        super(Resize, self).__init__(bigdl_type, resize_h, resize_w, resize_mode)

class Brightness(FeatureTransformer):
    """
    adjust the image brightness
    :param deltaLow brightness parameter: low bound
    :param deltaHigh brightness parameter: high bound
    """

    def __init__(self, delta_low, delta_high, bigdl_type="float"):
        super(Brightness, self).__init__(bigdl_type, delta_low, delta_high)

class ChannelOrder(FeatureTransformer):
    """
    random change the channel of an image
    """

    def __init__(self, bigdl_type="float"):
        super(ChannelOrder, self).__init__(bigdl_type)

class Contrast(FeatureTransformer):
    """
    Adjust the image contrast
    :param delta_low contrast parameter low bound
    :param delta_high contrast parameter high bound
    """

    def __init__(self, delta_low, delta_high, bigdl_type="float"):
        super(Contrast, self).__init__(bigdl_type, delta_low, delta_high)

class Saturation(FeatureTransformer):
    """
    Adjust image saturation
    """

    def __init__(self, delta_low, delta_high, bigdl_type="float"):
        super(Saturation, self).__init__(bigdl_type, delta_low, delta_high)

class Hue(FeatureTransformer):
    """
    Adjust image hue
    :param delta_low hue parameter: low bound
    :param delta_high hue parameter: high bound
    """

    def __init__(self, delta_low, delta_high, bigdl_type="float"):
        super(Hue, self).__init__(bigdl_type, delta_low, delta_high)

class ChannelNormalize(FeatureTransformer):
    """
    image channel normalize
    :param mean_r mean value in R channel
    :param mean_g mean value in G channel
    :param meanB_b mean value in B channel
    :param std_r std value in R channel
    :param std_g std value in G channel
    :param std_b std value in B channel
    """
    def __init__(self, mean_r, mean_b, mean_g, std_r=1.0, std_g=1.0, std_b=1.0, bigdl_type="float"):
        super(ChannelNormalize, self).__init__(bigdl_type, mean_r, mean_g, mean_b, std_r, std_g, std_b)

class PixelNormalize(FeatureTransformer):
    """
    Pixel level normalizer, data(i) = data(i) - mean(i)

    :param means pixel level mean, following H * W * C order
    """

    def __init__(self, means, bigdl_type="float"):
        super(PixelNormalize, self).__init__(bigdl_type, means)

class RandomCrop(FeatureTransformer):
    """
    Random crop a `cropWidth` x `cropHeight` patch from an image.
    The patch size should be less than the image size.

    :param crop_width width after crop
    :param crop_height height after crop
    :param is_clip whether to clip the roi to image boundaries
    """

    def __init__(self, crop_width, crop_height, is_clip=True, bigdl_type="float"):
        super(RandomCrop, self).__init__(bigdl_type, crop_width, crop_height, is_clip)

class CenterCrop(FeatureTransformer):
    """
    Crop a `cropWidth` x `cropHeight` patch from center of image.
    The patch size should be less than the image size.
    :param crop_width width after crop
    :param crop_height height after crop
    :param is_clip  clip cropping box boundary
    """

    def __init__(self, crop_width, crop_height, is_clip=True, bigdl_type="float"):
        super(CenterCrop, self).__init__(bigdl_type, crop_width, crop_height, is_clip)

class FixedCrop(FeatureTransformer):
    """
    Crop a fixed area of image

    :param x1 start in width
    :param y1 start in height
    :param x2 end in width
    :param y2 end in height
    :param normalized whether args are normalized, i.e. in range [0, 1]
    :param is_clip whether to clip the roi to image boundaries
    """

    def __init__(self, x1, y1, x2, y2, normalized=True, is_clip=True, bigdl_type="float"):
        super(FixedCrop, self).__init__(bigdl_type, x1, y1, x2, y2, normalized, is_clip)

class DetectionCrop(FeatureTransformer):
    """
    Crop from object detections, each image should has a tensor detection,
    which is stored in ImageFeature
    :param roi_key key that map a tensor detection
    :param normalized whether is detection is normalized, i.e. in range [0, 1]
    """

    def __init__(self, roi_key, normalized=True, bigdl_type="float"):
        super(DetectionCrop, self).__init__(bigdl_type, roi_key, normalized)

class Expand(FeatureTransformer):
    """
    expand image, fill the blank part with the meanR, meanG, meanB

    :param means_r means in R channel
    :param means_g means in G channel
    :param means_b means in B channel
    :param min_expand_ratio min expand ratio
    :param max_expand_ratio max expand ratio
    """

    def __init__(self, means_r=123, means_g=117, means_b=104,
                 min_expand_ratio=1.0,
                 max_expand_ratio=4.0, bigdl_type="float"):
        super(Expand, self).__init__(bigdl_type, means_r, means_g, means_b,
                                     min_expand_ratio, max_expand_ratio)

class Filler(FeatureTransformer):
    """
    Fill part of image with certain pixel value
    :param start_x start x ratio
    :param start_y start y ratio
    :param end_x end x ratio
    :param end_y end y ratio
    :param value filling value
    """

    def __init__(self, start_x, start_y, end_x, end_y, value = 255, bigdl_type="float"):
        super(Filler, self).__init__(bigdl_type, start_x,
                                     start_y,
                                     end_x,
                                     end_y,
                                     value)

class RandomTransformer(FeatureTransformer):
    """
    It is a wrapper for transformers to control the transform probability
    :param transformer transformer to apply randomness
    :param prob max prob
    """

    def __init__(self, transformer, prob, bigdl_type="float"):
        super(RandomTransformer, self).__init__(bigdl_type, transformer, prob)

class ColorJitter(FeatureTransformer):
    """
    Random adjust brightness, contrast, hue, saturation
    :param brightness_prob probability to adjust brightness
    :param brightness_delta brightness parameter
    :param contrast_prob probability to adjust contrast
    :param contrast_lower contrast lower parameter
    :param contrast_upper contrast upper parameter
    :param hue_prob probability to adjust hue
    :param hue_delta hue parameter
    :param saturation_prob probability to adjust saturation
    :param saturation_lower saturation lower parameter
    :param saturation_upper saturation upper parameter
    :param random_order_prob random order for different operation
    :param shuffle  shuffle the transformers
    """
    def __init__(self, brightness_prob = 0.5,
                 brightness_delta = 32.0,
                 contrast_prob = 0.5,
                 contrast_lower = 0.5,
                 contrast_upper = 1.5,
                 hue_prob = 0.5,
                 hue_delta = 18.0,
                 saturation_prob = 0.5,
                 saturation_lower = 0.5,
                 saturation_upper = 1.5,
                 random_order_prob = 0.0,
                 shuffle = False,
                 bigdl_type="float"):
        super(ColorJitter, self).__init__(bigdl_type, brightness_prob,
                                          brightness_delta,
                                          contrast_prob,
                                          contrast_lower,
                                          contrast_upper,
                                          hue_prob,
                                          hue_delta,
                                          saturation_prob,
                                          saturation_lower,
                                          saturation_upper,
                                          random_order_prob,
                                          shuffle)

class RandomSampler(FeatureTransformer):
    """
    Random sample a bounding box given some constraints and crop the image
    This is used in SSD training augmentation
    """

    def __init__(self):
        super(RandomSampler, self).__init__(bigdl_type)

class RoiProject(FeatureTransformer):
    """
    Project gt boxes onto the coordinate system defined by image boundary
    :param need_meet_center_constraint whether need to meet center constraint, i.e., the center of gt box need be within image boundary
    """

    def __init__(self, need_meet_center_constraint, bigdl_type="float"):
        super(RoiProject, self).__init__(bigdl_type, need_meet_center_constraint)

class RoiHFlip(FeatureTransformer):
    """
    horizontally flip the roi
    :param normalized whether the roi is normalized, i.e. in range [0, 1]
    """

    def __init__(self, normalized=True, bigdl_type="float"):
        super(RoiHFlip, self).__init__(bigdl_type, normalized)

class RoiResize(FeatureTransformer):
    """
    resize the roi according to scale
    :param normalized whether the roi is normalized, i.e. in range [0, 1]
    """
    def __init__(self, normalized=True, bigdl_type="float"):
        super(RoiResize, self).__init__(bigdl_type, normalized)

class RoiNormalize(FeatureTransformer):
    """
    Normalize Roi to [0, 1]
    """

    def __init__(self, bigdl_type="float"):
        super(RoiNormalize, self).__init__(bigdl_type)

class MatToFloats(FeatureTransformer):

    def __init__(self, valid_height=300, valid_width=300, valid_channel=300,
                 mean_r=-1.0, mean_g=-1.0, mean_b=-1.0, out_key = "floats", bigdl_type="float"):
        super(MatToFloats, self).__init__(bigdl_type, valid_height, valid_width, valid_channel,
                                          mean_r, mean_g, mean_b, out_key)
class AspectScale(FeatureTransformer):
    """
    Resize the image, keep the aspect ratio. scale according to the short edge
    :param scale scale size, apply to short edge
    :param scale_multiple_of make the scaled size multiple of some value
    :param max_size max size after scale
    """

    def __init__(self, scale, scale_multiple_of = 1, max_size = 1000, bigdl_type="float"):
        super(AspectScale, self).__init__(bigdl_type, scale, scale_multiple_of, max_size)

class RandomAspectScale(FeatureTransformer):
    """
    resize the image by randomly choosing a scale
    :param scales array of scale options that for random choice
    :param scaleMultipleOf Resize test images so that its width and height are multiples of
    :param maxSize Max pixel size of the longest side of a scaled input image
    """
    def __init__(self, scales, scale_multiple_of = 1, max_size = 1000, bigdl_type="float"):
        super(RandomAspectScale, self).__init__(bigdl_type, scales, scale_multiple_of, max_size)

class BytesToMat(FeatureTransformer):
    """
    Transform byte array(original image file in byte) to OpenCVMat
    """
    def __init__(self, bigdl_type="float"):
        super(BytesToMat, self).__init__(bigdl_type)

<|endoftext|>


<commit_before>import base64
import datetime
import logging
import urllib
import xml.etree.ElementTree as ET
from base64 import b64encode

from google.appengine.api import urlfetch
from google.appengine.api.app_identity import app_identity
from webapp2 import uri_for

from dontbelate import settings
from dontbelate.models import Profile
from generic.utils import utc_to_local

def _basic_auth_str(username, password):
    """Returns a Basic Auth string."""
    return 'Basic ' + b64encode('{}:{}'.format(username, password).encode('latin1')).strip()

def call_ns_api(station_from, station_to, departure_time):
    ns_api_base_url = 'http://webservices.ns.nl/ns-api-treinplanner'
    basic_auth = settings.NS_API_BASIC_AUTH
    num_previous_advices = 0
    include_high_speed = 'true'
    now = datetime.datetime.now()
    departure_time = now.replace(hour=departure_time.hour, minute=departure_time.minute).strftime('%Y-%m-%dT%H:%M')
    params = {
        'fromStation': station_from,
        'toStation': station_to,
        'hslAllowed': include_high_speed,
        'previousAdvices': num_previous_advices,
        'dateTime': departure_time,
    }
    url = '{}?{}'.format(ns_api_base_url, urllib.urlencode(params))
    response = urlfetch.fetch(
        url=url,
        headers={'Authorization': _basic_auth_str(*basic_auth)}
    )
    if response.status_code != 200:
        raise
    return parse_ns_api_result(response.content, station_from, station_to)

def parse_ns_api_result(xml, station_from, station_to):
    tree = ET.fromstring(xml)
    delays = []
    for element in tree.iter('ReisMogelijkheid'):
        print element.find('Status').text
        departure_planned = element.find('GeplandeVertrekTijd').text
        departure_planned = datetime.datetime.strptime(departure_planned[:-5], settings.NS_DATE_TIME_FORMAT)
        departure_actual = element.find('ActueleVertrekTijd').text
        departure_actual = datetime.datetime.strptime(departure_actual[:-5], settings.NS_DATE_TIME_FORMAT)
        delay = departure_actual - departure_planned
        if not delay.seconds:
            continue
        delay = delay.seconds / 60
        train_type = element.find('ReisDeel').find('VervoerType').text
        track_elem = element.find('ReisDeel').find('ReisStop').find('Spoor')
        track = track_elem.text
        track_change = False if track_elem.attrib.get('wijziging') == 'false' else True
        delays.append({
            'station_from': station_from,
            'station_to': station_to,
            'train_type': train_type,
            'track_change': track_change,
            'track': track,
            'departure_planned': departure_planned.strftime('%H:%M'),
            'departure_actual': departure_actual.strftime('%H:%M'),
            'delay': delay,
        })
    return delays

def create_push_message(delays):
    out = ''
    out_long = ''
    for i, delay in enumerate(delays):
        out += '{departure_planned}: (+{delay}) sp. {track}'.format(**delay)
        # add exclamation emoji if track changed
        if delay['track_change']:
            out += '\xE2\x9D\x97'
        out_long += '{station_from} - {station_to} {departure_planned}: (+{delay}) sp. {track} ({train_type})'.format(**delay)
        if i < len(delays) - 1:
            out += ', '
            out_long += '\n'
        else:
            out += '.'
    return out, out_long

def send_push_notification(message, message_long, access_token, url):
    boxcar_api_base_url = 'https://new.boxcar.io/api/notifications'
    params = {
        'user_credentials': access_token,
        'notification[title]': message,
        'notification[long_message]': message_long,
        'notification[sound]': 'boing',
        'notification[open_url]': url
    }
    response = urlfetch.fetch(
        url=boxcar_api_base_url,
        payload=urllib.urlencode(params),
        method=urlfetch.POST
    )
    if response.status_code != 201:
        logging.error('Error occured for push: {}'.format(response.content))

def check_routes(routes):
    all_delays = []
    for route in routes:
        if not route.profile.boxcar_send_push:
            continue
        if route.profile.silence_until and route.profile.silence_until > datetime.datetime.now():
            continue
        delays = call_ns_api(route.station_from, route.station_to, route.departure_time_from)
        if not delays:
            continue
        boxcar_access_token = route.profile.boxcar_access_token
        message, message_long = create_push_message(delays)
        hashed_message = base64.b64encode(message)
        if route.latest_push_message != hashed_message:
            route.latest_push_message = hashed_message
            route.put()
            url = 'https://{}.appspot.com{}'.format(app_identity.get_application_id(),
                                                    uri_for('profile_edit', obj_id=route.profile.key.id()))
            send_push_notification(message, message_long, boxcar_access_token, url)
        all_delays.extend(delays)
    return all_delays

def find_routes_to_check():
    profiles = Profile.query().fetch(limit=500)
    routes_to_check = []
    for profile in profiles:
        routes = profile.get_routes()
        for route in routes:
            if not route.key:
                continue
            now = datetime.datetime.now()
            local_now = utc_to_local(now)
            if route.departure_time_from_offset <= local_now.time() <= route.departure_time_until:
                route.profile = profile
                routes_to_check.append(route)
    return routes_to_check
<commit_msg>Only check on weekdays if weekdays_only is active<commit_after>import base64
import datetime
import logging
import urllib
import xml.etree.ElementTree as ET
from base64 import b64encode

from google.appengine.api import urlfetch
from google.appengine.api.app_identity import app_identity
from webapp2 import uri_for

from dontbelate import settings
from dontbelate.models import Profile
from generic.utils import utc_to_local

def _basic_auth_str(username, password):
    """Returns a Basic Auth string."""
    return 'Basic ' + b64encode('{}:{}'.format(username, password).encode('latin1')).strip()

def call_ns_api(station_from, station_to, departure_time):
    ns_api_base_url = 'http://webservices.ns.nl/ns-api-treinplanner'
    basic_auth = settings.NS_API_BASIC_AUTH
    num_previous_advices = 0
    include_high_speed = 'true'
    now = datetime.datetime.now()
    departure_time = now.replace(hour=departure_time.hour, minute=departure_time.minute).strftime('%Y-%m-%dT%H:%M')
    params = {
        'fromStation': station_from,
        'toStation': station_to,
        'hslAllowed': include_high_speed,
        'previousAdvices': num_previous_advices,
        'dateTime': departure_time,
    }
    url = '{}?{}'.format(ns_api_base_url, urllib.urlencode(params))
    response = urlfetch.fetch(
        url=url,
        headers={'Authorization': _basic_auth_str(*basic_auth)}
    )
    if response.status_code != 200:
        raise
    return parse_ns_api_result(response.content, station_from, station_to)

def parse_ns_api_result(xml, station_from, station_to):
    tree = ET.fromstring(xml)
    delays = []
    for element in tree.iter('ReisMogelijkheid'):
        print element.find('Status').text
        departure_planned = element.find('GeplandeVertrekTijd').text
        departure_planned = datetime.datetime.strptime(departure_planned[:-5], settings.NS_DATE_TIME_FORMAT)
        departure_actual = element.find('ActueleVertrekTijd').text
        departure_actual = datetime.datetime.strptime(departure_actual[:-5], settings.NS_DATE_TIME_FORMAT)
        delay = departure_actual - departure_planned
        if not delay.seconds:
            continue
        delay = delay.seconds / 60
        train_type = element.find('ReisDeel').find('VervoerType').text
        track_elem = element.find('ReisDeel').find('ReisStop').find('Spoor')
        track = track_elem.text
        track_change = False if track_elem.attrib.get('wijziging') == 'false' else True
        delays.append({
            'station_from': station_from,
            'station_to': station_to,
            'train_type': train_type,
            'track_change': track_change,
            'track': track,
            'departure_planned': departure_planned.strftime('%H:%M'),
            'departure_actual': departure_actual.strftime('%H:%M'),
            'delay': delay,
        })
    return delays

def create_push_message(delays):
    out = ''
    out_long = ''
    for i, delay in enumerate(delays):
        out += '{departure_planned}: (+{delay}) sp. {track}'.format(**delay)
        # add exclamation emoji if track changed
        if delay['track_change']:
            out += '\xE2\x9D\x97'
        out_long += '{station_from} - {station_to} {departure_planned}: (+{delay}) sp. {track} ({train_type})'.format(**delay)
        if i < len(delays) - 1:
            out += ', '
            out_long += '\n'
        else:
            out += '.'
    return out, out_long

def send_push_notification(message, message_long, access_token, url):
    boxcar_api_base_url = 'https://new.boxcar.io/api/notifications'
    params = {
        'user_credentials': access_token,
        'notification[title]': message,
        'notification[long_message]': message_long,
        'notification[sound]': 'boing',
        'notification[open_url]': url
    }
    response = urlfetch.fetch(
        url=boxcar_api_base_url,
        payload=urllib.urlencode(params),
        method=urlfetch.POST
    )
    if response.status_code != 201:
        logging.error('Error occured for push: {}'.format(response.content))

def check_routes(routes):
    all_delays = []
    for route in routes:
        if not route.profile.boxcar_send_push:
            continue
        if route.profile.silence_until and route.profile.silence_until > datetime.datetime.now():
            continue
        delays = call_ns_api(route.station_from, route.station_to, route.departure_time_from)
        if not delays:
            continue
        boxcar_access_token = route.profile.boxcar_access_token
        message, message_long = create_push_message(delays)
        hashed_message = base64.b64encode(message)
        if route.latest_push_message != hashed_message:
            route.latest_push_message = hashed_message
            route.put()
            url = 'https://{}.appspot.com{}'.format(app_identity.get_application_id(),
                                                    uri_for('profile_edit', obj_id=route.profile.key.id()))
            send_push_notification(message, message_long, boxcar_access_token, url)
        all_delays.extend(delays)
    return all_delays

def find_routes_to_check():
    profiles = Profile.query().fetch(limit=500)
    routes_to_check = []
    for profile in profiles:
        routes = profile.get_routes()
        for route in routes:
            if not route.key:
                continue
            now = datetime.datetime.now()
            local_now = utc_to_local(now)
            if route.weekdays_only and local_now.weekday() in [6, 7]:
                # it's weekend -> skip
                continue
            if route.departure_time_from_offset <= local_now.time() <= route.departure_time_until:
                route.profile = profile
                routes_to_check.append(route)
    return routes_to_check
<|endoftext|>


<commit_before>#!/usr/bin/python

__author__ = 'dust'

import codecs

from productinfo import Bluse

def main():
    f = codecs.open('./foobar.txt', 'r', 'utf-8')
    f.readline() #skip first line
    s = f.readline()
    print s
    b = Bluse(s)
    print b
    pass

if __name__ == '__main__':
    main()
<commit_msg>replaced wrong path with the correct one<commit_after>#!/usr/bin/python

__author__ = 'dust'

import codecs

from productinfo import Bluse

def main():
    f = codecs.open('./DamenBlusen.txt', 'r', 'utf-8')
    f.readline() #skip first line
    s = f.readline()
    print s
    b = Bluse(s)
    print b
    pass

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# This program generates Kamaelia's documentation directly from the
# source. I am currently running this as follows:
#
# mkdir ./pydoc/
# ./DocExtractor.py
#
# documentation is placed into the "pydoc" subdirectory

import textwrap
import inspect
import pprint
import time
from docutils import core
from docutils import nodes
import Kamaelia.Support.Data.Repository

from renderHTML      import RenderHTML
from renderPlaintext import RenderPlaintext

from Nodes import boxright

class docFormatter(object):
    def __init__(self, renderer=RenderPlaintext,debug=False):
        super(docFormatter,self).__init__()
        self.renderer = renderer
        self.debug = debug

    uid = 0

    def genUniqueRef(self):
        uid = str(self.uid)
        self.uid+=1
        return uid

    def boxes(self,componentName, label, boxes):
        items = []
        for box in boxes:
            try:
                description = boxes[box]
            except KeyError:
                description = ""
            except TypeError:
                description = "Code uses old style inbox/outbox description - no metadata available"
            items.append((str(box), str(description)))

        docTree= nodes.section('',
                ids   = ["component-"+componentName+"-"+label],
                names = ["component-"+componentName+"-"+label],
                *[ nodes.title('', label),
                   nodes.bullet_list('',
                      *[ nodes.list_item('', nodes.paragraph('', '',
                                                 nodes.strong('', boxname),
                                                 nodes.Text(" : "+boxdesc))
                                             )
                         for (boxname,boxdesc) in items
                       ]
                   ),
                ]
            )
        return docTree

    def docString(self,docstring, main=False):
        if docstring is None:
            docstring = " "
        lines = "\n".split(docstring)
        if len(lines)>1:
            line1 = textwrap.dedent(lines[0])
            rest = textwrap.dedent("\n".join(lines[1:]))
            docstring = line1+"\n"+rest
        else:
            docstring=textwrap.dedent(docstring)

        while docstring[0] == "\n":
            docstring = docstring[1:]
        while docstring[-1] == "\n":
            docstring = docstring[:-1]

        return nodes.section('', *core.publish_doctree(docstring).children)

    def formatArgSpec(self, argspec):
        return pprint.pformat(argspec[0]).replace("[","(").replace("]",")").replace("'","")

    def formatMethodDocStrings(self,X):
        docTree = nodes.section('') #self.emptyTree()

        for method in sorted([x for x in inspect.classify_class_attrs(X) if x[2] == X and x[1] == "method"]):
            if method[0][-7:] == "__super":
                continue
            methodHead = method[0]+self.formatArgSpec(inspect.getargspec(method[3]))

            docTree.append( nodes.section('',
                                ids   = ["component-"+X.__name__+"-method-"+method[0]],
                                names = ["component-"+X.__name__+"-method-"+method[0]],
                                * [ nodes.title('', methodHead) ]
                                  + self.docString(method[3].__doc__)
                            )
                          )

        return docTree

    def formatClassStatement(self, name, bases):
        return "class "+ name+"("+",".join([str(base)[8:-2] for base in bases])+")"

    def formatPrefabStatement(self, name):
        return "prefab: "+name

    def formatComponent(self, X, includeMethods=False):
        CLASSNAME = self.formatClassStatement(X.__name__, X.__bases__)
        CLASSDOC = self.docString(X.__doc__)
        INBOXES = self.boxes(X.__name__,"Inboxes", X.Inboxes)
        OUTBOXES = self.boxes(X.__name__,"Outboxes", X.Outboxes)

        if includeMethods:
            METHODS = [ nodes.section('',
                          nodes.title('', 'Methods defined here'),
                          boxright('',
                              nodes.paragraph('', '',
                                  nodes.strong('', nodes.Text("Warning!"))
                              ),
                              nodes.paragraph('', '',
                                  nodes.Text("You should be using the inbox/outbox interface, not these methods (except construction). This documentation is designed as a roadmap as to their functionalilty for maintainers and new component developers.")
                              ),
                          ),
                          * self.formatMethodDocStrings(X)
                        )
                      ]
        else:
            METHODS = []

        return nodes.container('',
                * [ nodes.title('', CLASSNAME, ids=["component-"+X.__name__]) ]
                  + CLASSDOC
                  + [ INBOXES, OUTBOXES ]
                  + METHODS
            )

    def formatPrefab(self, X):
        CLASSNAME = self.formatPrefabStatement(X.__name__)
        CLASSDOC = self.docString(X.__doc__)

        return nodes.container('',
                * [ nodes.title('', CLASSNAME, ids=["component-"+X.__name__]) ]
                  + CLASSDOC
            )

    def formatModuleTrail(self, moduleName):
        path = moduleName.split(".")

        trail = nodes.paragraph('')
        line = trail

        accum = ""
        firstPass=True
        for element in path:
            if not firstPass:
                accum += "."
            accum += element

            if not firstPass:
                line.append(nodes.Text(" . "))
            URI = self.renderer.makeURI(accum)
            line.append( nodes.reference('', element, refuri=URI) )

            firstPass=False

        return trail

    def componentList(self, componentList):
        return nodes.container('',
#            nodes.paragraph('', '', nodes.Text('Components and Prefabs')),
            nodes.bullet_list('',
                *[ nodes.list_item('',
                       nodes.paragraph('', '',
                         nodes.strong('', '',
                           nodes.reference('', COMPONENT, refid='component-'+COMPONENT))
                         )
                       )
                   for COMPONENT in componentList
                 ]
                )
            )

    def formatModule(self, moduleName, module, components, prefabs, tocDepth, includeMethods):

        trailTree = self.formatModuleTrail(moduleName)
        moduleTree = self.docString(module.__doc__, main=True)
        toc = self.buildTOC(moduleTree, depth=tocDepth)

        allDeclarations = []

        declarationTrees = {}
        for component in components:
            declarationTrees[component.__name__] = self.formatComponent(component, includeMethods)

        for prefab in prefabs:
            assert(prefab.__name__ not in declarationTrees)
            declarationTrees[prefab.__name__] = self.formatPrefab(prefab)

        for name in sorted(declarationTrees.keys()):
            allDeclarations.extend(declarationTrees[name])

        componentListTree = self.componentList( sorted(declarationTrees.keys()) )

        return nodes.section('',
            trailTree,
#            nodes.paragraph('', nodes.Text("Component/Prefab details")),
            componentListTree,
            nodes.paragraph('', nodes.Text("Explanations")),
            toc,
            nodes.transition(),
            moduleTree,
            nodes.transition(),
            nodes.section('',
                id = [ 'component-declarations' ],
                name = [ 'component-declarations' ],
                * [ nodes.title('', "Component/Prefab details"),
                    nodes.section('',
                        *allDeclarations
                    )
                  ]
                ),
            nodes.transition(),
            self.postscript(),
            )

    def buildTOC(self, srcTree, parent=None, depth=3):
        """Recurse through a source document tree, building a table of contents"""
        if parent is None:
            parent = nodes.bullet_list()

        if depth<=0:
            return parent

        items=nodes.section()

        for n in srcTree.children:
            if isinstance(n, nodes.title):
                refid = self.genUniqueRef()
                n.attributes['ids'].append(refid)
                newItem = nodes.list_item()
                newItem.append(nodes.paragraph('','', nodes.reference('', refid=refid, *n.children)))
                newItem.append(nodes.bullet_list())
                parent.append(newItem)
            elif isinstance(n, nodes.section):
                if len(parent)==0:
                    newItem = nodes.list_item()
                    newItem.append(nodes.bullet_list())
                    parent.append(newItem)
                self.buildTOC( n, parent[-1][-1], depth-1)

        return parent

    def formatIndex(self, indexName, subTree, depth):
        return nodes.section('',
#            nodes.title('', indexName),
            self.formatModuleTrail(indexName),
            self.generateIndex(subTree, depth=depth),
            nodes.transition(),
            self.postscript(),
            )

    def generateIndex(self, srcTree, parent=None, depth=99):
        if parent is None:
            parent = nodes.bullet_list()

        if depth<=0:
            return parent

        items=nodes.section()

        for name in sorted(srcTree.keys()):
            uri = self.renderer.makeURI(name)
            text = name.split(".")[-1]
            newItem = nodes.list_item('',
                nodes.paragraph('','', nodes.reference('', text, refuri=uri))
                )
            if srcTree[name]:  # if not empty, recurse
                newItem.append(nodes.bullet_list())
                self.generateIndex(srcTree[name], newItem[-1], depth-1)
            parent.append(newItem)

        return parent

    def postscript(self):
        return self.docString("""\
            Feedback
            --------
            Got a problem with the documentation? Something unclear that could be clearer?
            Want to help improve it? Constructive criticism is very welcome! (preferably in
            the form of suggested rewording)

            Please leave you feedback
            `here <http://kamaelia.sourceforge.net/cgi-bin/blog/blog.cgi?rm=viewpost&nodeid=1142023701>`_
            in reply to the documentation thread in the Kamaelia blog.
            """)

def generateDocumentationFiles(CONFIG):
    MODULES = dict(CONFIG.components.items())
    MODULES.update(dict(CONFIG.prefabs.items()))
    MODULES = [M for M in MODULES.keys() if CONFIG.filterPattern in M]
    for MODULE in MODULES:
        print "Processing: "+MODULE

        cNames = CONFIG.components.get(MODULE,[])
        pNames = CONFIG.prefabs.get(MODULE,[])

        module = __import__(MODULE, [], [], cNames+pNames)
        components = [ getattr(module, c) for c in cNames ]
        prefabs = [ getattr(module, p) for p in pNames ]

        doctree  = formatter.formatModule(MODULE, module, components, prefabs, CONFIG.tocDepth, CONFIG.includeMethods)
        filename = formatter.renderer.makeFilename(MODULE)
        output   = formatter.renderer.render(MODULE, doctree)

        F = open(CONFIG.docdir+"/"+filename, "w")
        F.write(output)
        F.close()

def generateIndices(CONFIG):

    # list of all leaf modules
    MODULES = dict(CONFIG.components.items())
    MODULES.update(dict(CONFIG.prefabs.items()))
    MODULES = MODULES.keys()

    # now build a list of all module sub paths, eg A, A.B, A.B.C
    MODULETREE = {}
    INDEX = {}
    for fullPath in MODULES:
        parts = fullPath.split(".")
        path = ""
        currentNode = MODULETREE
        for part in parts:
            if path!="":
                path+="."
            path+=part
            currentNode[path] = currentNode.get(path,{})
            currentNode = currentNode.get(path,{})
            if path not in INDEX and path not in MODULES:
                INDEX[path] = currentNode

    # MODULETREE now contains a tree structure; at each node is the full path
    # leading up to that point. This tree includes all leaves from MODULES

    # INDEX contains maps all partial paths, excluding modules, to subsets of
    # the module tree structure

    for indexName in INDEX.keys():
        print "Creating index: "+indexName

        doctree  = formatter.formatIndex(indexName, INDEX[indexName], CONFIG.treeDepth)
        filename = formatter.renderer.makeFilename(indexName)
        output   = formatter.renderer.render(indexName, doctree)

        F = open(CONFIG.docdir+"/"+filename, "w")
        F.write(output)
        F.close()

class DocGenConfig(object):
    def __init__(self):
        super(DocGenConfig,self).__init__()
        self.components = {}
        self.prefabs = {}
        self.debug=False
        self.filterPattern=""
        self.docdir="pydoc"
        self.treeDepth=99
        self.tocDepth=99
        self.includeMethods=False

if __name__ == "__main__":

    import sys

    formatter = docFormatter(RenderHTML(titlePrefix="Kamaelia docs : ",debug=False))
#    formatter = docFormatter(RenderPlaintext)

    config = DocGenConfig()

    debug = False
    if debug:
        COMPONENTS = {
            "Kamaelia.ReadFileAdaptor" : ("ReadFileAdaptor",)
        }
        PREFABS = {}
    else:
        C = Kamaelia.Support.Data.Repository.GetAllKamaeliaComponents()
        P = Kamaelia.Support.Data.Repository.GetAllKamaeliaPrefabs()
        COMPONENTS = {}
        for key in C.keys():
            COMPONENTS[".".join(key)] = C[key]
        PREFABS = {}
        for key in P.keys():
            PREFABS[".".join(key)] = P[key]

    config.components = COMPONENTS
    config.prefabs    = PREFABS
    config.docdir     = "pydoc"

    if len(sys.argv)>1:
        config.filterPattern = sys.argv[1]

    config.treeDepth=2
    config.tocDepth=3
    config.includeMethods=True

    generateDocumentationFiles(config)
    generateIndices(config)
<commit_msg>Re-introduced generation of separate pages for each component/prefab. Added breadcrumb trails to each component/prefab entry, both on individual pages and module level docs. Made breadcrumb trail more prominent (converted to heading)<commit_after>#!/usr/bin/python
#
# This program generates Kamaelia's documentation directly from the
# source. I am currently running this as follows:
#
# mkdir ./pydoc/
# ./DocExtractor.py
#
# documentation is placed into the "pydoc" subdirectory

import textwrap
import inspect
import pprint
import time
from docutils import core
from docutils import nodes
import Kamaelia.Support.Data.Repository

from renderHTML      import RenderHTML
from renderPlaintext import RenderPlaintext

from Nodes import boxright

class docFormatter(object):
    def __init__(self, renderer=RenderPlaintext,debug=False):
        super(docFormatter,self).__init__()
        self.renderer = renderer
        self.debug = debug

    uid = 0

    def genUniqueRef(self):
        uid = str(self.uid)
        self.uid+=1
        return uid

    def boxes(self,componentName, label, boxes):
        items = []
        for box in boxes:
            try:
                description = boxes[box]
            except KeyError:
                description = ""
            except TypeError:
                description = "Code uses old style inbox/outbox description - no metadata available"
            items.append((str(box), str(description)))

        docTree= nodes.section('',
                ids   = ["component-"+componentName+"-"+label],
                names = ["component-"+componentName+"-"+label],
                *[ nodes.title('', label),
                   nodes.bullet_list('',
                      *[ nodes.list_item('', nodes.paragraph('', '',
                                                 nodes.strong('', boxname),
                                                 nodes.Text(" : "+boxdesc))
                                             )
                         for (boxname,boxdesc) in items
                       ]
                   ),
                ]
            )
        return docTree

    def docString(self,docstring, main=False):
        if docstring is None:
            docstring = " "
        lines = "\n".split(docstring)
        if len(lines)>1:
            line1 = textwrap.dedent(lines[0])
            rest = textwrap.dedent("\n".join(lines[1:]))
            docstring = line1+"\n"+rest
        else:
            docstring=textwrap.dedent(docstring)

        while docstring[0] == "\n":
            docstring = docstring[1:]
        while docstring[-1] == "\n":
            docstring = docstring[:-1]

        return nodes.section('', *core.publish_doctree(docstring).children)

    def formatArgSpec(self, argspec):
        return pprint.pformat(argspec[0]).replace("[","(").replace("]",")").replace("'","")

    def formatMethodDocStrings(self,X):
        docTree = nodes.section('') #self.emptyTree()

        for method in sorted([x for x in inspect.classify_class_attrs(X) if x[2] == X and x[1] == "method"]):
            if method[0][-7:] == "__super":
                continue
            methodHead = method[0]+self.formatArgSpec(inspect.getargspec(method[3]))

            docTree.append( nodes.section('',
                                ids   = ["component-"+X.__name__+"-method-"+method[0]],
                                names = ["component-"+X.__name__+"-method-"+method[0]],
                                * [ nodes.title('', methodHead) ]
                                  + self.docString(method[3].__doc__)
                            )
                          )

        return docTree

    def formatClassStatement(self, name, bases):
        return "class "+ name+"("+",".join([str(base)[8:-2] for base in bases])+")"

    def formatPrefabStatement(self, name):
        return "prefab: "+name

    def formatComponent(self, X, includeMethods=False):
        CLASSNAME = self.formatClassStatement(X.__name__, X.__bases__)
        CLASSDOC = self.docString(X.__doc__)
        INBOXES = self.boxes(X.__name__,"Inboxes", X.Inboxes)
        OUTBOXES = self.boxes(X.__name__,"Outboxes", X.Outboxes)

        if includeMethods:
            METHODS = [ nodes.section('',
                          nodes.title('', 'Methods defined here'),
                          boxright('',
                              nodes.paragraph('', '',
                                  nodes.strong('', nodes.Text("Warning!"))
                              ),
                              nodes.paragraph('', '',
                                  nodes.Text("You should be using the inbox/outbox interface, not these methods (except construction). This documentation is designed as a roadmap as to their functionalilty for maintainers and new component developers.")
                              ),
                          ),
                          * self.formatMethodDocStrings(X)
                        )
                      ]
        else:
            METHODS = []

        trailTree = self.formatModuleTrail(X.__module__+"."+X.__name__)

        return \
                nodes.section('',
                * [ nodes.title('', CLASSNAME, ids=["component-"+X.__name__]) ]
                  + CLASSDOC
                  + [ INBOXES, OUTBOXES ]
                  + METHODS
                )

    def formatPrefab(self, X):
        CLASSNAME = self.formatPrefabStatement(X.__name__)
        CLASSDOC = self.docString(X.__doc__)

        return nodes.container('',
                * [ nodes.title('', CLASSNAME, ids=["component-"+X.__name__]) ]
                  + CLASSDOC
            )

    def formatModuleTrail(self, moduleName):
        path = moduleName.split(".")

        trail = nodes.paragraph('')
        line = trail

        accum = ""
        firstPass=True
        for element in path:
            if not firstPass:
                accum += "."
            accum += element

            if not firstPass:
                line.append(nodes.Text("."))
            URI = self.renderer.makeURI(accum)
            line.append( nodes.reference('', element, refuri=URI) )

            firstPass=False

        return trail

    def componentList(self, components):
        uris = {}
        for component in components:
            fullname = component.__module__ + "." + component.__name__
            uris[component.__name__] = self.renderer.makeURI(fullname)

        return nodes.container('',
#            nodes.paragraph('', '', nodes.Text('Components and Prefabs')),
            nodes.bullet_list('',
                *[ nodes.list_item('',
                       nodes.paragraph('', '',
                         nodes.strong('', '',
                           nodes.reference('', NAME, refuri=uris[NAME]))# refid='component-'+COMPONENT))
                         )
                       )
                   for NAME in sorted(uris.keys())
                 ]
                )
            )

    def formatComponentPage(self,moduleName, component, includeMethods):
        return self.formatDeclarationPage(moduleName, self.formatComponent, [component, includeMethods])

    def formatPrefabPage(self,moduleName, prefab):
        return self.formatDeclarationPage(moduleName, self.formatPrefab, [prefab])

    def formatDeclarationPage(self, name, method, args):
        parentURI = self.renderer.makeURI(".".join(name.split(".")[:-1]))
        trailTree = self.formatModuleTrail(name)
        declarationTree = method(*args)

        return nodes.section('',
            nodes.title('', '', *trailTree.children),
            nodes.paragraph('', '',
                nodes.Text("For examples and more explanations, see the "),
                nodes.reference('', 'module level docs.', refuri=parentURI)
                ),
            nodes.transition(),
            nodes.section('', *declarationTree),
            nodes.transition(),
            self.postscript(),
            )

    def formatModule(self, moduleName, module, components, prefabs, tocDepth, includeMethods):

        trailTree = self.formatModuleTrail(moduleName)
        moduleTree = self.docString(module.__doc__, main=True)
        toc = self.buildTOC(moduleTree, depth=tocDepth)

        allDeclarations = []

        declarationTrees = {}
        for component in components:
            cTrail = self.formatModuleTrail(moduleName+"."+component.__name__)
            declarationTrees[component.__name__] = nodes.container('',
                nodes.title('','', *cTrail.children),
                    self.formatComponent(component, includeMethods)
            )

        for prefab in prefabs:
            assert(prefab.__name__ not in declarationTrees)
            declarationTrees[prefab.__name__] = self.formatPrefab(prefab)

        for name in sorted(declarationTrees.keys()):
            allDeclarations.extend(declarationTrees[name])

        componentListTree = self.componentList( components + prefabs )

        return nodes.container('',
            nodes.section('',
                nodes.title('','', *trailTree.children),
                componentListTree,
                nodes.paragraph('', nodes.Text("Explanations")),
                toc,
            ),
            nodes.transition(),
            moduleTree,
            nodes.transition(),
            nodes.section('', *allDeclarations),
            nodes.transition(),
            self.postscript()
            )

    def buildTOC(self, srcTree, parent=None, depth=3):
        """Recurse through a source document tree, building a table of contents"""
        if parent is None:
            parent = nodes.bullet_list()

        if depth<=0:
            return parent

        items=nodes.section()

        for n in srcTree.children:
            if isinstance(n, nodes.title):
                refid = self.genUniqueRef()
                n.attributes['ids'].append(refid)
                newItem = nodes.list_item()
                newItem.append(nodes.paragraph('','', nodes.reference('', refid=refid, *n.children)))
                newItem.append(nodes.bullet_list())
                parent.append(newItem)
            elif isinstance(n, nodes.section):
                if len(parent)==0:
                    newItem = nodes.list_item()
                    newItem.append(nodes.bullet_list())
                    parent.append(newItem)
                self.buildTOC( n, parent[-1][-1], depth-1)

        return parent

    def formatIndex(self, indexName, subTree, depth):
        return nodes.section('',
#            nodes.title('', indexName),
            nodes.title('', '', *self.formatModuleTrail(indexName).children),
            self.generateIndex(subTree, depth=depth),
            nodes.transition(),
            self.postscript(),
            )

    def generateIndex(self, srcTree, parent=None, depth=99):
        if parent is None:
            parent = nodes.bullet_list()

        if depth<=0:
            return parent

        items=nodes.section()

        for name in sorted(srcTree.keys()):
            uri = self.renderer.makeURI(name)
            text = name.split(".")[-1]
            newItem = nodes.list_item('',
                nodes.paragraph('','', nodes.reference('', text, refuri=uri))
                )
            if srcTree[name]:  # if not empty, recurse
                newItem.append(nodes.bullet_list())
                self.generateIndex(srcTree[name], newItem[-1], depth-1)
            parent.append(newItem)

        return parent

    def postscript(self):
        return self.docString("""\
            Feedback
            --------
            Got a problem with the documentation? Something unclear that could be clearer?
            Want to help improve it? Constructive criticism is very welcome! (preferably in
            the form of suggested rewording)

            Please leave you feedback
            `here <http://kamaelia.sourceforge.net/cgi-bin/blog/blog.cgi?rm=viewpost&nodeid=1142023701>`_
            in reply to the documentation thread in the Kamaelia blog.
            """)

def generateDocumentationFiles(CONFIG):
    MODULES = dict(CONFIG.components.items())
    MODULES.update(dict(CONFIG.prefabs.items()))
    MODULES = [M for M in MODULES.keys() if CONFIG.filterPattern in M]
    for MODULE in MODULES:
        print "Processing: "+MODULE

        cNames = CONFIG.components.get(MODULE,[])
        pNames = CONFIG.prefabs.get(MODULE,[])

        module = __import__(MODULE, [], [], cNames+pNames)
        components = [ getattr(module, c) for c in cNames ]
        prefabs = [ getattr(module, p) for p in pNames ]

        doctree  = formatter.formatModule(MODULE, module, components, prefabs, CONFIG.tocDepth, CONFIG.includeMethods)
        filename = formatter.renderer.makeFilename(MODULE)
        output   = formatter.renderer.render(MODULE, doctree)

        F = open(CONFIG.docdir+"/"+filename, "w")
        F.write(output)
        F.close()

        for component in components:
            NAME = MODULE+"."+component.__name__
            print "    Component: "+NAME
            filename = formatter.renderer.makeFilename(NAME)
            doctree = formatter.formatComponentPage(NAME, component, CONFIG.includeMethods)
            output   = formatter.renderer.render(NAME, doctree)
            F = open(CONFIG.docdir+"/"+filename, "w")
            F.write(output)
            F.close()

        for prefab in prefabs:
            NAME = MODULE+"."+prefab.__name__
            print "    Prefab: "+NAME
            filename = formatter.renderer.makeFilename(NAME)
            doctree = formatter.formatPrefabPage(NAME, prefab)
            output   = formatter.renderer.render(NAME, doctree)
            F = open(CONFIG.docdir+"/"+filename, "w")
            F.write(output)
            F.close()

def generateIndices(CONFIG):

    # list of all leaf modules
    MODULES = dict(CONFIG.components.items())
    MODULES.update(dict(CONFIG.prefabs.items()))
    MODULES = MODULES.keys()

    # now build a list of all module sub paths, eg A, A.B, A.B.C
    MODULETREE = {}
    INDEX = {}
    for fullPath in MODULES:
        parts = fullPath.split(".")
        path = ""
        currentNode = MODULETREE
        for part in parts:
            if path!="":
                path+="."
            path+=part
            currentNode[path] = currentNode.get(path,{})
            currentNode = currentNode.get(path,{})
            if path not in INDEX and path not in MODULES:
                INDEX[path] = currentNode

    # MODULETREE now contains a tree structure; at each node is the full path
    # leading up to that point. This tree includes all leaves from MODULES

    # INDEX contains maps all partial paths, excluding modules, to subsets of
    # the module tree structure

    for indexName in INDEX.keys():
        print "Creating index: "+indexName

        doctree  = formatter.formatIndex(indexName, INDEX[indexName], CONFIG.treeDepth)
        filename = formatter.renderer.makeFilename(indexName)
        output   = formatter.renderer.render(indexName, doctree)

        F = open(CONFIG.docdir+"/"+filename, "w")
        F.write(output)
        F.close()

class DocGenConfig(object):
    def __init__(self):
        super(DocGenConfig,self).__init__()
        self.components = {}
        self.prefabs = {}
        self.debug=False
        self.filterPattern=""
        self.docdir="pydoc"
        self.treeDepth=99
        self.tocDepth=99
        self.includeMethods=False

if __name__ == "__main__":

    import sys

    formatter = docFormatter(RenderHTML(titlePrefix="Kamaelia docs : ",debug=False))
#    formatter = docFormatter(RenderPlaintext)

    config = DocGenConfig()

    debug = False
    if debug:
        COMPONENTS = {
            "Kamaelia.ReadFileAdaptor" : ("ReadFileAdaptor",)
        }
        PREFABS = {}
    else:
        C = Kamaelia.Support.Data.Repository.GetAllKamaeliaComponents()
        P = Kamaelia.Support.Data.Repository.GetAllKamaeliaPrefabs()
        COMPONENTS = {}
        for key in C.keys():
            COMPONENTS[".".join(key)] = C[key]
        PREFABS = {}
        for key in P.keys():
            PREFABS[".".join(key)] = P[key]

    config.components = COMPONENTS
    config.prefabs    = PREFABS
    config.docdir     = "pydoc"

    if len(sys.argv)>1:
        config.filterPattern = sys.argv[1]

    config.treeDepth=99
    config.tocDepth=3
    config.includeMethods=True

    generateDocumentationFiles(config)
    generateIndices(config)
<|endoftext|>