<commit_before>"""
Implementation from

Goodfellow, Ian, Jean Pouget-Abadie, Mehdi Mirza, Bing Xu, David Warde-Farley, Sherjil Ozair,
Aaron Courville, and Yoshua Bengio. "Generative adversarial nets." In Advances in Neural Information
 Processing Systems, pp. 2672-2680. 2014.
"""
from yann.special.gan import gan
from theano import tensor as T

def simple_gan ( dataset= None, verbose = 1 ):
    """
    This function is a demo example of a sparse autoencoder.
    This is an example code. You should study this code rather than merely run it.

    Args:
        dataset: Supply a dataset.
        verbose: Similar to the rest of the dataset.
    """
    optimizer_params =  {
                "momentum_type"       : 'nesterov',
                "momentum_params"     : (0.65, 0.95, 30),
                "regularization"      : (0.000, 0.000),
                "optimizer_type"      : 'rmsprop',
                "id"                  : "main"
                        }

    dataset_params  = {
                            "dataset"   : dataset,
                            "type"      : 'xy',
                            "id"        : 'data'
                    }

    visualizer_params = {
                    "root"       : '.',
                    "frequency"  : 1,
                    "sample_size": 225,
                    "rgb_filters": False,
                    "debug_functions" : False,
                    "debug_layers": True,
                    "id"         : 'main'
                        }

    # intitialize the network
    net = gan (      borrow = True,
                     verbose = verbose )

    net.add_module ( type = 'datastream',
                     params = dataset_params,
                     verbose = verbose )

    net.add_module ( type = 'visualizer',
                     params = visualizer_params,
                     verbose = verbose
                    )

    #z - latent space created by random gaussian layer
    net.add_layer(type = 'random',
                        id = 'z',
                        num_neurons = (500,64),
                        distribution = 'normal',
                        mu = 0,
                        sigma = 1,
                        verbose = verbose)

    #x - inputs come from dataset 1 X 784
    net.add_layer ( type = "input",
                    id = "x",
                    verbose = verbose,
                    datastream_origin = 'data', # if you didnt add a dataset module, now is
                                                 # the time.
                    mean_subtract = False )

    #G(z) contains params theta_g - 100 X 784 - creates images of 1 X 784
    net.add_layer ( type = "dot_product",
                    origin = "z",
                    id = "G(z)",
                    num_neurons = 784,
                    activation = 'relu',
                    verbose = verbose
                    )

    #D(x) - Contains params theta_d - 784 X 256 - first layer of D, creates features 1 X 256.
    net.add_layer ( type = "dot_product",
                    id = "D(x)",
                    origin = "x",
                    num_neurons = 512,
                    activation = ('maxout','maxout',2),
                    #activation = 'relu',
                    verbose = verbose
                    )

    #D(x) - Contains the same params theta_d from above
    #784 X 256 - Takes inputs from G(z) and outputs features 1 X 256.
    net.add_layer ( type = "dot_product",
                    id = "D(G(z))",
                    origin = "G(z)",
                    num_neurons = 512,
                    input_params = net.dropout_layers["D(x)"].params, # must be the same params,
                                                        # this way it remains the same network.
                    activation = ('maxout','maxout',2),
                    #activation = 'relu',
                    verbose = verbose
                    )

    #C(D(G(z))) fake - the classifier for fake/real that always predicts fake
    net.add_layer ( type = "dot_product",
                    id = "fake",
                    origin = "D(G(z))",
                    num_neurons = 1,
                    activation = 'sigmoid',
                    verbose = verbose
                    )

    #C(D(x)) - This is the opposite of C(D(G(z))), real
    net.add_layer ( type = "dot_product",
                    id = "real",
                    origin = "D(x)",
                    num_neurons = 1,
                    input_params = net.dropout_layers["fake"].params, # Again share their parameters
                    activation = 'sigmoid',
                    verbose = verbose
                    )

    #C(D(x)) - This is the opposite of C(D(G(z))), real
    net.add_layer ( type = "classifier",
                    id = "softmax",
                    origin = "D(x)",
                    num_classes = 10,
                    activation = 'softmax',
                    verbose = verbose
                    )

    # objective layers
    # fake objective
    net.add_layer ( type = "objective",
                    id = "fake_obj",
                    origin = "fake", # this is useless anyway.
                    layer_type = 'value',
                    objective = -T.mean(T.log(1-net.layers['fake'].output)),
                    datastream_origin = 'data',
                    verbose = verbose
                    )

    #real objective
    net.add_layer ( type = "objective",
                    id = "real_obj",
                    origin = "real", # this is useless anyway.
                    layer_type = 'value',
                    objective = -T.mean(T.log(net.layers['real'].output)),
                    datastream_origin = 'data',
                    verbose = verbose
                    )

    #softmax objective.
    net.add_layer ( type = "objective",
                    id = "classifier_obj",
                    origin = "softmax",
                    objective = "cce",
                    datastream_origin = 'data',
                    verbose = verbose
                    )

    from yann.utils.graph import draw_network
    draw_network(net.graph, filename = 'gan.png')
    net.pretty_print()

    net.cook (  objective_layers = ["classifier_obj","real_obj","fake_obj"],
                optimizer_params = optimizer_params,
                generator_layers = ["G(z)"],
                discriminator_layers = ["D(x)"],
                classifier_layers = ["D(x)","softmax"],
                softmax_layer = "softmax",
                verbose = verbose )

    learning_rates = (0.05, 0.01)

    net.train( epochs = (10),
               k = 30,  # refer to Ian Goodfellow's paper Algorithm 1.
               validate_after_epochs = 1,
               training_accuracy = True,
               show_progress = True,
               early_terminate = True,
               verbose = verbose)

if __name__ == '__main__':
    import sys
    dataset = None
    if len(sys.argv) > 1:
        if sys.argv[1] == 'create_dataset':
            from yann.special.datasets import cook_mnist
            data = cook_mnist (verbose = 2)
            dataset = data.dataset_location()
        else:
            dataset = sys.argv[1]
    else:
        print "provide dataset"

    if dataset is None:
        print " creating a new dataset to run through"
        from yann.special.datasets import cook_mnist
        data = cook_mnist (verbose = 2)
        dataset = data.dataset_location()

    simple_gan ( dataset, verbose = 2 )<commit_msg>update to gan<commit_after>"""
Implementation from

Goodfellow, Ian, Jean Pouget-Abadie, Mehdi Mirza, Bing Xu, David Warde-Farley, Sherjil Ozair,
Aaron Courville, and Yoshua Bengio. "Generative adversarial nets." In Advances in Neural Information
 Processing Systems, pp. 2672-2680. 2014.
"""
from yann.special.gan import gan
from theano import tensor as T

def simple_gan ( dataset= None, verbose = 1 ):
    """
    This function is a demo example of a sparse autoencoder.
    This is an example code. You should study this code rather than merely run it.

    Args:
        dataset: Supply a dataset.
        verbose: Similar to the rest of the dataset.
    """
    optimizer_params =  {
                "momentum_type"       : 'nesterov',
                "momentum_params"     : (0.65, 0.95, 30),
                "regularization"      : (0.000, 0.000),
                "optimizer_type"      : 'rmsprop',
                "id"                  : "main"
                        }

    dataset_params  = {
                            "dataset"   : dataset,
                            "type"      : 'xy',
                            "id"        : 'data'
                    }

    visualizer_params = {
                    "root"       : '.',
                    "frequency"  : 1,
                    "sample_size": 225,
                    "rgb_filters": False,
                    "debug_functions" : False,
                    "debug_layers": True,
                    "id"         : 'main'
                        }

    # intitialize the network
    net = gan (      borrow = True,
                     verbose = verbose )

    net.add_module ( type = 'datastream',
                     params = dataset_params,
                     verbose = verbose )

    net.add_module ( type = 'visualizer',
                     params = visualizer_params,
                     verbose = verbose
                    )

    #z - latent space created by random gaussian layer
    net.add_layer(type = 'random',
                        id = 'z',
                        num_neurons = (500,64),
                        distribution = 'normal',
                        mu = 0,
                        sigma = 1,
                        verbose = verbose)

    #x - inputs come from dataset 1 X 784
    net.add_layer ( type = "input",
                    id = "x",
                    verbose = verbose,
                    datastream_origin = 'data', # if you didnt add a dataset module, now is
                                                 # the time.
                    mean_subtract = False )

    #G(z) contains params theta_g - 100 X 784 - creates images of 1 X 784
    net.add_layer ( type = "dot_product",
                    origin = "z",
                    id = "G(z)",
                    num_neurons = 784,
                    activation = 'tanh',
                    verbose = verbose
                    )

    #D(x) - Contains params theta_d - 784 X 256 - first layer of D, creates features 1 X 256.
    net.add_layer ( type = "dot_product",
                    id = "D(x)",
                    origin = "x",
                    num_neurons = 512,
                    activation = ('maxout','maxout',2),
                    #activation = 'relu',
                    verbose = verbose
                    )

    #D(x) - Contains the same params theta_d from above
    #784 X 256 - Takes inputs from G(z) and outputs features 1 X 256.
    net.add_layer ( type = "dot_product",
                    id = "D(G(z))",
                    origin = "G(z)",
                    num_neurons = 512,
                    input_params = net.dropout_layers["D(x)"].params, # must be the same params,
                                                        # this way it remains the same network.
                    activation = ('maxout','maxout',2),
                    #activation = 'relu',
                    verbose = verbose
                    )

    #C(D(G(z))) fake - the classifier for fake/real that always predicts fake
    net.add_layer ( type = "dot_product",
                    id = "fake",
                    origin = "D(G(z))",
                    num_neurons = 1,
                    activation = 'sigmoid',
                    verbose = verbose
                    )

    #C(D(x)) - This is the opposite of C(D(G(z))), real
    net.add_layer ( type = "dot_product",
                    id = "real",
                    origin = "D(x)",
                    num_neurons = 1,
                    input_params = net.dropout_layers["fake"].params, # Again share their parameters
                    activation = 'sigmoid',
                    verbose = verbose
                    )

    #C(D(x)) - This is the opposite of C(D(G(z))), real
    net.add_layer ( type = "classifier",
                    id = "softmax",
                    origin = "D(x)",
                    num_classes = 10,
                    activation = 'softmax',
                    verbose = verbose
                    )

    # objective layers
    # fake objective
    net.add_layer ( type = "objective",
                    id = "fake_obj",
                    origin = "fake", # this is useless anyway.
                    layer_type = 'value',
                    objective = -T.mean(T.log(1-net.layers['fake'].output)),
                    datastream_origin = 'data',
                    verbose = verbose
                    )

    #real objective
    net.add_layer ( type = "objective",
                    id = "real_obj",
                    origin = "real", # this is useless anyway.
                    layer_type = 'value',
                    objective = -T.mean(T.log(net.layers['real'].output)),
                    datastream_origin = 'data',
                    verbose = verbose
                    )

    #softmax objective.
    net.add_layer ( type = "objective",
                    id = "classifier_obj",
                    origin = "softmax",
                    objective = "cce",
                    datastream_origin = 'data',
                    verbose = verbose
                    )

    from yann.utils.graph import draw_network
    draw_network(net.graph, filename = 'gan.png')
    net.pretty_print()

    net.cook (  objective_layers = ["classifier_obj","real_obj","fake_obj"],
                optimizer_params = optimizer_params,
                generator_layers = ["G(z)"],
                discriminator_layers = ["D(x)"],
                classifier_layers = ["D(x)","softmax"],
                softmax_layer = "softmax",
                verbose = verbose )

    learning_rates = (0.05, 0.01)

    net.train( epochs = (10),
               k = 10,  # refer to Ian Goodfellow's paper Algorithm 1.
               validate_after_epochs = 1,
               training_accuracy = True,
               show_progress = True,
               early_terminate = True,
               verbose = verbose)

if __name__ == '__main__':
    import sys
    dataset = None
    if len(sys.argv) > 1:
        if sys.argv[1] == 'create_dataset':
            from yann.special.datasets import cook_mnist
            data = cook_mnist (verbose = 2)
            dataset = data.dataset_location()
        else:
            dataset = sys.argv[1]
    else:
        print "provide dataset"

    if dataset is None:
        print " creating a new dataset to run through"
        from yann.special.datasets import cook_mnist
        data = cook_mnist (verbose = 2)
        dataset = data.dataset_location()

    simple_gan ( dataset, verbose = 2 )<|endoftext|>


<commit_before>import config
import cv2
import cv2.cv as cv
import numpy as np

class Frame:
    def __init__(self, original):
        self.original = original
        self.processed = None
        self.printed = False

    def process(self):
        # Frame can be processed only once...
        if self.processed is not None: return self.processed

        # convert to grayscale
        gray = cv2.cvtColor(self.original, cv2.COLOR_BGR2GRAY)

        # threshold the image
        # ret, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)

        thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY, 11, 2)

        # th2 = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
        #     cv2.THRESH_BINARY,11,2)

        self.processed = thresh

    def find_glints(self):
        # convert to grayscale
        self.processed = self.original
        img = cv2.cvtColor(self.processed, cv2.COLOR_BGR2GRAY)

        # get rid of too bright pixels
        # create gray background
        blank_image = np.zeros(img.shape, np.uint8)
        blank_image[:, :] = 100

        # create mask for too bright areas
        ret, mask = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        mask_inv = cv2.bitwise_not(mask)

        # paint all normal areas black on background
        img1_bg = cv2.bitwise_and(img, img, mask = mask_inv)

        # paint all bright areas black on foreground
        img2_fg = cv2.bitwise_and(blank_image, blank_image, mask = mask)

        # join two images
        img = cv2.add(img1_bg, img2_fg)

        cv2.imshow(config.WINDOW_NAME, img1_bg)

        # erode remaining white areas
        kernel = np.ones((10,10), np.uint8)
        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

        cv2.imshow('4', img)

        # find edges
        edges = cv2.Canny(img, 50, 130)
        cv2.imshow('2', edges)

        # find contours
        contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        # Find largest contour
        largest_area = 0
        contour = None
        for i in contours:
            area = cv2.contourArea(i)
            if area >= largest_area and area >= 50:
                contour = i
                largest_area = area

        if contour is not None and contour.any():
            (x,y),radius = cv2.minEnclosingCircle(contour)
            center = (int(x),int(y))
            radius = int(radius)
            img = cv2.circle(self.original,center,radius,(0,255,0),2)
            #
            # cv2.drawContours(self.processed, contour, -1, (0,255,0), 3)
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(self.original, 'Largest contour area: ' + str(largest_area), (10,10), font, 0.5, (255,255,255), 2)
            cv2.putText(self.original, 'Center of the contour: (' + str(x) + ', ' + str(y) + ')', (10,30), font, 0.5, (255,255,255), 2)

        cv2.imshow('3', self.original)

        # cv2.imshow(config.WINDOW_NAME, img)
        # edges = cv2.Canny(img, 0, 200)
        # #
        # cv2.imshow(config.WINDOW_NAME, edges)

        # ret, thresh = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        # #
        # contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
        # cv2.drawContours(self.processed, contours, -1, (0,255,0), 3)
        #
        # if not self.printed:
        #     print len(contours)
        # self.printed = True

        # img = img[c1:c1+25,r1:r1+25]

    def show(self):
        cv2.imshow(config.WINDOW_NAME, self.processed)

    def show_original(self):
        cv2.imshow(config.WINDOW_NAME, self.original)
<commit_msg>Update<commit_after>import config
import cv2
import cv2.cv as cv
import numpy as np

class Frame:
    def __init__(self, original):
        self.original = original
        self.processed = None
        self.printed = False

    def process(self):
        # Frame can be processed only once...
        if self.processed is not None: return self.processed

        # convert to grayscale
        gray = cv2.cvtColor(self.original, cv2.COLOR_BGR2GRAY)

        # threshold the image
        # ret, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)

        thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY, 11, 2)

        # th2 = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
        #     cv2.THRESH_BINARY,11,2)

        self.processed = thresh

    def find_glints(self):
        # convert to grayscale
        self.processed = self.original
        img = cv2.cvtColor(self.processed, cv2.COLOR_BGR2GRAY)

        # get rid of too bright pixels
        # create gray background
        blank_image = np.zeros(img.shape, np.uint8)
        blank_image[:, :] = 100

        # create mask for too bright areas
        ret, mask = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        mask_inv = cv2.bitwise_not(mask)

        # paint all normal areas black on background
        img1_bg = cv2.bitwise_and(img, img, mask = mask_inv)

        # paint all bright areas black on foreground
        img2_fg = cv2.bitwise_and(blank_image, blank_image, mask = mask)

        # join two images
        img = cv2.add(img1_bg, img2_fg)

        cv2.imshow(config.WINDOW_NAME, img1_bg)

        # erode remaining white areas
        kernel = np.ones((10,10), np.uint8)
        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

        cv2.imshow('4', img)

        # find edges
        edges = cv2.Canny(img, 50, 130)
        cv2.imshow('2', edges)

        # find contours
        contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        # Find largest contour
        largest_area = 0
        contour = None
        for i in contours:
            area = cv2.contourArea(i)
            if area >= largest_area and area >= 50:
                contour = i
                largest_area = area

        if contour is not None and contour.any():
            (x,y),radius = cv2.minEnclosingCircle(contour)
            center = (int(x),int(y))
            radius = int(radius)
            img = cv2.circle(self.original,center,radius,(0,255,0),2)
            #
            # cv2.drawContours(self.processed, contour, -1, (0,255,0), 3)
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(self.original, 'Largest contour area: ' + str(largest_area), (10,10), font, 0.3, (255,255,255), 2)
            cv2.putText(self.original, 'Center of the contour: (' + str(x) + ', ' + str(y) + ')', (10,30), font, 0.3, (255,255,255), 2)

        cv2.imshow('3', self.original)

        # cv2.imshow(config.WINDOW_NAME, img)
        # edges = cv2.Canny(img, 0, 200)
        # #
        # cv2.imshow(config.WINDOW_NAME, edges)

        # ret, thresh = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        # #
        # contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
        # cv2.drawContours(self.processed, contours, -1, (0,255,0), 3)
        #
        # if not self.printed:
        #     print len(contours)
        # self.printed = True

        # img = img[c1:c1+25,r1:r1+25]

    def show(self):
        cv2.imshow(config.WINDOW_NAME, self.processed)

    def show_original(self):
        cv2.imshow(config.WINDOW_NAME, self.original)
<|endoftext|>


<commit_before>import config
import cv2
import cv2.cv as cv
import numpy as np

class Frame:
    def __init__(self, original):
        self.original = original
        self.processed = None
        self.printed = False

    def process(self):
        # Frame can be processed only once...
        if self.processed is not None: return self.processed

        # convert to grayscale
        gray = cv2.cvtColor(self.original, cv2.COLOR_BGR2GRAY)

        # threshold the image
        # ret, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)

        thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY, 11, 2)

        # th2 = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
        #     cv2.THRESH_BINARY,11,2)

        self.processed = thresh

    def find_glints(self):
        # convert to grayscale
        self.processed = self.original
        img = cv2.cvtColor(self.processed, cv2.COLOR_BGR2GRAY)

        # get rid of too bright pixels
        create gray background
        blank_image = np.zeros(img.shape, np.uint8)
        blank_image[:, :] = 100

        # create mask for too bright areas
        ret, mask = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        mask_inv = cv2.bitwise_not(mask)

        # paint all normal areas black on background
        img1_bg = cv2.bitwise_and(img, img, mask = mask_inv)

        # paint all bright areas black on foreground
        img2_fg = cv2.bitwise_and(blank_image, blank_image, mask = mask)

        # join two images
        img = cv2.add(img1_bg, img2_fg)

        cv2.imshow(config.WINDOW_NAME, img1_bg)

        # erode remaining white areas
        kernel = np.ones((10,10), np.uint8)
        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

        cv2.imshow('4', img)

        # find edges
        edges = cv2.Canny(img, 50, 130)
        cv2.imshow('2', edges)

        # find contours
        contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        # Find largest contour
        largest_area = 0
        contour = None
        for i in contours:
            area = cv2.contourArea(i)
            if area >= largest_area and area >= 50:
                contour = i
                largest_area = area

        if contour is not None and contour.any():
            (x,y),radius = cv2.minEnclosingCircle(contour)
            center = (int(x),int(y))
            radius = int(radius)
            img = cv2.circle(self.original,center,radius,(0,255,0),2)
            #
            # cv2.drawContours(self.processed, contour, -1, (0,255,0), 3)
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(self.original, 'Largest contour area: ' + str(largest_area), (10,10), font, 0.3, (255,255,255), 2)
            cv2.putText(self.original, 'Center of the contour: (' + str(x) + ', ' + str(y) + ')', (10,30), font, 0.3, (255,255,255), 2)

        cv2.imshow('3', self.original)

        # cv2.imshow(config.WINDOW_NAME, img)
        # edges = cv2.Canny(img, 0, 200)
        # #
        # cv2.imshow(config.WINDOW_NAME, edges)

        # ret, thresh = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        # #
        # contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
        # cv2.drawContours(self.processed, contours, -1, (0,255,0), 3)
        #
        # if not self.printed:
        #     print len(contours)
        # self.printed = True

        # img = img[c1:c1+25,r1:r1+25]

    def show(self):
        cv2.imshow(config.WINDOW_NAME, self.processed)

    def show_original(self):
        cv2.imshow(config.WINDOW_NAME, self.original)
<commit_msg>Update<commit_after>import config
import cv2
import cv2.cv as cv
import numpy as np

class Frame:
    def __init__(self, original):
        self.original = original
        self.processed = None
        self.printed = False

    def process(self):
        # Frame can be processed only once...
        if self.processed is not None: return self.processed

        # convert to grayscale
        gray = cv2.cvtColor(self.original, cv2.COLOR_BGR2GRAY)

        # threshold the image
        # ret, thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)

        thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY, 11, 2)

        # th2 = cv2.adaptiveThreshold(gray,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
        #     cv2.THRESH_BINARY,11,2)

        self.processed = thresh

    def find_glints(self):
        # convert to grayscale
        self.processed = self.original
        img = cv2.cvtColor(self.processed, cv2.COLOR_BGR2GRAY)

        # get rid of too bright pixels
        # create gray background
        blank_image = np.zeros(img.shape, np.uint8)
        blank_image[:, :] = 100

        # create mask for too bright areas
        ret, mask = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        mask_inv = cv2.bitwise_not(mask)

        # paint all normal areas black on background
        img1_bg = cv2.bitwise_and(img, img, mask = mask_inv)

        # paint all bright areas black on foreground
        img2_fg = cv2.bitwise_and(blank_image, blank_image, mask = mask)

        # join two images
        img = cv2.add(img1_bg, img2_fg)

        cv2.imshow(config.WINDOW_NAME, img1_bg)

        # erode remaining white areas
        kernel = np.ones((10,10), np.uint8)
        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

        cv2.imshow('4', img)

        # find edges
        edges = cv2.Canny(img, 50, 130)
        cv2.imshow('2', edges)

        # find contours
        contours, hierarchy = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        # Find largest contour
        largest_area = 0
        contour = None
        for i in contours:
            area = cv2.contourArea(i)
            if area >= largest_area and area >= 50:
                contour = i
                largest_area = area

        if contour is not None and contour.any():
            (x,y),radius = cv2.minEnclosingCircle(contour)
            center = (int(x),int(y))
            radius = int(radius)
            img = cv2.circle(self.original,center,radius,(0,255,0),2)
            #
            # cv2.drawContours(self.processed, contour, -1, (0,255,0), 3)
            font = cv2.FONT_HERSHEY_SIMPLEX
            cv2.putText(self.original, 'Largest contour area: ' + str(largest_area), (10,10), font, 0.3, (255,255,255), 2)
            cv2.putText(self.original, 'Center of the contour: (' + str(x) + ', ' + str(y) + ')', (10,30), font, 0.3, (255,255,255), 2)

        cv2.imshow('3', self.original)

        # cv2.imshow(config.WINDOW_NAME, img)
        # edges = cv2.Canny(img, 0, 200)
        # #
        # cv2.imshow(config.WINDOW_NAME, edges)

        # ret, thresh = cv2.threshold(img, 200, 255, cv2.THRESH_BINARY)
        # #
        # contours, hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
        # cv2.drawContours(self.processed, contours, -1, (0,255,0), 3)
        #
        # if not self.printed:
        #     print len(contours)
        # self.printed = True

        # img = img[c1:c1+25,r1:r1+25]

    def show(self):
        cv2.imshow(config.WINDOW_NAME, self.processed)

    def show_original(self):
        cv2.imshow(config.WINDOW_NAME, self.original)
<|endoftext|>


<commit_before>from flask import render_template, Blueprint

main_blueprint  = Blueprint("main", __name__)

@main_blueprint.route('/')
def index():
	return render_template('index.html')

@papertalk.route('/about')
def about():
	return render_template('about.html')

@papertalk.route('/get-involved')
def get_involved():
	return render_template('get-involved.html')

<commit_msg>fix the routes to work with changes reflected in master<commit_after>from flask import render_template, Blueprint

main_blueprint  = Blueprint("main", __name__)

@main_blueprint.route('/')
def index():
	return render_template('index.html')

@main_blueprint.route('/about')
def about():
	return render_template('about.html')

@main_blueprint.route('/get-involved')
def get_involved():
	return render_template('get-involved.html')

<|endoftext|>


<commit_before># -*- coding: latin-1 -*-
# Copyright (c) 2010 Australian Government, Department of Environment, Heritage, Water and the Arts
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

'''
base 64 encoded gif images for the GUI buttons
'''

class app_img:
    format='gif'
    data='''R0lGODlhEAAQAMZdAAAAADYePzQhSjkiNDslKDImSD4mOTYrNjkrSkYsLSw1QzA1QSE+JSY9Iz80
            Oj81RyhCIkc4LydDKjk/IjhANCdHKSVJJUc+PCZKKipKKSpLKE89SihMLStMKSdNKyhNKidPLCxP
            LSlQLCdRKCdRLCxQLClTK1NESFxDVW1FQ0tRYGBOY1RVVVtWWL4znME2j7k4k7RGGMI1mqNOE1xg
            ZrtAlKpSFlhkVrdBnbNPI7pDlKtVIcRFjXdhYK1WMF1ocL1Ik2hmabtLkcpJmc1Kl8RMqLpSl7tT
            jHBubr9fMchTo8xUo89Tp3F3ZsZZoMFoQbpsNcJqQsNgp8NsRrl1QcB1Tr94TcGBW8aDbL+KWsWK
            XMCNWs2JZP//////////////////////////////////////////////////////////////////
            /////////////////////////////////////////////////////////////////////////yH5
            BAEKAH8ALAAAAAAQABAAAAeQgH+Cg4SFggCIiYqGfwApGwYTBwIDJwCMAEgPDgEFCxQ0l4OKPREJ
            CAoEF0Gihw0VAE03KiwoLSs/rY0MGYq+l4ojHBgdISUmFroAUkoAEh4aJCIgHxDKTkS/i4pDLjhG
            QEc8S8pcWABFMDo1Qi8yTMpbV9qJhABaT1NRVlVUWbqjksTI4WPHDBtQAB6iByAQADs='''

class shp_img:
    format='gif'
    data='''R0lGODlhEAAQAMIFABAQEIBnII+HgLS0pfDwsC8gIC8gIC8gICH5BAEKAAcALAAAAAAQABAAAAND
            eLrcJzBKqcQIN+MtwAvTNHTPSJwoQAigxwpouo4urZ7364I4cM8kC0x20n2GRGEtJGl9NFBMkBny
            HHzYrNbB7XoXCQA7'''

class dir_img:
    format='gif',
    data='''R0lGODlhEAAQAMZUABAQEB8QEB8YEC8gIC8vIEA4ME9IQF9IIFpTSWBXQHBfUFBoj3NlRoBnII9v
            IIBwUGB3kH93YIZ5UZ94IJB/YIqAcLB/EI+IcICHn4+HgMCHEI6Oe4CPn4+PgMCQANCHEJ+PgICX
            r9CQANCQEJ+XgJKanaCgkK+fgJykoaKjo7CgkKimk+CfIKKoo6uoleCgMLCnkNCnUKuwpLSvkrSv
            mfCoMLWyn7+wkM+vcLS0pfCwML+4kPC3QNDAgM+/kPDAQP+/UODIgP/IUODQoP/QUPDQgP/QYP/P
            cPDYgP/XYP/XcP/YgPDgkP/ggP/gkPDnoP/noPDwoPDwsP/woP//////////////////////////
            ////////////////////////////////////////////////////////////////////////////
            /////////////////////////////////////////////////////////////////////////yH5
            BAEKAH8ALAAAAAAQABAAAAe1gH+Cg4SFhoQyHBghKIeEECV/ORwtEDYwmJg0hikLCzBDUlJTUCoz
            hZ4LKlGjUFBKJiQkIB0XgypPpFBLSb2+toImT643N5gnJ7IgIBkXJExQQTBN1NVNSkoxFc9OMDtK
            vkZEQjwvDC4gSNJNR0lGRkI/PDoNEn8gRTA+Su9CQPM1PhxY8SdDj2nw4umowWJEAwSCLqjAIaKi
            Bw0WLExwcGBDRAoRHihIYKAAgQECAARwxFJQIAA7'''

class xls_img:
    format='gif'
    data='''R0lGODlhEAAQAPcAAAAAAIAAAACAAICAAAAAgIAAgACAgICAgMDAwP8AAAD/AP//AAAA//8A/wD/
            /////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
            AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMwAAZgAAmQAAzAAA/wAzAAAzMwAzZgAzmQAzzAAz/wBm
            AABmMwBmZgBmmQBmzABm/wCZAACZMwCZZgCZmQCZzACZ/wDMAADMMwDMZgDMmQDMzADM/wD/AAD/
            MwD/ZgD/mQD/zAD//zMAADMAMzMAZjMAmTMAzDMA/zMzADMzMzMzZjMzmTMzzDMz/zNmADNmMzNm
            ZjNmmTNmzDNm/zOZADOZMzOZZjOZmTOZzDOZ/zPMADPMMzPMZjPMmTPMzDPM/zP/ADP/MzP/ZjP/
            mTP/zDP//2YAAGYAM2YAZmYAmWYAzGYA/2YzAGYzM2YzZmYzmWYzzGYz/2ZmAGZmM2ZmZmZmmWZm
            zGZm/2aZAGaZM2aZZmaZmWaZzGaZ/2bMAGbMM2bMZmbMmWbMzGbM/2b/AGb/M2b/Zmb/mWb/zGb/
            /5kAAJkAM5kAZpkAmZkAzJkA/5kzAJkzM5kzZpkzmZkzzJkz/5lmAJlmM5lmZplmmZlmzJlm/5mZ
            AJmZM5mZZpmZmZmZzJmZ/5nMAJnMM5nMZpnMmZnMzJnM/5n/AJn/M5n/Zpn/mZn/zJn//8wAAMwA
            M8wAZswAmcwAzMwA/8wzAMwzM8wzZswzmcwzzMwz/8xmAMxmM8xmZsxmmcxmzMxm/8yZAMyZM8yZ
            ZsyZmcyZzMyZ/8zMAMzMM8zMZszMmczMzMzM/8z/AMz/M8z/Zsz/mcz/zMz///8AAP8AM/8AZv8A
            mf8AzP8A//8zAP8zM/8zZv8zmf8zzP8z//9mAP9mM/9mZv9mmf9mzP9m//+ZAP+ZM/+ZZv+Zmf+Z
            zP+Z///MAP/MM//MZv/Mmf/MzP/M////AP//M///Zv//mf//zP///ywAAAAAEAAQAAAIngBfuUKF
            ipBBg4MS9umTJYsrBAheSZwokGBBhwgeaNzIUSOhLKgydhz5EdWrB4oOelT5kdDJLwgUKRpEKOUX
            Gtpannzw5ZVNQje15czicmNPg1lwCtW5EeirQV+IEtI2iOjOmh9dQc2SimqWQa4efGzYcGZUr4NQ
            ddSWimwWr33UahRKly61qn0Iza1rl9qXKVIPIkyY8Mtft4gTTwkIADs='''

class xsl_img:
    format='gif'
    data='''R0lGODdhEAAQAOMPAAAAAAAAgAAAmQAA/zNmmQCAgDNm/zOZAIaGhjOZ/zPM/8DAwKbK8DP///Hx
            8f///ywBAAAADwAQAAAEWBDJSeW76Or9Vn4f5zzOAp5kOo5AC2QOMxaFQcrP+zDCUzyNROAhkL14
            pEJDcQiMijqkIXEYDIsOXWwU6N5Yn5VKpSWYz2fwRcwmldFo9bidhc3Hrrw+HwEAOw=='''

class log_img:
    format='gif'
    data='''R0lGODlhEAAQAIQQAG9s0oJ5eatyP6tycpePj6ulP6ulctWeOaulpdWentXSOcvHx9XS0v/MzP//
            zP///y8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gICH5BAEK
            ABAALAAAAAAQABAAAAViICSOUNMwjEOOhyIUyhAbzMoAgJAQi9EjtRGAIXgUjw9CUDR8OJ9OJakJ
            fUqFjCSBZ11CqNWkt7ndLqLjbFg8zZa5bOw6znSfoVfm3clYIP5eEH4EAQFlCAsrEH2ICygoJCEA
            Ow=='''
<commit_msg>Icon updated<commit_after># -*- coding: latin-1 -*-
# Copyright (c) 2010 Australian Government, Department of Environment, Heritage, Water and the Arts
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

'''
base 64 encoded gif images for the GUI buttons
'''

class app_img:
    format='gif'
    data='''R0lGODlhEAAQAOeRACcLIiAbCSAjCjMdMzsfMjUkGUcmRjwwJ0YqRj4xJVwoUFguRkU2MS0/LzQ8
            PC8/LzM+QTJCMDJCQTpCQCxIME1CIXQyYW48KTpLO1REPEpKSktKS01KSkpLSkxLTE1LS0VNUDtS
            PD9PT0tMTExMTE1MTUxNTU1NTU5NTUFUQFFOTkZRU1BPTU9QUUVTVF9PO1JUVVRVSnlNMEVeRlZX
            W1ZYVVZYWF5XVFBdUkpfX2RZXIZMgVtdX11eX1tfW1xfW1tfXqZEkFtgW2NfYWZgW2tdal9iXk9m
            Z19iYk9pTqZIn5lNlU1rTp1XOF9lZVxnXF5oXlNrZ59eM1FzU1dyVcVItJJmSl5ycq1Wp1t0cLlU
            tWB1eF52dmKBY12DX9RWwGN/f+RSzaVzTdNbxmaEhLlzRdFhs2WJZWeJZmOMZ7Z2UXGGhm2IiGqJ
            iKV+VmuKimyKi26Ojm2ScnGQkGuWb22Wb3OTk+xp2+dr5eF73Pl154SfoMKYeIampoimptiYbPuB
            8viD8I2sq/KJ7pOtrZGuruebbpGvr/+I/Ja1tdqrf9i3i/iweviwhP+zhf/Hif/Lpf//////////
            ////////////////////////////////////////////////////////////////////////////
            ////////////////////////////////////////////////////////////////////////////
            ////////////////////////////////////////////////////////////////////////////
            ////////////////////////////////////////////////////////////////////////////
            ////////////////////////////////////////////////////////////////////////////
            /////////////////////////////////////////////////////yH5BAEKAP8ALAAAAAAQABAA
            AAjRAP8JHEiwoMGBGk6MOChQgwYgEnJwcdGjoAYbIo5EyQIGjh02axyYIOjkSqI4bci8mdPnECEk
            Ggi2WFHIj6A9WyDQgEFiYIcfKR5MAMHDhJAQTCLUIGgEQ5cZDZKgqUMnDRUfMQVu8ADFi5wzUyjg
            KLEh6z8PCAZhGfIEBQALZgAtMUCwyI48Y6roQRToThglAzYMZEFkgRY8X4Io0CEgBkENByDxYUAg
            QAU3jB6JKUBQxYtFigw5avSnjBQZN8wKTGBFTZMLGRwy/Mfhg2qCAQEAOw=='''

class shp_img:
    format='gif'
    data='''R0lGODlhEAAQAMIFABAQEIBnII+HgLS0pfDwsC8gIC8gIC8gICH5BAEKAAcALAAAAAAQABAAAAND
            eLrcJzBKqcQIN+MtwAvTNHTPSJwoQAigxwpouo4urZ7364I4cM8kC0x20n2GRGEtJGl9NFBMkBny
            HHzYrNbB7XoXCQA7'''

class dir_img:
    format='gif',
    data='''R0lGODlhEAAQAMZUABAQEB8QEB8YEC8gIC8vIEA4ME9IQF9IIFpTSWBXQHBfUFBoj3NlRoBnII9v
            IIBwUGB3kH93YIZ5UZ94IJB/YIqAcLB/EI+IcICHn4+HgMCHEI6Oe4CPn4+PgMCQANCHEJ+PgICX
            r9CQANCQEJ+XgJKanaCgkK+fgJykoaKjo7CgkKimk+CfIKKoo6uoleCgMLCnkNCnUKuwpLSvkrSv
            mfCoMLWyn7+wkM+vcLS0pfCwML+4kPC3QNDAgM+/kPDAQP+/UODIgP/IUODQoP/QUPDQgP/QYP/P
            cPDYgP/XYP/XcP/YgPDgkP/ggP/gkPDnoP/noPDwoPDwsP/woP//////////////////////////
            ////////////////////////////////////////////////////////////////////////////
            /////////////////////////////////////////////////////////////////////////yH5
            BAEKAH8ALAAAAAAQABAAAAe1gH+Cg4SFhoQyHBghKIeEECV/ORwtEDYwmJg0hikLCzBDUlJTUCoz
            hZ4LKlGjUFBKJiQkIB0XgypPpFBLSb2+toImT643N5gnJ7IgIBkXJExQQTBN1NVNSkoxFc9OMDtK
            vkZEQjwvDC4gSNJNR0lGRkI/PDoNEn8gRTA+Su9CQPM1PhxY8SdDj2nw4umowWJEAwSCLqjAIaKi
            Bw0WLExwcGBDRAoRHihIYKAAgQECAARwxFJQIAA7'''

class xls_img:
    format='gif'
    data='''R0lGODlhEAAQAPcAAAAAAIAAAACAAICAAAAAgIAAgACAgICAgMDAwP8AAAD/AP//AAAA//8A/wD/
            /////wAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
            AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAMwAAZgAAmQAAzAAA/wAzAAAzMwAzZgAzmQAzzAAz/wBm
            AABmMwBmZgBmmQBmzABm/wCZAACZMwCZZgCZmQCZzACZ/wDMAADMMwDMZgDMmQDMzADM/wD/AAD/
            MwD/ZgD/mQD/zAD//zMAADMAMzMAZjMAmTMAzDMA/zMzADMzMzMzZjMzmTMzzDMz/zNmADNmMzNm
            ZjNmmTNmzDNm/zOZADOZMzOZZjOZmTOZzDOZ/zPMADPMMzPMZjPMmTPMzDPM/zP/ADP/MzP/ZjP/
            mTP/zDP//2YAAGYAM2YAZmYAmWYAzGYA/2YzAGYzM2YzZmYzmWYzzGYz/2ZmAGZmM2ZmZmZmmWZm
            zGZm/2aZAGaZM2aZZmaZmWaZzGaZ/2bMAGbMM2bMZmbMmWbMzGbM/2b/AGb/M2b/Zmb/mWb/zGb/
            /5kAAJkAM5kAZpkAmZkAzJkA/5kzAJkzM5kzZpkzmZkzzJkz/5lmAJlmM5lmZplmmZlmzJlm/5mZ
            AJmZM5mZZpmZmZmZzJmZ/5nMAJnMM5nMZpnMmZnMzJnM/5n/AJn/M5n/Zpn/mZn/zJn//8wAAMwA
            M8wAZswAmcwAzMwA/8wzAMwzM8wzZswzmcwzzMwz/8xmAMxmM8xmZsxmmcxmzMxm/8yZAMyZM8yZ
            ZsyZmcyZzMyZ/8zMAMzMM8zMZszMmczMzMzM/8z/AMz/M8z/Zsz/mcz/zMz///8AAP8AM/8AZv8A
            mf8AzP8A//8zAP8zM/8zZv8zmf8zzP8z//9mAP9mM/9mZv9mmf9mzP9m//+ZAP+ZM/+ZZv+Zmf+Z
            zP+Z///MAP/MM//MZv/Mmf/MzP/M////AP//M///Zv//mf//zP///ywAAAAAEAAQAAAIngBfuUKF
            ipBBg4MS9umTJYsrBAheSZwokGBBhwgeaNzIUSOhLKgydhz5EdWrB4oOelT5kdDJLwgUKRpEKOUX
            Gtpannzw5ZVNQje15czicmNPg1lwCtW5EeirQV+IEtI2iOjOmh9dQc2SimqWQa4efGzYcGZUr4NQ
            ddSWimwWr33UahRKly61qn0Iza1rl9qXKVIPIkyY8Mtft4gTTwkIADs='''

class xsl_img:
    format='gif'
    data='''R0lGODdhEAAQAOMPAAAAAAAAgAAAmQAA/zNmmQCAgDNm/zOZAIaGhjOZ/zPM/8DAwKbK8DP///Hx
            8f///ywBAAAADwAQAAAEWBDJSeW76Or9Vn4f5zzOAp5kOo5AC2QOMxaFQcrP+zDCUzyNROAhkL14
            pEJDcQiMijqkIXEYDIsOXWwU6N5Yn5VKpSWYz2fwRcwmldFo9bidhc3Hrrw+HwEAOw=='''

class log_img:
    format='gif'
    data='''R0lGODlhEAAQAIQQAG9s0oJ5eatyP6tycpePj6ulP6ulctWeOaulpdWentXSOcvHx9XS0v/MzP//
            zP///y8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gIC8gICH5BAEK
            ABAALAAAAAAQABAAAAViICSOUNMwjEOOhyIUyhAbzMoAgJAQi9EjtRGAIXgUjw9CUDR8OJ9OJakJ
            fUqFjCSBZ11CqNWkt7ndLqLjbFg8zZa5bOw6znSfoVfm3clYIP5eEH4EAQFlCAsrEH2ICygoJCEA
            Ow=='''
<|endoftext|>


<commit_before>"""
 I parse the Iowa Road Condition reports AFOS PIL: STOIA
 I also generate a shapefile of the parsed data
"""
# Twisted Python imports
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/stoia_parser', facility=LOG_LOCAL2)
from twisted.python import log
from twisted.internet import reactor
from pyldm import ldmbridge
from pyiem.nws.product import TextProduct
from pyiem import wellknowntext

import re
import pytz
import dbflib
import shapelib
import zipfile
import os
import shutil
import subprocess

import common

DBPOOL = common.get_database("postgis", cp_max=1)
CONDITIONS = {}
ROADS = {}

# Changedir to /tmp
os.chdir("/tmp")

def shutdown():
    """ Down we go! """
    log.msg("Stopping...")
    reactor.callWhenRunning(reactor.stop)

class MyProductIngestor(ldmbridge.LDMProductReceiver):
    """ I receive products from ldmbridge and process them 1 by 1 :) """

    def connectionLost(self, reason):
        """ called when the connection is lost """
        log.msg('connectionLost')
        log.err(reason)
        reactor.callLater(5, shutdown)

    def process_data(self, buf):
        """ Process the product """
        defer = DBPOOL.runInteraction(real_parser, buf)
        defer.addErrback(common.email_error, buf)
        defer.addErrback(log.err)

def figureCondition(txn, condition):
    """Conversion of condition text into a code

    Args:
      txn (cursor): psycopg2 database cursor
      condition (str): condition string to convert to a code

    Returns:
      code (int): the converted code
    """
    # shortcut
    if condition in CONDITIONS:
        return CONDITIONS[condition]
    # If we find this string, we want this to be 51 (Important)
    if (condition.find("TRAVEL NOT ADVISED") > -1 or
            condition.find("TRAVEL ADVISORY") > -1):
        CONDITIONS[condition] = 51
        return 51
    # Now we prioritize what we can look for
    for typ in ["Closed", "CC Ice", "CC Snow", "CC Slush",
                "CC Mixed", "MC Ice", "MC Snow", "MC Slush", "MC Mixed",
                "PC Ice", "PC Snow", "PC Slush", "PC Mixed", "CC Frost",
                "MC Frost", "PC Frost", "Wet", "Normal",
                "No Conditions Reproted"]:
        if condition.find(typ.upper()) == -1:
            continue
        if typ.upper() not in CONDITIONS:
            log.msg("Unknown Condition: |%s| |%s|\n" % (typ, condition))
            txn.execute("SELECT max(code) as m from roads_conditions")
            row = txn.fetchone()
            if row['m'] is None:
                newID = 1
            else:
                newID = int(row['m']) + 1
            txn.execute("INSERT into roads_conditions VALUES (%s, %s) ",
                        (newID, typ))
            CONDITIONS[typ.upper()] = newID

        return CONDITIONS[typ.upper()]

    return CONDITIONS.get("NORMAL", 0)

def init_dicts(txn):
    """ Setup what we need to process this file """
    txn.execute("SELECT label, code from roads_conditions")
    for row in txn:
        CONDITIONS[row['label'].upper()] = row['code']
    log.msg("Loaded %s conditions" % (len(CONDITIONS),))

    # Load up dictionary of roads...
    txn.execute("SELECT major, minor, longname, segid from roads_base")
    for row in txn:
        ROADS[row['longname']] = {'segid': row['segid'], 'major': row['major'],
                                  'minor': row['minor']}
    log.msg("Loaded %s road segments" % (len(ROADS),))

def real_parser(txn, raw):
    """Actually do the heavy lifting of parsing this product

    Args:
      txn (cursor): psycopg2 database transaction
      raw (str): the raw text that needs parsing
    """
    # Load up dictionary of Possible Road Conditions
    if len(ROADS) == 0:
        log.msg("Initializing ROADS and CONDITIONS dicts...")
        init_dicts(txn)

    tp = TextProduct(raw)
    log.msg("PROCESSING STOIA: %s" % (tp.valid,))

    # Lets start our processing by looking for the first * and then
    # processing after finding it
    lines = re.split("\n", raw[raw.find("*"):])
    for line in lines:
        if len(line) < 40 or line[0] == "*" or line[30:40].strip() == '':
            continue
        data = line[7:]
        # Find the right most ) and chomp everything up until it
        pos = data.rfind(")")
        meat = data[:pos+1].replace(", ", " ")
        condition = data[(pos+1):].upper().strip()
        if meat.strip() == '':
            continue
        if meat not in ROADS:
            log.msg("Unknown road: %s\n" % (meat, ))
            continue

        road_code = figureCondition(txn, condition)
        towingProhibited = (condition.find("TOWING PROHIBITED") > -1)
        limitedVis = (condition.find("LIMITED VIS.") > -1)
        segid = ROADS[meat]['segid']

        txn.execute("""
            UPDATE roads_current SET cond_code = %s, valid = %s,
            towing_prohibited = %s, limited_vis = %s, raw = %s
            WHERE segid = %s
            """, (road_code, tp.valid, towingProhibited, limitedVis,
                  condition, segid))

    # Copy the currents table over to the log... HARD CODED
    txn.execute("""
        INSERT into roads_%s_log
        SELECT * from roads_current
        """ % (tp.valid.year, ))

    # Now we generate a shapefile....
    dbf = dbflib.create("iaroad_cond")
    dbf.add_field("SEGID", dbflib.FTInteger, 4, 0)
    dbf.add_field("MAJOR", dbflib.FTString, 10, 0)
    dbf.add_field("MINOR", dbflib.FTString, 128, 0)
    dbf.add_field("US1", dbflib.FTInteger, 4, 0)
    dbf.add_field("ST1", dbflib.FTInteger, 4, 0)
    dbf.add_field("INT1", dbflib.FTInteger, 4, 0)
    dbf.add_field("TYPE", dbflib.FTInteger, 4, 0)
    dbf.add_field("VALID", dbflib.FTString, 12, 0)
    dbf.add_field("COND_CODE", dbflib.FTInteger, 4, 0)
    dbf.add_field("COND_TXT", dbflib.FTString, 120, 0)
    dbf.add_field("BAN_TOW", dbflib.FTString, 1, 0)
    dbf.add_field("LIM_VIS", dbflib.FTString, 1, 0)

    shp = shapelib.create("iaroad_cond", shapelib.SHPT_ARC)

    txn.execute("""select b.*, c.*, ST_astext(b.geom) as bgeom from
         roads_base b, roads_current c WHERE b.segid = c.segid
         and valid is not null and b.geom is not null""")
    i = 0
    for row in txn:
        s = row["bgeom"]
        f = wellknowntext.convert_well_known_text(s)
        valid = row["valid"]
        d = {}
        d["SEGID"] = row["segid"]
        d["MAJOR"] = row["major"]
        d["MINOR"] = row["minor"]
        d["US1"] = row["us1"]
        d["ST1"] = row["st1"]
        d["INT1"] = row["int1"]
        d["TYPE"] = row["type"]
        d["VALID"] = valid.strftime("%Y%m%d%H%M")
        d["COND_CODE"] = row["cond_code"]
        d["COND_TXT"] = row["raw"]
        d["BAN_TOW"] = str(row["towing_prohibited"])[0]
        d["LIM_VIS"] = str(row["limited_vis"])[0]

        obj = shapelib.SHPObject(shapelib.SHPT_ARC, 1, f)
        shp.write_object(-1, obj)
        dbf.write_record(i, d)

        del(obj)
        i += 1

    del(shp)
    del(dbf)
    z = zipfile.ZipFile("iaroad_cond.zip", 'w')
    z.write("iaroad_cond.shp")
    z.write("iaroad_cond.shx")
    z.write("iaroad_cond.dbf")
    shutil.copyfile("/mesonet/data/gis/meta/26915.prj", "iaroad_cond.prj")
    z.write("iaroad_cond.prj")
    z.close()

    utc = tp.valid.astimezone(pytz.timezone("UTC"))
    subprocess.call(("/home/ldm/bin/pqinsert -p 'zip ac %s "
                     "gis/shape/26915/ia/iaroad_cond.zip "
                     "GIS/iaroad_cond_%s.zip zip' iaroad_cond.zip"
                     "") % (utc.strftime("%Y%m%d%H%M"),
                            utc.strftime("%Y%m%d%H%M")), shell=True)

    for suffix in ['shp', 'shx', 'dbf', 'prj', 'zip']:
        os.unlink("iaroad_cond.%s" % (suffix,))

if __name__ == "__main__":
    # main
    ldmbridge.LDMProductFactory(MyProductIngestor())
    reactor.run()
<commit_msg>write epsg26915 from string, instead of voliate FS copy<commit_after>"""
 I parse the Iowa Road Condition reports AFOS PIL: STOIA
 I also generate a shapefile of the parsed data
"""
# Twisted Python imports
from syslog import LOG_LOCAL2
from twisted.python import syslog
syslog.startLogging(prefix='pyWWA/stoia_parser', facility=LOG_LOCAL2)
from twisted.python import log
from twisted.internet import reactor
from pyldm import ldmbridge
from pyiem.nws.product import TextProduct
from pyiem import wellknowntext

import re
import pytz
import dbflib
import shapelib
import zipfile
import os
import shutil
import subprocess

import common

EPSG26915 = """PROJCS["NAD_1983_UTM_Zone_15N",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-93.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]"""
DBPOOL = common.get_database("postgis", cp_max=1)
CONDITIONS = {}
ROADS = {}

# Changedir to /tmp
os.chdir("/tmp")

def shutdown():
    """ Down we go! """
    log.msg("Stopping...")
    reactor.callWhenRunning(reactor.stop)

class MyProductIngestor(ldmbridge.LDMProductReceiver):
    """ I receive products from ldmbridge and process them 1 by 1 :) """

    def connectionLost(self, reason):
        """ called when the connection is lost """
        log.msg('connectionLost')
        log.err(reason)
        reactor.callLater(5, shutdown)

    def process_data(self, buf):
        """ Process the product """
        defer = DBPOOL.runInteraction(real_parser, buf)
        defer.addErrback(common.email_error, buf)
        defer.addErrback(log.err)

def figureCondition(txn, condition):
    """Conversion of condition text into a code

    Args:
      txn (cursor): psycopg2 database cursor
      condition (str): condition string to convert to a code

    Returns:
      code (int): the converted code
    """
    # shortcut
    if condition in CONDITIONS:
        return CONDITIONS[condition]
    # If we find this string, we want this to be 51 (Important)
    if (condition.find("TRAVEL NOT ADVISED") > -1 or
            condition.find("TRAVEL ADVISORY") > -1):
        CONDITIONS[condition] = 51
        return 51
    # Now we prioritize what we can look for
    for typ in ["Closed", "CC Ice", "CC Snow", "CC Slush",
                "CC Mixed", "MC Ice", "MC Snow", "MC Slush", "MC Mixed",
                "PC Ice", "PC Snow", "PC Slush", "PC Mixed", "CC Frost",
                "MC Frost", "PC Frost", "Wet", "Normal",
                "No Conditions Reproted"]:
        if condition.find(typ.upper()) == -1:
            continue
        if typ.upper() not in CONDITIONS:
            log.msg("Unknown Condition: |%s| |%s|\n" % (typ, condition))
            txn.execute("SELECT max(code) as m from roads_conditions")
            row = txn.fetchone()
            if row['m'] is None:
                newID = 1
            else:
                newID = int(row['m']) + 1
            txn.execute("INSERT into roads_conditions VALUES (%s, %s) ",
                        (newID, typ))
            CONDITIONS[typ.upper()] = newID

        return CONDITIONS[typ.upper()]

    return CONDITIONS.get("NORMAL", 0)

def init_dicts(txn):
    """ Setup what we need to process this file """
    txn.execute("SELECT label, code from roads_conditions")
    for row in txn:
        CONDITIONS[row['label'].upper()] = row['code']
    log.msg("Loaded %s conditions" % (len(CONDITIONS),))

    # Load up dictionary of roads...
    txn.execute("SELECT major, minor, longname, segid from roads_base")
    for row in txn:
        ROADS[row['longname']] = {'segid': row['segid'], 'major': row['major'],
                                  'minor': row['minor']}
    log.msg("Loaded %s road segments" % (len(ROADS),))

def real_parser(txn, raw):
    """Actually do the heavy lifting of parsing this product

    Args:
      txn (cursor): psycopg2 database transaction
      raw (str): the raw text that needs parsing
    """
    # Load up dictionary of Possible Road Conditions
    if len(ROADS) == 0:
        log.msg("Initializing ROADS and CONDITIONS dicts...")
        init_dicts(txn)

    tp = TextProduct(raw)
    log.msg("PROCESSING STOIA: %s" % (tp.valid,))

    # Lets start our processing by looking for the first * and then
    # processing after finding it
    lines = re.split("\n", raw[raw.find("*"):])
    for line in lines:
        if len(line) < 40 or line[0] == "*" or line[30:40].strip() == '':
            continue
        data = line[7:]
        # Find the right most ) and chomp everything up until it
        pos = data.rfind(")")
        meat = data[:pos+1].replace(", ", " ")
        condition = data[(pos+1):].upper().strip()
        if meat.strip() == '':
            continue
        if meat not in ROADS:
            log.msg("Unknown road: %s\n" % (meat, ))
            continue

        road_code = figureCondition(txn, condition)
        towingProhibited = (condition.find("TOWING PROHIBITED") > -1)
        limitedVis = (condition.find("LIMITED VIS.") > -1)
        segid = ROADS[meat]['segid']

        txn.execute("""
            UPDATE roads_current SET cond_code = %s, valid = %s,
            towing_prohibited = %s, limited_vis = %s, raw = %s
            WHERE segid = %s
            """, (road_code, tp.valid, towingProhibited, limitedVis,
                  condition, segid))

    # Copy the currents table over to the log... HARD CODED
    txn.execute("""
        INSERT into roads_%s_log
        SELECT * from roads_current
        """ % (tp.valid.year, ))

    # Now we generate a shapefile....
    dbf = dbflib.create("iaroad_cond")
    dbf.add_field("SEGID", dbflib.FTInteger, 4, 0)
    dbf.add_field("MAJOR", dbflib.FTString, 10, 0)
    dbf.add_field("MINOR", dbflib.FTString, 128, 0)
    dbf.add_field("US1", dbflib.FTInteger, 4, 0)
    dbf.add_field("ST1", dbflib.FTInteger, 4, 0)
    dbf.add_field("INT1", dbflib.FTInteger, 4, 0)
    dbf.add_field("TYPE", dbflib.FTInteger, 4, 0)
    dbf.add_field("VALID", dbflib.FTString, 12, 0)
    dbf.add_field("COND_CODE", dbflib.FTInteger, 4, 0)
    dbf.add_field("COND_TXT", dbflib.FTString, 120, 0)
    dbf.add_field("BAN_TOW", dbflib.FTString, 1, 0)
    dbf.add_field("LIM_VIS", dbflib.FTString, 1, 0)

    shp = shapelib.create("iaroad_cond", shapelib.SHPT_ARC)

    txn.execute("""select b.*, c.*, ST_astext(b.geom) as bgeom from
         roads_base b, roads_current c WHERE b.segid = c.segid
         and valid is not null and b.geom is not null""")
    i = 0
    for row in txn:
        s = row["bgeom"]
        f = wellknowntext.convert_well_known_text(s)
        valid = row["valid"]
        d = {}
        d["SEGID"] = row["segid"]
        d["MAJOR"] = row["major"]
        d["MINOR"] = row["minor"]
        d["US1"] = row["us1"]
        d["ST1"] = row["st1"]
        d["INT1"] = row["int1"]
        d["TYPE"] = row["type"]
        d["VALID"] = valid.strftime("%Y%m%d%H%M")
        d["COND_CODE"] = row["cond_code"]
        d["COND_TXT"] = row["raw"]
        d["BAN_TOW"] = str(row["towing_prohibited"])[0]
        d["LIM_VIS"] = str(row["limited_vis"])[0]

        obj = shapelib.SHPObject(shapelib.SHPT_ARC, 1, f)
        shp.write_object(-1, obj)
        dbf.write_record(i, d)

        del(obj)
        i += 1

    del(shp)
    del(dbf)
    z = zipfile.ZipFile("iaroad_cond.zip", 'w')
    z.write("iaroad_cond.shp")
    z.write("iaroad_cond.shx")
    z.write("iaroad_cond.dbf")
    o = open('iaroad_cond.prj', 'w')
    o.write(EPSG26915)
    o.close()
    z.write("iaroad_cond.prj")
    z.close()

    utc = tp.valid.astimezone(pytz.timezone("UTC"))
    subprocess.call(("/home/ldm/bin/pqinsert -p 'zip ac %s "
                     "gis/shape/26915/ia/iaroad_cond.zip "
                     "GIS/iaroad_cond_%s.zip zip' iaroad_cond.zip"
                     "") % (utc.strftime("%Y%m%d%H%M"),
                            utc.strftime("%Y%m%d%H%M")), shell=True)

    for suffix in ['shp', 'shx', 'dbf', 'prj', 'zip']:
        os.unlink("iaroad_cond.%s" % (suffix,))

if __name__ == "__main__":
    # main
    ldmbridge.LDMProductFactory(MyProductIngestor())
    reactor.run()
<|endoftext|>


<commit_before>from scipy.io.wavfile import write
import numpy as np
import scipy

import math
from pax import plugin

class WavOutput(plugin.OutputPlugin):
    """Convert sum waveforms of event and dataset to WAV file

    If we don't find dark matter, at least we'll have contemporary music.
    """

    def startup(self):
        self.filename = self.config['wav_file']

        # Used for building waveform for entire dataset
        self.all_data = {}

        self.start_time = None
        self.n = None
        self.rate = 44100
        self.single_events = False

    def write_event(self, event):
        self.log.debug('Writing event')

        # Make sum waveform without PMT info
        data = event.pmt_waveforms.sum(0)
        self.n = len(data)

        # Note that // is an integer divide
        self.all_data[event.start_time//event.sample_duration] = data.copy()

        if self.single_events:
            # Write a file
            write('song_%d.wav' % event.event_number,
                  self.rate, # Frequency
                  data)

    def shutdown(self):
        # Determine how long dataset music should be
        start = min(self.all_data.keys())
        end = max(self.all_data.keys())

        if self.n == None:
            self.log.error("Not known how many samples per event!")

        data = np.zeros(end - start + self.n, dtype=np.int16)
        for key, value in self.all_data.items():
            key -= start
            key = int(key)
            data[key : key + len(value)] += value

        # Resample such that data plays at live speed
        R = 10**8 // self.rate
        pad_size = math.ceil(float(data.size)/R)*R - data.size
        b_padded = np.append(data, np.zeros(pad_size)*np.NaN)
        new_data = scipy.nanmean(b_padded.reshape(-1,R), axis=1)

        # Output
        write(self.filename,
              self.rate,
              new_data)

<commit_msg>Optimize memory usage<commit_after>from scipy.io.wavfile import write
import numpy as np
import scipy

import math
from pax import plugin

class WavOutput(plugin.OutputPlugin):
    """Convert sum waveforms of event and dataset to WAV file

    If we don't find dark matter, at least we'll have contemporary music.
    """

    def startup(self):
        self.filename = self.config['wav_file']

        # Used for building waveform for entire dataset
        self.all_data = {}

        self.start_time = None
        self.n = None
        self.rate = 44100
        self.single_events = False

    def write_event(self, event):
        self.log.debug('Writing event')

        # Make sum waveform without PMT info
        data = event.pmt_waveforms.sum(0)
        self.n = len(data)

        # Note that // is an integer divide
        self.all_data[event.start_time//event.sample_duration] = data.copy()

        if self.single_events:
            # Write a file
            write('song_%d.wav' % event.event_number,
                  self.rate, # Frequency
                  data)

    def shutdown(self):
        # Determine how long dataset music should be
        start = min(self.all_data.keys())
        end = max(self.all_data.keys())

        if self.n == None:
            self.log.error("Not known how many samples per event!")

        data = np.zeros(end - start + self.n, dtype=np.int16)
        for key, value in self.all_data.items():
            key -= start
            key = int(key)
            data[key : key + len(value)] += value

        # Resample such that data plays at live speed
        R = 10**8 // self.rate
        end = math.floor(float(data.size)/R) * R
        data = data[:end]
        data = scipy.nanmean(data.reshape(-1,R),
                             axis=1)

        # Output
        write(self.filename,
              self.rate,
              data)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# PCEF - Python/Qt Code Editing Framework
# Copyright 2013, Colin Duquesnoy <colin.duquesnoy@gmail.com>
#
# This software is released under the LGPLv3 license.
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
"""
Contains the mode that control the editor zoom.
"""
from pcef.qt import QtCore
from pcef.core import constants
from pcef.core.mode import Mode

class ZoomMode(Mode):
    """
    Zoom in/out mode. (the editor font is increased/decreased)
    """
    #: Mode identifier
    IDENTIFIER = "editorZoom"
    #: Mode description
    DESCRIPTION = "Zoom the editor with ctrl+mouse wheel"

    def __init__(self):
        super(ZoomMode, self).__init__()
        self.prev_delta = 0
        self.default_font_size = constants.FONT_SIZE

    def onStateChanged(self, state):
        """
        Connects/Disconnects to the mouseWheelActivated and keyPressed event
        """
        if state is True:
            self.editor.mouseWheelActivated.connect(
                self.__onWheelEvent)
            self.editor.keyPressed.connect(self.__onKeyPressed)
        else:
            self.editor.mouseWheelActivated.disconnect(
                self.__onWheelEvent)
            self.editor.keyPressed.disconnect(self.__onKeyPressed)

    def __onKeyPressed(self, event):
        """
        Resets editor font size to the default font size

        :param event: wheelEvent
        :type event: QKeyEvent
        """
        if int(event.modifiers()) & QtCore.Qt.ControlModifier > 0:
            if event.key() == QtCore.Qt.Key_0:
                self.editor.resetZoom()
                event.accept()
            if event.key() == QtCore.Qt.Key_Plus:
                self.editor.zoomIn()
                event.accept()
            if event.key() == QtCore.Qt.Key_Minus:
                self.editor.zoomOut()
                event.accept()

    def __onWheelEvent(self, event):
        """
        Increments or decrements editor fonts settings on mouse wheel event
        if ctrl modifier is on.

        :param event: wheel event
        :type event: QWheelEvent
        """
        delta = event.delta()
        if int(event.modifiers()) & QtCore.Qt.ControlModifier > 0:
            if delta < self.prev_delta:
                self.editor.zoomOut()
                event.accept()
            else:
                self.editor.zoomIn()
                event.accept()
<commit_msg>Minor refactorisation<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# PCEF - Python/Qt Code Editing Framework
# Copyright 2013, Colin Duquesnoy <colin.duquesnoy@gmail.com>
#
# This software is released under the LGPLv3 license.
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
"""
Contains the mode that control the editor zoom.
"""
from pcef.qt import QtCore
from pcef.core import constants
from pcef.core.mode import Mode

class ZoomMode(Mode):
    """
    Zoom in/out mode. (the editor font is increased/decreased)
    """
    #: Mode identifier
    IDENTIFIER = "editorZoom"
    #: Mode description
    DESCRIPTION = "Zoom the editor with ctrl+mouse wheel"

    def __init__(self):
        super(ZoomMode, self).__init__()
        self.prev_delta = 0
        self.default_font_size = constants.FONT_SIZE

    def onStateChanged(self, state):
        """
        Connects/Disconnects to the mouseWheelActivated and keyPressed event
        """
        if state:
            self.editor.mouseWheelActivated.connect(
                self.__onWheelEvent)
            self.editor.keyPressed.connect(self.__onKeyPressed)
        else:
            self.editor.mouseWheelActivated.disconnect(
                self.__onWheelEvent)
            self.editor.keyPressed.disconnect(self.__onKeyPressed)

    def __onKeyPressed(self, event):
        """
        Resets editor font size to the default font size

        :param event: wheelEvent
        :type event: QKeyEvent
        """
        if int(event.modifiers()) & QtCore.Qt.ControlModifier > 0:
            if event.key() == QtCore.Qt.Key_0:
                self.editor.resetZoom()
                event.accept()
            if event.key() == QtCore.Qt.Key_Plus:
                self.editor.zoomIn()
                event.accept()
            if event.key() == QtCore.Qt.Key_Minus:
                self.editor.zoomOut()
                event.accept()

    def __onWheelEvent(self, event):
        """
        Increments or decrements editor fonts settings on mouse wheel event
        if ctrl modifier is on.

        :param event: wheel event
        :type event: QWheelEvent
        """
        delta = event.delta()
        if int(event.modifiers()) & QtCore.Qt.ControlModifier > 0:
            if delta < self.prev_delta:
                self.editor.zoomOut()
                event.accept()
            else:
                self.editor.zoomIn()
                event.accept()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -- Content-Encoding: UTF-8 --
"""
Definition of Factory and Component context classes

:author: Thomas Calmant
:copyright: Copyright 2013, isandlaTech
:license: GPLv3
:version: 0.5.5
:status: Alpha

..

    This file is part of iPOPO.

    iPOPO is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    iPOPO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with iPOPO. If not, see <http://www.gnu.org/licenses/>.
"""

# Module version
__version_info__ = (0, 5, 5)
__version__ = ".".join(str(x) for x in __version_info__)

# Documentation strings format
__docformat__ = "restructuredtext en"

# ------------------------------------------------------------------------------

# Pelix beans
from pelix.framework import BundleContext

# Pelix utilities
from pelix.constants import OBJECTCLASS
from pelix.utilities import is_string
import pelix.ldapfilter as ldapfilter

# iPOPO constants
import pelix.ipopo.constants as constants

# Standard library
import copy

# ------------------------------------------------------------------------------

class Requirement(object):
    """
    Represents a component requirement
    """
    # The dictionary form fields (filter is a special case)
    __stored_fields__ = ('specification', 'aggregate', 'optional')

    def __init__(self, specification, aggregate=False, optional=False,
                 spec_filter=None):
        """
        Sets up the requirement

        :param specification: The requirement specification, which must be
                              unique and can't be None
        :param aggregate: If true, this requirement represents a list
        :param optional: If true, this requirement is optional
        :param spec_filter: A filter to select dependencies

        :raise TypeError: A parameter has an invalid type
        :raise ValueError: An error occurred while parsing the filter
        """
        if not is_string(specification):
            raise TypeError("A Requirement specification must be a string")

        if not specification:
            raise ValueError("No specification given")

        self.aggregate = aggregate
        self.optional = optional
        self.specification = specification

        # Original filter keeper
        self.__original_filter = None

        # Full filter (with the specification test)
        self.__full_filter = None

        # Set up the requirement filter (after setting up self.specification)
        self.filter = None
        self.set_filter(spec_filter)

    def __eq__(self, other):
        """
        Equality test
        """
        if other is self:
            # Identity check
            return True

        if not isinstance(other, Requirement):
            # Different types
            return False

        if self.aggregate != other.aggregate or self.optional != other.optional:
            # Different flags
            return False

        if self.specification != other.specification:
            # Different specifications
            return False

        if self.filter != other.filter:
            # Different filters (therefore different specifications)
            return False

        return True

    def __ne__(self, other):
        """
        Inequality test
        """
        return not self.__eq__(other)

    def copy(self):
        """
        Returns a copy of this instance

        :return: A copy of this instance
        """
        return Requirement(self.specification, self.aggregate, self.optional,
                           self.__original_filter)

    @classmethod
    def from_dictionary_form(cls, dictionary):
        """
        Sets up an instance with the given dictionary form

        :param dictionary: The dictionary form
        :return: A configured requirement instance
        :raise ValueError: An attribute is missing in the dictionary form
        :raise TypeError: Invalid form type (only dictionaries are accepted)
        """
        if not isinstance(dictionary, dict):
            raise TypeError("Invalid form type '{0}'".format(
                                                     type(dictionary).__name__))

        if not "specification" in dictionary:
            raise ValueError("Missing specification in the dictionary form")

        specification = dictionary["specification"]
        aggregate = dictionary.get("aggregate", False)
        optional = dictionary.get("optional", False)
        spec_filter = ldapfilter.get_ldap_filter(dictionary.get("filter", None))

        return cls(specification, aggregate, optional, spec_filter)

    def matches(self, properties):
        """
        Tests if the given _StoredInstance matches this requirement

        :param properties: Service properties
        :return: True if the instance matches this requirement
        """
        if properties is None:
            # No properties : invalid service
            return False

        # Properties filter test
        return self.__full_filter.matches(properties)

    @property
    def full_filter(self):
        """
        The filter that tests both specification and properties
        """
        return self.__full_filter

    @property
    def original_filter(self):
        """
        The original requirement filter string, not the computed one
        """
        if self.__original_filter is None:
            return ""

        return str(self.__original_filter)

    def set_filter(self, props_filter):
        """
        Changes the current filter for the given one

        :param props_filter: The new requirement filter on service properties
        :raise TypeError: Unknown filter type
        """
        if props_filter is not None \
        and not (is_string(props_filter) \
                 or isinstance(props_filter, (ldapfilter.LDAPFilter,
                                              ldapfilter.LDAPCriteria))):
            # Unknown type
            raise TypeError("Invalid filter type {0}" \
                            .format(type(props_filter).__name__))

        if props_filter is not None:
            # Filter given, keep its string form
            self.__original_filter = str(props_filter)
        else:
            # No filter
            self.__original_filter = None

        # Parse the filter
        self.filter = ldapfilter.get_ldap_filter(props_filter)

        # Prepare the full filter
        spec_filter = "({0}={1})".format(OBJECTCLASS, self.specification)
        self.__full_filter = ldapfilter.combine_filters((spec_filter,
                                                         self.filter))

    def to_dictionary_form(self):
        """
        Returns a dictionary form of the current object

        :raise AttributeError: A field to store is missing in the instance
        """
        result = {}
        for field in self.__stored_fields__:
            result[field] = getattr(self, field)

        # Special case: store the original filter
        result['filter'] = self.__original_filter

        return result

# ------------------------------------------------------------------------------

class FactoryContext(object):
    """
    Represents the data stored in a component factory (class)
    """

    __basic_fields = ('callbacks', 'field_callbacks', 'name', 'properties',
                      'properties_fields', 'provides', 'completed')

    def __init__(self):
        """
        Sets up the factory context
        """
        # Factory bundle context
        self.bundle_context = None

        # Callbacks : Kind -> callback method
        self.callbacks = {}

        # Field callbacks: Field -> {Kind -> Callback}
        self.field_callbacks = {}

        # The factory name
        self.name = None

        # Properties : Name -> Value
        self.properties = {}

        # Properties fields : Field name -> Property name
        self.properties_fields = {}

        # Provided specifications:
        # Array of tuples (specifications(arrays of strings), controller name)
        self.provides = []

        # Requirements : Field name -> Requirement object
        self.requirements = {}

        # The factory manipulation has been completed
        self.completed = False

        # Handler ID -> configuration
        self.__handlers = {}

    def __eq__(self, other):
        """
        Equality test
        """
        if other is self:
            # Identity
            return True

        if not isinstance(other, FactoryContext):
            # Different types
            return False

        # Do not compare the bundle context, as it must not be stored
        for field in ('name', 'callbacks', 'field_callbacks', 'properties',
                      'properties_fields', 'requirements'):
            # Comparable fields
            if getattr(self, field, None) != getattr(other, field, None):
                return False

        # Treat the list differently
        if len(self.provides) != len(other.provides):
            return False

        for provided in self.provides:
            if provided not in other.provides:
                # Missing a provided service
                return False

        return True

    def __ne__(self, other):
        """
        Inequality test
        """
        return not self.__eq__(other)

    def get_handlers_ids(self):
        """
        Retrieves the IDs of the handlers to instantiate for this component
        """
        return list(self.__handlers.keys())

    def copy(self):
        """
        Returns a copy of the current FactoryContext instance
        """
        return copy.deepcopy(self)
#
#         context = FactoryContext()
#
#         direct = ("bundle_context", "name", "completed")
#         copied = ("callbacks", "field_callbacks", "properties",
#                   "properties_fields", "requirements")
#         lists = ("provides",)
#
#         # Direct copy of primitive values
#         for entry in direct:
#             setattr(context, entry, getattr(self, entry))
#
#         # Copy "complex" values
#         for entry in copied:
#             value = getattr(self, entry)
#             if value is not None:
#                 value = value.copy()
#
#             setattr(context, entry, value)
#
#         # Copy lists
#         for entry in lists:
#             value = getattr(self, entry)
#             if value is not None:
#                 value = value[:]
#
#             setattr(context, entry, value)
#
#         return context

    def get_handler(self, handler_id, default=None):
        """
        Retrieves the configuration associated to the given handler

        :param handler_id: The ID of the configured handler
        :param default: The default configuration value
        :return: The existing configuration or the given default
        """
        return self.__handlers.setdefault(handler_id, default)

    def set_handler(self, handler_id, configuration):
        """
        Stores the configuration of the given handler

        :param handler_id: The ID of the configured handler
        :param configuration: The complete configuration of the handler
        """
        self.__handlers[handler_id] = configuration

    @classmethod
    def from_dictionary_form(cls, dictionary):
        """
        Sets up this instance with the given dictionary form

        :param dictionary: The dictionary form
        :raise ValueError: An attribute is missing in the dictionary form
        :raise TypeError: Invalid form type (only dictionaries are accepted)
        """
        if not isinstance(dictionary, dict):
            raise TypeError("Invalid form type '{0}'".format(
                                                    type(dictionary).__name__))

        # Prepare the instance, initializing it
        instance = cls()

        # Basic fields
        for field in cls.__basic_fields:
            if field not in dictionary:
                raise ValueError("Incomplete dictionary form: missing {0}" \
                                 .format(field))

            setattr(instance, field, dictionary[field])

        # Requirements field
        if 'requirements' not in dictionary:
            raise ValueError("Incomplete dictionary form: missing " \
                             "'requirements'")

        requirements = dictionary['requirements']
        if not isinstance(requirements, dict):
            raise TypeError("Only dictionaries are handled for 'requirements'")

        for field, requirement_dict in requirements.items():
            instance.requirements[field] = Requirement.from_dictionary_form(\
                                                            requirement_dict)

        return instance

    def set_bundle_context(self, bundle_context):
        """
        Sets up the bundle context associated to this factory context

        :param bundle_context: The factory bundle context
        """
        if self.bundle_context is None:
            assert isinstance(bundle_context, BundleContext)
            self.bundle_context = bundle_context

    def to_dictionary_form(self):
        """
        Returns a dictionary form of the current object

        :raise AttributeError: A field to store in missing in the instance
        """
        result = {}

        # Fields with standard Python types (no conversion needed)
        for entry in self.__basic_fields:
            result[entry] = getattr(self, entry)

        # Requirements field
        requirements = {}
        for field, requirement in self.requirements.items():
            requirements[field] = requirement.to_dictionary_form()

        result['requirements'] = requirements
        return result

# ------------------------------------------------------------------------------

class ComponentContext(object):
    """
    Represents the data stored in a component instance
    """

    # Try to reduce memory footprint (stored __instances)
    __slots__ = ('factory_context', 'name', 'properties')

    def __init__(self, factory_context, name, properties):
        """
        Sets up the context

        :param factory_context: The parent factory context
        :param properties: The component properties
        """
        assert isinstance(factory_context, FactoryContext)
        assert isinstance(properties, dict)

        self.factory_context = factory_context
        self.name = name

        # Force the instance name property
        properties[constants.IPOPO_INSTANCE_NAME] = name

        self.properties = factory_context.properties.copy()
        self.properties.update(properties)

    def get_bundle_context(self):
        """
        Retrieves the bundle context

        :return: The component bundle context
        """
        return self.factory_context.bundle_context

    def get_callback(self, event):
        """
        Retrieves the registered method for the given event. Returns None if not
        found

        :param event: A component life cycle event
        :return: The callback associated to the given event
        """
        return self.factory_context.callbacks.get(event, None)

    def get_field_callback(self, field, event):
        """
        Retrieves the registered method for the given event. Returns None if not
        found

        :param field: Name of the dependency field
        :param event: A component life cycle event
        :return: The callback associated to the given event
        """
        return self.factory_context.field_callbacks.get(field,
                                                        {}).get(event, None)

    def get_factory_name(self):
        """
        Retrieves the component factory name

        :return: The component factory name
        """
        return self.factory_context.name

    def get_handler(self, handler_id):
        """
        Retrieves the configuration for the given handler from the factory
        context

        :param handler_id: The ID of the configured handler
        :return: The handler configuration, or None
        """
        return self.factory_context.get_handler(handler_id, None)

    def get_provides(self):
        """
        Retrieves the services provided by this component.
        Returns an array containing arrays of specifications.

        :return: An array of tuples (specifications, controller)
        """
        return self.factory_context.provides
<commit_msg>Contexts: code clean up<commit_after>#!/usr/bin/env python
# -- Content-Encoding: UTF-8 --
"""
Definition of Factory and Component context classes

:author: Thomas Calmant
:copyright: Copyright 2013, isandlaTech
:license: GPLv3
:version: 0.5.5
:status: Alpha

..

    This file is part of iPOPO.

    iPOPO is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    iPOPO is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with iPOPO. If not, see <http://www.gnu.org/licenses/>.
"""

# Module version
__version_info__ = (0, 5, 5)
__version__ = ".".join(str(x) for x in __version_info__)

# Documentation strings format
__docformat__ = "restructuredtext en"

# ------------------------------------------------------------------------------

# Pelix beans
from pelix.framework import BundleContext

# Pelix utilities
from pelix.constants import OBJECTCLASS
from pelix.utilities import is_string
import pelix.ldapfilter as ldapfilter

# iPOPO constants
import pelix.ipopo.constants as constants

# Standard library
import copy

# ------------------------------------------------------------------------------

class Requirement(object):
    """
    Represents a component requirement
    """
    # The dictionary form fields (filter is a special case)
    __stored_fields__ = ('specification', 'aggregate', 'optional')

    def __init__(self, specification, aggregate=False, optional=False,
                 spec_filter=None):
        """
        Sets up the requirement

        :param specification: The requirement specification, which must be
                              unique and can't be None
        :param aggregate: If true, this requirement represents a list
        :param optional: If true, this requirement is optional
        :param spec_filter: A filter to select dependencies

        :raise TypeError: A parameter has an invalid type
        :raise ValueError: An error occurred while parsing the filter
        """
        if not is_string(specification):
            raise TypeError("A Requirement specification must be a string")

        if not specification:
            raise ValueError("No specification given")

        self.specification = specification
        self.aggregate = aggregate
        self.optional = optional

        # Original filter keeper
        self.__original_filter = None

        # Full filter (with the specification test)
        self.__full_filter = None

        # Set up the requirement filter (after setting up self.specification)
        self.filter = None
        self.set_filter(spec_filter)

    def __eq__(self, other):
        """
        Equality test
        """
        if other is self:
            # Identity check
            return True

        if not isinstance(other, Requirement):
            # Different types
            return False

        if self.aggregate != other.aggregate or self.optional != other.optional:
            # Different flags
            return False

        if self.specification != other.specification:
            # Different specifications
            return False

        if self.filter != other.filter:
            # Different filters (therefore different specifications)
            return False

        return True

    def __ne__(self, other):
        """
        Inequality test
        """
        return not self.__eq__(other)

    def __deepcopy__(self, memo):
        """
        Called by copy.deepcopy()
        """
        return self.copy()

    def copy(self):
        """
        Returns a copy of this instance

        :return: A copy of this instance
        """
        return Requirement(self.specification, self.aggregate, self.optional,
                           self.__original_filter)

    def matches(self, properties):
        """
        Tests if the given _StoredInstance matches this requirement

        :param properties: Service properties
        :return: True if the instance matches this requirement
        """
        if properties is None:
            # No properties : invalid service
            return False

        # Properties filter test
        return self.__full_filter.matches(properties)

    @property
    def full_filter(self):
        """
        The filter that tests both specification and properties
        """
        return self.__full_filter

    @property
    def original_filter(self):
        """
        The original requirement filter string, not the computed one
        """
        if self.__original_filter is None:
            return ""

        return str(self.__original_filter)

    def set_filter(self, props_filter):
        """
        Changes the current filter for the given one

        :param props_filter: The new requirement filter on service properties
        :raise TypeError: Unknown filter type
        """
        if props_filter is not None \
        and not (is_string(props_filter) \
                 or isinstance(props_filter, (ldapfilter.LDAPFilter,
                                              ldapfilter.LDAPCriteria))):
            # Unknown type
            raise TypeError("Invalid filter type {0}" \
                            .format(type(props_filter).__name__))

        if props_filter is not None:
            # Filter given, keep its string form
            self.__original_filter = str(props_filter)
        else:
            # No filter
            self.__original_filter = None

        # Parse the filter
        self.filter = ldapfilter.get_ldap_filter(props_filter)

        # Prepare the full filter
        spec_filter = "({0}={1})".format(OBJECTCLASS, self.specification)
        self.__full_filter = ldapfilter.combine_filters((spec_filter,
                                                         self.filter))

# ------------------------------------------------------------------------------

class FactoryContext(object):
    """
    Represents the data stored in a component factory (class)
    """
    def __init__(self):
        """
        Sets up the factory context
        """
        # Factory bundle context
        self.bundle_context = None

        # Callbacks : Kind -> callback method
        self.callbacks = {}

        # Field callbacks: Field -> {Kind -> Callback}
        self.field_callbacks = {}

        # The factory name
        self.name = None

        # Properties : Name -> Value
        self.properties = {}

        # Properties fields : Field name -> Property name
        self.properties_fields = {}

        # The factory manipulation has been completed
        self.completed = False

        # Handler ID -> configuration
        self.__handlers = {}

    def __eq__(self, other):
        """
        Equality test
        """
        if other is self:
            # Identity
            return True

        if not isinstance(other, FactoryContext):
            # Different types
            return False

        # Name-based equality
        return self.name == other.name

    def __ne__(self, other):
        """
        Inequality test
        """
        return not self.__eq__(other)

    def get_handlers_ids(self):
        """
        Retrieves the IDs of the handlers to instantiate for this component
        """
        return list(self.__handlers.keys())

    def copy(self):
        """
        Returns a deep copy of the current FactoryContext instance
        """
        return copy.deepcopy(self)

    def get_handler(self, handler_id, default=None):
        """
        Retrieves the configuration associated to the given handler

        :param handler_id: The ID of the configured handler
        :param default: The default configuration value
        :return: The existing configuration or the given default
        """
        return self.__handlers.setdefault(handler_id, default)

    def set_handler(self, handler_id, configuration):
        """
        Stores the configuration of the given handler

        :param handler_id: The ID of the configured handler
        :param configuration: The complete configuration of the handler
        """
        self.__handlers[handler_id] = configuration

    def set_bundle_context(self, bundle_context):
        """
        Sets up the bundle context associated to this factory context

        :param bundle_context: The factory bundle context
        """
        if self.bundle_context is None:
            assert isinstance(bundle_context, BundleContext)
            self.bundle_context = bundle_context

# ------------------------------------------------------------------------------

class ComponentContext(object):
    """
    Represents the data stored in a component instance
    """
    # Try to reduce memory footprint (many instances)
    __slots__ = ('factory_context', 'name', 'properties')

    def __init__(self, factory_context, name, properties):
        """
        Sets up the context

        :param factory_context: The parent factory context
        :param properties: The component properties
        """
        assert isinstance(factory_context, FactoryContext)
        assert isinstance(properties, dict)

        self.factory_context = factory_context
        self.name = name

        # Force the instance name property
        properties[constants.IPOPO_INSTANCE_NAME] = name

        self.properties = factory_context.properties.copy()
        self.properties.update(properties)

    def get_bundle_context(self):
        """
        Retrieves the bundle context

        :return: The component bundle context
        """
        return self.factory_context.bundle_context

    def get_callback(self, event):
        """
        Retrieves the registered method for the given event. Returns None if not
        found

        :param event: A component life cycle event
        :return: The callback associated to the given event
        """
        return self.factory_context.callbacks.get(event, None)

    def get_field_callback(self, field, event):
        """
        Retrieves the registered method for the given event. Returns None if not
        found

        :param field: Name of the dependency field
        :param event: A component life cycle event
        :return: The callback associated to the given event
        """
        return self.factory_context.field_callbacks.get(field,
                                                        {}).get(event, None)

    def get_factory_name(self):
        """
        Retrieves the component factory name

        :return: The component factory name
        """
        return self.factory_context.name

    def get_handler(self, handler_id):
        """
        Retrieves the configuration for the given handler from the factory
        context

        :param handler_id: The ID of the configured handler
        :return: The handler configuration, or None
        """
        return self.factory_context.get_handler(handler_id, None)
<|endoftext|>