<commit_before>#!/usr/bin/python

# A Basic SimpleCV interactive shell tutorial

#load required libraries
from SimpleCV import *

from subprocess import call
from code import InteractiveInterpreter
import platform

lb = "\n" #linebreak
tb = "\t" #tab
tutorial_interpreter = InteractiveInterpreter(globals())
logo = None
img = None
clone = None
thumb = None
eroded = None
cropped = None

#Command to clear the shell screen
def shellclear():
    if platform.system() == "Windows":
        return
    call("clear")

def attempt(variable_name, desired_class):
    prompt_and_run()
    variable = globals().get(variable_name)

    if isinstance(variable,desired_class):
        if desired_class == Image:
            if variable.isEmpty():
                print lb
                print "Although you can create empty Images on SimpleCV, let's not"
                print "play with that now!"
                print lb
                return False

        return True

    return False

def prompt_and_run():

    command = raw_input("SimpleCV:> ")
    tutorial_interpreter.runsource(command)
    return command

def request_show_command():
    while True:
        if prompt_and_run().endswith('.show()'):
            return

def end_tutorial():
    print lb
    print "Type 'quit' to leave the tutorials, or press Enter to move on!"
    command = raw_input("SimpleCV:> ")
    return command.lower() == 'quit'

def end_of_tutorial():
    print lb
    print "This is the end of our tutorial!"
    print lb
    print "For more help, go to www.simplecv.org, and don't forget about the"
    print "help function!"
    print lb

def command_loop(command, desired_tuple):
    while True:
        print command
        print lb

        if attempt(desired_tuple[0], desired_tuple[1]):
            return

        print lb
        print "Oops! %s is still not %s" % (desired_tuple[0], str(desired_tuple[1]))

def tutorial_image():
    shellclear()
    print "SimpleCV Image tutorial"
    print "-----------------------"
    print lb
    print "Using images are simple, in SimpleCV."
    print lb
    print "First thing we are going to do is load an image. Try it yourself:"
    print lb

    cmd = "logo = Image(\"simplecv\")"
    desired_tuple = ('logo', Image)
    command_loop(cmd, desired_tuple)

    print lb
    print "Correct! You just loaded SimpleCV logo into memory."
    print "Let's try it to use one of your images. There are different ways to"
    print "do that. You can try, for example:"
    print lb
    print "img = Image(URL_TO_MY_PICTURE) or img = Image(PATH_TO_MY_PICTURE)"
    print lb
    cmd =  "Example: img = Image('http://simplecv.org/logo.jpg')"

    desired_tuple = ('img', Image)
    command_loop(cmd, desired_tuple)

    print lb
    print "Perfect! Now we want to see it:"
    print lb
    cmd = "img.show()"
    print cmd
    print lb

    request_show_command()

    print lb
    print "Alright! This was tutorial 1/6."
    print "Next tutorial: Saving Images"
    if not end_tutorial():
        tutorial_save()
    return

def tutorial_save():
    shellclear()
    print "Saving Images"
    print lb
    print "Once you have an Image Object loaded in memory you can"
    print "now save it to disk."
    print lb
    raw_input("[Press enter to continue]")
    print lb
    print "Saving an image is very simple, pardon the pun. Once it's loaded"
    print "into memory, it's literally just:"
    print "img.save()"
    print lb
    print "This will save the image back to the location it was loaded from"
    print "so if you did img = Image('/tmp/test.jpg'), then it would save"
    print "it back there, otherwise you can do:"
    print "img.save('/any/path/you/want')"
    print lb
    print "So try it now and save an image somewhere on your system"
    print lb

    if platform.system() == "Windows":
        print "img.save('C:/myimg.jpg')"
    else:
        print "img.save('/tmp/new.jpg')"

    print lb

    while True:
        if prompt_and_run().startswith('img.save'):
            break
        print "Please try to save img!"
        print lb

    print "Correct, you just saved a new copy of your image!"
    print "As you can see in SimpleCV most of the functions are intuitive."

    print lb
    print "Alright! This was tutorial 2/6."
    print "Next tutorial: Camera"
    if not end_tutorial():
        tutorial_camera()
    return

def tutorial_camera():
    shellclear()
    print "Camera"
    print lb
    print "As long as your camera driver is supported then you shouldn't have a"
    print "problem. Type 'skip' to skip the camera tutorial, or press Enter to"
    print "continue."
    print lb

    command = raw_input("SimpleCV:> ")
    if command.lower() != 'skip':
        print lb
        print "To load the camera, just type:"
        print lb

        cmd = "cam = Camera()"
        desired_tuple = ('cam', Camera)
        command_loop(cmd, desired_tuple)

        print lb
        print "Next, to grab an image from the Camera we type:"
        cmd = "img = cam.getImage()"
        tutorial_interpreter.runsource("del(img)")
        desired_tuple = ('img', Image)
        command_loop(cmd, desired_tuple)

        print "Just as before, if we want to display it, we just type:"
        print lb
        print "img.show()"
        print lb

        request_show_command()

    print lb
    print "Alright! This was tutorial 3/6."
    print "Next tutorial: Copying Images"
    if not end_tutorial():
        tutorial_copy()
    return

def tutorial_copy():
    shellclear()
    print "Copying Images"
    print lb
    print "If you need a copy of an image, this is also very simple:"
    print "Let's try to clone img, which we already have."

    global img
    if not img:
        img = Image("lenna")

    print lb
    cmd = "clone = img.copy()"
    desired_tuple = ('clone', Image)

    while True:
        command_loop(cmd, desired_tuple)
        if clone != img: #Returns False if they have different addresses.
            break

        print "You have to use the copy() function!"

    print lb
    print "Correct, you just cloned an image into memory."
    print "You need to be careful when using this method though as using as a"
    print "reference vs. a copy.  For instance, if you just typed:"
    print lb
    print "clone = img"
    print lb
    print "clone would actually point at the same thing in memory as img."

    print lb
    print "Alright! This was tutorial 4/6."
    print "Next tutorial: Manipulating Images"
    if not end_tutorial():
        tutorial_manipulation()
    return

def tutorial_manipulation():
    shellclear()
    print "Manipulating Images"
    print lb
    print "Now we can easily load and save images. It's time to start doing some"
    print "image processing with them. Let's make img, which we already have, a"
    print "90x90 thumbnail:"

    global img
    if not img:
        img = Image("lenna")

    print lb
    cmd = "thumb = img.scale(90,90)"
    desired_tuple = ('thumb', Image)

    while True:
        command_loop(cmd, desired_tuple)
        if thumb.size() == (90,90):
            break

        print "Your thumbnail's size isn't 90x90! Try again!"

    print lb
    print "Now display it with thumb.show()"
    print lb
    request_show_command()

    print lb
    print "Now let's erode the picture some:"
    print lb
    cmd = "eroded = img.erode()"
    desired_tuple = ('eroded', Image)
    command_loop(cmd, desired_tuple)

    print lb
    print "Display it with eroded.show(). It should look almost as if the image"
    print "was made if ink and had water spoiled on it."
    print lb
    request_show_command()

    print lb
    print "Last but not least, let's crop a section of the image out:"
    print lb
    cmd = "cropped = img.crop(100, 100, 50, 50)"
    desired_tuple = ('cropped', Image)
    command_loop(cmd, desired_tuple)

    print lb
    print "Use cropped.show() to display it."
    print lb
    request_show_command()
    print lb
    print "That went from the coordinate in (X,Y), which is (0,0) and is the"
    print "top left corner of the picture, to coordinates (100,100) in the"
    print "(X,Y) and cropped a picture from that which is 50 pixels by 50 pixels."

    print lb
    print "Alright! This was tutorial 5/6."
    print "Next tutorial: Features"
    if not end_tutorial():
        tutorial_features()
    return

def tutorial_slicing():
    shellclear()
    print "Slicing Images"
    print lb
    print "Slicing is sort of a new paradigm to access parts of an image."
    print "Typically in vision a region of interest (ROI) is given.  "
    print "In this case, slicing is a very powerful way to access parts"
    print "of an image, or basically any matrix in SimpleCV in general."
    print lb
    print "This is done by using:"
    print "section = img[1:10,1:10]"
    print lb
    print "What is returned is an image object with that window."
    print "the slicing basically acts like a ROI but returns an image"
    print "so if you wanted to say run edge detection on a 20x20 box"
    print "in the picture that started at x=5,y=10 you use:"
    print "foundedges = img[5:25,10:30].edges()"
    print lb
    raw_input("[Press enter to continue]")
    shellclear()
    in_text = ""
    shouldbe = "ROI = img[1:6,1:6]"
    print "Please type this now:"
    print shouldbe
    print lb
    while (in_text != shouldbe):
        in_text = raw_input("SimpleCV:>")
        if(in_text != shouldbe):
            print "sorry, that is incorrect"
            print "please type:"
            print shouldbe
    shellclear()
    print "Correct, you just returned a 5 pixel by 5 pixel image object"
    print lb
    return

def tutorial_features():
    shellclear()
    print "Features"
    print lb
    print "Features are things you are looking for in the picture. They can be"
    print "blobs, corners, lines, etc. Features are sometimes referred to as a"
    print "fiducial in computer vision. These features are something that is"
    print "measurable, and something that makes images unique. Features are"
    print "something like when comparing things like fruit. In this case the"
    print "features could be the shape and the color, amongst others."
    print lb
    print "What features are in SimpleCV is an abstract representation of that."
    print "You take your image, then perform a function on it, and get back"
    print "features or another image with them applied. The crop example is"
    print "a case where an image is returned after we perform something."
    print lb
    print "In a simple example we will use the famous 'lenna' image, and find"
    print "corners in the picture."
    print lb
    tutorial_interpreter.runsource("img = Image('lenna')")
    print "img = Image('lenna') (already done for you)"
    print lb
    print "Try it yourself:"
    print lb

    cmd = "corners = img.findCorners()"
    desired_tuple = ('corners', FeatureSet)
    command_loop(cmd, desired_tuple)

    print lb
    print "Correct, you just got a featureset object which contains"
    print "feature objects.  These feature objects contain data from the"
    print "found corners"
    print lb

    print "Tip: If your are unsure what parameters to pass, you can always use"
    print "the built in help support by typing help(Image.findCorners). Keep in"
    print "mind that this help works for all of the functions available in"
    print "SimpleCV"
    print lb

    print "We can also do that with blobs. Try it:"
    print lb

    cmd = "blobs = img.findBlobs()"
    desired_tuple = ('blobs', FeatureSet)
    command_loop(cmd, desired_tuple)

    print lb
    print "Great, but..."
    print "When we show the image we won't notice anything different. This"
    print "is because we have to actually tell the blobs to draw themselves"
    print "on the image:"
    print lb
    print "blobs.draw()"
    print lb

    while True:
        if prompt_and_run().endswith('.draw()'):
            break
        print "No blobs have been drawn!"
        print lb

    print "Now use img.show() to see the changes!"
    print lb
    request_show_command()
    print lb
    raw_input("[Press enter to continue]")

    print lb
    print lb
    print "There's also a small trick built into SimpleCV to do this even faster"
    print lb
    tutorial_interpreter.runsource("img = Image('lenna')")
    print "img = Image('lenna') (already done for you)"
    print lb

    while True:
        print "img.findBlobs().show()"
        print lb
        if prompt_and_run().endswith('.show()'):
            break
        print "Nothing has been shown!"
        print lb

    print lb
    print "Alright! This was tutorial 6/6."
    #print "Next tutorial: ..."
    return

def magic_tutorial(self,arg):
    tutorials_dict = {'image': tutorial_image, 'save': tutorial_save,
                     'camera': tutorial_camera, 'manipulation': tutorial_manipulation,
                     'copy': tutorial_copy, 'features': tutorial_features}

    if (arg == ""):
        shellclear()
        print "+--------------------------------+"
        print " Welcome to the SimpleCV tutorial "
        print "+--------------------------------+"
        print lb
        print "At anytime on the SimpleCV Interactive Shell you can type tutorial,"
        print "then press the tab key and it will autocomplete any tutorial that"
        print "is currently available."
        print lb
        print "Let's start off with Loading and Saving images!"
        print lb
        print lb
        raw_input("[Press enter to continue]")
        tutorial_image()
        end_of_tutorial()
        return
    else:
        if arg in tutorials_dict:
            tutorials_dict[arg]()
        else:
            print "%s is not a tutorial!" % arg
<commit_msg>Corrected typo using images are => using images is<commit_after>#!/usr/bin/python

# A Basic SimpleCV interactive shell tutorial

#load required libraries
from SimpleCV import *

from subprocess import call
from code import InteractiveInterpreter
import platform

lb = "\n" #linebreak
tb = "\t" #tab
tutorial_interpreter = InteractiveInterpreter(globals())
logo = None
img = None
clone = None
thumb = None
eroded = None
cropped = None

#Command to clear the shell screen
def shellclear():
    if platform.system() == "Windows":
        return
    call("clear")

def attempt(variable_name, desired_class):
    prompt_and_run()
    variable = globals().get(variable_name)

    if isinstance(variable,desired_class):
        if desired_class == Image:
            if variable.isEmpty():
                print lb
                print "Although you can create empty Images on SimpleCV, let's not"
                print "play with that now!"
                print lb
                return False

        return True

    return False

def prompt_and_run():

    command = raw_input("SimpleCV:> ")
    tutorial_interpreter.runsource(command)
    return command

def request_show_command():
    while True:
        if prompt_and_run().endswith('.show()'):
            return

def end_tutorial():
    print lb
    print "Type 'quit' to leave the tutorials, or press Enter to move on!"
    command = raw_input("SimpleCV:> ")
    return command.lower() == 'quit'

def end_of_tutorial():
    print lb
    print "This is the end of our tutorial!"
    print lb
    print "For more help, go to www.simplecv.org, and don't forget about the"
    print "help function!"
    print lb

def command_loop(command, desired_tuple):
    while True:
        print command
        print lb

        if attempt(desired_tuple[0], desired_tuple[1]):
            return

        print lb
        print "Oops! %s is still not %s" % (desired_tuple[0], str(desired_tuple[1]))

def tutorial_image():
    shellclear()
    print "SimpleCV Image tutorial"
    print "-----------------------"
    print lb
    print "Using images is simple, in SimpleCV."
    print lb
    print "First thing we are going to do is load an image. Try it yourself:"
    print lb

    cmd = "logo = Image(\"simplecv\")"
    desired_tuple = ('logo', Image)
    command_loop(cmd, desired_tuple)

    print lb
    print "Correct! You just loaded SimpleCV logo into memory."
    print "Let's try it to use one of your images. There are different ways to"
    print "do that. You can try, for example:"
    print lb
    print "img = Image(URL_TO_MY_PICTURE) or img = Image(PATH_TO_MY_PICTURE)"
    print lb
    cmd =  "Example: img = Image('http://simplecv.org/logo.jpg')"

    desired_tuple = ('img', Image)
    command_loop(cmd, desired_tuple)

    print lb
    print "Perfect! Now we want to see it:"
    print lb
    cmd = "img.show()"
    print cmd
    print lb

    request_show_command()

    print lb
    print "Alright! This was tutorial 1/6."
    print "Next tutorial: Saving Images"
    if not end_tutorial():
        tutorial_save()
    return

def tutorial_save():
    shellclear()
    print "Saving Images"
    print lb
    print "Once you have an Image Object loaded in memory you can"
    print "now save it to disk."
    print lb
    raw_input("[Press enter to continue]")
    print lb
    print "Saving an image is very simple, pardon the pun. Once it's loaded"
    print "into memory, it's literally just:"
    print "img.save()"
    print lb
    print "This will save the image back to the location it was loaded from"
    print "so if you did img = Image('/tmp/test.jpg'), then it would save"
    print "it back there, otherwise you can do:"
    print "img.save('/any/path/you/want')"
    print lb
    print "So try it now and save an image somewhere on your system"
    print lb

    if platform.system() == "Windows":
        print "img.save('C:/myimg.jpg')"
    else:
        print "img.save('/tmp/new.jpg')"

    print lb

    while True:
        if prompt_and_run().startswith('img.save'):
            break
        print "Please try to save img!"
        print lb

    print "Correct, you just saved a new copy of your image!"
    print "As you can see in SimpleCV most of the functions are intuitive."

    print lb
    print "Alright! This was tutorial 2/6."
    print "Next tutorial: Camera"
    if not end_tutorial():
        tutorial_camera()
    return

def tutorial_camera():
    shellclear()
    print "Camera"
    print lb
    print "As long as your camera driver is supported then you shouldn't have a"
    print "problem. Type 'skip' to skip the camera tutorial, or press Enter to"
    print "continue."
    print lb

    command = raw_input("SimpleCV:> ")
    if command.lower() != 'skip':
        print lb
        print "To load the camera, just type:"
        print lb

        cmd = "cam = Camera()"
        desired_tuple = ('cam', Camera)
        command_loop(cmd, desired_tuple)

        print lb
        print "Next, to grab an image from the Camera we type:"
        cmd = "img = cam.getImage()"
        tutorial_interpreter.runsource("del(img)")
        desired_tuple = ('img', Image)
        command_loop(cmd, desired_tuple)

        print "Just as before, if we want to display it, we just type:"
        print lb
        print "img.show()"
        print lb

        request_show_command()

    print lb
    print "Alright! This was tutorial 3/6."
    print "Next tutorial: Copying Images"
    if not end_tutorial():
        tutorial_copy()
    return

def tutorial_copy():
    shellclear()
    print "Copying Images"
    print lb
    print "If you need a copy of an image, this is also very simple:"
    print "Let's try to clone img, which we already have."

    global img
    if not img:
        img = Image("lenna")

    print lb
    cmd = "clone = img.copy()"
    desired_tuple = ('clone', Image)

    while True:
        command_loop(cmd, desired_tuple)
        if clone != img: #Returns False if they have different addresses.
            break

        print "You have to use the copy() function!"

    print lb
    print "Correct, you just cloned an image into memory."
    print "You need to be careful when using this method though as using as a"
    print "reference vs. a copy.  For instance, if you just typed:"
    print lb
    print "clone = img"
    print lb
    print "clone would actually point at the same thing in memory as img."

    print lb
    print "Alright! This was tutorial 4/6."
    print "Next tutorial: Manipulating Images"
    if not end_tutorial():
        tutorial_manipulation()
    return

def tutorial_manipulation():
    shellclear()
    print "Manipulating Images"
    print lb
    print "Now we can easily load and save images. It's time to start doing some"
    print "image processing with them. Let's make img, which we already have, a"
    print "90x90 thumbnail:"

    global img
    if not img:
        img = Image("lenna")

    print lb
    cmd = "thumb = img.scale(90,90)"
    desired_tuple = ('thumb', Image)

    while True:
        command_loop(cmd, desired_tuple)
        if thumb.size() == (90,90):
            break

        print "Your thumbnail's size isn't 90x90! Try again!"

    print lb
    print "Now display it with thumb.show()"
    print lb
    request_show_command()

    print lb
    print "Now let's erode the picture some:"
    print lb
    cmd = "eroded = img.erode()"
    desired_tuple = ('eroded', Image)
    command_loop(cmd, desired_tuple)

    print lb
    print "Display it with eroded.show(). It should look almost as if the image"
    print "was made if ink and had water spoiled on it."
    print lb
    request_show_command()

    print lb
    print "Last but not least, let's crop a section of the image out:"
    print lb
    cmd = "cropped = img.crop(100, 100, 50, 50)"
    desired_tuple = ('cropped', Image)
    command_loop(cmd, desired_tuple)

    print lb
    print "Use cropped.show() to display it."
    print lb
    request_show_command()
    print lb
    print "That went from the coordinate in (X,Y), which is (0,0) and is the"
    print "top left corner of the picture, to coordinates (100,100) in the"
    print "(X,Y) and cropped a picture from that which is 50 pixels by 50 pixels."

    print lb
    print "Alright! This was tutorial 5/6."
    print "Next tutorial: Features"
    if not end_tutorial():
        tutorial_features()
    return

def tutorial_slicing():
    shellclear()
    print "Slicing Images"
    print lb
    print "Slicing is sort of a new paradigm to access parts of an image."
    print "Typically in vision a region of interest (ROI) is given.  "
    print "In this case, slicing is a very powerful way to access parts"
    print "of an image, or basically any matrix in SimpleCV in general."
    print lb
    print "This is done by using:"
    print "section = img[1:10,1:10]"
    print lb
    print "What is returned is an image object with that window."
    print "the slicing basically acts like a ROI but returns an image"
    print "so if you wanted to say run edge detection on a 20x20 box"
    print "in the picture that started at x=5,y=10 you use:"
    print "foundedges = img[5:25,10:30].edges()"
    print lb
    raw_input("[Press enter to continue]")
    shellclear()
    in_text = ""
    shouldbe = "ROI = img[1:6,1:6]"
    print "Please type this now:"
    print shouldbe
    print lb
    while (in_text != shouldbe):
        in_text = raw_input("SimpleCV:>")
        if(in_text != shouldbe):
            print "sorry, that is incorrect"
            print "please type:"
            print shouldbe
    shellclear()
    print "Correct, you just returned a 5 pixel by 5 pixel image object"
    print lb
    return

def tutorial_features():
    shellclear()
    print "Features"
    print lb
    print "Features are things you are looking for in the picture. They can be"
    print "blobs, corners, lines, etc. Features are sometimes referred to as a"
    print "fiducial in computer vision. These features are something that is"
    print "measurable, and something that makes images unique. Features are"
    print "something like when comparing things like fruit. In this case the"
    print "features could be the shape and the color, amongst others."
    print lb
    print "What features are in SimpleCV is an abstract representation of that."
    print "You take your image, then perform a function on it, and get back"
    print "features or another image with them applied. The crop example is"
    print "a case where an image is returned after we perform something."
    print lb
    print "In a simple example we will use the famous 'lenna' image, and find"
    print "corners in the picture."
    print lb
    tutorial_interpreter.runsource("img = Image('lenna')")
    print "img = Image('lenna') (already done for you)"
    print lb
    print "Try it yourself:"
    print lb

    cmd = "corners = img.findCorners()"
    desired_tuple = ('corners', FeatureSet)
    command_loop(cmd, desired_tuple)

    print lb
    print "Correct, you just got a featureset object which contains"
    print "feature objects.  These feature objects contain data from the"
    print "found corners"
    print lb

    print "Tip: If your are unsure what parameters to pass, you can always use"
    print "the built in help support by typing help(Image.findCorners). Keep in"
    print "mind that this help works for all of the functions available in"
    print "SimpleCV"
    print lb

    print "We can also do that with blobs. Try it:"
    print lb

    cmd = "blobs = img.findBlobs()"
    desired_tuple = ('blobs', FeatureSet)
    command_loop(cmd, desired_tuple)

    print lb
    print "Great, but..."
    print "When we show the image we won't notice anything different. This"
    print "is because we have to actually tell the blobs to draw themselves"
    print "on the image:"
    print lb
    print "blobs.draw()"
    print lb

    while True:
        if prompt_and_run().endswith('.draw()'):
            break
        print "No blobs have been drawn!"
        print lb

    print "Now use img.show() to see the changes!"
    print lb
    request_show_command()
    print lb
    raw_input("[Press enter to continue]")

    print lb
    print lb
    print "There's also a small trick built into SimpleCV to do this even faster"
    print lb
    tutorial_interpreter.runsource("img = Image('lenna')")
    print "img = Image('lenna') (already done for you)"
    print lb

    while True:
        print "img.findBlobs().show()"
        print lb
        if prompt_and_run().endswith('.show()'):
            break
        print "Nothing has been shown!"
        print lb

    print lb
    print "Alright! This was tutorial 6/6."
    #print "Next tutorial: ..."
    return

def magic_tutorial(self,arg):
    tutorials_dict = {'image': tutorial_image, 'save': tutorial_save,
                     'camera': tutorial_camera, 'manipulation': tutorial_manipulation,
                     'copy': tutorial_copy, 'features': tutorial_features}

    if (arg == ""):
        shellclear()
        print "+--------------------------------+"
        print " Welcome to the SimpleCV tutorial "
        print "+--------------------------------+"
        print lb
        print "At anytime on the SimpleCV Interactive Shell you can type tutorial,"
        print "then press the tab key and it will autocomplete any tutorial that"
        print "is currently available."
        print lb
        print "Let's start off with Loading and Saving images!"
        print lb
        print lb
        raw_input("[Press enter to continue]")
        tutorial_image()
        end_of_tutorial()
        return
    else:
        if arg in tutorials_dict:
            tutorials_dict[arg]()
        else:
            print "%s is not a tutorial!" % arg
<|endoftext|>


<commit_before>from blckur.constants import *
from blckur.exceptions import *
from blckur.helpers import *
from blckur import filter
from blckur import formatter
from blckur import handler
from blckur import test_case

import time
import uuid
import requests

class Base(object):
    _instance = None
    base_url = None
    filter = filter.ReportFilter()
    formatter = formatter.ReportFormatter()
    handler = handler.ReportHandler()
    request_kwargs = None

    def __init__(self):
        self.objects = {}
        self.requests = requests

    @static_property
    def TestCase(cls):
        cls._instance = cls._instance or cls()
        return type('TestCase', (test_case.TestCase,), {
            'id': uuid.uuid4().hex,
            'base': cls._instance,
        })

    def setup(self):
        pass

    def run_all(self):
        module = __import__('__main__')

        for name in dir(module):
            obj = getattr(module, name)
            if isinstance(obj, type) and \
                    issubclass(obj, test_case.TestCase) and \
                    obj.base == self and obj.id not in self.objects:
                obj()

    def handle_request(self, method, url, **kwargs):
        if self.request_kwargs:
            kwargs.update(self.request_kwargs)

        start = time.time()
        response = self.request(method, url, **kwargs)
        response.duration = int((time.time() - start) * 1000)

        return response

    def request(self, method, url, headers=None, json=None,
            params=None, data=None, **kwargs):
        return self.requests.request(
            method,
            url,
            headers=headers,
            json=json,
            params=params,
            data=data,
            **kwargs
        )

    def log_error(self, tst_case, error_msg):
        if not self.filter.error(self):
            return

        report = self.formatter.error(tst_case, error_msg)
        self.handler.error(report)

    def log_response(self, tst_case):
        if not self.filter.response(self):
            return

        report = self.formatter.response(tst_case)
        self.handler.response(report)

    @classmethod
    def main(cls):
        cls._instance.setup()
        cls._instance.run_all()

class SessionBase(Base):
    def __init__(self):
        Base.__init__(self)
        self.requests = requests.Session()

    def _get_attr(self, name, default=None):
        if hasattr(self, name):
            return getattr(self, name)
        return default

    def setup(self):
        test_case = type('SessionInitTestCase', (self.TestCase,), {
            'base': self,
            'require': self._get_attr('require'),
            'required': self._get_attr('required', True),
            'method': self._get_attr('method'),
            'path': self._get_attr('path'),
            'expect_status': self._get_attr('expect_status'),
            'input_headers': self._get_attr('input_headers'),
            'expect_headers': self._get_attr('expect_headers'),
            'input_json': self._get_attr('input_json'),
            'input_params': self._get_attr('input_params'),
            'input_data': self._get_attr('input_data'),
            'expect_json': self._get_attr('expect_json'),
            'request_kwargs': self._get_attr('request_kwargs'),
        })
        test_case()

def append_to(base_cls):
    def _wrapped(cls):
        if not hasattr(base_cls, 'test_cases'):
            base_cls.test_cases = collections.deque()
        base_cls.test_cases.append(cls)
        return cls
    return _wrapped

def prepend_to(base_cls):
    def _wrapped(cls):
        if not hasattr(base_cls, 'test_cases'):
            base_cls.test_cases = collections.deque()
        base_cls.test_cases.appendleft(cls)
        return cls
    return _wrapped
<commit_msg>Update session setup<commit_after>from blckur.constants import *
from blckur.exceptions import *
from blckur.helpers import *
from blckur import filter
from blckur import formatter
from blckur import handler
from blckur import test_case

import time
import uuid
import requests

class Base(object):
    _instance = None
    base_url = None
    filter = filter.ReportFilter()
    formatter = formatter.ReportFormatter()
    handler = handler.ReportHandler()
    request_kwargs = None

    def __init__(self):
        self.objects = {}
        self.requests = requests

    @static_property
    def TestCase(cls):
        cls._instance = cls._instance or cls()
        return type('TestCase', (test_case.TestCase,), {
            'id': uuid.uuid4().hex,
            'base': cls._instance,
        })

    def setup(self):
        pass

    def run_all(self):
        module = __import__('__main__')

        for name in dir(module):
            obj = getattr(module, name)
            if isinstance(obj, type) and \
                    issubclass(obj, test_case.TestCase) and \
                    obj.base == self and obj.id not in self.objects:
                obj()

    def handle_request(self, method, url, **kwargs):
        if self.request_kwargs:
            kwargs.update(self.request_kwargs)

        start = time.time()
        response = self.request(method, url, **kwargs)
        response.duration = int((time.time() - start) * 1000)

        return response

    def request(self, method, url, headers=None, json=None,
            params=None, data=None, **kwargs):
        return self.requests.request(
            method,
            url,
            headers=headers,
            json=json,
            params=params,
            data=data,
            **kwargs
        )

    def log_error(self, tst_case, error_msg):
        if not self.filter.error(self):
            return

        report = self.formatter.error(tst_case, error_msg)
        self.handler.error(report)

    def log_response(self, tst_case):
        if not self.filter.response(self):
            return

        report = self.formatter.response(tst_case)
        self.handler.response(report)

    @classmethod
    def main(cls):
        cls._instance.setup()
        cls._instance.run_all()

class SessionBase(Base):
    def __init__(self):
        Base.__init__(self)
        self.requests = requests.Session()

    def _get_attr(self, name, default=None):
        if hasattr(self, name):
            return getattr(self, name)
        return default

    def setup(self):
        test_case = type('SessionInitTestCase', (self.TestCase,), {
            'base': self,
            'require': self._get_attr('require'),
            'required': self._get_attr('required', True),
            'method': self._get_attr('method'),
            'path': self._get_attr('path'),
            'expect_status': self._get_attr('expect_status'),
            'input_headers': self._get_attr('input_headers'),
            'expect_headers': self._get_attr('expect_headers'),
            'input_json': self._get_attr('input_json'),
            'input_params': self._get_attr('input_params'),
            'input_data': self._get_attr('input_data'),
            'expect_json': self._get_attr('expect_json'),
            'request_kwargs': self._get_attr('request_kwargs'),
        })
        self.test_cases.appendleft(tst_case)

def append_to(base_cls):
    def _wrapped(cls):
        if not hasattr(base_cls, 'test_cases'):
            base_cls.test_cases = collections.deque()
        base_cls.test_cases.append(cls)
        return cls
    return _wrapped

def prepend_to(base_cls):
    def _wrapped(cls):
        if not hasattr(base_cls, 'test_cases'):
            base_cls.test_cases = collections.deque()
        base_cls.test_cases.appendleft(cls)
        return cls
    return _wrapped
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

import pytest
import asyncio
from unittest.mock import patch, call, MagicMock
from foglamp.common import logger
from foglamp.common.storage_client.storage_client import ReadingsStorageClient, StorageClient
from foglamp.common.statistics import Statistics
from foglamp.tasks.purge.purge import Purge
from foglamp.common.process import FoglampProcess
from foglamp.common.configuration_manager import ConfigurationManager
from foglamp.common.audit_logger import AuditLogger

__author__ = "Vaibhav Singhal"
__copyright__ = "Copyright (c) 2017 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"

@pytest.allure.feature("unit")
@pytest.allure.story("tasks", "purge")
class TestPurge:
    """Test the units of purge.py"""

    def test_init(self, event_loop):
        """Test that creating an instance of Purge calls init of FoglampProcess and creates loggers"""
        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)
        with patch.object(FoglampProcess, "__init__") as mock_process:
            with patch.object(logger, "setup") as log:
                with patch.object(mockAuditLogger, "__init__", return_value=None):
                    p = Purge(loop=event_loop)
                assert isinstance(p, Purge)
                assert isinstance(p._audit, AuditLogger)
            log.assert_called_once_with("Data Purge")
        mock_process.assert_called_once_with()

    def test_write_statistics(self, event_loop):
        """Test that write_statistics calls update statistics with defined keys and value increments"""

        @asyncio.coroutine
        def mock_s_update():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)
        with patch.object(FoglampProcess, '__init__'):
            with patch.object(Statistics, 'update', return_value=mock_s_update()) as mock_stats_update:
                with patch.object(mockAuditLogger, "__init__", return_value=None):
                    p = Purge(loop=event_loop)
                    p._storage = mockStorageClient
                    p.write_statistics(1, 2)
                    mock_stats_update.assert_has_calls([call('PURGED', 1), call('UNSNPURGED', 2)])

    def test_set_configuration(self, event_loop):
        """Test that purge's set_configuration returns configuration item with key 'PURGE_READ' """

        @asyncio.coroutine
        def mock_cm_return():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)
        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                p._storage = MagicMock(spec=StorageClient)
                mock_cm = ConfigurationManager(p._storage)
                with patch.object(mock_cm, 'create_category', return_value=mock_cm_return()) as mock_create_cat:
                    with patch.object(mock_cm, 'get_category_all_items', return_value=mock_cm_return()) \
                            as mock_get_cat:
                        p.set_configuration()
                        mock_get_cat.assert_called_once_with('PURGE_READ')
                    args, kwargs = mock_create_cat.call_args
                    assert len(args) == 3
                    assert args[0] == 'PURGE_READ'

    @pytest.fixture()
    def store_purge(self, **kwargs):
        if kwargs.get('age') == '-1' or kwargs.get('size') == '-1':
            return {"message": "409 Conflict"}
        return {"readings": 10, "removed": 1, "unsentPurged": 2, "unsentRetained": 7}

    config = {"purgeAgeSize": {"retainUnsent": {"value": "False"}, "age": {"value": "72"}, "size": {"value": "20"}},
              "purgeAge": {"retainUnsent": {"value": "False"}, "age": {"value": "72"}, "size": {"value": "0"}},
              "purgeSize": {"retainUnsent": {"value": "False"}, "age": {"value": "0"}, "size": {"value": "100"}},
              "retainAgeSize": {"retainUnsent": {"value": "True"}, "age": {"value": "72"}, "size": {"value": "20"}},
              "retainAge": {"retainUnsent": {"value": "True"}, "age": {"value": "72"}, "size": {"value": "0"}},
              "retainSize": {"retainUnsent": {"value": "True"}, "age": {"value": "0"}, "size": {"value": "100"}}}

    @pytest.mark.parametrize("conf, expected_return, expected_calls", [
        (config["purgeAgeSize"], (2, 4), {'sent_id': 0, 'size': '20', 'flag': 'purge'}),
        (config["purgeAge"], (1, 2), {'sent_id': 0, 'age': '72', 'flag': 'purge'}),
        (config["purgeSize"], (1, 2), {'sent_id': 0, 'size': '100', 'flag': 'purge'}),
        (config["retainAgeSize"], (2, 4), {'sent_id': 0, 'size': '20', 'flag': 'retain'}),
        (config["retainAge"], (1, 2), {'sent_id': 0, 'age': '72', 'flag': 'retain'}),
        (config["retainSize"], (1, 2), {'sent_id': 0, 'size': '100', 'flag': 'retain'})
    ])
    def test_purge_data(self, event_loop, conf, expected_return, expected_calls):
        """Test that purge_data calls Storage's purge with defined configuration"""

        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                p._logger = logger
                p._logger.info = MagicMock()
                p._logger.error = MagicMock()
                p._storage = MagicMock(spec=StorageClient)
                p._readings_storage = MagicMock(spec=ReadingsStorageClient)
                audit = p._audit
                with patch.object(p._readings_storage, 'purge', side_effect=self.store_purge) as mock_storage_purge:
                    with patch.object(audit, 'information', return_value=mock_audit_info()) as audit_info:
                        # Test the positive case when all if conditions in purge_data pass
                        assert expected_return == p.purge_data(conf)
                        assert audit_info.called
                        args, kwargs = mock_storage_purge.call_args
                        assert kwargs == expected_calls

    @pytest.mark.parametrize("conf, expected_return", [
        ({"retainUnsent": {"value": "False"}, "age": {"value": "0"}, "size": {"value": "0"}}, (0, 0)),
        ({"retainUnsent": {"value": "True"}, "age": {"value": "0"}, "size": {"value": "0"}}, (0, 0))
    ])
    def test_purge_data_no_data_purged(self, event_loop, conf, expected_return):
        """Test that purge_data logs message when no data was purged"""

        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                p._logger = logger
                p._logger.info = MagicMock()
                p._logger.error = MagicMock()
                p._storage = MagicMock(spec=StorageClient)
                p._readings_storage = MagicMock(spec=ReadingsStorageClient)
                audit = p._audit
                with patch.object(p._readings_storage, 'purge', side_effect=self.store_purge):
                    with patch.object(audit, 'information', return_value=mock_audit_info()):
                        assert expected_return == p.purge_data(conf)
                        p._logger.info.assert_called_once_with("No rows purged")

    @pytest.mark.parametrize("conf, expected_return", [
        ({"retainUnsent": {"value": "True"}, "age": {"value": "-1"}, "size": {"value": "-1"}}, (0, 0))
    ])
    def test_purge_error_storage_response(self, event_loop, conf, expected_return):
        """Test that purge_data logs error when storage purge returns an error response"""

        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                p._logger = logger
                p._logger.info = MagicMock()
                p._logger.error = MagicMock()
                p._storage = MagicMock(spec=StorageClient)
                p._readings_storage = MagicMock(spec=ReadingsStorageClient)
                audit = p._audit
                with patch.object(p._readings_storage, 'purge', side_effect=self.store_purge):
                    with patch.object(audit, 'information', return_value=mock_audit_info()):
                        assert expected_return == p.purge_data(conf)
                        p._logger.error.assert_called_with('Purge failed: %s', '409 Conflict')

    @pytest.mark.parametrize("conf, expected_error_key",
                             [({"retainUnsent": {"value": "True"}, "age": {"value": "bla"}, "size": {"value": "0"}},
                               "age"),
                              ({"retainUnsent": {"value": "True"}, "age": {"value": "0"}, "size": {"value": "bla"}},
                               "size")])
    def test_purge_data_invalid_conf(self, event_loop, conf, expected_error_key):
        """Test that purge_data raises exception when called with invalid configuration"""

        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                p._logger = logger
                p._logger.info = MagicMock()
                p._logger.error = MagicMock()
                p._storage = MagicMock(spec=StorageClient)
                p._readings_storage = MagicMock(spec=ReadingsStorageClient)
                audit = p._audit
                with patch.object(p._readings_storage, 'purge', side_effect=self.store_purge):
                    with patch.object(audit, 'information', return_value=mock_audit_info()):
                        # Test the code block when purge failed because of invalid configuration
                        with pytest.raises(ValueError):
                            p.purge_data(conf)
                        p._logger.error.assert_called_with('Configuration item {} bla should be integer!'.
                                                           format(expected_error_key))

    def test_run(self, event_loop):
        """Test that run calls all units of purge process"""
        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                config = "Some config"
                p._logger.exception = MagicMock()
                with patch.object(p, 'set_configuration', return_value=config) as mock_set_config:
                    with patch.object(p, 'purge_data', return_value=(1, 2)) as mock_purge_data:
                        with patch.object(p, 'write_statistics') as mock_write_stats:
                            p.run()
                # Test the positive case when no error in try block
                mock_set_config.assert_called_once_with()
                mock_purge_data.assert_called_once_with(config)
                mock_write_stats.assert_called_once_with(1, 2)

    def test_run_exception(self, event_loop):
        """Test that run calls all units of purge process and checks the exception handling"""
        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                config = "Some config"
                p._logger.exception = MagicMock()
                with patch.object(p, 'set_configuration', return_value=config):
                    with patch.object(p, 'purge_data', return_value=Exception()):
                        with patch.object(p, 'write_statistics'):
                            p.run()
                # Test the negative case when function purge_data raise some exception
                p._logger.exception.assert_called_once_with("'Exception' object is not iterable")
<commit_msg>Test fixed with code change<commit_after># -*- coding: utf-8 -*-

# FOGLAMP_BEGIN
# See: http://foglamp.readthedocs.io/
# FOGLAMP_END

import pytest
import asyncio
from unittest.mock import patch, call, MagicMock
from foglamp.common import logger
from foglamp.common.storage_client.storage_client import ReadingsStorageClient, StorageClient
from foglamp.common.statistics import Statistics
from foglamp.tasks.purge.purge import Purge
from foglamp.common.process import FoglampProcess
from foglamp.common.configuration_manager import ConfigurationManager
from foglamp.common.audit_logger import AuditLogger
from foglamp.common.storage_client.exceptions import *

__author__ = "Vaibhav Singhal"
__copyright__ = "Copyright (c) 2017 OSIsoft, LLC"
__license__ = "Apache 2.0"
__version__ = "${VERSION}"

@pytest.allure.feature("unit")
@pytest.allure.story("tasks", "purge")
class TestPurge:
    """Test the units of purge.py"""

    def test_init(self, event_loop):
        """Test that creating an instance of Purge calls init of FoglampProcess and creates loggers"""
        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)
        with patch.object(FoglampProcess, "__init__") as mock_process:
            with patch.object(logger, "setup") as log:
                with patch.object(mockAuditLogger, "__init__", return_value=None):
                    p = Purge(loop=event_loop)
                assert isinstance(p, Purge)
                assert isinstance(p._audit, AuditLogger)
            log.assert_called_once_with("Data Purge")
        mock_process.assert_called_once_with()

    def test_write_statistics(self, event_loop):
        """Test that write_statistics calls update statistics with defined keys and value increments"""

        @asyncio.coroutine
        def mock_s_update():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)
        with patch.object(FoglampProcess, '__init__'):
            with patch.object(Statistics, 'update', return_value=mock_s_update()) as mock_stats_update:
                with patch.object(mockAuditLogger, "__init__", return_value=None):
                    p = Purge(loop=event_loop)
                    p._storage = mockStorageClient
                    p.write_statistics(1, 2)
                    mock_stats_update.assert_has_calls([call('PURGED', 1), call('UNSNPURGED', 2)])

    def test_set_configuration(self, event_loop):
        """Test that purge's set_configuration returns configuration item with key 'PURGE_READ' """

        @asyncio.coroutine
        def mock_cm_return():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)
        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                p._storage = MagicMock(spec=StorageClient)
                mock_cm = ConfigurationManager(p._storage)
                with patch.object(mock_cm, 'create_category', return_value=mock_cm_return()) as mock_create_cat:
                    with patch.object(mock_cm, 'get_category_all_items', return_value=mock_cm_return()) \
                            as mock_get_cat:
                        p.set_configuration()
                        mock_get_cat.assert_called_once_with('PURGE_READ')
                    args, kwargs = mock_create_cat.call_args
                    assert len(args) == 3
                    assert args[0] == 'PURGE_READ'

    @pytest.fixture()
    def store_purge(self, **kwargs):
        if kwargs.get('age') == '-1' or kwargs.get('size') == '-1':
            raise StorageServerError(400, "Bla", "Some Error")
        return {"readings": 10, "removed": 1, "unsentPurged": 2, "unsentRetained": 7}

    config = {"purgeAgeSize": {"retainUnsent": {"value": "False"}, "age": {"value": "72"}, "size": {"value": "20"}},
              "purgeAge": {"retainUnsent": {"value": "False"}, "age": {"value": "72"}, "size": {"value": "0"}},
              "purgeSize": {"retainUnsent": {"value": "False"}, "age": {"value": "0"}, "size": {"value": "100"}},
              "retainAgeSize": {"retainUnsent": {"value": "True"}, "age": {"value": "72"}, "size": {"value": "20"}},
              "retainAge": {"retainUnsent": {"value": "True"}, "age": {"value": "72"}, "size": {"value": "0"}},
              "retainSize": {"retainUnsent": {"value": "True"}, "age": {"value": "0"}, "size": {"value": "100"}}}

    @pytest.mark.parametrize("conf, expected_return, expected_calls", [
        (config["purgeAgeSize"], (2, 4), {'sent_id': 0, 'size': '20', 'flag': 'purge'}),
        (config["purgeAge"], (1, 2), {'sent_id': 0, 'age': '72', 'flag': 'purge'}),
        (config["purgeSize"], (1, 2), {'sent_id': 0, 'size': '100', 'flag': 'purge'}),
        (config["retainAgeSize"], (2, 4), {'sent_id': 0, 'size': '20', 'flag': 'retain'}),
        (config["retainAge"], (1, 2), {'sent_id': 0, 'age': '72', 'flag': 'retain'}),
        (config["retainSize"], (1, 2), {'sent_id': 0, 'size': '100', 'flag': 'retain'})
    ])
    def test_purge_data(self, event_loop, conf, expected_return, expected_calls):
        """Test that purge_data calls Storage's purge with defined configuration"""

        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                p._logger = logger
                p._logger.info = MagicMock()
                p._logger.error = MagicMock()
                p._storage = MagicMock(spec=StorageClient)
                p._readings_storage = MagicMock(spec=ReadingsStorageClient)
                audit = p._audit
                with patch.object(p._readings_storage, 'purge', side_effect=self.store_purge) as mock_storage_purge:
                    with patch.object(audit, 'information', return_value=mock_audit_info()) as audit_info:
                        # Test the positive case when all if conditions in purge_data pass
                        assert expected_return == p.purge_data(conf)
                        assert audit_info.called
                        args, kwargs = mock_storage_purge.call_args
                        assert kwargs == expected_calls

    @pytest.mark.parametrize("conf, expected_return", [
        ({"retainUnsent": {"value": "False"}, "age": {"value": "0"}, "size": {"value": "0"}}, (0, 0)),
        ({"retainUnsent": {"value": "True"}, "age": {"value": "0"}, "size": {"value": "0"}}, (0, 0))
    ])
    def test_purge_data_no_data_purged(self, event_loop, conf, expected_return):
        """Test that purge_data logs message when no data was purged"""

        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                p._logger = logger
                p._logger.info = MagicMock()
                p._logger.error = MagicMock()
                p._storage = MagicMock(spec=StorageClient)
                p._readings_storage = MagicMock(spec=ReadingsStorageClient)
                audit = p._audit
                with patch.object(p._readings_storage, 'purge', side_effect=self.store_purge):
                    with patch.object(audit, 'information', return_value=mock_audit_info()):
                        assert expected_return == p.purge_data(conf)
                        p._logger.info.assert_called_once_with("No rows purged")

    @pytest.mark.parametrize("conf, expected_return", [
        ({"retainUnsent": {"value": "True"}, "age": {"value": "-1"}, "size": {"value": "-1"}}, (0, 0))
    ])
    def test_purge_error_storage_response(self, event_loop, conf, expected_return):
        """Test that purge_data logs error when storage purge returns an error response"""

        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                p._logger = logger
                p._logger.info = MagicMock()
                p._logger.error = MagicMock()
                p._storage = MagicMock(spec=StorageClient)
                p._readings_storage = MagicMock(spec=ReadingsStorageClient)
                audit = p._audit
                with patch.object(p._readings_storage, 'purge', side_effect=self.store_purge):
                    with patch.object(audit, 'information', return_value=mock_audit_info()):
                        assert expected_return == p.purge_data(conf)

    @pytest.mark.parametrize("conf, expected_error_key",
                             [({"retainUnsent": {"value": "True"}, "age": {"value": "bla"}, "size": {"value": "0"}},
                               "age"),
                              ({"retainUnsent": {"value": "True"}, "age": {"value": "0"}, "size": {"value": "bla"}},
                               "size")])
    def test_purge_data_invalid_conf(self, event_loop, conf, expected_error_key):
        """Test that purge_data raises exception when called with invalid configuration"""

        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                p._logger = logger
                p._logger.info = MagicMock()
                p._logger.error = MagicMock()
                p._storage = MagicMock(spec=StorageClient)
                p._readings_storage = MagicMock(spec=ReadingsStorageClient)
                audit = p._audit
                with patch.object(p._readings_storage, 'purge', side_effect=self.store_purge):
                    with patch.object(audit, 'information', return_value=mock_audit_info()):
                        # Test the code block when purge failed because of invalid configuration
                        p.purge_data(conf)
                        p._logger.error.assert_called_with('Configuration item {} bla should be integer!'.
                                                           format(expected_error_key))

    def test_run(self, event_loop):
        """Test that run calls all units of purge process"""
        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                config = "Some config"
                p._logger.exception = MagicMock()
                with patch.object(p, 'set_configuration', return_value=config) as mock_set_config:
                    with patch.object(p, 'purge_data', return_value=(1, 2)) as mock_purge_data:
                        with patch.object(p, 'write_statistics') as mock_write_stats:
                            p.run()
                # Test the positive case when no error in try block
                mock_set_config.assert_called_once_with()
                mock_purge_data.assert_called_once_with(config)
                mock_write_stats.assert_called_once_with(1, 2)

    def test_run_exception(self, event_loop):
        """Test that run calls all units of purge process and checks the exception handling"""
        @asyncio.coroutine
        def mock_audit_info():
            return ""

        mockStorageClient = MagicMock(spec=StorageClient)
        mockAuditLogger = AuditLogger(mockStorageClient)

        with patch.object(FoglampProcess, '__init__'):
            with patch.object(mockAuditLogger, "__init__", return_value=None):
                p = Purge(loop=event_loop)
                config = "Some config"
                p._logger.exception = MagicMock()
                with patch.object(p, 'set_configuration', return_value=config):
                    with patch.object(p, 'purge_data', return_value=Exception()):
                        with patch.object(p, 'write_statistics'):
                            p.run()
                # Test the negative case when function purge_data raise some exception
                p._logger.exception.assert_called_once_with("'Exception' object is not iterable")
<|endoftext|>


<commit_before>"""models.py."""
from django.db import models
from django.utils import timezone

class Category(models.Model):
    """カテゴリー."""

    name = models.CharField('カテゴリ名', max_length=255)
    description = models.TextField('概要', blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.name

class Tag(models.Model):
    """タグ."""

    name = models.CharField('タグ名', max_length=255)
    description = models.TextField('概要', blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.name

class Post(models.Model):
    """ブログのポスト."""

    title = models.CharField('タイトル', max_length=255)
    text = models.TextField('本文')
    category = models.ForeignKey(
        Category, verbose_name='カテゴリ', on_delete=models.PROTECT, null=True)
    tag = models.ManyToManyField(Tag, blank=True, verbose_name='タグ')
    thumnail = models.ImageField(
        'サムネイル', upload_to='thumnail/', blank=True, null=True)
    is_publick = models.BooleanField('公開可能か?', default=True)
    friend_posts = models.ManyToManyField(
        'self', verbose_name='関連記事', blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.title

    def get_next(self):
        """次の記事."""
        next_post = Post.objects.filter(
            is_publick=True, created_at__gt=self.created_at)
        if next_post:
            return next_post.first()
        return None

    def get_prev(self):
        """前の記事."""
        prev_post = Post.objects.filter(
            is_publick=True, created_at__lt=self.created_at)
        if prev_post:
            return prev_post.last()
        return None

class Comment(models.Model):
    """コメント."""

    name = models.CharField('名前', max_length=255, default='名無し')
    text = models.TextField('コメント')
    icon = models.ImageField(
        'サムネイル', upload_to='com_icon/', blank=True, null=True)
    target = models.ForeignKey(
        Post, on_delete=models.CASCADE, verbose_name='対象記事')
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.text[:10]

class ReComment(models.Model):
    """返信コメント."""

    name = models.CharField('名前', max_length=255, default='名無し')
    text = models.TextField('コメント')
    icon = models.ImageField(
        'サムネイル', upload_to='com_icon/', blank=True, null=True)
    target = models.ForeignKey(
        Comment, on_delete=models.CASCADE, verbose_name='対象コメント')
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.text[:10]

class Link(models.Model):
    """リンク."""

    name = models.CharField('リンク名', max_length=255)
    adrs = models.CharField('アドレス', max_length=255)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.name

class Analytics(models.Model):
    """アナリティクスの情報."""

    name = models.CharField('アナリティクス', max_length=255, blank=True)
    html = models.TextField('アナリティクスHTML', blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.name

class Ads(models.Model):
    """広告関連."""

    name = models.CharField('広告名', max_length=255, blank=True)
    html = models.TextField('広告HTML', blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.name

class SiteDetail(models.Model):
    """サイトの詳細."""

    title = models.CharField('タイトル', max_length=255, blank=True)
    description = models.CharField('サイトの説明', max_length=255, blank=True)
    author = models.CharField('管理者', max_length=255, blank=True)
    author_mail = models.EmailField('管理者アドレス', max_length=255, blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.author

class PopularPost(models.Model):
    """人気記事."""

    title = models.CharField('タイトル', max_length=255)
    url = models.CharField('URL', max_length=255)
    page_view = models.IntegerField('ページビュー数')

    def __str__(self):
        return '{0} - {1} - {2}'.format(
            self.url, self.title, self.page_view)
<commit_msg>次記事・前記事の並び順を調整<commit_after>"""models.py."""
from django.db import models
from django.utils import timezone

class Category(models.Model):
    """カテゴリー."""

    name = models.CharField('カテゴリ名', max_length=255)
    description = models.TextField('概要', blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.name

class Tag(models.Model):
    """タグ."""

    name = models.CharField('タグ名', max_length=255)
    description = models.TextField('概要', blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.name

class Post(models.Model):
    """ブログのポスト."""

    title = models.CharField('タイトル', max_length=255)
    text = models.TextField('本文')
    category = models.ForeignKey(
        Category, verbose_name='カテゴリ', on_delete=models.PROTECT, null=True)
    tag = models.ManyToManyField(Tag, blank=True, verbose_name='タグ')
    thumnail = models.ImageField(
        'サムネイル', upload_to='thumnail/', blank=True, null=True)
    is_publick = models.BooleanField('公開可能か?', default=True)
    friend_posts = models.ManyToManyField(
        'self', verbose_name='関連記事', blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.title

    def get_next(self):
        """次の記事."""
        next_post = Post.objects.filter(
            is_publick=True, created_at__gt=self.created_at)
        if next_post:
            return next_post.last()
        return None

    def get_prev(self):
        """前の記事."""
        prev_post = Post.objects.filter(
            is_publick=True, created_at__lt=self.created_at)
        if prev_post:
            return prev_post.first()
        return None

class Comment(models.Model):
    """コメント."""

    name = models.CharField('名前', max_length=255, default='名無し')
    text = models.TextField('コメント')
    icon = models.ImageField(
        'サムネイル', upload_to='com_icon/', blank=True, null=True)
    target = models.ForeignKey(
        Post, on_delete=models.CASCADE, verbose_name='対象記事')
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.text[:10]

class ReComment(models.Model):
    """返信コメント."""

    name = models.CharField('名前', max_length=255, default='名無し')
    text = models.TextField('コメント')
    icon = models.ImageField(
        'サムネイル', upload_to='com_icon/', blank=True, null=True)
    target = models.ForeignKey(
        Comment, on_delete=models.CASCADE, verbose_name='対象コメント')
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.text[:10]

class Link(models.Model):
    """リンク."""

    name = models.CharField('リンク名', max_length=255)
    adrs = models.CharField('アドレス', max_length=255)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.name

class Analytics(models.Model):
    """アナリティクスの情報."""

    name = models.CharField('アナリティクス', max_length=255, blank=True)
    html = models.TextField('アナリティクスHTML', blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.name

class Ads(models.Model):
    """広告関連."""

    name = models.CharField('広告名', max_length=255, blank=True)
    html = models.TextField('広告HTML', blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.name

class SiteDetail(models.Model):
    """サイトの詳細."""

    title = models.CharField('タイトル', max_length=255, blank=True)
    description = models.CharField('サイトの説明', max_length=255, blank=True)
    author = models.CharField('管理者', max_length=255, blank=True)
    author_mail = models.EmailField('管理者アドレス', max_length=255, blank=True)
    created_at = models.DateTimeField('作成日', default=timezone.now)

    def __str__(self):
        """str."""
        return self.author

class PopularPost(models.Model):
    """人気記事."""

    title = models.CharField('タイトル', max_length=255)
    url = models.CharField('URL', max_length=255)
    page_view = models.IntegerField('ページビュー数')

    def __str__(self):
        return '{0} - {1} - {2}'.format(
            self.url, self.title, self.page_view)
<|endoftext|>


<commit_before>import logging
import sys
import os

import vcf

from sv_interval import SVInterval

logger = logging.getLogger(__name__)

mydir = os.path.dirname(os.path.realpath(__file__))

'''
from http://gmt.genome.wustl.edu/packages/pindel/user-manual.html

There is a head line for each variant reported, followed by the alignment of supporting reads to the reference on the
second line. The example variants are a 1671bp deletion and a 10bp insertion on chr20. The breakpoints are specified
after "BP". Due to microhomology around the breakpoints, the breakpoint coordinates may shift both upstream and
downstream,'BP_range' is used to indicate the range of this shift. The header line contains the following data:

1) The index of the indel/SV (57 means that 57 insertions precede this insertion in the file)

2) The type of indel/SV: I for insertion, D for deletion, INV for inversion, TD for tandem duplication

3) The length of the SV

4) "NT" (to indicate that the next number is the length of non-template sequences inserted; insertions are fully covered
 by the NT-fields, deletions can have NT bases if the deletion is not 'pure', meaning that while bases have been
 deleted, some bases have been inserted between the breakpoints)

5) the length(s) of the NT fragment(s)

6) the sequence(s) of the NT fragment(s)

7-8) the identifier of the chromosome the read was found on

9-10-11) BP: the start and end positions of the SV

12-13-14) BP_range: if the exact position of the SV is unclear since bases at the edge of one read-half could equally
well be appended to the other read-half. In the deletion example, ACA could be on any side of the gap, so the original
deletion could have been between 1337143 and 1338815, between 1337144 and 1338816, or between 1337145 and 133817, or
between 1337146 and 133818. BP-range is used to indicate this range.

15) "Supports": announces that the total count of reads supporting the SV follow.

16) The number of reads supporting the SV

17) The number of unique reads supporting the SV (so not counting duplicate reads)

18) +: supports from reads whose anchors are upstream of the SV

19-20) total number of supporting reads and unique number of supporting reads whose anchors are upstream of the SV.

21) -: supports from reads whose anchors are downstream of the SV

22-23) total number of supporting reads and unique number of supporting reads whose anchors are downstream of the SV

24-25) S1: a simple score, ("# +" + 1)* ("# -" + 1) ;

26-27) SUM_MS: sum of mapping qualities of anchor reads, The reads with variants or unmapped are called split-read,
whose mate is called anchor reads. We use anchor reads to narrow down the search space to speed up and increase
sensitivity;

28) the number of different samples scanned

29-30-31) NumSupSamples?: the number of samples supporting the SV, as well as the number of samples having unique
reads supporting the SV (in practice, these numbers are the same)

32+) Per sample: the sample name, followed by the total number of supporting reads whose anchors are upstream, the
total number of unique supporting reads whose anchors are upstream, the total number of supporting reads whose anchors
 are downstream, and finally the total number of unique supporting reads whose anchors are downstream.

----

The reported larger insertion (LI) record is rather different than other types of variants. Here is the format:
1) index,
2) type(LI),
3) "ChrID",
4) chrName,
5) left breakpoint,
6) "+"
7) number of supporting reads for the left coordinate,
8) right breakpoint,
9) "-"
10) number of supporting reads for the right coordinate.

Following lines are repeated for each sample
11) Sample name
12) "+"
13) upstream supporting reads
14) "-"
15) downstream supporting reads

'''

pindel_name = "Pindel"
pindel_source = set([pindel_name])
min_coverage = 10
het_cutoff = 0.2
hom_cutoff = 0.8

GT_REF = "0/0"
GT_HET = "0/1"
GT_HOM = "1/1"

PINDEL_TO_SV_TYPE = {"I": "INS", "D": "DEL", "LI": "INS", "TD": "DUP:TANDEM", "INV": "INV"}

class PindelRecord:
    def __init__(self, record_string, reference_handle=None):
        fields = record_string.split()
        self.sv_type = fields[1]
        self.name = pindel_name

        self.sv_len = None
        self.num_nt_added = None
        self.nt_added = None
        self.chromosome = None
        self.start_pos = None
        self.end_pos = None
        self.bp_range = None
        self.read_supp = None
        self.uniq_read_supp = None
        self.up_read_supp = None
        self.up_uniq_read_supp = None
        self.down_read_supp = None
        self.down_uniq_read_supp = None
        self.simple_score = None
        self.sum_mapq = None
        self.num_sample = None
        self.num_sample_supp = None
        self.num_sample_uniq_supp = None
        self.homlen = None
        self.homseq = None
        self.samples = None

        if self.sv_type != "LI":
            self.sv_len = int(fields[2])
            self.num_nt_added = map(int, fields[4].split(":"))
            self.nt_added = map(lambda x: x.replace('"', ''), fields[5].split(":"))
            self.chromosome = fields[7]
            self.start_pos = int(fields[9])
            self.end_pos = int(fields[10]) - 1
            self.bp_range = (int(fields[12]), int(fields[13]))
            self.read_supp = int(fields[15])  # The number of reads supporting the SV
            self.uniq_read_supp = int(
                fields[16])  # The number of unique reads supporting SV (not count duplicate reads)
            self.up_read_supp = int(fields[18])  # upstream
            self.up_uniq_read_supp = int(fields[19])
            self.down_read_supp = int(fields[21])  # downstream
            self.down_uniq_read_supp = int(fields[22])
            self.simple_score = int(fields[24])
            self.sum_mapq = int(fields[26])  # sum of mapping qualities of anchor reads
            self.num_sample = int(fields[27])  # number of samples
            self.num_sample_supp = int(fields[29])  # number of samples with supporting reads
            self.num_sample_uniq_supp = int(fields[30])  # number of sample with unique supporting readas
            self.homlen = self.bp_range[1] - self.end_pos
            self.homseq = reference_handle.fetch(self.chromosome, self.end_pos - 1,
                                                 self.bp_range[1] - 1) if reference_handle else ""
            pindel024u_or_later = len(fields) > 31 + 5 * self.num_sample
            if pindel024u_or_later:
                self.samples = self.samples = [{"name": fields[i], "ref_support_at_start": int(fields[i + 1]),
                                                "ref_support_at_end": int(fields[i + 2]),
                                                "plus_support": sum(map(int, fields[i + 3:i + 5])),
                                                "minus_support": sum(map(int, fields[i + 5:i + 7]))} for i in
                                               xrange(31, len(fields), 5)]
            else:
                self.samples = [{"name": fields[i], "ref_support_at_start": int(fields[i + 1]),
                                 "ref_support_at_end": int(fields[i + 2]), "plus_support": int(fields[i + 3]),
                                 "minus_support": int(fields[i + 4])} for i in xrange(31, len(fields), 5)]
        else:
            self.sv_len = 0
            self.chromosome = fields[3]
            self.start_pos = min(int(fields[4]), int(fields[7]))
            self.up_read_supp = int(fields[6])  # upstream
            self.end_pos = self.start_pos + 1
            self.down_read_supp = int(fields[9])  # downstream
            self.bp_range = (self.start_pos, self.end_pos)
            self.homlen = 0
            self.homseq = ""
            self.samples = [{"name": fields[i], "plus_support": int(fields[i + 2]), "minus_support": int(fields[i + 4])}
                            for i in xrange(10, len(fields), 5)]

        self.info = {
            "END": self.end_pos,
            "PD_NUM_NT_ADDED": self.num_nt_added,
            "PD_NT_ADDED": self.nt_added,
            "PD_BP_RANGE_START": self.bp_range[0],
            "PD_BP_RANGE_END": self.bp_range[1],
            "PD_READ_SUPP": self.read_supp,
            "PD_UNIQ_READ_SUPP": self.uniq_read_supp,
            "PD_UP_READ_SUPP": self.up_read_supp,
            "PD_UP_UNIQ_READ_SUPP": self.up_uniq_read_supp,
            "PD_DOWN_READ_SUPP": self.down_read_supp,
            "PD_DOWN_UNIQ_READ_SUPP": self.down_uniq_read_supp,
            "PD_SIMPLE_SCORE": self.simple_score,
            "PD_SUM_MAPQ": self.sum_mapq,
            "PD_NUM_SAMPLE": self.num_sample,
            "PD_NUM_SAMPLE_SUPP": self.num_sample_supp,
            "PD_NUM_SAMPLE_UNIQ_SUPP": self.num_sample_uniq_supp,
            "PD_HOMLEN": self.homlen,
            "PD_HOMSEQ": self.homseq
        }

        self.derive_genotype()

    def derive_genotype(self):
        if self.sv_type == "LI" or self.sv_type == "I":
            self.gt = GT_HET if (self.up_read_supp + self.down_read_supp) > 0 else GT_REF
            return

        total_event_reads = self.uniq_read_supp
        total_ref_reads = self.up_uniq_read_supp + self.down_uniq_read_supp
        if total_event_reads + total_ref_reads < min_coverage:
            self.gt = GT_REF
            return

        allele_fraction = float(total_event_reads) / (float(total_event_reads) + float(total_ref_reads))
        if allele_fraction < het_cutoff:
            self.gt = GT_REF
        elif allele_fraction < hom_cutoff:
            self.gt = GT_HET
        else:
            self.gt = GT_HOM

    def to_sv_interval(self):
        sv_type = PINDEL_TO_SV_TYPE[self.sv_type]
        if sv_type != "INS":
            return SVInterval(self.chromosome,
                              self.start_pos,
                              self.end_pos,
                              name=self.name,
                              sv_type=sv_type,
                              length=self.sv_len,
                              sources=pindel_source,
                              info=self.info,
                              native_sv=self)
        else:
            return SVInterval(self.chromosome,
                              self.start_pos,
                              self.start_pos,
                              self.name,
                              sv_type=sv_type,
                              length=self.sv_len,
                              sources=pindel_source,
                              native_sv=self,
                              wiggle=100,
                              info=self.info,
                              gt=self.gt)

    def to_vcf_record(self, sample):
        alt = ["<%s>" % (PINDEL_TO_SV_TYPE[self.sv_type])]
        info = {"SVLEN": self.sv_len,
                "SVTYPE": self.sv_type
        }

        info.update(self.info)

        vcf_record = vcf.model._Record(self.chromosome,
                                       self.start_pos - 1,
                                       ".",
                                       "N",
                                       alt,
                                       ".",
                                       ".",
                                       info,
                                       "GT",
                                       [0],
                                       [vcf.model._Call(None, sample, vcf.model.make_calldata_tuple("GT")(GT=self.gt))])
        return vcf_record

    def __str__(self):
        return str(self.__dict__)

class PindelReader:
    def __init__(self, file_name, reference_handle=None):
        logger.info("File is " + str(file_name))
        self.file_fd = open(file_name) if file_name is not None else sys.stdin
        self.reference_handle = reference_handle

    def __iter__(self):
        return self

    def next(self):
        while True:
            line = self.file_fd.next()
            if line.find("ChrID") >= 1:
                return PindelRecord(line.strip(), self.reference_handle)

<commit_msg>Fix number of entries per sample to 7 for current pindel<commit_after>import logging
import sys
import os

import vcf

from sv_interval import SVInterval

logger = logging.getLogger(__name__)

mydir = os.path.dirname(os.path.realpath(__file__))

'''
from http://gmt.genome.wustl.edu/packages/pindel/user-manual.html

There is a head line for each variant reported, followed by the alignment of supporting reads to the reference on the
second line. The example variants are a 1671bp deletion and a 10bp insertion on chr20. The breakpoints are specified
after "BP". Due to microhomology around the breakpoints, the breakpoint coordinates may shift both upstream and
downstream,'BP_range' is used to indicate the range of this shift. The header line contains the following data:

1) The index of the indel/SV (57 means that 57 insertions precede this insertion in the file)

2) The type of indel/SV: I for insertion, D for deletion, INV for inversion, TD for tandem duplication

3) The length of the SV

4) "NT" (to indicate that the next number is the length of non-template sequences inserted; insertions are fully covered
 by the NT-fields, deletions can have NT bases if the deletion is not 'pure', meaning that while bases have been
 deleted, some bases have been inserted between the breakpoints)

5) the length(s) of the NT fragment(s)

6) the sequence(s) of the NT fragment(s)

7-8) the identifier of the chromosome the read was found on

9-10-11) BP: the start and end positions of the SV

12-13-14) BP_range: if the exact position of the SV is unclear since bases at the edge of one read-half could equally
well be appended to the other read-half. In the deletion example, ACA could be on any side of the gap, so the original
deletion could have been between 1337143 and 1338815, between 1337144 and 1338816, or between 1337145 and 133817, or
between 1337146 and 133818. BP-range is used to indicate this range.

15) "Supports": announces that the total count of reads supporting the SV follow.

16) The number of reads supporting the SV

17) The number of unique reads supporting the SV (so not counting duplicate reads)

18) +: supports from reads whose anchors are upstream of the SV

19-20) total number of supporting reads and unique number of supporting reads whose anchors are upstream of the SV.

21) -: supports from reads whose anchors are downstream of the SV

22-23) total number of supporting reads and unique number of supporting reads whose anchors are downstream of the SV

24-25) S1: a simple score, ("# +" + 1)* ("# -" + 1) ;

26-27) SUM_MS: sum of mapping qualities of anchor reads, The reads with variants or unmapped are called split-read,
whose mate is called anchor reads. We use anchor reads to narrow down the search space to speed up and increase
sensitivity;

28) the number of different samples scanned

29-30-31) NumSupSamples?: the number of samples supporting the SV, as well as the number of samples having unique
reads supporting the SV (in practice, these numbers are the same)

32+) Per sample: the sample name, followed by the total number of supporting reads whose anchors are upstream, the
total number of unique supporting reads whose anchors are upstream, the total number of supporting reads whose anchors
 are downstream, and finally the total number of unique supporting reads whose anchors are downstream.

----

The reported larger insertion (LI) record is rather different than other types of variants. Here is the format:
1) index,
2) type(LI),
3) "ChrID",
4) chrName,
5) left breakpoint,
6) "+"
7) number of supporting reads for the left coordinate,
8) right breakpoint,
9) "-"
10) number of supporting reads for the right coordinate.

Following lines are repeated for each sample
11) Sample name
12) "+"
13) upstream supporting reads
14) "-"
15) downstream supporting reads

'''

pindel_name = "Pindel"
pindel_source = set([pindel_name])
min_coverage = 10
het_cutoff = 0.2
hom_cutoff = 0.8

GT_REF = "0/0"
GT_HET = "0/1"
GT_HOM = "1/1"

PINDEL_TO_SV_TYPE = {"I": "INS", "D": "DEL", "LI": "INS", "TD": "DUP:TANDEM", "INV": "INV"}

class PindelRecord:
    def __init__(self, record_string, reference_handle=None):
        fields = record_string.split()
        self.sv_type = fields[1]
        self.name = pindel_name

        self.sv_len = None
        self.num_nt_added = None
        self.nt_added = None
        self.chromosome = None
        self.start_pos = None
        self.end_pos = None
        self.bp_range = None
        self.read_supp = None
        self.uniq_read_supp = None
        self.up_read_supp = None
        self.up_uniq_read_supp = None
        self.down_read_supp = None
        self.down_uniq_read_supp = None
        self.simple_score = None
        self.sum_mapq = None
        self.num_sample = None
        self.num_sample_supp = None
        self.num_sample_uniq_supp = None
        self.homlen = None
        self.homseq = None
        self.samples = None

        if self.sv_type != "LI":
            self.sv_len = int(fields[2])
            self.num_nt_added = map(int, fields[4].split(":"))
            self.nt_added = map(lambda x: x.replace('"', ''), fields[5].split(":"))
            self.chromosome = fields[7]
            self.start_pos = int(fields[9])
            self.end_pos = int(fields[10]) - 1
            self.bp_range = (int(fields[12]), int(fields[13]))
            self.read_supp = int(fields[15])  # The number of reads supporting the SV
            self.uniq_read_supp = int(
                fields[16])  # The number of unique reads supporting SV (not count duplicate reads)
            self.up_read_supp = int(fields[18])  # upstream
            self.up_uniq_read_supp = int(fields[19])
            self.down_read_supp = int(fields[21])  # downstream
            self.down_uniq_read_supp = int(fields[22])
            self.simple_score = int(fields[24])
            self.sum_mapq = int(fields[26])  # sum of mapping qualities of anchor reads
            self.num_sample = int(fields[27])  # number of samples
            self.num_sample_supp = int(fields[29])  # number of samples with supporting reads
            self.num_sample_uniq_supp = int(fields[30])  # number of sample with unique supporting readas
            self.homlen = self.bp_range[1] - self.end_pos
            self.homseq = reference_handle.fetch(self.chromosome, self.end_pos - 1,
                                                 self.bp_range[1] - 1) if reference_handle else ""
            pindel024u_or_later = len(fields) > 31 + 5 * self.num_sample
            if pindel024u_or_later:
                self.samples = self.samples = [{"name": fields[i], "ref_support_at_start": int(fields[i + 1]),
                                                "ref_support_at_end": int(fields[i + 2]),
                                                "plus_support": sum(map(int, fields[i + 3:i + 5])),
                                                "minus_support": sum(map(int, fields[i + 5:i + 7]))} for i in
                                               xrange(31, len(fields), 7)]
            else:
                self.samples = [{"name": fields[i], "ref_support_at_start": int(fields[i + 1]),
                                 "ref_support_at_end": int(fields[i + 2]), "plus_support": int(fields[i + 3]),
                                 "minus_support": int(fields[i + 4])} for i in xrange(31, len(fields), 5)]
        else:
            self.sv_len = 0
            self.chromosome = fields[3]
            self.start_pos = min(int(fields[4]), int(fields[7]))
            self.up_read_supp = int(fields[6])  # upstream
            self.end_pos = self.start_pos + 1
            self.down_read_supp = int(fields[9])  # downstream
            self.bp_range = (self.start_pos, self.end_pos)
            self.homlen = 0
            self.homseq = ""
            self.samples = [{"name": fields[i], "plus_support": int(fields[i + 2]), "minus_support": int(fields[i + 4])}
                            for i in xrange(10, len(fields), 5)]

        self.info = {
            "END": self.end_pos,
            "PD_NUM_NT_ADDED": self.num_nt_added,
            "PD_NT_ADDED": self.nt_added,
            "PD_BP_RANGE_START": self.bp_range[0],
            "PD_BP_RANGE_END": self.bp_range[1],
            "PD_READ_SUPP": self.read_supp,
            "PD_UNIQ_READ_SUPP": self.uniq_read_supp,
            "PD_UP_READ_SUPP": self.up_read_supp,
            "PD_UP_UNIQ_READ_SUPP": self.up_uniq_read_supp,
            "PD_DOWN_READ_SUPP": self.down_read_supp,
            "PD_DOWN_UNIQ_READ_SUPP": self.down_uniq_read_supp,
            "PD_SIMPLE_SCORE": self.simple_score,
            "PD_SUM_MAPQ": self.sum_mapq,
            "PD_NUM_SAMPLE": self.num_sample,
            "PD_NUM_SAMPLE_SUPP": self.num_sample_supp,
            "PD_NUM_SAMPLE_UNIQ_SUPP": self.num_sample_uniq_supp,
            "PD_HOMLEN": self.homlen,
            "PD_HOMSEQ": self.homseq
        }

        self.derive_genotype()

    def derive_genotype(self):
        if self.sv_type == "LI" or self.sv_type == "I":
            self.gt = GT_HET if (self.up_read_supp + self.down_read_supp) > 0 else GT_REF
            return

        total_event_reads = self.uniq_read_supp
        total_ref_reads = self.up_uniq_read_supp + self.down_uniq_read_supp
        if total_event_reads + total_ref_reads < min_coverage:
            self.gt = GT_REF
            return

        allele_fraction = float(total_event_reads) / (float(total_event_reads) + float(total_ref_reads))
        if allele_fraction < het_cutoff:
            self.gt = GT_REF
        elif allele_fraction < hom_cutoff:
            self.gt = GT_HET
        else:
            self.gt = GT_HOM

    def to_sv_interval(self):
        sv_type = PINDEL_TO_SV_TYPE[self.sv_type]
        if sv_type != "INS":
            return SVInterval(self.chromosome,
                              self.start_pos,
                              self.end_pos,
                              name=self.name,
                              sv_type=sv_type,
                              length=self.sv_len,
                              sources=pindel_source,
                              info=self.info,
                              native_sv=self)
        else:
            return SVInterval(self.chromosome,
                              self.start_pos,
                              self.start_pos,
                              self.name,
                              sv_type=sv_type,
                              length=self.sv_len,
                              sources=pindel_source,
                              native_sv=self,
                              wiggle=100,
                              info=self.info,
                              gt=self.gt)

    def to_vcf_record(self, sample):
        alt = ["<%s>" % (PINDEL_TO_SV_TYPE[self.sv_type])]
        info = {"SVLEN": self.sv_len,
                "SVTYPE": self.sv_type
        }

        info.update(self.info)

        vcf_record = vcf.model._Record(self.chromosome,
                                       self.start_pos - 1,
                                       ".",
                                       "N",
                                       alt,
                                       ".",
                                       ".",
                                       info,
                                       "GT",
                                       [0],
                                       [vcf.model._Call(None, sample, vcf.model.make_calldata_tuple("GT")(GT=self.gt))])
        return vcf_record

    def __str__(self):
        return str(self.__dict__)

class PindelReader:
    def __init__(self, file_name, reference_handle=None):
        logger.info("File is " + str(file_name))
        self.file_fd = open(file_name) if file_name is not None else sys.stdin
        self.reference_handle = reference_handle

    def __iter__(self):
        return self

    def next(self):
        while True:
            line = self.file_fd.next()
            if line.find("ChrID") >= 1:
                return PindelRecord(line.strip(), self.reference_handle)

<|endoftext|>


<commit_before>import os
import sys
from PySide import QtGui, QtCore
DIR = os.path.dirname(__file__)

class OScanViewer(QtGui.QStackedWidget):
    '''
    Creates the overscan viewer.
    '''
    def __init__(self, width=720, height=480):
        super(OScanViewer, self).__init__()

        self.oScanX = 1.2
        self.oScanY = 1.2

        self.width = width
        self.height = height

        self.oWidth = int(self.width * self.oScanX)
        self.oHeight = int(self.height * self.oScanY)

        self.centerX = int((self.oWidth - self.width)/2.0)
        self.centerY = int((self.oHeight - self.height)/2.0)

        self.imgPath = os.path.join(DIR, "images/ponyo.jpeg")

        self.initUI()

    def initUI(self):
        '''
        Initilizes the ui.
        '''
        #Center the window.
        qr = self.frameGeometry()
        cp = QtGui.QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

        #Resize.
        self.resize(self.oWidth, self.oHeight)
        self.setWindowTitle('OScan Viewer')
        self.show()

def main():
    app = QtGui.QApplication(sys.argv)
    ex = OScanViewer(width=720, height=480)
    sys.exit(app.exec_())

if __name__ == '__main__':
    main()
<commit_msg>Updated ui.<commit_after>import os
import sys
from PySide import QtGui, QtCore
DIR = os.path.dirname(__file__)

class OScanViewer(QtGui.QDialog):
    '''
    Creates the overscan viewer.
    '''
    def __init__(self, parent=None, width=720, height=480):

        super(OScanViewer, self).__init__(parent)

        self.oScanX = 1.2
        self.oScanY = 1.2

        self.width = width
        self.height = height

        self.oWidth = int(self.width * self.oScanX)
        self.oHeight = int(self.height * self.oScanY)

        self.centerX = int((self.oWidth - self.width)/2.0)
        self.centerY = int((self.oHeight - self.height)/2.0)

        self.imgPath = os.path.join(DIR, "images/ponyo.jpeg")

        #Resize.
        self.setGeometry(0, 0, self.oWidth, self.oHeight)
        self.setWindowTitle('OScan Viewer')

        #Init ui.
        self.initUI()

        #Create layout.
        self.create_layout()

        #Delete on close.
        self.setAttribute(QtCore.Qt.WA_DeleteOnClose)
        self.show()

    def initUI(self):
        '''
        Initilizes the ui.
        '''
        #Center the window.
        qr = self.frameGeometry()
        cp = QtGui.QDesktopWidget().availableGeometry().center()
        qr.moveCenter(cp)
        self.move(qr.topLeft())

        #central layout.
        self.central_boxLayout = QtGui.QVBoxLayout()
        # self.central_boxLayout.setSpacing(10)
        # self.central_boxLayout.setContentsMargins(10, 10, 10, 10)
        self.setLayout(self.central_boxLayout)

    def create_layout(self):
        '''
        Creating a layout.
        '''
        #Our main layout
        self.oScan_Border = QtGui.QWidget()
        self.oScan_Border.setFixedWidth(self.oWidth)
        self.oScan_Border.setFixedHeight(self.oHeight)

        #Resolution center.
        self.res_Lbl = QtGui.QLabel()

        #Image.
        self.pixmap = QtGui.QPixmap(self.imgPath)
        self.pixmap = self.pixmap.scaled(
            QtCore.QSize(self.width, self.height),
            QtCore.Qt.KeepAspectRatio, QtCore.Qt.SmoothTransformation)
        self.res_Lbl.setPixmap(self.pixmap)
        self.res_Lbl.setFixedWidth(self.width)
        self.res_Lbl.setFixedHeight(self.height)
        self.res_Lbl.setAlignment(
            QtCore.Qt.AlignHCenter | QtCore.Qt.AlignVCenter)
        self.res_Lbl.setScaledContents(True)

        #Stacked widget.
        self.main_Stack = QtGui.QStackedWidget()
        self.main_Stack.addWidget(self.res_Lbl)
        self.main_Stack.addWidget(self.oScan_Border)
        self.central_boxLayout.addWidget(self.main_Stack)

def main():
    app = QtGui.QApplication(sys.argv)
    ex = OScanViewer(width=720, height=480)
    sys.exit(app.exec_())

if __name__ == '__main__':
    main()
<|endoftext|>