<commit_before>
#!/usr/bin/env python

import os
import sys
import json
import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk, Gdk, Gio, GdkPixbuf

from helpers import rgb_to_hex, hex_to_rgb, random_rgb

class App(Gtk.Window):
    def __init__(self):
        """Initialize app"""
        self.app = "coulr"
        self.app_name = "Coulr"
        Gtk.Window.__init__(self, title=self.app_name, border_width=10)
        self.set_size_request(600, -1)
        self.set_position(Gtk.WindowPosition.CENTER)

        # Main vars
        self.rgb_color = None
        self.clipboard = Gtk.Clipboard.get(Gdk.SELECTION_CLIPBOARD)

        # Paths
        home_path = os.path.expanduser("~")
        if os.path.basename(sys.argv[0]) == self.app:
            logo_path = "/usr/share/{name}/{name}.png".format(name=self.app)
        else:
            logo_path = "img/{}.png".format(self.app)
        self.save_file = "{}/.config/{}.json".format(home_path, self.app)
        self.logo = GdkPixbuf.Pixbuf.new_from_file(logo_path)

        # Icon
        self.set_icon(self.logo)

        # Header bar
        header_bar = Gtk.HeaderBar()
        header_bar.set_show_close_button(True)
        header_bar.props.title = self.app_name
        header_bar.set_subtitle("Enjoy colors and feel happy !")
        self.set_titlebar(header_bar)

        # About button
        button_about = Gtk.Button()
        icon_about = Gio.ThemedIcon(name="help-about-symbolic")
        image_about = Gtk.Image.new_from_gicon(icon_about, Gtk.IconSize.BUTTON)
        button_about.add(image_about)
        button_about.connect("clicked", self.about_dialog)
        header_bar.pack_end(button_about)

        # Copy button
        button_copy = Gtk.Button()
        icon_copy = Gio.ThemedIcon(name="edit-copy-symbolic")
        image_copy = Gtk.Image.new_from_gicon(icon_copy, Gtk.IconSize.BUTTON)
        button_copy.add(image_copy)
        button_copy.connect("clicked", self.copy_output)
        header_bar.pack_end(button_copy)

        # Main wrapper
        main_box = Gtk.Grid(column_spacing=6)

        # Second layout level
        layout1 = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, vexpand=False)
        layout2 = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=6)

        main_box.attach(layout1, 0, 0, 1, 1)
        main_box.attach(layout2, 1, 0, 1, 1)
        self.add(main_box)

        # Layout 1
        self.notebook_input = Gtk.Notebook()

        # RGB tab
        grid_rgb = Gtk.Grid(row_spacing=6, column_spacing=6, border_width=10)

        # Red spinner
        adj = Gtk.Adjustment(0, 0, 255, 1, 10, 0)
        self.spinbutton_r = Gtk.SpinButton(adjustment=adj)
        self.spinbutton_r.connect("value-changed", self.rgb_spin_changed)
        grid_rgb.attach(self.spinbutton_r, 0, 0, 1, 1)

        # Red slider
        adj = Gtk.Adjustment(0, 0, 255, 2, 10, 0)
        self.slider_r = Gtk.Scale(adjustment=adj, draw_value=False)
        self.slider_r.set_vexpand(True)
        self.slider_r.set_hexpand(True)
        self.slider_r.connect("value-changed", self.rgb_slider_moved)
        grid_rgb.attach(self.slider_r, 1, 0, 2, 1)

        # Green spinner
        adj = Gtk.Adjustment(0, 0, 255, 1, 10, 0)
        self.spinbutton_g = Gtk.SpinButton(adjustment=adj)
        self.spinbutton_g.connect("value-changed", self.rgb_spin_changed)
        grid_rgb.attach(self.spinbutton_g, 0, 1, 1, 1)

        # Green slider
        adj = Gtk.Adjustment(0, 0, 255, 2, 10, 0)
        self.slider_g = Gtk.Scale(adjustment=adj, draw_value=False)
        self.slider_g.set_vexpand(True)
        self.slider_g.set_hexpand(True)
        self.slider_g.connect("value-changed", self.rgb_slider_moved)
        grid_rgb.attach(self.slider_g, 1, 1, 2, 1)

        # Blue spinner
        adj = Gtk.Adjustment(0, 0, 255, 1, 10, 0)
        self.spinbutton_b = Gtk.SpinButton(adjustment=adj)
        self.spinbutton_b.connect("value-changed", self.rgb_spin_changed)
        grid_rgb.attach(self.spinbutton_b, 0, 2, 1, 1)

        # Blue slider
        adj = Gtk.Adjustment(0, 0, 255, 2, 10, 0)
        self.slider_b = Gtk.Scale(adjustment=adj, draw_value=False)
        self.slider_b.set_vexpand(True)
        self.slider_b.set_hexpand(True)
        self.slider_b.connect("value-changed", self.rgb_slider_moved)
        grid_rgb.attach(self.slider_b, 1, 2, 2, 1)

        # Hex tab
        box_hex = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, border_width=10)
        self.entry_hex = Gtk.Entry(max_length=7)
        self.entry_hex.connect("changed", self.hex_entry_changed)
        box_hex.add(self.entry_hex)

        # Lucky tab
        box_luck = Gtk.Box(orientation=Gtk.Orientation.VERTICAL,
                           border_width=10)
        self.button_luck = Gtk.Button("Feel lucky !")
        self.button_luck.connect("clicked", self.luck_button_clicked)
        box_luck.add(self.button_luck)

        self.notebook_input.append_page(grid_rgb, Gtk.Label("RGB"))
        self.notebook_input.append_page(box_hex, Gtk.Label("Hexadecimal"))
        self.notebook_input.append_page(box_luck, Gtk.Label("Lucky mode"))

        # Layout 2
        # Output mode
        self.combo_output = Gtk.ComboBoxText()
        self.combo_output.append("hex", "Hexadecimal")
        self.combo_output.append("rgb", "RGB")
        self.combo_output.set_active(0)
        self.combo_output.connect("changed", self.change_output)

        # Output label
        self.output = Gtk.Label(selectable=True)

        # Preview color with square
        self.square = Gtk.Frame()
        self.square.set_size_request(150, 150)

        layout1.add(self.notebook_input)
        layout2.add(self.square)
        layout2.add(self.combo_output)
        layout2.add(self.output)

        # Initialize color
        if os.path.exists(self.save_file):
            try:
                with open(self.save_file, "r") as save_file:
                    data = json.load(save_file)
                    color = hex_to_rgb(data["color"].lstrip("#"))
            except (OSError, json.JSONDecodeError):
                print("Error when trying to read save file.")
        else:
            color = random_rgb()
        self.change_color(color)

    def change_color(self, rgb):
        """Refresh preview and set values of all fields.
        :param rgb: rgb color values
        :type rgb: tuple
        """

        rgba = Gdk.RGBA()
        rgba.parse("rgb({},{},{})".format(*rgb))
        self.square.override_background_color(Gtk.StateType.NORMAL, rgba)

        self.spinbutton_r.set_value(rgb[0])
        self.slider_r.set_value(rgb[0])

        self.slider_g.set_value(rgb[1])
        self.spinbutton_g.set_value(rgb[1])

        self.slider_b.set_value(rgb[2])
        self.spinbutton_b.set_value(rgb[2])

        self.entry_hex.set_text(rgb_to_hex(rgb))

        self.rgb_color = rgb
        self.change_output()

    def change_output(self, event=None):
        """Set output field"""
        combo_id = self.combo_output.get_active_id()
        if combo_id == "hex":
            output = rgb_to_hex(self.rgb_color)
        elif combo_id == "rgb":
            output = "rgb({},{},{})".format(*self.rgb_color)

        self.output.set_text(output)

    def rgb_spin_changed(self, event):
        """RGB spinners values changed"""
        spin_red = self.spinbutton_r.get_value_as_int()
        spin_green = self.spinbutton_g.get_value_as_int()
        spin_blue = self.spinbutton_b.get_value_as_int()

        self.change_color((spin_red, spin_green, spin_blue))

    def rgb_slider_moved(self, event):
        """RGB sliders values changed"""
        slider_red = int(self.slider_r.get_value())
        slider_green = int(self.slider_g.get_value())
        slider_blue = int(self.slider_b.get_value())

        self.change_color((slider_red, slider_green, slider_blue))

    def hex_entry_changed(self, event):
        """Hex entry value changed"""
        value = self.entry_hex.get_text().lstrip("#")

        if len(value) == 6:
            rgb = hex_to_rgb(value)
            self.change_color(rgb)

    def luck_button_clicked(self, event):
        """Luck button clicked"""
        self.change_color(random_rgb())

    def copy_output(self, event):
        """Copy current output"""
        self.clipboard.set_text(self.output.get_text(), -1)

    def about_dialog(self, event):
        """About dialog"""
        about_dialog = Gtk.AboutDialog(self)
        about_dialog.set_program_name(self.app_name)
        about_dialog.set_version("1.3.0")
        about_dialog.set_copyright("Hugo Posnic")
        about_dialog.set_comments("Enjoy colors and feel happy !")
        about_dialog.set_website("https://github.com/Huluti/{}"
                                    .format(self.app_name))
        about_dialog.set_website_label("Github")
        about_dialog.set_authors(["Hugo Posnic"])
        about_dialog.set_logo(self.logo)
        about_dialog.set_license("{} is under MIT Licence."
                                    .format(self.app_name))
        about_dialog.set_transient_for(self)
        about_dialog.run()
        about_dialog.destroy()

    def close(self, event, data):
        """Save last color then quit app"""
        try:
            with open(self.save_file, "w+") as save_file:
                try:
                    data = json.load(save_file)
                except ValueError:
                    data = dict()
                data["color"] = rgb_to_hex(self.rgb_color)
                json.dump(data, save_file)
        except (OSError, json.JSONDecodeError):
            print("Error when trying to set save file.")
        Gtk.main_quit()

app = App()
app.connect("delete-event", app.close)
app.show_all()
Gtk.main()
<commit_msg>Fix path test<commit_after>
#!/usr/bin/env python

import os
import sys
import json
import gi
gi.require_version("Gtk", "3.0")
from gi.repository import Gtk, Gdk, Gio, GdkPixbuf

from helpers import rgb_to_hex, hex_to_rgb, random_rgb

class App(Gtk.Window):
    def __init__(self):
        """Initialize app"""
        self.app = "coulr"
        self.app_name = "Coulr"
        Gtk.Window.__init__(self, title=self.app_name, border_width=10)
        self.set_size_request(600, -1)
        self.set_position(Gtk.WindowPosition.CENTER)

        # Main vars
        self.rgb_color = None
        self.clipboard = Gtk.Clipboard.get(Gdk.SELECTION_CLIPBOARD)

        # Paths
        home_path = os.path.expanduser("~")
        run_path = os.path.realpath(__file__)
        if run_path[:4] == "/usr":
            logo_path = "/usr/share/{name}/{name}.png".format(name=self.app)
        else:
            logo_path = "img/{}.png".format(self.app)
        self.save_file = "{}/.config/{}.json".format(home_path, self.app)
        self.logo = GdkPixbuf.Pixbuf.new_from_file(logo_path)

        # Icon
        self.set_icon(self.logo)

        # Header bar
        header_bar = Gtk.HeaderBar()
        header_bar.set_show_close_button(True)
        header_bar.props.title = self.app_name
        header_bar.set_subtitle("Enjoy colors and feel happy !")
        self.set_titlebar(header_bar)

        # About button
        button_about = Gtk.Button()
        icon_about = Gio.ThemedIcon(name="help-about-symbolic")
        image_about = Gtk.Image.new_from_gicon(icon_about, Gtk.IconSize.BUTTON)
        button_about.add(image_about)
        button_about.connect("clicked", self.about_dialog)
        header_bar.pack_end(button_about)

        # Copy button
        button_copy = Gtk.Button()
        icon_copy = Gio.ThemedIcon(name="edit-copy-symbolic")
        image_copy = Gtk.Image.new_from_gicon(icon_copy, Gtk.IconSize.BUTTON)
        button_copy.add(image_copy)
        button_copy.connect("clicked", self.copy_output)
        header_bar.pack_end(button_copy)

        # Main wrapper
        main_box = Gtk.Grid(column_spacing=6)

        # Second layout level
        layout1 = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, vexpand=False)
        layout2 = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=6)

        main_box.attach(layout1, 0, 0, 1, 1)
        main_box.attach(layout2, 1, 0, 1, 1)
        self.add(main_box)

        # Layout 1
        self.notebook_input = Gtk.Notebook()

        # RGB tab
        grid_rgb = Gtk.Grid(row_spacing=6, column_spacing=6, border_width=10)

        # Red spinner
        adj = Gtk.Adjustment(0, 0, 255, 1, 10, 0)
        self.spinbutton_r = Gtk.SpinButton(adjustment=adj)
        self.spinbutton_r.connect("value-changed", self.rgb_spin_changed)
        grid_rgb.attach(self.spinbutton_r, 0, 0, 1, 1)

        # Red slider
        adj = Gtk.Adjustment(0, 0, 255, 2, 10, 0)
        self.slider_r = Gtk.Scale(adjustment=adj, draw_value=False)
        self.slider_r.set_vexpand(True)
        self.slider_r.set_hexpand(True)
        self.slider_r.connect("value-changed", self.rgb_slider_moved)
        grid_rgb.attach(self.slider_r, 1, 0, 2, 1)

        # Green spinner
        adj = Gtk.Adjustment(0, 0, 255, 1, 10, 0)
        self.spinbutton_g = Gtk.SpinButton(adjustment=adj)
        self.spinbutton_g.connect("value-changed", self.rgb_spin_changed)
        grid_rgb.attach(self.spinbutton_g, 0, 1, 1, 1)

        # Green slider
        adj = Gtk.Adjustment(0, 0, 255, 2, 10, 0)
        self.slider_g = Gtk.Scale(adjustment=adj, draw_value=False)
        self.slider_g.set_vexpand(True)
        self.slider_g.set_hexpand(True)
        self.slider_g.connect("value-changed", self.rgb_slider_moved)
        grid_rgb.attach(self.slider_g, 1, 1, 2, 1)

        # Blue spinner
        adj = Gtk.Adjustment(0, 0, 255, 1, 10, 0)
        self.spinbutton_b = Gtk.SpinButton(adjustment=adj)
        self.spinbutton_b.connect("value-changed", self.rgb_spin_changed)
        grid_rgb.attach(self.spinbutton_b, 0, 2, 1, 1)

        # Blue slider
        adj = Gtk.Adjustment(0, 0, 255, 2, 10, 0)
        self.slider_b = Gtk.Scale(adjustment=adj, draw_value=False)
        self.slider_b.set_vexpand(True)
        self.slider_b.set_hexpand(True)
        self.slider_b.connect("value-changed", self.rgb_slider_moved)
        grid_rgb.attach(self.slider_b, 1, 2, 2, 1)

        # Hex tab
        box_hex = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, border_width=10)
        self.entry_hex = Gtk.Entry(max_length=7)
        self.entry_hex.connect("changed", self.hex_entry_changed)
        box_hex.add(self.entry_hex)

        # Lucky tab
        box_luck = Gtk.Box(orientation=Gtk.Orientation.VERTICAL,
                           border_width=10)
        self.button_luck = Gtk.Button("Feel lucky !")
        self.button_luck.connect("clicked", self.luck_button_clicked)
        box_luck.add(self.button_luck)

        self.notebook_input.append_page(grid_rgb, Gtk.Label("RGB"))
        self.notebook_input.append_page(box_hex, Gtk.Label("Hexadecimal"))
        self.notebook_input.append_page(box_luck, Gtk.Label("Lucky mode"))

        # Layout 2
        # Output mode
        self.combo_output = Gtk.ComboBoxText()
        self.combo_output.append("hex", "Hexadecimal")
        self.combo_output.append("rgb", "RGB")
        self.combo_output.set_active(0)
        self.combo_output.connect("changed", self.change_output)

        # Output label
        self.output = Gtk.Label(selectable=True)

        # Preview color with square
        self.square = Gtk.Frame()
        self.square.set_size_request(150, 150)

        layout1.add(self.notebook_input)
        layout2.add(self.square)
        layout2.add(self.combo_output)
        layout2.add(self.output)

        # Initialize color
        if os.path.exists(self.save_file):
            try:
                with open(self.save_file, "r") as save_file:
                    data = json.load(save_file)
                    color = hex_to_rgb(data["color"].lstrip("#"))
            except (OSError, json.JSONDecodeError):
                print("Error when trying to read save file.")
        else:
            color = random_rgb()
        self.change_color(color)

    def change_color(self, rgb):
        """Refresh preview and set values of all fields.
        :param rgb: rgb color values
        :type rgb: tuple
        """

        rgba = Gdk.RGBA()
        rgba.parse("rgb({},{},{})".format(*rgb))
        self.square.override_background_color(Gtk.StateType.NORMAL, rgba)

        self.spinbutton_r.set_value(rgb[0])
        self.slider_r.set_value(rgb[0])

        self.slider_g.set_value(rgb[1])
        self.spinbutton_g.set_value(rgb[1])

        self.slider_b.set_value(rgb[2])
        self.spinbutton_b.set_value(rgb[2])

        self.entry_hex.set_text(rgb_to_hex(rgb))

        self.rgb_color = rgb
        self.change_output()

    def change_output(self, event=None):
        """Set output field"""
        combo_id = self.combo_output.get_active_id()
        if combo_id == "hex":
            output = rgb_to_hex(self.rgb_color)
        elif combo_id == "rgb":
            output = "rgb({},{},{})".format(*self.rgb_color)

        self.output.set_text(output)

    def rgb_spin_changed(self, event):
        """RGB spinners values changed"""
        spin_red = self.spinbutton_r.get_value_as_int()
        spin_green = self.spinbutton_g.get_value_as_int()
        spin_blue = self.spinbutton_b.get_value_as_int()

        self.change_color((spin_red, spin_green, spin_blue))

    def rgb_slider_moved(self, event):
        """RGB sliders values changed"""
        slider_red = int(self.slider_r.get_value())
        slider_green = int(self.slider_g.get_value())
        slider_blue = int(self.slider_b.get_value())

        self.change_color((slider_red, slider_green, slider_blue))

    def hex_entry_changed(self, event):
        """Hex entry value changed"""
        value = self.entry_hex.get_text().lstrip("#")

        if len(value) == 6:
            rgb = hex_to_rgb(value)
            self.change_color(rgb)

    def luck_button_clicked(self, event):
        """Luck button clicked"""
        self.change_color(random_rgb())

    def copy_output(self, event):
        """Copy current output"""
        self.clipboard.set_text(self.output.get_text(), -1)

    def about_dialog(self, event):
        """About dialog"""
        about_dialog = Gtk.AboutDialog(self)
        about_dialog.set_program_name(self.app_name)
        about_dialog.set_version("1.3.0")
        about_dialog.set_copyright("Hugo Posnic")
        about_dialog.set_comments("Enjoy colors and feel happy !")
        about_dialog.set_website("https://github.com/Huluti/{}"
                                    .format(self.app_name))
        about_dialog.set_website_label("Github")
        about_dialog.set_authors(["Hugo Posnic"])
        about_dialog.set_logo(self.logo)
        about_dialog.set_license("{} is under MIT Licence."
                                    .format(self.app_name))
        about_dialog.set_transient_for(self)
        about_dialog.run()
        about_dialog.destroy()

    def close(self, event, data):
        """Save last color then quit app"""
        try:
            with open(self.save_file, "w+") as save_file:
                try:
                    data = json.load(save_file)
                except ValueError:
                    data = dict()
                data["color"] = rgb_to_hex(self.rgb_color)
                json.dump(data, save_file)
        except (OSError, json.JSONDecodeError):
            print("Error when trying to set save file.")
        Gtk.main_quit()

app = App()
app.connect("delete-event", app.close)
app.show_all()
Gtk.main()
<|endoftext|>


<commit_before>#!/usr/bin/python

def enum(collection,st):
    #just like "enumerate" but you define your own starting position.
    #this returns indices RELATIVE TO ORIGINAL LIST
    i = st
    while i < len(collection):
        yield (i,collection[i])
        i += 1

def getfirstindex(L,st,value,K):
    for pos,t in enum(L,st):
        if t[1] > value-K:
            return pos #returns first read in the list that CAN contain given bp
    return 0

def getlastindex(L,st,value):
    for pos,t in enum(L,st):
        if t[1] > value:
            return pos #returns first read in the list that CANNOT contain given bp
    return len(L)

# ^these 2 were inspired by code here: http://stackoverflow.com/questions/946860/using-pythons-list-index-method-on-a-list-of-tuples-or-objects (see answer labeled "10", superperformant)

# function for converting SAM "flags" to bits:
def to_bin(n):
    return bin(n)[2:].zfill(11)

def countReadlets(fname, outfname, k, chromosome, stranded, rev):
    import pysam
    #from datetime import datetime #for debugging
    samfile = pysam.Samfile(fname,"rb")
    id_start_end = []
    cigar = []
    maxpos = 0
    minpos = 3000000000
    for read in samfile.fetch(chromosome):
        if to_bin(read.flag)[6] == '1' and not rev:
            strand = "-"
        elif to_bin(read.flag)[6] == '1' and rev:
            strand = "+"
        elif to_bin(read.flag)[6] == '0' and not rev:
            strand = "+"
        else:
            strand = "-"
        readstart = read.pos+1
        readend = read.aend
        if len(read.cigar)>1:
            currentloc = readstart
            for a in range(len(read.cigar)):
                if read.cigar[a][0] == 0: #match: count this segment as covered, obviously
                    id_start_end.append([read.qname, currentloc, currentloc+read.cigar[a][1]-1])
                    currentloc = currentloc + read.cigar[a][1]
                if read.cigar[a][0] == 1: #insertion - not in reference, so our location doesn't move
                    continue
                if read.cigar[a][0] == 2: #deletion - count this segment as covered, since a read overlaps it.
                    id_start_end.append([read.qname, currentloc, currentloc+read.cigar[a][1]-1])
                    currentloc = currentloc + read.cigar[a][1]
                if read.cigar[a][0] == 3: #skipped region: don't count and move the location
                    currentloc = currentloc + read.cigar[a][1]
                if read.cigar[a][0] > 3: #I don't think the rest of these flags exist in these alignments, but just make sure.
                    print("WARNING: you have a cigar flag in here that you haven't accounted for.")
        else:
            id_start_end.append([read.qname, readstart, readend, strand])
        if read.aend > maxpos:
            maxpos = read.aend
        if read.pos+1 < minpos:
            minpos = read.pos+1
    # sort list by starting points, since we've now introduced extra complication...:
    import operator
    id_start_end.sort(key=operator.itemgetter(1))
    # credit: http://stackoverflow.com/questions/5201191/sorting-a-list-of-lists-in-python

    if stranded:
        fplus = open(outfname+'_plus', 'w')
        fminus = open(outfname+'_minus', 'w')
        first = 0
        last = 0
        for z in xrange(1,minpos):
            fplus.write("%s\t%s\n" % (z,0))
            fminus.write("%s\t%s\n" % (z,0))
        for i in xrange(minpos, maxpos+1):
            last = getlastindex(id_start_end,first,i)
            first = getfirstindex(id_start_end,first,i,k)
            if first == last:
                fplus.write("%s\t%s\n" % (i,0))
                fminus.write("%s\t%s\n" % (i,0))
                continue
            overlaps = id_start_end[first:last]
            readnames_plus = set()
            readnames_minus = set()
            for j in xrange(len(overlaps)):
                if i>=overlaps[j][1] and i<=overlaps[j][2]:
                    if overlaps[j][3] == "+":
                        readnames_plus.add(overlaps[j][0])
                    else:
                        readnames_minus.add(overlaps[j][0])
            numreads_plus = len(readnames_plus) #count readlets only once per read
            numreads_minus = len(readnames_minus) #count readlets only once per read
            fplus.write("%s\t%s\n" % (i,numreads_plus))
            fminus.write("%s\t%s\n" % (i,numreads_minus))
        fplus.close()
        fminus.close()

    else:
        f = open(outfname, 'w')
        #g = open('keeptrackofiterations','w') #for debugging
        first = 0
        last = 0
        #npos = 0 #for debugging
        for z in xrange(1,minpos):
            f.write("%s\t%s\n" % (z,0))
        for i in xrange(minpos, maxpos+1):
        #for i in xrange(maxpos-2999,maxpos+1): #for debugging
            #npos = npos+1 #for debugging
            #if npos % 10000 == 0: g.write("Did 10000 "+str(datetime.now())+"\n") #for debugging
            last = getlastindex(id_start_end,first,i)
            first = getfirstindex(id_start_end,first,i,k)
            if first == last:
                f.write("%s\t%s\n" % (i,0))
                continue
            overlaps = id_start_end[first:last]
            readnames = set()
            for j in xrange(len(overlaps)):
                if i>=overlaps[j][1] and i<=overlaps[j][2]:
                    readnames.add(overlaps[j][0])
            numreads = len(readnames) #count readlets only once per read
            f.write("%s\t%s\n" % (i,numreads))
        f.close()

    return None

# get arguments from command line
from optparse import OptionParser
opts = OptionParser()
opts.add_option("--file","-f",type="string",help="input file name (must be .bam)")
opts.add_option("--output","-o",type="string",help="output file name")
opts.add_option("--kmer","-k",type="int",help="kmer length")
opts.add_option("--chrom","-c",type="string",help="chromosome to parse")
opts.add_option("--stranded","-s",type="string",help="stranded or reverse-stranded protocol?",default="FALSE")
options,arguments = opts.parse_args()

if options.stranded == "TRUE":
    stranded = True
    rev = False
elif options.stranded == "REVERSE":
    stranded = True
    rev = True
elif options.stranded != "FALSE":
    ValueError('stranded must either be TRUE, REVERSE, or FALSE')
else:
    stranded = False
    rev = False

countReadlets(options.file, options.output, options.kmer, options.chrom, stranded, rev)
<commit_msg>small bug fix (strand was not always added to read info)<commit_after>#!/usr/bin/python

def enum(collection,st):
    #just like "enumerate" but you define your own starting position.
    #this returns indices RELATIVE TO ORIGINAL LIST
    i = st
    while i < len(collection):
        yield (i,collection[i])
        i += 1

def getfirstindex(L,st,value,K):
    for pos,t in enum(L,st):
        if t[1] > value-K:
            return pos #returns first read in the list that CAN contain given bp
    return 0

def getlastindex(L,st,value):
    for pos,t in enum(L,st):
        if t[1] > value:
            return pos #returns first read in the list that CANNOT contain given bp
    return len(L)

# ^these 2 were inspired by code here: http://stackoverflow.com/questions/946860/using-pythons-list-index-method-on-a-list-of-tuples-or-objects (see answer labeled "10", superperformant)

# function for converting SAM "flags" to bits:
def to_bin(n):
    return bin(n)[2:].zfill(11)

def countReadlets(fname, outfname, k, chromosome, stranded, rev):
    import pysam
    #from datetime import datetime #for debugging
    samfile = pysam.Samfile(fname,"rb")
    id_start_end = []
    cigar = []
    maxpos = 0
    minpos = 3000000000
    for read in samfile.fetch(chromosome):
        if to_bin(read.flag)[6] == '1' and not rev:
            strand = "-"
        elif to_bin(read.flag)[6] == '1' and rev:
            strand = "+"
        elif to_bin(read.flag)[6] == '0' and not rev:
            strand = "+"
        else:
            strand = "-"
        readstart = read.pos+1
        readend = read.aend
        if len(read.cigar)>1:
            currentloc = readstart
            for a in range(len(read.cigar)):
                if read.cigar[a][0] == 0: #match: count this segment as covered, obviously
                    id_start_end.append([read.qname, currentloc, currentloc+read.cigar[a][1]-1, strand])
                    currentloc = currentloc + read.cigar[a][1]
                if read.cigar[a][0] == 1: #insertion - not in reference, so our location doesn't move
                    continue
                if read.cigar[a][0] == 2: #deletion - count this segment as covered, since a read overlaps it.
                    id_start_end.append([read.qname, currentloc, currentloc+read.cigar[a][1]-1, strand])
                    currentloc = currentloc + read.cigar[a][1]
                if read.cigar[a][0] == 3: #skipped region: don't count and move the location
                    currentloc = currentloc + read.cigar[a][1]
                if read.cigar[a][0] > 3: #I don't think the rest of these flags exist in these alignments, but just make sure.
                    print("WARNING: you have a cigar flag in here that you haven't accounted for.")
        else:
            id_start_end.append([read.qname, readstart, readend, strand])
        if read.aend > maxpos:
            maxpos = read.aend
        if read.pos+1 < minpos:
            minpos = read.pos+1
    # sort list by starting points, since we've now introduced extra complication...:
    import operator
    id_start_end.sort(key=operator.itemgetter(1))
    # credit: http://stackoverflow.com/questions/5201191/sorting-a-list-of-lists-in-python

    if stranded:
        fplus = open(outfname+'_plus', 'w')
        fminus = open(outfname+'_minus', 'w')
        first = 0
        last = 0
        for z in xrange(1,minpos):
            fplus.write("%s\t%s\n" % (z,0))
            fminus.write("%s\t%s\n" % (z,0))
        for i in xrange(minpos, maxpos+1):
            last = getlastindex(id_start_end,first,i)
            first = getfirstindex(id_start_end,first,i,k)
            if first == last:
                fplus.write("%s\t%s\n" % (i,0))
                fminus.write("%s\t%s\n" % (i,0))
                continue
            overlaps = id_start_end[first:last]
            readnames_plus = set()
            readnames_minus = set()
            for j in xrange(len(overlaps)):
                if i>=overlaps[j][1] and i<=overlaps[j][2]:
                    if overlaps[j][3] == "+":
                        readnames_plus.add(overlaps[j][0])
                    else:
                        readnames_minus.add(overlaps[j][0])
            numreads_plus = len(readnames_plus) #count readlets only once per read
            numreads_minus = len(readnames_minus) #count readlets only once per read
            fplus.write("%s\t%s\n" % (i,numreads_plus))
            fminus.write("%s\t%s\n" % (i,numreads_minus))
        fplus.close()
        fminus.close()

    else:
        f = open(outfname, 'w')
        #g = open('keeptrackofiterations','w') #for debugging
        first = 0
        last = 0
        #npos = 0 #for debugging
        for z in xrange(1,minpos):
            f.write("%s\t%s\n" % (z,0))
        for i in xrange(minpos, maxpos+1):
        #for i in xrange(maxpos-2999,maxpos+1): #for debugging
            #npos = npos+1 #for debugging
            #if npos % 10000 == 0: g.write("Did 10000 "+str(datetime.now())+"\n") #for debugging
            last = getlastindex(id_start_end,first,i)
            first = getfirstindex(id_start_end,first,i,k)
            if first == last:
                f.write("%s\t%s\n" % (i,0))
                continue
            overlaps = id_start_end[first:last]
            readnames = set()
            for j in xrange(len(overlaps)):
                if i>=overlaps[j][1] and i<=overlaps[j][2]:
                    readnames.add(overlaps[j][0])
            numreads = len(readnames) #count readlets only once per read
            f.write("%s\t%s\n" % (i,numreads))
        f.close()

    return None

# get arguments from command line
from optparse import OptionParser
opts = OptionParser()
opts.add_option("--file","-f",type="string",help="input file name (must be .bam)")
opts.add_option("--output","-o",type="string",help="output file name")
opts.add_option("--kmer","-k",type="int",help="kmer length")
opts.add_option("--chrom","-c",type="string",help="chromosome to parse")
opts.add_option("--stranded","-s",type="string",help="stranded or reverse-stranded protocol?",default="FALSE")
options,arguments = opts.parse_args()

if options.stranded == "TRUE":
    stranded = True
    rev = False
elif options.stranded == "REVERSE":
    stranded = True
    rev = True
elif options.stranded != "FALSE":
    ValueError('stranded must either be TRUE, REVERSE, or FALSE')
else:
    stranded = False
    rev = False

countReadlets(options.file, options.output, options.kmer, options.chrom, stranded, rev)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) Techland. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.

import sys
import argparse
import cmd
import os
import logging
import traceback
from cppbuildprofiler import *

class Interpreter(cmd.Cmd):

    _CSV_COLUMNS = {
        Analyser.ABSOLUTE_PATH_KEY: { 'title': 'absolute path', 'default': None },
        Analyser.COMPILATION_COMMAND_KEY: { 'title': 'compilation command', 'default': '' },
        Analyser.BUILD_TIME_KEY: { 'title': 'build time [s]', 'default': 0.0 },
        Analyser.FILE_SIZE_KEY: { 'title': 'file size [B]', 'default': 0.0 },
        Analyser.TOTAL_SIZE_KEY: { 'title': 'total size [B]', 'default': 0.0 },
        Analyser.TOTAL_BUILD_TIME_KEY: { 'title': 'total build time of dependants [s]', 'default': 0.0 },
        Analyser.TRANSLATION_UNITS_KEY: { 'title': 'number of dependent translation units', 'default': 0 },
        Analyser.TU_BUILD_TIME_TO_SIZE_RATIO: { 'title': 'translation unit build time divided by total size sum [s/B]', 'default': 0 },
        }

    _ALL_METRICS = [
        Analyser.ABSOLUTE_PATH_KEY,
        Analyser.TRANSLATION_UNITS_KEY,
        Analyser.FILE_SIZE_KEY,
        Analyser.TOTAL_SIZE_KEY,
        Analyser.BUILD_TIME_KEY,
        Analyser.TOTAL_BUILD_TIME_KEY,
        Analyser.TU_BUILD_TIME_TO_SIZE_RATIO,
        ]

    def __init__(self):
        super().__init__()
        self.prompt = 'c++bp$ '
        self.use_rawinput = True

    def _argv(self, param_string):
        return filter(bool, param_string.split(' '))

    def emptyline(self):
        pass

    def onecmd(self, line):
        try:
            super().onecmd(line)
        except Exception:
            print(traceback.format_exc())

    def do_shell(self, line):
        out = os.popen(line).read()
        print(out)

    def _set_verbosity_argparser(self):
        parser = argparse.ArgumentParser('sets verbosity for the logging '
                                         'system')
        parser.add_argument(
            'level',
            action='store',
            choices=['DEBUG', 'INFO'])
        return parser

    def help_set_verbosity(self):
        self._set_verbosity_argparser().print_help()

    def do_set_verbosity(self, params):
        parser = self._set_verbosity_argparser()
        try:
            opts = parser.parse_args(self._argv(params))
            if opts.level == 'DEBUG':
                logging.getLogger().setLevel(logging.DEBUG)
            elif opts.level == 'INFO':
                logging.getLogger().setLevel(logging.INFO)
        except SystemExit:
            return

    def _parse_vs_log_argparser(self):
        parser = argparse.ArgumentParser('parses a visual studio build log '
                                         'and creates a dependency graph')
        parser.add_argument(
            'path',
            action='store',
            help='path to the log file to parse')
        return parser

    def help_parse_vs_log(self):
        self._parse_vs_log_argparser().print_help()

    def do_parse_vs_log(self, params):
        parser = self._parse_vs_log_argparser()
        try:
            opts = parser.parse_args(self._argv(params))
            self._depgraph = parse_vs_log(opts.path)
            self._depgraph.log_stats('Parsed %s into a dependency graph' %
                                     opts.path)
        except SystemExit:
            return

    def _load_argparser(self):
        parser = argparse.ArgumentParser('Loads a dependency graph from a file')
        parser.add_argument(
            'path',
            action='store',
            help='path to the file to load from')
        return parser

    def help_load(self):
        self._load_argparser().print_help()

    def do_load(self, params):
        parser = self._load_argparser()
        try:
            opts = parser.parse_args(self._argv(params))
            self._depgraph = DependencyGraph.read(opts.path)
            self._depgraph.log_stats('Loaded dependency grap from %s' %
                                     opts.path)
        except SystemExit:
            return

    def _store_argparser(self):
        parser = argparse.ArgumentParser('Stores a dependency graph in a file')
        parser.add_argument(
            'path',
            action='store',
            help='path to the file to write to')
        return parser

    def help_store(self):
        self._store_argparser().print_help()

    def do_store(self, params):
        parser = self._store_argparser()
        try:
            opts = parser.parse_args(self._argv(params))
            self._depgraph.write(opts.path)
            self._depgraph.log_stats('Stored dependency graph in %s' %
                                     opts.path)
        except SystemExit:
            return

    def do_analyse(self, params):
        Analyser(self._depgraph).run_full_analysis()

    def do_remove_pch(self, params):
        Analyser(self._depgraph).remove_pch()

    def _remove_thirdparty_dependencies_argparser(self):
        parser = argparse.ArgumentParser('removes third-party header '
                                         'dependencies')
        parser.add_argument(
            'codebase_root',
            action='store',
            help='path to the root directory of the codebase')
        return parser

    def help_remove_thirdparty_dependencies(self):
        self._remove_thirdparty_dependencies_argparser().print_help()

    def do_remove_thirdparty_dependencies(self, params):
        try:
            parser = self._remove_thirdparty_dependencies_argparser()
            opts = parser.parse_args(self._argv(params))
            Analyser(self._depgraph).remove_thirdparty_dependencies(
                opts.codebase_root)
        except SystemExit:
            return

    def _remove_node_argparser(self):
        parser = argparse.ArgumentParser('removes a node from the dependency graph')
        parser.add_argument(
            'label',
            action='store',
            help='label of the node to remove')
        return parser

    def help_remove_node(self):
        self._remove_node_argparser().print_help()

    def do_remove_node(self, params):
        try:
            parser = self._remove_node_argparser()
            opts = parser.parse_args(self._argv(params))

            self._depgraph.remove_nodes([opts.label])
            self._depgraph.remove_orphans()
        except SystemExit:
            return

    def _subgraph_argparser(self):
        parser = argparse.ArgumentParser('creates a subgraph of the '
                                         'dependency graph')
        parser.add_argument('--origin', '-o',
                            action='store',
                            help='label of the origin node')
        parser.add_argument('--dependencies', action='store_true',
                            help='include nodes origin depends on')
        parser.add_argument('--dependants', action='store_true',
                            help='include nodes that depend on origin')
        return parser

    def help_subgraph(self):
        self._subgraph_argparser().print_help()

    def do_subgraph(self, params):
        parser = self._subgraph_argparser()
        try:
            opts = parser.parse_args(self._argv(params))
            self._depgraph = DependencyGraph(self._depgraph.get_subgraph(
                opts.origin, opts.dependencies, opts.dependants))
            self._depgraph.log_stats('Created subgraph of %s' % opts.origin)
        except SystemExit:
            return

    def _print_argparser(self):
        parser = argparse.ArgumentParser('prints the dependency graph')
        parser.add_argument('--out', '-o',
                            action='store',
                            help='file to print out to',
                            required=False)
        parser.add_argument('--metrics', '-m',
                            help='metrics to print (space separated)',
                            nargs='*',
                            choices=self._ALL_METRICS)
        parser.add_argument('--all-metrics', '-M',
                            action='store_const',
                            help='print all available metrics '
                                 '(without the compilation command)',
                            const=self._ALL_METRICS,
                            dest='metrics')
        parser.add_argument('--column-separator',
                            action='store',
                            help='column separator (defaults to ";")',
                            default=';')
        return parser

    def help_print(self):
        self._print_argparser().print_help()

    def do_print(self, params):
        parser = self._print_argparser()
        try:
            opts = parser.parse_args(self._argv(params))

            if opts.metrics is not None:
                metrics = opts.metrics
            else:
                raise RuntimeError('Specify --all-metrics to print all metrics '
                                   'or a list of metrics after --metrics')

            columns = { metric : self._CSV_COLUMNS[metric] for metric in metrics }

            if opts.out:
                stream = open(opts.out, 'w')
            else:
                stream = os.fdopen(os.dup(sys.stdout.fileno()), 'w')

            with stream:
                self._depgraph.print_csv(
                    stream,
                    columns,
                    opts.column_separator)
        except SystemExit:
            return

    def do_quit(self, params):
        sys.exit(0)

def main(args=None):
    logging.basicConfig(level=logging.INFO)

    if args is None:
        args = sys.argv[1:]

    cl = Interpreter()

    argline = ' '.join(args).strip('\'" \\t')

    if len(argline) > 0:
        for command in argline.split(';'):
            command = command.strip()
            logging.info('Running "%s"', command)
            cl.onecmd(command)
    else:
        cl.cmdloop('Welcome to the C++ build profile command line interface. '
                   'Use "help topic" to print help on a given topic')

if __name__ == '__main__':
    main()
<commit_msg>Added pattern option to remove_pch<commit_after>#!/usr/bin/env python

# Copyright (c) Techland. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.

import sys
import argparse
import cmd
import os
import logging
import traceback
from cppbuildprofiler import *

class Interpreter(cmd.Cmd):

    _CSV_COLUMNS = {
        Analyser.ABSOLUTE_PATH_KEY: { 'title': 'absolute path', 'default': None },
        Analyser.COMPILATION_COMMAND_KEY: { 'title': 'compilation command', 'default': '' },
        Analyser.BUILD_TIME_KEY: { 'title': 'build time [s]', 'default': 0.0 },
        Analyser.FILE_SIZE_KEY: { 'title': 'file size [B]', 'default': 0.0 },
        Analyser.TOTAL_SIZE_KEY: { 'title': 'total size [B]', 'default': 0.0 },
        Analyser.TOTAL_BUILD_TIME_KEY: { 'title': 'total build time of dependants [s]', 'default': 0.0 },
        Analyser.TRANSLATION_UNITS_KEY: { 'title': 'number of dependent translation units', 'default': 0 },
        Analyser.TU_BUILD_TIME_TO_SIZE_RATIO: { 'title': 'translation unit build time divided by total size sum [s/B]', 'default': 0 },
        }

    _ALL_METRICS = [
        Analyser.ABSOLUTE_PATH_KEY,
        Analyser.TRANSLATION_UNITS_KEY,
        Analyser.FILE_SIZE_KEY,
        Analyser.TOTAL_SIZE_KEY,
        Analyser.BUILD_TIME_KEY,
        Analyser.TOTAL_BUILD_TIME_KEY,
        Analyser.TU_BUILD_TIME_TO_SIZE_RATIO,
        ]

    def __init__(self):
        super().__init__()
        self.prompt = 'c++bp$ '
        self.use_rawinput = True

    def _argv(self, param_string):
        return filter(bool, param_string.split(' '))

    def emptyline(self):
        pass

    def onecmd(self, line):
        try:
            super().onecmd(line)
        except Exception:
            print(traceback.format_exc())

    def do_shell(self, line):
        out = os.popen(line).read()
        print(out)

    def _set_verbosity_argparser(self):
        parser = argparse.ArgumentParser('sets verbosity for the logging '
                                         'system')
        parser.add_argument(
            'level',
            action='store',
            choices=['DEBUG', 'INFO'])
        return parser

    def help_set_verbosity(self):
        self._set_verbosity_argparser().print_help()

    def do_set_verbosity(self, params):
        parser = self._set_verbosity_argparser()
        try:
            opts = parser.parse_args(self._argv(params))
            if opts.level == 'DEBUG':
                logging.getLogger().setLevel(logging.DEBUG)
            elif opts.level == 'INFO':
                logging.getLogger().setLevel(logging.INFO)
        except SystemExit:
            return

    def _parse_vs_log_argparser(self):
        parser = argparse.ArgumentParser('parses a visual studio build log '
                                         'and creates a dependency graph')
        parser.add_argument(
            'path',
            action='store',
            help='path to the log file to parse')
        return parser

    def help_parse_vs_log(self):
        self._parse_vs_log_argparser().print_help()

    def do_parse_vs_log(self, params):
        parser = self._parse_vs_log_argparser()
        try:
            opts = parser.parse_args(self._argv(params))
            self._depgraph = parse_vs_log(opts.path)
            self._depgraph.log_stats('Parsed %s into a dependency graph' %
                                     opts.path)
        except SystemExit:
            return

    def _load_argparser(self):
        parser = argparse.ArgumentParser('Loads a dependency graph from a file')
        parser.add_argument(
            'path',
            action='store',
            help='path to the file to load from')
        return parser

    def help_load(self):
        self._load_argparser().print_help()

    def do_load(self, params):
        parser = self._load_argparser()
        try:
            opts = parser.parse_args(self._argv(params))
            self._depgraph = DependencyGraph.read(opts.path)
            self._depgraph.log_stats('Loaded dependency grap from %s' %
                                     opts.path)
        except SystemExit:
            return

    def _store_argparser(self):
        parser = argparse.ArgumentParser('Stores a dependency graph in a file')
        parser.add_argument(
            'path',
            action='store',
            help='path to the file to write to')
        return parser

    def help_store(self):
        self._store_argparser().print_help()

    def do_store(self, params):
        parser = self._store_argparser()
        try:
            opts = parser.parse_args(self._argv(params))
            self._depgraph.write(opts.path)
            self._depgraph.log_stats('Stored dependency graph in %s' %
                                     opts.path)
        except SystemExit:
            return

    def do_analyse(self, params):
        Analyser(self._depgraph).run_full_analysis()

    def _remove_pch_argparser(self):
        parser = argparse.ArgumentParser('removes precompiled headers from the graph')
        parser.add_argument(
            'label_pattern',
            action='store',
            help='a regex pattern that will be looked for in graph labels to '
                 'determine whether they denote a precompiled header (defaults '
                 'to "stdafx\\.h"',
            default=r'stdafx\.h',
            nargs='?')
        return parser

    def help_remove_pch(self):
        self._remove_pch_argparser().print_help()

    def do_remove_pch(self, params):
        parser = self._remove_pch_argparser()
        try:
            opts = parser.parse_args(self._argv(params))
            Analyser(self._depgraph).remove_pch(opts.label_pattern)
        except SystemExit:
            return

    def _remove_thirdparty_dependencies_argparser(self):
        parser = argparse.ArgumentParser('removes third-party header '
                                         'dependencies')
        parser.add_argument(
            'codebase_root',
            action='store',
            help='path to the root directory of the codebase')
        return parser

    def help_remove_thirdparty_dependencies(self):
        self._remove_thirdparty_dependencies_argparser().print_help()

    def do_remove_thirdparty_dependencies(self, params):
        try:
            parser = self._remove_thirdparty_dependencies_argparser()
            opts = parser.parse_args(self._argv(params))
            Analyser(self._depgraph).remove_thirdparty_dependencies(
                opts.codebase_root)
        except SystemExit:
            return

    def _remove_node_argparser(self):
        parser = argparse.ArgumentParser('removes a node from the dependency graph')
        parser.add_argument(
            'label',
            action='store',
            help='label of the node to remove')
        return parser

    def help_remove_node(self):
        self._remove_node_argparser().print_help()

    def do_remove_node(self, params):
        try:
            parser = self._remove_node_argparser()
            opts = parser.parse_args(self._argv(params))

            self._depgraph.remove_nodes([opts.label])
            self._depgraph.remove_orphans()
        except SystemExit:
            return

    def _subgraph_argparser(self):
        parser = argparse.ArgumentParser('creates a subgraph of the '
                                         'dependency graph')
        parser.add_argument('--origin', '-o',
                            action='store',
                            help='label of the origin node')
        parser.add_argument('--dependencies', action='store_true',
                            help='include nodes origin depends on')
        parser.add_argument('--dependants', action='store_true',
                            help='include nodes that depend on origin')
        return parser

    def help_subgraph(self):
        self._subgraph_argparser().print_help()

    def do_subgraph(self, params):
        parser = self._subgraph_argparser()
        try:
            opts = parser.parse_args(self._argv(params))
            self._depgraph = DependencyGraph(self._depgraph.get_subgraph(
                opts.origin, opts.dependencies, opts.dependants))
            self._depgraph.log_stats('Created subgraph of %s' % opts.origin)
        except SystemExit:
            return

    def _print_argparser(self):
        parser = argparse.ArgumentParser('prints the dependency graph')
        parser.add_argument('--out', '-o',
                            action='store',
                            help='file to print out to',
                            required=False)
        parser.add_argument('--metrics', '-m',
                            help='metrics to print (space separated)',
                            nargs='*',
                            choices=self._ALL_METRICS)
        parser.add_argument('--all-metrics', '-M',
                            action='store_const',
                            help='print all available metrics '
                                 '(without the compilation command)',
                            const=self._ALL_METRICS,
                            dest='metrics')
        parser.add_argument('--column-separator',
                            action='store',
                            help='column separator (defaults to ";")',
                            default=';')
        return parser

    def help_print(self):
        self._print_argparser().print_help()

    def do_print(self, params):
        parser = self._print_argparser()
        try:
            opts = parser.parse_args(self._argv(params))

            if opts.metrics is not None:
                metrics = opts.metrics
            else:
                raise RuntimeError('Specify --all-metrics to print all metrics '
                                   'or a list of metrics after --metrics')

            columns = { metric : self._CSV_COLUMNS[metric] for metric in metrics }

            if opts.out:
                stream = open(opts.out, 'w')
            else:
                stream = os.fdopen(os.dup(sys.stdout.fileno()), 'w')

            with stream:
                self._depgraph.print_csv(
                    stream,
                    columns,
                    opts.column_separator)
        except SystemExit:
            return

    def do_quit(self, params):
        sys.exit(0)

def main(args=None):
    logging.basicConfig(level=logging.INFO)

    if args is None:
        args = sys.argv[1:]

    cl = Interpreter()

    argline = ' '.join(args).strip('\'" \\t')

    if len(argline) > 0:
        for command in argline.split(';'):
            command = command.strip()
            logging.info('Running "%s"', command)
            cl.onecmd(command)
    else:
        cl.cmdloop('Welcome to the C++ build profile command line interface. '
                   'Use "help topic" to print help on a given topic')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from subprocess import call
import sys
import subprocess
import os
import time

# input arguments
fuzzer = None
seed_corp_dir = None
out_corp_dir = None
# runtime
start_time = None
results_dir = None
TIME_INTERVAL = 60 * 20  # in seconds

''' Functions '''

def parse_arguments():
    # python2.7 covtracker.py lf cmin_ossfuzz lf_out
    global seed_corp_dir, out_corp_dir, results_dir, fuzzer
    time.sleep(1)
    if not (len(sys.argv) == 4):
        print '\ncovtracker.py:\tUsage:\t\tpython2.7 covtracker.py <afl_or_lf> <seed_corp_dir> <out_corp_dir>'
        print 'covtracker.py:\tUsage example:\tpython2.7 covtracker.py lf cmin_seed_dir out_dir\n'
        sys.exit()
    else:
        fuzzer = str(sys.argv[1])
        seed_corp_dir = str(sys.argv[2])
        out_corp_dir = str(sys.argv[3])
    results_dir = 'r_' + out_corp_dir

def run_coverage_tracker():
    global seed_corp_dir, out_corp_dir, results_dir, start_time, fuzzer
    dev_null = open(os.devnull, 'w')
    time_counter = 0
    call('rm -rf ' + results_dir, stdout=dev_null, stderr=subprocess.STDOUT, shell=True)
    call('mkdir ' + results_dir, stdout=dev_null, stderr=subprocess.STDOUT, shell=True)
    print 'covtracker.py:\t\tUsing ' + results_dir + ' as output folder.'
    if fuzzer == 'afl':
        out_corp_dir = out_corp_dir + '/1/queue'
    start_time = int('%.0f' % time.time())
    while True:
        try:
            if time_counter == 0:
                save_current_corpus(time_counter, dev_null, seed_corp_dir)
            else:
                save_current_corpus(time_counter,  dev_null, out_corp_dir)
            time.sleep(TIME_INTERVAL)
            time_counter += TIME_INTERVAL
        except KeyboardInterrupt:
            print 'covtracker.py:\t\tReceived interrupt...'
            save_current_corpus('final', dev_null, out_corp_dir)
            print 'covtracker.py:\t\tFinal corpus saved.\n'
            dev_null.close()
            quit()

def save_current_corpus(time_counter, dev_null, corpus_to_save):
    global results_dir, start_time, seed_corp_dir
    if time_counter == 'final':
        current_time = int('%.0f' % time.time())
        time_counter = current_time - start_time
    save_location = './' + results_dir + '/' + str(time_counter)
    call('mkdir ' + save_location, stdout=dev_null, stderr=subprocess.STDOUT, shell=True)
    copy_command = 'cp ./' + corpus_to_save + '/* ' + save_location
    call(copy_command, stdout=dev_null, stderr=subprocess.STDOUT, shell=True)
    #
    copy_command2 = 'cp ./' + seed_corp_dir + '/* ' + save_location
    call(copy_command2, stdout=dev_null, stderr=subprocess.STDOUT, shell=True)
    #
    print 'covtracker.py:\t\tSaved current corpus at ' + str(time_counter) + ' seconds.'

''' Program '''

parse_arguments()
run_coverage_tracker()

<commit_msg>Delete covtracker.py<commit_after><|endoftext|>


<commit_before># This file houses all default settings for the Crawler
# to override please use a custom localsettings.py file

# Specify the host and port to use when connecting to Redis.
REDIS_HOST = 'localhost'
REDIS_PORT = '6379'

# Kafka server information
KAFKA_HOSTS = 'locahost:9092'
KAFKA_TOPIC_PREFIX = 'demo'
KAFKA_APPID_TOPICS = False

ZOOKEEPER_ASSIGN_PATH = '/scrapy-cluster/crawler/'
ZOOKEEPER_ID = 'all'
ZOOKEEPER_HOSTS =  'localhost:2181'

PUBLIC_IP_URL = 'http://ip.42.pl/raw'

# Scrapy Settings
# ~~~~~~~~~~~~~~~
DOWNLOADER_CLIENTCONTEXTFACTORY = 'crawling.contextfactory.MyClientContextFactory'

# Scrapy settings for distributed_crawling project
#
BOT_NAME = 'crawling'

SPIDER_MODULES = ['crawling.spiders']
NEWSPIDER_MODULE = 'crawling.spiders'

# Enables scheduling storing requests queue in redis.
SCHEDULER = "crawling.distributed_scheduler.DistributedScheduler"

# Don't cleanup redis queues, allows to pause/resume crawls.
SCHEDULER_PERSIST = True

# Schedule requests using a priority queue. (default)
SCHEDULER_QUEUE_CLASS = 'crawling.redis_queue.RedisPriorityQueue'

# seconds to wait between seeing new queues, cannot be faster than spider_idle time of 5
SCHEDULER_QUEUE_REFRESH = 10

# throttled queue defaults per domain, x hits in a y second window
QUEUE_HITS = 10
QUEUE_WINDOW = 60

# we want the queue to produce a consistent pop flow
QUEUE_MODERATED = True

# how long we want the duplicate timeout queues to stick around in seconds
DUPEFILTER_TIMEOUT = 600

# how often to refresh the ip address of the scheduler
SCHEDULER_IP_REFRESH = 60

'''
----------------------------------------
The below parameters configure how spiders throttle themselves across the cluster
All throttling is based on the TLD of the page you are requesting, plus any of the
following parameters:

Type: You have different spider types and want to limit how often a given type of
spider hits a domain

IP: Your crawlers are spread across different IP's, and you want each IP crawler clump
to throttle themselves for a given domain

Combinations for any given Top Level Domain:
None - all spider types and all crawler ips throttle themselves from one tld queue
Type only - all spiders throttle themselves based off of their own type-based tld queue,
    regardless of crawler ip address
IP only - all spiders throttle themselves based off of their public ip address, regardless
    of spider type
Type and IP - every spider's throttle queue is determined by the spider type AND the
    ip address, allowing the most fined grained control over the throttling mechanism
'''
# add Spider type to throttle mechanism
SCHEDULER_TYPE_ENABLED = True

# add ip address to throttle mechanism
SCHEDULER_IP_ENABLED = True

'''
----------------------------------------
'''

# how many times to retry getting an item from the queue before the spider is considered idle
SCHEUDLER_ITEM_RETRIES = 3

# Store scraped item in redis for post-processing.
ITEM_PIPELINES = {
    'crawling.pipelines.KafkaPipeline': 100,
    'crawling.pipelines.LoggingBeforePipeline': 1,
    'crawling.pipelines.LoggingAfterPipeline': 101,
}

SPIDER_MIDDLEWARES = {
    # disable built-in DepthMiddleware, since we do our own
    # depth management per crawl request
    'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': None,
}

DOWNLOADER_MIDDLEWARES = {
    # Handle timeout retries with the redis scheduler and logger
    'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware' : None,
    'crawling.redis_retry_middleware.RedisRetryMiddleware': 510,
    # exceptions processed in reverse order
    'crawling.log_retry_middleware.LogRetryMiddleware': 520,
    # custom cookies to not persist across crawl requests
    'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware' : None,
    'crawling.custom_cookies.CustomCookiesMiddleware' :700,
}

# Disable the built in logging in production
LOG_ENABLED = False

# Allow all return codes
HTTPERROR_ALLOW_ALL = True

RETRY_TIMES = 3

DOWNLOAD_TIMEOUT = 10

# log setup scrapy cluster crawler
SC_LOG_STDOUT = True
SC_LOG_DIR = 'logs'
SC_LOG_FILE = 'main.log'
SC_LOG_MAX_BYTES = '10MB'
SC_LOG_JSON = False
SC_LOG_LEVEL = 'DEBUG'

# Avoid in-memory DNS cache. See Advanced topics of docs for info
DNSCACHE_ENABLED = False

# Local Overrides
# ~~~~~~~~~~~~~~~

try:
    from localsettings import *
except ImportError:
    pass
<commit_msg>Fix typos in settings file<commit_after># This file houses all default settings for the Crawler
# to override please use a custom localsettings.py file

# Specify the host and port to use when connecting to Redis.
REDIS_HOST = 'localhost'
REDIS_PORT = '6379'

# Kafka server information
KAFKA_HOSTS = 'localhost:9092'
KAFKA_TOPIC_PREFIX = 'demo'
KAFKA_APPID_TOPICS = False

ZOOKEEPER_ASSIGN_PATH = '/scrapy-cluster/crawler/'
ZOOKEEPER_ID = 'all'
ZOOKEEPER_HOSTS =  'localhost:2181'

PUBLIC_IP_URL = 'http://ip.42.pl/raw'

# Scrapy Settings
# ~~~~~~~~~~~~~~~
DOWNLOADER_CLIENTCONTEXTFACTORY = 'crawling.contextfactory.MyClientContextFactory'

# Scrapy settings for distributed_crawling project
#
BOT_NAME = 'crawling'

SPIDER_MODULES = ['crawling.spiders']
NEWSPIDER_MODULE = 'crawling.spiders'

# Enables scheduling storing requests queue in redis.
SCHEDULER = "crawling.distributed_scheduler.DistributedScheduler"

# Don't cleanup redis queues, allows to pause/resume crawls.
SCHEDULER_PERSIST = True

# Schedule requests using a priority queue. (default)
SCHEDULER_QUEUE_CLASS = 'crawling.redis_queue.RedisPriorityQueue'

# seconds to wait between seeing new queues, cannot be faster than spider_idle time of 5
SCHEDULER_QUEUE_REFRESH = 10

# throttled queue defaults per domain, x hits in a y second window
QUEUE_HITS = 10
QUEUE_WINDOW = 60

# we want the queue to produce a consistent pop flow
QUEUE_MODERATED = True

# how long we want the duplicate timeout queues to stick around in seconds
DUPEFILTER_TIMEOUT = 600

# how often to refresh the ip address of the scheduler
SCHEDULER_IP_REFRESH = 60

'''
----------------------------------------
The below parameters configure how spiders throttle themselves across the cluster
All throttling is based on the TLD of the page you are requesting, plus any of the
following parameters:

Type: You have different spider types and want to limit how often a given type of
spider hits a domain

IP: Your crawlers are spread across different IP's, and you want each IP crawler clump
to throttle themselves for a given domain

Combinations for any given Top Level Domain:
None - all spider types and all crawler ips throttle themselves from one tld queue
Type only - all spiders throttle themselves based off of their own type-based tld queue,
    regardless of crawler ip address
IP only - all spiders throttle themselves based off of their public ip address, regardless
    of spider type
Type and IP - every spider's throttle queue is determined by the spider type AND the
    ip address, allowing the most fined grained control over the throttling mechanism
'''
# add Spider type to throttle mechanism
SCHEDULER_TYPE_ENABLED = True

# add ip address to throttle mechanism
SCHEDULER_IP_ENABLED = True

'''
----------------------------------------
'''

# how many times to retry getting an item from the queue before the spider is considered idle
SCHEUDLER_ITEM_RETRIES = 3

# Store scraped item in redis for post-processing.
ITEM_PIPELINES = {
    'crawling.pipelines.KafkaPipeline': 100,
    'crawling.pipelines.LoggingBeforePipeline': 1,
    'crawling.pipelines.LoggingAfterPipeline': 101,
}

SPIDER_MIDDLEWARES = {
    # disable built-in DepthMiddleware, since we do our own
    # depth management per crawl request
    'scrapy.contrib.spidermiddleware.depth.DepthMiddleware': None,
}

DOWNLOADER_MIDDLEWARES = {
    # Handle timeout retries with the redis scheduler and logger
    'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware' : None,
    'crawling.redis_retry_middleware.RedisRetryMiddleware': 510,
    # exceptions processed in reverse order
    'crawling.log_retry_middleware.LogRetryMiddleware': 520,
    # custom cookies to not persist across crawl requests
    'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware' : None,
    'crawling.custom_cookies.CustomCookiesMiddleware' :700,
}

# Disable the built in logging in production
LOG_ENABLED = False

# Allow all return codes
HTTPERROR_ALLOW_ALL = True

RETRY_TIMES = 3

DOWNLOAD_TIMEOUT = 10

# log setup scrapy cluster crawler
SC_LOG_STDOUT = True
SC_LOG_DIR = 'logs'
SC_LOG_FILE = 'main.log'
SC_LOG_MAX_BYTES = '10MB'
SC_LOG_JSON = False
SC_LOG_LEVEL = 'INFO'

# Avoid in-memory DNS cache. See Advanced topics of docs for info
DNSCACHE_ENABLED = False

# Local Overrides
# ~~~~~~~~~~~~~~~

try:
    from localsettings import *
except ImportError:
    pass
<|endoftext|>


<commit_before>#####################################################################################
#
#  Copyright (C) Tavendo GmbH
#
#  Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you
#  have purchased a commercial license), the license terms below apply.
#
#  Should you enter into a separate license agreement after having received a copy of
#  this software, then the terms of such license agreement replace the terms below at
#  the time at which such license agreement becomes effective.
#
#  In case a separate license agreement ends, and such agreement ends without being
#  replaced by another separate license agreement, the license terms below apply
#  from the time at which said agreement ends.
#
#  LICENSE TERMS
#
#  This program is free software: you can redistribute it and/or modify it under the
#  terms of the GNU Affero General Public License, version 3, as published by the
#  Free Software Foundation. This program is distributed in the hope that it will be
#  useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
#  See the GNU Affero General Public License Version 3 for more details.
#
#  You should have received a copy of the GNU Affero General Public license along
#  with this program. If not, see <http://www.gnu.org/licenses/agpl-3.0.en.html>.
#
#####################################################################################

from __future__ import absolute_import, division, print_function

import json

from six import StringIO as NativeStringIO

from twisted.trial.unittest import TestCase

from io import StringIO

from mock import Mock

from twisted.logger import formatTime

from crossbar._logging import make_logger, CrossbarLogger, LogLevel
from crossbar import _logging

_log = make_logger("info", logger=Mock)

def _makelog():
    log = make_logger("info", logger=Mock)
    return log

class _InitLoggerMaker(object):
    def __init__(self):
        self.log = make_logger("info", logger=Mock)

class _ClassDefLoggerMaker(object):
    log = make_logger("info", logger=Mock)

class LoggerModuleTests(TestCase):

    def setUp(self):
        self.existing_level = _logging._loglevel

    def tearDown(self):
        _logging.set_global_log_level(self.existing_level)

    def test_set_global(self):
        """
        Setting the global log level via the function changes it.
        """
        _logging.set_global_log_level("warn")
        self.assertEqual(_logging._loglevel, "warn")

    def test_set_global_changes_loggers(self):
        """
        Setting the global log level changes the level of all loggers that were
        not instantiated with a level.
        """
        log = make_logger()
        self.assertEqual(log._log_level, "info")
        _logging.set_global_log_level("warn")
        self.assertEqual(log._log_level, "warn")

    def test_set_global_does_not_change_explicit_loggers(self):
        """
        Setting the global log level does not change loggers that have an
        explicit level set.
        """
        log = make_logger("info")
        self.assertEqual(log._log_level, "info")
        _logging.set_global_log_level("warn")
        self.assertEqual(log._log_level, "info")

class CrossbarLoggerTests(TestCase):

    def test_disallow_direct_instantiation(self):
        """
        The developer shouldn't call CrossbarLogger directly, but use
        make_logger.
        """
        with self.assertRaises(AssertionError):
            CrossbarLogger("warn")

    def test_set_level(self):
        """
        The log level needs to be one of the accepted log levels.
        """
        with self.assertRaises(ValueError):
            make_logger("not a suitable level")

    def test_logger_emits(self):
        """
        A Logger emits messages through to its child logger.
        """
        log = make_logger("trace", logger=Mock)

        log.error("Foo happened!!!")
        log.logger.error.assert_called_with("Foo happened!!!")

        log.warn("Stuff", foo="bar")
        log.logger.warn.assert_called_with("Stuff", foo="bar")

        log.trace("Stuff that's trace", foo="bar")
        log.logger.debug.assert_called_with("Stuff that's trace",
                                            foo="bar", cb_trace=1)

    def test_logger_emits_if_higher(self):
        """
        A Logger that has a log level of a higher severity will not emit
        messages of a lower severity.
        """
        log = make_logger("info", logger=Mock)

        log.error("Error!")
        log.debug("Debug!")
        log.info("Info!")
        log.trace("Trace!")
        log.emit(LogLevel.info, "Infoooo!")

        self.assertEqual(log.logger.failure.call_count, 0)
        self.assertEqual(log.logger.critical.call_count, 0)
        self.assertEqual(log.logger.error.call_count, 1)
        self.assertEqual(log.logger.warn.call_count, 0)
        self.assertEqual(log.logger.info.call_count, 2)
        self.assertEqual(log.logger.debug.call_count, 0)
        self.assertEqual(log.logger.trace.call_count, 0)

    def test_logger_namespace_init(self):
        """
        The namespace of the Logger is of the creator when using __init__.
        """
        lm = _InitLoggerMaker()

        self.assertEqual(lm.log.logger.namespace,
                         "crossbar.test.test_logger._InitLoggerMaker")

    def test_logger_namespace_classdef(self):
        """
        The namespace of the Logger is of the creator when using it in a class
        definition.
        """
        lm = _ClassDefLoggerMaker()

        self.assertEqual(lm.log.logger.namespace,
                         "crossbar.test.test_logger._ClassDefLoggerMaker")

    def test_logger_namespace_moduledef(self):
        """
        The namespace of the Logger is the creator module when it is made in a
        module.
        """
        self.assertEqual(_log.logger.namespace,
                         "crossbar.test.test_logger")

    def test_logger_namespace_function(self):
        """
        The namespace of the Logger is the creator function when it is made in
        a function outside of a class.
        """
        log = _makelog()
        self.assertEqual(log.logger.namespace,
                         "crossbar.test.test_logger._makelog")

    def test_logger_failure(self):
        """
        The failure method catches the in-flight exception.
        """
        log = make_logger("info", logger=Mock)

        try:
            1 / 0
        except:
            log.failure("Failure happened!")

        log.logger.failure.assert_called_once()

    # def test_logger_failure_not_called(self):
    #     """
    #     The failure method isn't called under 'none'.
    #     """
    #     log = make_logger("none", logger=Mock)

    #     try:
    #         1 / 0
    #     except:
    #         log.failure("Failure happened!")

    #     self.assertEqual(log.logger.failure.call_count, 0)

class JSONObserverTests(TestCase):

    def test_basic(self):
        """
        The JSON observer outputs a stream of log events.
        """
        stream = StringIO()
        observer = _logging.make_JSON_observer(stream)
        log = make_logger(observer=observer)

        log.info("Hello")

        result = stream.getvalue()
        log_entry = json.loads(result[:-1])

        self.assertEqual(result[-1], _logging.record_separator)
        self.assertEqual(len(log_entry.keys()), 4)
        self.assertEqual(log_entry["level"], u"info")
        self.assertEqual(log_entry["text"], u"Hello")

    def test_failure(self):
        """
        Failures include the stacktrace.
        """
        stream = StringIO()
        observer = _logging.make_JSON_observer(stream)
        log = make_logger(observer=observer)

        try:
            1 / 0
        except:
            log.failure("Oh no")

        result = stream.getvalue()
        log_entry = json.loads(result[:-1])

        self.assertEqual(result[-1], _logging.record_separator)
        self.assertEqual(len(log_entry.keys()), 4)
        self.assertIn(u"ZeroDivisionError", log_entry["text"])
        self.assertIn(u"Oh no", log_entry["text"])
        self.assertEqual(log_entry["level"], u"critical")

class StdoutObserverTests(TestCase):

    def test_basic(self):

        stream = NativeStringIO()
        observer = _logging.make_stdout_observer(_file=stream)
        log = make_logger(observer=observer)

        log.info("Hi!", log_system="foo")

        result = stream.getvalue()
        self.assertIn(u"[foo]", result)

    def test_output_nocolour(self):
        """
        The output format is the time, the system in square brackets, and the
        message.
        """
        stream = NativeStringIO()
        observer = _logging.make_stdout_observer(_file=stream,
                                                 format="nocolour")
        event = {'log_level': LogLevel.info,
                 'log_namespace': 'crossbar.test.test_logger.StdoutObserverTests',
                 'log_source': None, 'log_format': 'Hi there!',
                 'log_system': 'foo', 'log_time': 1434099813.77449}

        observer(event)

        result = stream.getvalue()
        self.assertEqual(result[:-1],
                         formatTime(event["log_time"]) + " [foo] Hi there!")

    def test_output_syslogd(self):
        """
        The syslogd output format is the system in square brackets, and the
        message.
        """
        stream = NativeStringIO()
        observer = _logging.make_stdout_observer(_file=stream,
                                                 format="syslogd")
        event = {'log_level': LogLevel.info,
                 'log_namespace': 'crossbar.test.test_logger.StdoutObserverTests',
                 'log_source': None, 'log_format': 'Hi there!',
                 'log_system': 'foo', 'log_time': 1434099813.77449}

        observer(event)

        result = stream.getvalue()
        self.assertEqual(result[:-1], "[foo] Hi there!")
<commit_msg>fix travis<commit_after>#####################################################################################
#
#  Copyright (C) Tavendo GmbH
#
#  Unless a separate license agreement exists between you and Tavendo GmbH (e.g. you
#  have purchased a commercial license), the license terms below apply.
#
#  Should you enter into a separate license agreement after having received a copy of
#  this software, then the terms of such license agreement replace the terms below at
#  the time at which such license agreement becomes effective.
#
#  In case a separate license agreement ends, and such agreement ends without being
#  replaced by another separate license agreement, the license terms below apply
#  from the time at which said agreement ends.
#
#  LICENSE TERMS
#
#  This program is free software: you can redistribute it and/or modify it under the
#  terms of the GNU Affero General Public License, version 3, as published by the
#  Free Software Foundation. This program is distributed in the hope that it will be
#  useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
#  See the GNU Affero General Public License Version 3 for more details.
#
#  You should have received a copy of the GNU Affero General Public license along
#  with this program. If not, see <http://www.gnu.org/licenses/agpl-3.0.en.html>.
#
#####################################################################################

from __future__ import absolute_import, division, print_function

import json

from six import StringIO as NativeStringIO

from twisted.trial.unittest import TestCase

from io import StringIO

from mock import Mock

from twisted.logger import formatTime

from crossbar._logging import make_logger, CrossbarLogger, LogLevel
from crossbar import _logging

_log = make_logger("info", logger=Mock)

def _makelog():
    log = make_logger("info", logger=Mock)
    return log

class _InitLoggerMaker(object):
    def __init__(self):
        self.log = make_logger("info", logger=Mock)

class _ClassDefLoggerMaker(object):
    log = make_logger("info", logger=Mock)

class LoggerModuleTests(TestCase):

    def setUp(self):
        self.existing_level = _logging._loglevel

    def tearDown(self):
        _logging.set_global_log_level(self.existing_level)

    def test_set_global(self):
        """
        Setting the global log level via the function changes it.
        """
        _logging.set_global_log_level("warn")
        self.assertEqual(_logging._loglevel, "warn")

    def test_set_global_changes_loggers(self):
        """
        Setting the global log level changes the level of all loggers that were
        not instantiated with a level.
        """
        log = make_logger()
        self.assertEqual(log._log_level, "info")
        _logging.set_global_log_level("warn")
        self.assertEqual(log._log_level, "warn")

    def test_set_global_does_not_change_explicit_loggers(self):
        """
        Setting the global log level does not change loggers that have an
        explicit level set.
        """
        log = make_logger("info")
        self.assertEqual(log._log_level, "info")
        _logging.set_global_log_level("warn")
        self.assertEqual(log._log_level, "info")

class CrossbarLoggerTests(TestCase):

    def test_disallow_direct_instantiation(self):
        """
        The developer shouldn't call CrossbarLogger directly, but use
        make_logger.
        """
        with self.assertRaises(AssertionError):
            CrossbarLogger("warn")

    def test_set_level(self):
        """
        The log level needs to be one of the accepted log levels.
        """
        with self.assertRaises(ValueError):
            make_logger("not a suitable level")

    def test_logger_emits(self):
        """
        A Logger emits messages through to its child logger.
        """
        log = make_logger("trace", logger=Mock)

        log.error("Foo happened!!!")
        log.logger.error.assert_called_with("Foo happened!!!")

        log.warn("Stuff", foo="bar")
        log.logger.warn.assert_called_with("Stuff", foo="bar")

        log.trace("Stuff that's trace", foo="bar")
        log.logger.debug.assert_called_with("Stuff that's trace",
                                            foo="bar", cb_trace=1)

    def test_logger_emits_if_higher(self):
        """
        A Logger that has a log level of a higher severity will not emit
        messages of a lower severity.
        """
        log = make_logger("info", logger=Mock)

        log.error("Error!")
        log.debug("Debug!")
        log.info("Info!")
        log.trace("Trace!")
        log.emit(LogLevel.info, "Infoooo!")

        self.assertEqual(log.logger.failure.call_count, 0)
        self.assertEqual(log.logger.critical.call_count, 0)
        self.assertEqual(log.logger.error.call_count, 1)
        self.assertEqual(log.logger.warn.call_count, 0)
        self.assertEqual(log.logger.info.call_count, 2)
        self.assertEqual(log.logger.debug.call_count, 0)
        self.assertEqual(log.logger.trace.call_count, 0)

    def test_logger_namespace_init(self):
        """
        The namespace of the Logger is of the creator when using __init__.
        """
        lm = _InitLoggerMaker()

        self.assertEqual(lm.log.logger.namespace,
                         "crossbar.test.test_logger._InitLoggerMaker")

    def test_logger_namespace_classdef(self):
        """
        The namespace of the Logger is of the creator when using it in a class
        definition.
        """
        lm = _ClassDefLoggerMaker()

        self.assertEqual(lm.log.logger.namespace,
                         "crossbar.test.test_logger._ClassDefLoggerMaker")

    def test_logger_namespace_moduledef(self):
        """
        The namespace of the Logger is the creator module when it is made in a
        module.
        """
        self.assertEqual(_log.logger.namespace,
                         "crossbar.test.test_logger")

    def test_logger_namespace_function(self):
        """
        The namespace of the Logger is the creator function when it is made in
        a function outside of a class.
        """
        log = _makelog()
        self.assertEqual(log.logger.namespace,
                         "crossbar.test.test_logger._makelog")

    # def test_logger_failure(self):
    #     """
    #     The failure method catches the in-flight exception.
    #     """
    #     log = make_logger("info", logger=Mock)

    #     try:
    #         1 / 0
    #     except:
    #         log.failure("Failure happened!")

    #     log.logger.failure.assert_called_once()

    # def test_logger_failure_not_called(self):
    #     """
    #     The failure method isn't called under 'none'.
    #     """
    #     log = make_logger("none", logger=Mock)

    #     try:
    #         1 / 0
    #     except:
    #         log.failure("Failure happened!")

    #     self.assertEqual(log.logger.failure.call_count, 0)

class JSONObserverTests(TestCase):

    def test_basic(self):
        """
        The JSON observer outputs a stream of log events.
        """
        stream = StringIO()
        observer = _logging.make_JSON_observer(stream)
        log = make_logger(observer=observer)

        log.info("Hello")

        result = stream.getvalue()
        log_entry = json.loads(result[:-1])

        self.assertEqual(result[-1], _logging.record_separator)
        self.assertEqual(len(log_entry.keys()), 4)
        self.assertEqual(log_entry["level"], u"info")
        self.assertEqual(log_entry["text"], u"Hello")

    def test_failure(self):
        """
        Failures include the stacktrace.
        """
        stream = StringIO()
        observer = _logging.make_JSON_observer(stream)
        log = make_logger(observer=observer)

        try:
            1 / 0
        except:
            log.failure("Oh no")

        result = stream.getvalue()
        log_entry = json.loads(result[:-1])

        self.assertEqual(result[-1], _logging.record_separator)
        self.assertEqual(len(log_entry.keys()), 4)
        self.assertIn(u"ZeroDivisionError", log_entry["text"])
        self.assertIn(u"Oh no", log_entry["text"])
        self.assertEqual(log_entry["level"], u"critical")

class StdoutObserverTests(TestCase):

    def test_basic(self):

        stream = NativeStringIO()
        observer = _logging.make_stdout_observer(_file=stream)
        log = make_logger(observer=observer)

        log.info("Hi!", log_system="foo")

        result = stream.getvalue()
        self.assertIn(u"[foo]", result)

    def test_output_nocolour(self):
        """
        The output format is the time, the system in square brackets, and the
        message.
        """
        stream = NativeStringIO()
        observer = _logging.make_stdout_observer(_file=stream,
                                                 format="nocolour")
        event = {'log_level': LogLevel.info,
                 'log_namespace': 'crossbar.test.test_logger.StdoutObserverTests',
                 'log_source': None, 'log_format': 'Hi there!',
                 'log_system': 'foo', 'log_time': 1434099813.77449}

        observer(event)

        result = stream.getvalue()
        self.assertEqual(result[:-1],
                         formatTime(event["log_time"]) + " [foo] Hi there!")

    def test_output_syslogd(self):
        """
        The syslogd output format is the system in square brackets, and the
        message.
        """
        stream = NativeStringIO()
        observer = _logging.make_stdout_observer(_file=stream,
                                                 format="syslogd")
        event = {'log_level': LogLevel.info,
                 'log_namespace': 'crossbar.test.test_logger.StdoutObserverTests',
                 'log_source': None, 'log_format': 'Hi there!',
                 'log_system': 'foo', 'log_time': 1434099813.77449}

        observer(event)

        result = stream.getvalue()
        self.assertEqual(result[:-1], "[foo] Hi there!")
<|endoftext|>