<commit_before>#! /usr/bin/env python

import os
import sys
import fnmatch
import time
import shutil

def commonJob(f, suf):
    return "universe  = vanilla\n\
    request_cpus = 1\n\
    request_memory = 20G\n\
    request_disk = 10G\n\
    output = logs/" + f + "." + suf + ".condout\n\
    error = logs/" + f + "." + suf + ".conderr\n\
    logs = logs/" + f + "." + suf + ".condlog\n\
    should_transfer_files   = YES\n\
    when_to_transfer_output = ON_EXIT\n\
    queue\n"

def checkJobConditions(f, suf, tf):
    res = True;
    if suf == "DT" or suf == "SD" or suf == "FD":
        if not os.path.isfile("data/" + f + ".cnf"):
            print "# CNF file data/" + f + ".cnf missing while creating job " + f + "." + suf
            res = False
        if not os.path.isfile("data/" + f + ".drat"):
            print "# DRAT file data/" + f + ".drat missing while creating job " + f + "." + suf
            res = False
    elif (suf == "vDT" or suf == "cDT") and tf == "FALSE":
        res = False
    elif tf == "TRUE":
        if not os.path.isfile("data/" + f + ".cnf"):
            print "# CNF file data/" + f + ".cnf missing while creating job " + f + "." + suf
            res = False
        if not os.path.isfile("witness/" + f + "." + suf[1:2] + ".lrat"):
            print "# LRAT file witness/" + f + "." + suf[1:2] + ".lrat missing while creating job " + f + "." + suf
            res = False
    elif tf == "FALSE":
        if not os.path.isfile("data/" + f + ".cnf"):
            print "# CNF file data/" + f + ".cnf missing while creating job " + f + "." + suf
            res = False
        if not os.path.isfile("data/" + f + ".drat"):
            print "# DRAT file data/" + f + ".drat missing while creating job " + f + "." + suf
            res = False
        if not os.path.isfile("witness/" + f + "." + suf[1:2] + ".sick"):
            print "# LRAT file witness/" + f + "." + suf[1:2] + ".sick missing while creating job " + f + "." + suf
            res = False
    elif tf == "NONE":
        print "# execution over " + f + "." + suf[1:2] + "failed, skipping verification"
        res = False
    return res

def createJob(f, suf, res):
    common = commonJob(f, suf)
    write = checkJobConditions(f, suf, res)
    if write == True:
        if suf == "DT" or suf == "SD" or suf == "FD":
            transLine = "transfer_input_files = data/" + f + ".cnf,data/" + f + ".drat\n"
        if res == "FALSE" and (suf == "vSD" or suf == "vFD" or suf == "cSD" or suf == "cFD"):
            transLine = "transfer_input_files = data/" + f + ".cnf,data/" + f + ".drat,witness/" + f + "." + suf[1:2] + ".sick\n"
            execLine = "executable = bin/sickcheck\n"
            argsLine = "arguments = " + f + ".cnf " + f + ".drat " + f + "." + suf[1:2] + ".sick\n"
        if res == "FALSE" and (suf == "cSD" or suf == "cFD"):
            transLine = "transfer_input_files = data/" + f + ".cnf,data/" + f + ".drat,witness/" + f + "." + suf[1:2] + ".sick\n"
            # TODO: coq-sick-check executable & arguments
            # execLine = "executable = bin/sickcheck\n"
            # argsLine = "arguments = " + f + ".cnf " + f + ".drat " + f + "." + suf[1:2] + ".sick\n"
        if res == "TRUE" and (suf == "vDT" or suf == "vSD" or suf == "vFD"):
            transLine = "transfer_input_files = data/" + f + ".cnf,witness/" + f + "." + suf[1:2] + ".lrat\n"
            execLine = "executable = bin/lratcheck\n"
            argsLine = "arguments = " + f + ".cnf " + f + "." + suf[1:2] + ".lrat\n"
        if res == "TRUE" and (suf == "cDT" or suf == "cSD" or suf == "cFD"):
            transLine = "transfer_input_files = data/" + f + ".cnf,witness/" + f + "." + suf[1:2] + ".lrat\n"
            # TODO: coq-lrat-check executable & arguments
            # execLine = "executable = bin/lratcheck\n"
            # argsLine = "arguments = " + f + ".cnf " + f + "." + suf[1:2] + ".lrat\n"
        if suf == "DT":
            execLine = "executable = bin/drat-trim\n"
            argsLine = "arguments = " + f + ".cnf " + f + ".drat -L " + f + "." + suf + ".lrat -t 5000\n"
        if suf == "SD" or suf == "FD":
            execLine = "executable = bin/rupee\n"
            if suf == "SD":
                mode = "-skip-deletion"
            if suf == "FD":
                mode = "-full-deletion"
            argsLine = "arguments = " + f + ".cnf " + f + ".drat -lrat " + f + "." +  suf + ".lrat -recheck " + f + "." + suf + ".sick " + mode + "\n"
        wr = open("jobs/" + f + "." + suf + ".sub", "w")
        wr.write(execLine + argsLine + transLine + common)
        wr.close()

def submitJob(f, suf):
    if os.path.isfile("jobs/" + f + "." + suf + ".sub"):
        os.popen("condor_submit jobs/" + f + "." + suf + ".sub")
    else:
        print "# SUB job description for file " f + "." +  suf + " missing"

def parseResult(f):
    result = "NONE"
    if os.path.isfile("logs/" + f + ".condout"):
        o = open("logs/" + f  + ".condout")
        lines = o.read().splitlines()
        o.close()
        for line  in lines:
            if line.find("s VERIFIED") != -1:
                result = "TRUE"
            if line.find("s INCORRECT") != -1:
                result = "FALSE"
            if line.find("s NOT VERIFIED") != -1:
                result = "FALSE"
    return result

def getFiles(x):
    o = open(sys.argv[1])
    l = o.read().splitlines()
    o.close()
    return l

def moveWitnesses(files):
    for file in files:
        for suf in ["DT", "SD", "FD"]:
            for wit in [".lrat", ".sick"]:
                if os.path.isfile(f + "." + suf + wit):
                    os.rename(f + "." + suf + wit, "witness/" + f + "." + suf + wit)

def cleanFolder(folder):
    shutil.rmtree(folder)
    os.mkdir(folder)

masterList = sys.argv[1]
stage = sys.argv[2]
fileList = getFiles(masterList)
if stage == "-check":
    cleanFolder("jobs")
    cleanFolder("logs")
    cleanFolder("witness")
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            createJob(file, suf, "")
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            submitJob(file, suf)
elif stage == "-verify:":
    moveWitnesses(fileList)
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            res = parseResult(file + "." + suf[1:2])
            createJob(file, "v" + suf, res)
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            submitJob(file, "v" + suf)
elif stage == "-coqverify":
    moveWitnesses(fileList)
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            res = parseResult(file + "." + suf[1:2])
            createJob(file, "c" + suf, res)
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            submitJob(file, "c" + suf)
<commit_msg>fixed script bug<commit_after>#! /usr/bin/env python

import os
import sys
import fnmatch
import time
import shutil

def commonJob(f, suf):
    return "universe  = vanilla\n\
    request_cpus = 1\n\
    request_memory = 20G\n\
    request_disk = 10G\n\
    output = logs/" + f + "." + suf + ".condout\n\
    error = logs/" + f + "." + suf + ".conderr\n\
    logs = logs/" + f + "." + suf + ".condlog\n\
    should_transfer_files   = YES\n\
    when_to_transfer_output = ON_EXIT\n\
    queue\n"

def checkJobConditions(f, suf, tf):
    res = True;
    if suf == "DT" or suf == "SD" or suf == "FD":
        if not os.path.isfile("data/" + f + ".cnf"):
            print "# CNF file data/" + f + ".cnf missing while creating job " + f + "." + suf
            res = False
        if not os.path.isfile("data/" + f + ".drat"):
            print "# DRAT file data/" + f + ".drat missing while creating job " + f + "." + suf
            res = False
    elif (suf == "vDT" or suf == "cDT") and tf == "FALSE":
        res = False
    elif tf == "TRUE":
        if not os.path.isfile("data/" + f + ".cnf"):
            print "# CNF file data/" + f + ".cnf missing while creating job " + f + "." + suf
            res = False
        if not os.path.isfile("witness/" + f + "." + suf[1:2] + ".lrat"):
            print "# LRAT file witness/" + f + "." + suf[1:2] + ".lrat missing while creating job " + f + "." + suf
            res = False
    elif tf == "FALSE":
        if not os.path.isfile("data/" + f + ".cnf"):
            print "# CNF file data/" + f + ".cnf missing while creating job " + f + "." + suf
            res = False
        if not os.path.isfile("data/" + f + ".drat"):
            print "# DRAT file data/" + f + ".drat missing while creating job " + f + "." + suf
            res = False
        if not os.path.isfile("witness/" + f + "." + suf[1:2] + ".sick"):
            print "# LRAT file witness/" + f + "." + suf[1:2] + ".sick missing while creating job " + f + "." + suf
            res = False
    elif tf == "NONE":
        print "# execution over " + f + "." + suf[1:2] + "failed, skipping verification"
        res = False
    return res

def createJob(f, suf, res):
    common = commonJob(f, suf)
    write = checkJobConditions(f, suf, res)
    if write == True:
        if suf == "DT" or suf == "SD" or suf == "FD":
            transLine = "transfer_input_files = data/" + f + ".cnf,data/" + f + ".drat\n"
        if res == "FALSE" and (suf == "vSD" or suf == "vFD" or suf == "cSD" or suf == "cFD"):
            transLine = "transfer_input_files = data/" + f + ".cnf,data/" + f + ".drat,witness/" + f + "." + suf[1:2] + ".sick\n"
            execLine = "executable = bin/sickcheck\n"
            argsLine = "arguments = " + f + ".cnf " + f + ".drat " + f + "." + suf[1:2] + ".sick\n"
        if res == "FALSE" and (suf == "cSD" or suf == "cFD"):
            transLine = "transfer_input_files = data/" + f + ".cnf,data/" + f + ".drat,witness/" + f + "." + suf[1:2] + ".sick\n"
            # TODO: coq-sick-check executable & arguments
            # execLine = "executable = bin/sickcheck\n"
            # argsLine = "arguments = " + f + ".cnf " + f + ".drat " + f + "." + suf[1:2] + ".sick\n"
        if res == "TRUE" and (suf == "vDT" or suf == "vSD" or suf == "vFD"):
            transLine = "transfer_input_files = data/" + f + ".cnf,witness/" + f + "." + suf[1:2] + ".lrat\n"
            execLine = "executable = bin/lratcheck\n"
            argsLine = "arguments = " + f + ".cnf " + f + "." + suf[1:2] + ".lrat\n"
        if res == "TRUE" and (suf == "cDT" or suf == "cSD" or suf == "cFD"):
            transLine = "transfer_input_files = data/" + f + ".cnf,witness/" + f + "." + suf[1:2] + ".lrat\n"
            # TODO: coq-lrat-check executable & arguments
            # execLine = "executable = bin/lratcheck\n"
            # argsLine = "arguments = " + f + ".cnf " + f + "." + suf[1:2] + ".lrat\n"
        if suf == "DT":
            execLine = "executable = bin/drat-trim\n"
            argsLine = "arguments = " + f + ".cnf " + f + ".drat -L " + f + "." + suf + ".lrat -t 5000\n"
        if suf == "SD" or suf == "FD":
            execLine = "executable = bin/rupee\n"
            if suf == "SD":
                mode = "-skip-deletion"
            if suf == "FD":
                mode = "-full-deletion"
            argsLine = "arguments = " + f + ".cnf " + f + ".drat -lrat " + f + "." +  suf + ".lrat -recheck " + f + "." + suf + ".sick " + mode + "\n"
        wr = open("jobs/" + f + "." + suf + ".sub", "w")
        wr.write(execLine + argsLine + transLine + common)
        wr.close()

def submitJob(f, suf):
    if os.path.isfile("jobs/" + f + "." + suf + ".sub"):
        os.popen("condor_submit jobs/" + f + "." + suf + ".sub")
    else:
        print "# SUB job description for file "+  f + "." +  suf + " missing"

def parseResult(f):
    result = "NONE"
    if os.path.isfile("logs/" + f + ".condout"):
        o = open("logs/" + f  + ".condout")
        lines = o.read().splitlines()
        o.close()
        for line  in lines:
            if line.find("s VERIFIED") != -1:
                result = "TRUE"
            if line.find("s INCORRECT") != -1:
                result = "FALSE"
            if line.find("s NOT VERIFIED") != -1:
                result = "FALSE"
    return result

def getFiles(x):
    o = open(sys.argv[1])
    l = o.read().splitlines()
    o.close()
    return l

def moveWitnesses(files):
    for file in files:
        for suf in ["DT", "SD", "FD"]:
            for wit in [".lrat", ".sick"]:
                if os.path.isfile(f + "." + suf + wit):
                    os.rename(f + "." + suf + wit, "witness/" + f + "." + suf + wit)

def cleanFolder(folder):
    shutil.rmtree(folder)
    os.mkdir(folder)

masterList = sys.argv[1]
stage = sys.argv[2]
fileList = getFiles(masterList)
if stage == "-check":
    cleanFolder("jobs")
    cleanFolder("logs")
    cleanFolder("witness")
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            createJob(file, suf, "")
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            submitJob(file, suf)
elif stage == "-verify:":
    moveWitnesses(fileList)
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            res = parseResult(file + "." + suf[1:2])
            createJob(file, "v" + suf, res)
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            submitJob(file, "v" + suf)
elif stage == "-coqverify":
    moveWitnesses(fileList)
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            res = parseResult(file + "." + suf[1:2])
            createJob(file, "c" + suf, res)
    for file in fileList:
        for suf in ["DT", "SD", "FD"]:
            submitJob(file, "c" + suf)
<|endoftext|>


<commit_before>'''
Dataset source: https://nlds.soe.ucsc.edu/sarcasm2

By Shereen Oraby, Vrindavan Harrison, Lena Reed, Ernesto Hernandez, Ellen Riloff and Marilyn Walker. "Creating and Characterizing a Diverse Corpus of Sarcasm in Dialogue." In The 17th Annual SIGdial Meeting on Discourse and Dialogue (SIGDIAL), Los Angeles, California, USA, 2016.
'''
import numpy as np
import pandas
import re

def main():
	data = clean_data()

def clean_data():
	'''
	Steps:
	1. Load data from csv file
	2. Remove unnecessary quotes
	3. Remove emoticons
	3. Fix all typos using Microsoft's Bing Spell Check API: https://www.microsoft.com/cognitive-services/en-us/bing-spell-check-api
	'''
	# step 1
	data = pandas.read_csv('sarcasm_v2.csv').as_matrix()
	# print(data.shape)
	data[:, 0] = np.array([find_category(x) for x in data[:, 0]])
	data[:, 1] = np.array([sarcasm(x) for x in data[:, 1]])
	# print(data[0,1]) # should be 1 for sarcasm

	# step 2
	data[:, 3] = np.array([clean_quotes(x) for x in data[:, 3]])
	data[:, 4] = np.array([clean_quotes(x) for x in data[:, 4]])

	# step 3
	data[:, 3] = np.array([remove_emoticons(x) for x in data[:, 3]])
	data[:, 4] = np.array([remove_emoticons(x) for x in data[:, 4]])

	return data

def find_category(category):
	return {
        'GEN': 0, # general
        'HYP': 1, # hyperbole
		'RQ': 2 # rhetorical question
    }[category]

def sarcasm(sarcasm_value):
	return {
		'sarc': 1, # true for sarcasm
		'notsarc': 0 # false for sarcasm
	}[sarcasm_value]

def get_data_index(ID):
	'''find the index of the data point. Corresponds to 1234 in GEN_sarc_1234 under ID in data.
	'''
	# TODO: given a string as shown in the comment, extract the number in it, possibly with regex.
        # Test

def clean_quotes(string):
	'''
	clean_quotes a string and removes beginning and closing quotation marks. Also replaces all duplicated double quotation marks with just one set of double quotation marks, e.g. ""hello"" with "hello"
	'''
	if string[0] == '"':
		string = string[1:] # first character
	if string[-1] == '"':
		string = string[:-1] # last character
	string = re.sub('\"\"', "\"", string)
	# triple quotes should not exist after removing first and last quotes from beginning and end, respectively, of the original string.
	# quadruple quotes will also be removed by replacing all duplicated double quotes "" with a single double quote ". This way, back-to-back duplicated double quotes (e.g. ""hello""""world"" will become "hello""world") will be taken care of as well.
	return string

if __name__ == '__main__':
	main()
<commit_msg>Create remove_emoticons function<commit_after>'''
Dataset source: https://nlds.soe.ucsc.edu/sarcasm2

By Shereen Oraby, Vrindavan Harrison, Lena Reed, Ernesto Hernandez, Ellen Riloff and Marilyn Walker. "Creating and Characterizing a Diverse Corpus of Sarcasm in Dialogue." In The 17th Annual SIGdial Meeting on Discourse and Dialogue (SIGDIAL), Los Angeles, California, USA, 2016.
'''
import numpy as np
import pandas
import re

def main():
	data = clean_data()

def clean_data():
	'''
	Steps:
	1. Load data from csv file
	2. Remove unnecessary quotes
	3. Remove emoticons
	3. Fix all typos using Microsoft's Bing Spell Check API: https://www.microsoft.com/cognitive-services/en-us/bing-spell-check-api
	'''
	# step 1
	data = pandas.read_csv('sarcasm_v2.csv').as_matrix()
	# print(data.shape)
	data[:, 0] = np.array([find_category(x) for x in data[:, 0]])
	data[:, 1] = np.array([sarcasm(x) for x in data[:, 1]])
	# print(data[0,1]) # should be 1 for sarcasm

	# step 2
	data[:, 3] = np.array([clean_quotes(x) for x in data[:, 3]])
	data[:, 4] = np.array([clean_quotes(x) for x in data[:, 4]])

	# step 3
	data[:, 3] = np.array([remove_emoticons(x) for x in data[:, 3]])
	data[:, 4] = np.array([remove_emoticons(x) for x in data[:, 4]])

	return data

def find_category(category):
	return {
        'GEN': 0, # general
        'HYP': 1, # hyperbole
		'RQ': 2 # rhetorical question
    }[category]

def sarcasm(sarcasm_value):
	return {
		'sarc': 1, # true for sarcasm
		'notsarc': 0 # false for sarcasm
	}[sarcasm_value]

def get_data_index(ID):
	'''find the index of the data point. Corresponds to 1234 in GEN_sarc_1234 under ID in data.
	'''
	# TODO: given a string as shown in the comment, extract the number in it, possibly with regex.
        # Test

def clean_quotes(string):
	'''
	clean_quotes a string and removes beginning and closing quotation marks. Also replaces all duplicated double quotation marks with just one set of double quotation marks, e.g. ""hello"" with "hello"
	'''
	if string[0] == '"':
		string = string[1:] # first character
	if string[-1] == '"':
		string = string[:-1] # last character
	string = re.sub('\"\"', "\"", string)
	# triple quotes should not exist after removing first and last quotes from beginning and end, respectively, of the original string.
	# quadruple quotes will also be removed by replacing all duplicated double quotes "" with a single double quote ". This way, back-to-back duplicated double quotes (e.g. ""hello""""world"" will become "hello""world") will be taken care of as well.
	return string

def remove_emoticons(string):
	''' removes emoticons from the strings.'''

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>#from https://subliminal.readthedocs.org/en/latest/user/usage.html#cli

from __future__ import unicode_literals
import os
from babelfish import *
from subliminal import *

video = Video.fromname('The.Big.Bang.Theory.S05E18.HDTV.x264-LOL.mp4')
print video
print video.video_codec
print video.release_group

my_region = region.configure('dogpile.cache.memory')

best_subtitles = download_best_subtitles([video], {Language('eng')}, providers=['podnapisi'])
best_subtitles[video]
best_subtitle = best_subtitles[video][0]
print best_subtitle.content.split(b'\n')[2]<commit_msg>working on it..<commit_after>#from https://subliminal.readthedocs.org/en/latest/user/usage.html#cli

from __future__ import unicode_literals
import os
from babelfish import *
from subliminal import *
import urlparse
import os.path
import sys
import xmlrpc_client

print len(sys.argv)
if len(sys.argv) > 1:
	moviePath = sys.argv[1]
else:
	moviePath = "/Users/badescuga/Downloads/Rick.and.Morty.S02E02.HDTV.x264-BATV.mp4"
print "eeee"

movieName = os.path.basename(urlparse.urlsplit(moviePath).path)
video = Video.fromname(movieName)
print video
print video.video_codec
print video.release_group

my_region = region.configure('dogpile.cache.memory')

best_subtitles = download_best_subtitles([video], {Language('eng')}, providers=['podnapisi'])
best_subtitles[video]
best_subtitle = best_subtitles[video][0]

#save subtitle

subPath = os.path.splitext(moviePath)[0] + ".srt"
print subPath
file_ = open(subPath, 'w')
file_.write(best_subtitle.content)
file_.close()

#print best_subtitle.content.split(b'\n')[2]

#os.system("alias vlc='/Applications/VLC.app/Contents/MacOS/VLC'")
os.system("/Applications/VLC.app/Contents/MacOS/VLC " + moviePath + " --sub-file "+subPath)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Author:  csiu
# Created:
import argparse
from ConfigParser import SafeConfigParser
import sys
import os
import re
import linecache

from utils import get_value_from_keycolonvalue_list, ensure_dir, random_string
import tc_normalization
import correlation
import mirna_proximity
import features
import promi2

usage = """
- you will need to set parameters in tc-normalization
"""
def _reformat_infile_gff2tcnorm(infile, outfile):
    with open(outfile, 'w') as out:
        with open(infile) as f:
            for l in f:
                l = l.strip().split('\t')
                chrom = l[0]
                start = l[3]
                stop  = l[4]
                strand = l[6]

                newline = 'chr%s:%s..%s,%s' % (chrom, start, stop, strand)
                out.write(newline + '\n')
    return

def _index_tcnorm(f_ids):
    tc_index = {}
    with open(f_ids) as f:
        c = 5
        for tc_id in f:
            tc_id = tc_id.strip()
            c += 1

            tc_index[tc_id] = ['%s:%s' % (tc_id, c)]
    return tc_index

def _interpret_tss_mirna_pairings(gff_infile, gff_mirna, pair_pos):
    mirna_lookup = {}
    with open(gff_mirna) as f:
        for l in f:
            chrom, _, mirbaseid, start, stop, _, strand, r, info = l.strip().split('\t')
            srnaseqid = info.split(':')[1]

            mirna_annot = ','.join(['title=' + srnaseqid,
                                    'mirbase_id='  + mirbaseid,
                                    'mirna_start=' + start,
                                    'mirna_stop='  + stop
                                    ])
            mirna_lookup[srnaseqid] = [r, mirna_annot]
    mirna_lookup_keys =  mirna_lookup.keys()

    with open(pair_pos, 'w') as out:
        with open(gff_infile) as f:
            n = 0
            for l in f:
                n += 1

                chrom, _, _, start, stop, _, strand, _, mirna = l.strip().split('\t')
                tss_annot = 'title=chr%s:%s..%s,%s' % (chrom, start, stop, strand)

                mirna = mirna.lower()
                for k in [m for m in mirna_lookup_keys if re.match('%s(-.*$|$)' % mirna, m)]:
                    r, mirna_annot = mirna_lookup[k]
                    pairid = 'pair_id=' + '.'.join([chrom, start, stop, strand,
                                                    re.sub('-[35]p$', '', k)])

                    newline = '\t'.join([tss_annot, str(n),
                                         ','.join([mirna_annot, pairid]), r])

                    out.write(newline + '\n')
    return

def _index_corr_pairid(gff_corr):
    pairid_index = {}
    with open(gff_corr) as f:
        c = 0
        for l in f:
            c += 1

            _, info = l.strip().rsplit('\t', 1)
            info    = info.split(';')

            pid = get_value_from_keycolonvalue_list('pair_id', info)

            try:
                pairid_index[pid].append(c)
            except KeyError:
                pairid_index[pid] = [c]

    return pairid_index

def _reformat_tss_to_1kb(gff_infile, gff1kb_infile):
    with open(gff1kb_infile, 'w') as out:
        with open(gff_infile) as f:
            for l in f:
                chrom, _, _, start, stop, _, strand, _, _ = l.strip().split()

                newinfo = ';'.join(['start:' + start,
                                    'stop:'  + stop])

                start = int(start)
                stop  = int(stop)
                mid = (start + stop) / 2

                newline = '\t'.join([chrom, '.', '.',
                                     str(mid - 500),
                                     str(mid  + 500),
                                     '.', strand, '.', newinfo])
                out.write(newline + '\n')
    return

def _index_1kbfeatures(gff_1kbfeatures):
    feature_index = {}
    with open(gff_1kbfeatures) as f:
        c = 0
        for l in f:
            c += 1
            chrom, _, _, start, stop, _, strand, features, _ = l.split('\t')

            pos = '.'.join([chrom, start, stop, strand])

            try:
                feature_index[pos].append(c)
            except KeyError:
                feature_index[pos] = [c]

    return feature_index

def extractFeatures_given_posPairs(config, gff_infile, outdir):
    cparser = SafeConfigParser()
    cparser.read(config)

    tc_config = cparser.get('configs', 'tcconfig')
    m_mirna   = cparser.get('correlation', 'srnaseqmatrix')

    ## PART1: tc normalization
    ## 1a. setup infile
    outdir_tc = os.path.join(outdir, 'tc-norm')
    f1_pos    = os.path.join(outdir, 'f1_pos.txt')
    ensure_dir(outdir_tc)

    ## 1b. reformat infile so that can be read by tc-quantify
    _reformat_infile_gff2tcnorm(gff_infile, f1_pos)

    ## 1c. run
    fo_bed = tc_normalization.main(tc_config, f1_pos, outdir_tc)

    ncount_dict = {}
    with open(fo_bed) as f:
        for l in f:
            l = l.strip().split('\t')
            _, chrom, start, _, stop, strand = re.split('[r:.,]', l[3])
            pos = '.'.join([chrom, start, stop, strand])
            ncount_dict[pos] = l[6]

    ## 1d. setup outfile
    f_rle = re.sub('max_tpm.bed$', 'tpm_rle.matrix', fo_bed)

    tcparser = SafeConfigParser()
    tcparser.read(tc_config)
    f_ids = tcparser.get('tc_normalization', 'ids')

    ## PART2: correlation setup
    outdir_corr = os.path.join(outdir, 'corr/')
    ensure_dir(outdir_corr)

    mirbase_gff2 = cparser.get('mirbase', 'gff2')
    corrmethod   = cparser.get('correlation', 'corrmethod')
    gff_mirna = os.path.join(outdir_corr, '4corr_mirna.gff')
    gff_tss   = os.path.join(outdir_corr, '4corr_tss.gff')
    pair_pos    = os.path.join(outdir_corr, '4corrPair_row_pos_tss-mirna.txt')
    pair_sample = os.path.join(outdir_corr, '4corrPair_col_sample_CAGE-sRNAseq.txt')
    fo_corr = os.path.join(outdir_corr, 'features_correlation-%s.gff' % corrmethod)

    ## position pair:
    correlation._find_miRNA_pos(m_mirna, mirbase_gff2, gff_mirna)
    correlation._get_tss_pos(f1_pos, gff_tss)
    _interpret_tss_mirna_pairings(gff_infile, gff_mirna, pair_pos)

    ## sample pair:
    srnaseq_index = correlation._index_srnaseq(m_mirna)
    cage_index    = _index_tcnorm(f_ids)
    correlation._get_sample_pairings(cage_index, srnaseq_index, pair_sample)

    ## compute correlation:
    correlation._compute_correlation(pair_pos, pair_sample,
                                     f_rle, m_mirna,
                                     fo_corr, corrmethod, '.')

    pairid_index = _index_corr_pairid(fo_corr)

    ## PART3: compute features
    ## compute cpg, cons, tata ...
    outdir_seqfeatures = os.path.join(outdir, 'seqfeatures/')
    ensure_dir(outdir_seqfeatures)

    f_fasta      = cparser.get('genome','fasta')
    f_chromsizes = cparser.get('genome','chromsizes')
    d_phastcons  = cparser.get('cons','phastcons')
    TRAP         = cparser.get('tata','trap')
    f_psemmatrix = cparser.get('tata','psem')

    gff1kb_infile   = os.path.join(outdir_seqfeatures, 'infile_1kbseq.gff')
    gff_1kbfeatures = os.path.join(outdir_seqfeatures, 'features_1kbseq.gff')

    _reformat_tss_to_1kb(gff_infile, gff1kb_infile)

    features.main(gff1kb_infile, outdir_seqfeatures,
                  f_fasta, f_chromsizes, d_phastcons, TRAP, f_psemmatrix,
                  gff_1kbfeatures)

    features_index = _index_1kbfeatures(gff_1kbfeatures)

    ## start consolidating features ...
    gff_allfeatures = os.path.join(outdir, 'features.gff')
    with open(gff_allfeatures, 'w') as out:
        with open(gff_infile) as f:
            for l in f:
                chrom, _, _, start, stop, _, strand, _, mirna = l.strip().split('\t')
                mirna = mirna.lower()

                pairid = '.'.join([chrom, start, stop, strand, mirna])
                tssid  = '.'.join([chrom, start, stop, strand])

                ncount = ncount_dict[tssid]

                for n in pairid_index[pairid]:

                    ## feature: correlation (corr)
                    cline =  linecache.getline(fo_corr, n).strip().split('\t')
                    corr  = cline[5]
                    cinfo = cline[8].split(';')

                    ## feature: mirna (mprox)
                    mstart = int(get_value_from_keycolonvalue_list('mirna_start', cinfo))
                    mstop = int(get_value_from_keycolonvalue_list('mirna_stop', cinfo))

                    d = mirna_proximity.calculate_distance(start, stop, mstart, mstop, strand)
                    mprox = str(mirna_proximity.distance_score(d))

                    ## features: cpg, cons, tata (cct)
                    for t in features_index[tssid]:
                        fline = linecache.getline(gff_1kbfeatures, t).strip().split('\t')
                        cct = fline[7]
                        info_region = fline[8]

                        ## write out!
                        newfeatures = ';'.join([cct,
                                                'mirna_prox:' + mprox,
                                                'corr:'       + corr])

                        newinfo = ';'.join([info_region,
                                            ';'.join(cinfo[:-1]),
                                            'distance:' + str(d)])

                        newline = '\t'.join([chrom, '.', '.',
                                             start, stop, ncount,
                                             strand, newfeatures, newinfo])
                        out.write(newline + '\n')

    return gff_allfeatures

def main(f_config, gff_infile, outdir):
    outdir = '../Testout-custom'
    ensure_dir(outdir)

    '''
    cparser = SafeConfigParser()
    cparser.read(f_config)
    f_params       = cparser.get('promi2', 'params')
    listoffeatures = cparser.get('promi2', 'features').split(',')
    '''

    ## Extract features
    gff_allfeatures = extractFeatures_given_posPairs(f_config, gff_infile, outdir)

    '''
    ## Run Promirna
    fo_predictions = os.path.join(outdir,
                                  'Predictions.%s.txt' % os.path.basename(gff_infile))
    promi2.promi2(f_params, listoffeatures, gff_allfeatures, fo_predictions)
    '''

    ## Plot
    print 'good day'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=usage,
                                     formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('-i', '--infile', dest='infile',
                        required=True,
                        help='''Path to input gff input file.
Tab-separated columns should be like:
  1. chrom
  2. "." (source; not used)
  3. "." (feature; not used)
  4. putative_tss start
  5. putative_tss stop
  6. "." (score; not used)
  7. strand
  8. "." (frame; not used)
  9. "." (or miRNA if you used '-m' flag)
''')

    parser.add_argument('-m', dest='has_mirna',
                        action='store_true',
                        help='''Flag to designate that INFILE has miRNA in column 9
e.g. tss-mirna pairing is already provided &
program does not need to look for pairs

''')

    parser.add_argument('-c', '--config', dest='f_config',
                        default='config.ini',
                        help='Path to config file; default="config.ini"')

    parser.add_argument('-o', '--outdir', dest='outdir',
                        help='Specify output directory')

    ##get at the arguments
    args = parser.parse_args()

    ## do something..
    main(args.f_config, args.infile, args.outdir)
<commit_msg>made it so that script runs promirna via promi2 wrapper<commit_after>#!/usr/bin/env python
# Author:  csiu
# Created:
import argparse
from ConfigParser import SafeConfigParser
import sys
import os
import re
import linecache

from utils import get_value_from_keycolonvalue_list, ensure_dir, random_string
import tc_normalization
import correlation
import mirna_proximity
import features
import promi2

usage = """
- you will need to set parameters in tc-normalization
"""
def _reformat_infile_gff2tcnorm(infile, outfile):
    with open(outfile, 'w') as out:
        with open(infile) as f:
            for l in f:
                l = l.strip().split('\t')
                chrom = l[0]
                start = l[3]
                stop  = l[4]
                strand = l[6]

                newline = 'chr%s:%s..%s,%s' % (chrom, start, stop, strand)
                out.write(newline + '\n')
    return

def _index_tcnorm(f_ids):
    tc_index = {}
    with open(f_ids) as f:
        c = 5
        for tc_id in f:
            tc_id = tc_id.strip()
            c += 1

            tc_index[tc_id] = ['%s:%s' % (tc_id, c)]
    return tc_index

def _interpret_tss_mirna_pairings(gff_infile, gff_mirna, pair_pos):
    mirna_lookup = {}
    with open(gff_mirna) as f:
        for l in f:
            chrom, _, mirbaseid, start, stop, _, strand, r, info = l.strip().split('\t')
            srnaseqid = info.split(':')[1]

            mirna_annot = ','.join(['title=' + srnaseqid,
                                    'mirbase_id='  + mirbaseid,
                                    'mirna_start=' + start,
                                    'mirna_stop='  + stop
                                    ])
            mirna_lookup[srnaseqid] = [r, mirna_annot]
    mirna_lookup_keys =  mirna_lookup.keys()

    with open(pair_pos, 'w') as out:
        with open(gff_infile) as f:
            n = 0
            for l in f:
                n += 1

                chrom, _, _, start, stop, _, strand, _, mirna = l.strip().split('\t')
                tss_annot = 'title=chr%s:%s..%s,%s' % (chrom, start, stop, strand)

                mirna = mirna.lower()
                for k in [m for m in mirna_lookup_keys if re.match('%s(-.*$|$)' % mirna, m)]:
                    r, mirna_annot = mirna_lookup[k]
                    pairid = 'pair_id=' + '.'.join([chrom, start, stop, strand,
                                                    re.sub('-[35]p$', '', k)])

                    newline = '\t'.join([tss_annot, str(n),
                                         ','.join([mirna_annot, pairid]), r])

                    out.write(newline + '\n')
    return

def _index_corr_pairid(gff_corr):
    pairid_index = {}
    with open(gff_corr) as f:
        c = 0
        for l in f:
            c += 1

            _, info = l.strip().rsplit('\t', 1)
            info    = info.split(';')

            pid = get_value_from_keycolonvalue_list('pair_id', info)

            try:
                pairid_index[pid].append(c)
            except KeyError:
                pairid_index[pid] = [c]

    return pairid_index

def _reformat_tss_to_1kb(gff_infile, gff1kb_infile):
    with open(gff1kb_infile, 'w') as out:
        with open(gff_infile) as f:
            for l in f:
                chrom, _, _, start, stop, _, strand, _, _ = l.strip().split()

                newinfo = ';'.join(['start:' + start,
                                    'stop:'  + stop])

                start = int(start)
                stop  = int(stop)
                mid = (start + stop) / 2

                newline = '\t'.join([chrom, '.', '.',
                                     str(mid - 500),
                                     str(mid  + 500),
                                     '.', strand, '.', newinfo])
                out.write(newline + '\n')
    return

def _index_1kbfeatures(gff_1kbfeatures):
    feature_index = {}
    with open(gff_1kbfeatures) as f:
        c = 0
        for l in f:
            c += 1
            chrom, _, _, start, stop, _, strand, features, _ = l.split('\t')

            pos = '.'.join([chrom, start, stop, strand])

            try:
                feature_index[pos].append(c)
            except KeyError:
                feature_index[pos] = [c]

    return feature_index

def extractFeatures_given_posPairs(config, gff_infile, outdir):
    cparser = SafeConfigParser()
    cparser.read(config)

    tc_config = cparser.get('configs', 'tcconfig')
    m_mirna   = cparser.get('correlation', 'srnaseqmatrix')

    ## PART1: tc normalization
    ## 1a. setup infile
    outdir_tc = os.path.join(outdir, 'tc-norm')
    f1_pos    = os.path.join(outdir, 'f1_pos.txt')
    ensure_dir(outdir_tc)

    ## 1b. reformat infile so that can be read by tc-quantify
    _reformat_infile_gff2tcnorm(gff_infile, f1_pos)

    ## 1c. run
    fo_bed = tc_normalization.main(tc_config, f1_pos, outdir_tc)

    ncount_dict = {}
    with open(fo_bed) as f:
        for l in f:
            l = l.strip().split('\t')
            _, chrom, start, _, stop, strand = re.split('[r:.,]', l[3])
            pos = '.'.join([chrom, start, stop, strand])
            ncount_dict[pos] = l[6]

    ## 1d. setup outfile
    f_rle = re.sub('max_tpm.bed$', 'tpm_rle.matrix', fo_bed)

    tcparser = SafeConfigParser()
    tcparser.read(tc_config)
    f_ids = tcparser.get('tc_normalization', 'ids')

    ## PART2: correlation setup
    outdir_corr = os.path.join(outdir, 'corr/')
    ensure_dir(outdir_corr)

    mirbase_gff2 = cparser.get('mirbase', 'gff2')
    corrmethod   = cparser.get('correlation', 'corrmethod')
    gff_mirna = os.path.join(outdir_corr, '4corr_mirna.gff')
    gff_tss   = os.path.join(outdir_corr, '4corr_tss.gff')
    pair_pos    = os.path.join(outdir_corr, '4corrPair_row_pos_tss-mirna.txt')
    pair_sample = os.path.join(outdir_corr, '4corrPair_col_sample_CAGE-sRNAseq.txt')
    fo_corr = os.path.join(outdir_corr, 'features_correlation-%s.gff' % corrmethod)

    ## position pair:
    correlation._find_miRNA_pos(m_mirna, mirbase_gff2, gff_mirna)
    correlation._get_tss_pos(f1_pos, gff_tss)
    _interpret_tss_mirna_pairings(gff_infile, gff_mirna, pair_pos)

    ## sample pair:
    srnaseq_index = correlation._index_srnaseq(m_mirna)
    cage_index    = _index_tcnorm(f_ids)
    correlation._get_sample_pairings(cage_index, srnaseq_index, pair_sample)

    ## compute correlation:
    correlation._compute_correlation(pair_pos, pair_sample,
                                     f_rle, m_mirna,
                                     fo_corr, corrmethod, '.')

    pairid_index = _index_corr_pairid(fo_corr)

    ## PART3: compute features
    ## compute cpg, cons, tata ...
    outdir_seqfeatures = os.path.join(outdir, 'seqfeatures/')
    ensure_dir(outdir_seqfeatures)

    f_fasta      = cparser.get('genome','fasta')
    f_chromsizes = cparser.get('genome','chromsizes')
    d_phastcons  = cparser.get('cons','phastcons')
    TRAP         = cparser.get('tata','trap')
    f_psemmatrix = cparser.get('tata','psem')

    gff1kb_infile   = os.path.join(outdir_seqfeatures, 'infile_1kbseq.gff')
    gff_1kbfeatures = os.path.join(outdir_seqfeatures, 'features_1kbseq.gff')

    _reformat_tss_to_1kb(gff_infile, gff1kb_infile)

    features.main(gff1kb_infile, outdir_seqfeatures,
                  f_fasta, f_chromsizes, d_phastcons, TRAP, f_psemmatrix,
                  gff_1kbfeatures)

    features_index = _index_1kbfeatures(gff_1kbfeatures)

    ## start consolidating features ...
    gff_allfeatures = os.path.join(outdir, 'features.gff')
    with open(gff_allfeatures, 'w') as out:
        with open(gff_infile) as f:
            for l in f:
                chrom, _, _, start, stop, _, strand, _, mirna = l.strip().split('\t')
                mirna = mirna.lower()

                pairid = '.'.join([chrom, start, stop, strand, mirna])
                tssid  = '.'.join([chrom, start, stop, strand])

                ncount = ncount_dict[tssid]

                for n in pairid_index[pairid]:

                    ## feature: correlation (corr)
                    cline =  linecache.getline(fo_corr, n).strip().split('\t')
                    corr  = cline[5]
                    cinfo = cline[8].split(';')

                    ## feature: mirna (mprox)
                    mstart = int(get_value_from_keycolonvalue_list('mirna_start', cinfo))
                    mstop = int(get_value_from_keycolonvalue_list('mirna_stop', cinfo))

                    d = mirna_proximity.calculate_distance(start, stop, mstart, mstop, strand)
                    mprox = str(mirna_proximity.distance_score(d))

                    ## features: cpg, cons, tata (cct)
                    for t in features_index[tssid]:
                        fline = linecache.getline(gff_1kbfeatures, t).strip().split('\t')
                        cct = fline[7]
                        info_region = fline[8]

                        ## write out!
                        newfeatures = ';'.join([cct,
                                                'mirna_prox:' + mprox,
                                                'corr:'       + corr])

                        newinfo = ';'.join([info_region,
                                            ';'.join(cinfo[:-1]),
                                            'distance:' + str(d)])

                        newline = '\t'.join([chrom, '.', '.',
                                             start, stop, ncount,
                                             strand, newfeatures, newinfo])
                        out.write(newline + '\n')

    return gff_allfeatures

def main(f_config, gff_infile, outdir):
    outdir = '../Testout-custom'
    ensure_dir(outdir)

    cparser = SafeConfigParser()
    cparser.read(f_config)
    f_params       = cparser.get('promi2', 'params')
    listoffeatures = cparser.get('promi2', 'features').split(',')

    ## Extract features
    gff_allfeatures = extractFeatures_given_posPairs(f_config, gff_infile, outdir)

    ## Run Promirna
    fo_predictions = os.path.join(outdir,
                                  'Predictions.%s.txt' % os.path.basename(gff_infile))
    promi2.promi2(f_params, listoffeatures, gff_allfeatures, fo_predictions)

    ## Plot
    print 'good day'

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=usage,
                                     formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('-i', '--infile', dest='infile',
                        required=True,
                        help='''Path to input gff input file.
Tab-separated columns should be like:
  1. chrom
  2. "." (source; not used)
  3. "." (feature; not used)
  4. putative_tss start
  5. putative_tss stop
  6. "." (score; not used)
  7. strand
  8. "." (frame; not used)
  9. "." (or miRNA if you used '-m' flag)
''')

    parser.add_argument('-m', dest='has_mirna',
                        action='store_true',
                        help='''Flag to designate that INFILE has miRNA in column 9
e.g. tss-mirna pairing is already provided &
program does not need to look for pairs

''')

    parser.add_argument('-c', '--config', dest='f_config',
                        default='config.ini',
                        help='Path to config file; default="config.ini"')

    parser.add_argument('-o', '--outdir', dest='outdir',
                        help='Specify output directory')

    ##get at the arguments
    args = parser.parse_args()

    ## do something..
    main(args.f_config, args.infile, args.outdir)
<|endoftext|>


<commit_before>"""
LocalQServer
"""

import time
import threading
import localq
from operator import methodcaller

class LocalQServer():
    def __init__(self, num_cores_available, interval, priority_method, stderr="/dev/null", stdout="/dev/null"):
        self.num_cores_available = num_cores_available
        self.interval = float(interval)
        self.priority_method = priority_method
        self.stdout = stdout
        self.stderr = stderr
        self.queue = []
        self.running = []
        self.completed = []
        self.failed = []
        self.next_jobid = 1

    def add(self, cmd, num_cores, rundir=None, stdout=None, stderr=None):
        job = localq.Job(cmd, num_cores, stdout=stdout, stderr=stderr, rundir=rundir)
        job.set_jobid(self.next_jobid)
        self.queue.append(job)
        self.next_jobid += 1
        return job.jobid

    def list_queue(self):
        retstr = ""
        retstr += "JOBID\tPRIO\tSTATUS\tNUM_CORES\tPRIO_METHOD\tCMD\n"
        for j in self.running:
            retstr += j.info() + "\n"
        for j in self.queue:
            retstr += j.info() + "\n"
        for j in self.completed:
            retstr += j.info() + "\n"
        for j in self.failed:
            retstr += j.info() + "\n"
        return retstr.strip("\n")

    def get_status(self, jobid):
        """
        Check in which state (list) a job is.
        Jobs are moved between lists every self.interval seconds (see LocalQServer.run() for implementation)
        :param jobid: a job id to search for
        :return: Any of Job.PENDING, RUNNING, COMPLETED or FAILED. If no job with the given ID is found, None is returned.
        """
        if jobid in [j.jobid for j in self.queue]:
            return localq.Job.PENDING
        if jobid in [j.jobid for j in self.running]:
            return localq.Job.RUNNING
        if jobid in [j.jobid for j in self.completed]:
            return localq.Job.COMPLETED
        if jobid in [j.jobid for j in self.failed]:
            return localq.Job.FAILED
        else:
            return None

    def run(self):
        print "Starting localqd with {} available cores".format(self.num_cores_available)
        print "Checking queue every {} seconds".format(self.interval)

        def check_queue():
            while True:

                # check which running jobs are completed
                for job in self.running:
                    # if job is completed, move to self.completed
                    if job.status() == localq.Job.COMPLETED:
                        self.running.remove(job)
                        self.completed.append(job)

                    # if job failed, move to self.failed
                    elif job.status() == localq.Job.FAILED:
                        self.running.remove(job)
                        self.failed.append(job)
                    else:  # if job is still running, keep it in self.running
                        pass

                sortedqueue = sorted(self.queue, key=methodcaller('priority'), reverse=True)

                # check if new jobs can be started
                for job in sortedqueue:
                    n_cores_busy = 0
                    for j in self.running:
                        n_cores_busy += j.num_cores
                    cores_available = self.num_cores_available - n_cores_busy

                    if cores_available >= job.num_cores:
                        job.run()
                        self.running.append(job)
                        self.queue.remove(job)
                    else:  # If highest-priority job can't start, don't try to start next job until next cycle
                        continue

                time.sleep(self.interval)

        thread = threading.Thread(target=check_queue)
        thread.setDaemon(True)
        thread.start()
<commit_msg>renamed self.queue to self.pending to match job states in local.Job<commit_after>"""
LocalQServer
"""

import time
import threading
import localq
from operator import methodcaller

class LocalQServer():
    def __init__(self, num_cores_available, interval, priority_method, stderr="/dev/null", stdout="/dev/null"):
        self.num_cores_available = num_cores_available
        self.interval = float(interval)
        self.priority_method = priority_method
        self.stdout = stdout
        self.stderr = stderr
        self.pending = []
        self.running = []
        self.completed = []
        self.failed = []
        self.next_jobid = 1

    def add(self, cmd, num_cores, rundir=None, stdout=None, stderr=None):
        job = localq.Job(cmd, num_cores, stdout=stdout, stderr=stderr, rundir=rundir)
        job.set_jobid(self.next_jobid)
        self.pending.append(job)
        self.next_jobid += 1
        return job.jobid

    def list_queue(self):
        retstr = ""
        retstr += "JOBID\tPRIO\tSTATUS\tNUM_CORES\tPRIO_METHOD\tCMD\n"
        for j in self.running:
            retstr += j.info() + "\n"
        for j in self.pending:
            retstr += j.info() + "\n"
        for j in self.completed:
            retstr += j.info() + "\n"
        for j in self.failed:
            retstr += j.info() + "\n"
        return retstr.strip("\n")

    def get_status(self, jobid):
        """
        Check in which state (list) a job is.
        Jobs are moved between lists every self.interval seconds (see LocalQServer.run() for implementation)
        :param jobid: a job id to search for
        :return: Any of Job.PENDING, RUNNING, COMPLETED or FAILED. If no job with the given ID is found, None is returned.
        """
        if jobid in [j.jobid for j in self.pending]:
            return localq.Job.PENDING
        if jobid in [j.jobid for j in self.running]:
            return localq.Job.RUNNING
        if jobid in [j.jobid for j in self.completed]:
            return localq.Job.COMPLETED
        if jobid in [j.jobid for j in self.failed]:
            return localq.Job.FAILED
        else:
            return None

    def run(self):
        print "Starting localqd with {} available cores".format(self.num_cores_available)
        print "Checking queue every {} seconds".format(self.interval)

        def check_queue():
            while True:

                # check which running jobs are completed
                for job in self.running:
                    # if job is completed, move to self.completed
                    if job.status() == localq.Job.COMPLETED:
                        self.running.remove(job)
                        self.completed.append(job)

                    # if job failed, move to self.failed
                    elif job.status() == localq.Job.FAILED:
                        self.running.remove(job)
                        self.failed.append(job)
                    else:  # if job is still running, keep it in self.running
                        pass

                sortedqueue = sorted(self.pending, key=methodcaller('priority'), reverse=True)

                # check if new jobs can be started
                for job in sortedqueue:
                    n_cores_busy = 0
                    for j in self.running:
                        n_cores_busy += j.num_cores
                    cores_available = self.num_cores_available - n_cores_busy

                    if cores_available >= job.num_cores:
                        job.run()
                        self.running.append(job)
                        self.pending.remove(job)
                    else:  # If highest-priority job can't start, don't try to start next job until next cycle
                        continue

                time.sleep(self.interval)

        thread = threading.Thread(target=check_queue)
        thread.setDaemon(True)
        thread.start()
<|endoftext|>


<commit_before>#!/usr/env python
# coding: utf8
'''
Created on 20.09.2011

p is a paste service. As the brother of s it is designed to be fast and small.

@author: Kevin Zuber
'''
# python standard library
import sys
import os

# 3rd party python module

# packed 3rd party python module
sys.path.append(os.path.join(os.path.dirname(__file__), "lib"))
from bottle import route, run, static_file, view, abort, redirect, request

sys.path.append(os.path.dirname(__file__))
# own python module
from service.paster import Paster
from service import formater

paste = Paster("p.db")

@route("/")
@view("index")
def index():
    return {"format_default": formater.format_default,
            "formats_long_to_short": formater.formats_long_to_short,
            "formats_long": formater.formats_long}

@route("/", method="POST")
def add_paste():
    f = request.forms
    if f.format not in formater.formats_short:
        f.format = formater.format_default
    if f.title and f.content and f.format:
        key = paste.add(f.title, f.content, f.format, request['REMOTE_ADDR'])
        redirect("/" + key)
    else:
        abort(400, "You did not provide all required fields.")

@route("/:key")
@view("view_paste")
def view_paste(key):
    try:
        thing = paste.thing[key]
        format = thing[2]
        if format not in formater.formats_short:
            format = formater.default_format
        return {"title": thing[0],
            "content": thing[1],
            "content_html": formater.format(thing[1], format),
            "format_used": thing[2],
            "time": thing[4],
            "formats_long_to_short": formater.formats_long_to_short,
            "formats_long": formater.formats_long,
            "key": key}
    except KeyError:
        abort(404, "Key " + key + " not found")

@route("/:key/raw")
def view_paste_raw(key):
    try:
        thing = paste.thing[key]
        return thing[1]
    except KeyError:
        abort(404, "Key " + key + " not found")

@route('/static/:path#.+#')
def server_static(path):
    return static_file(path, root='./static')

if __name__ == '__main__':
    import bottle
    bottle.debug(True)
    run(host="localhost", port=8080, reloader=True)
else:
    import bottle
    bottle.TEMPLATE_PATH.insert(0,
        os.path.join(os.path.dirname(__file__), 'views/'))
    application = bottle.default_app()
<commit_msg>[fix] set content type "text/plain" for raw data<commit_after>#!/usr/env python
# coding: utf8
'''
Created on 20.09.2011

p is a paste service. As the brother of s it is designed to be fast and small.

@author: Kevin Zuber
'''
# python standard library
import sys
import os

# 3rd party python module

# packed 3rd party python module
sys.path.append(os.path.join(os.path.dirname(__file__), "lib"))
from bottle import route, run, static_file, view, abort, redirect, request
from bottle import response

sys.path.append(os.path.dirname(__file__))
# own python module
from service.paster import Paster
from service import formater

paste = Paster("p.db")

@route("/")
@view("index")
def index():
    return {"format_default": formater.format_default,
            "formats_long_to_short": formater.formats_long_to_short,
            "formats_long": formater.formats_long}

@route("/", method="POST")
def add_paste():
    f = request.forms
    if f.format not in formater.formats_short:
        f.format = formater.format_default
    if f.title and f.content and f.format:
        key = paste.add(f.title, f.content, f.format, request['REMOTE_ADDR'])
        redirect("/" + key)
    else:
        abort(400, "You did not provide all required fields.")

@route("/:key")
@view("view_paste")
def view_paste(key):
    try:
        thing = paste.thing[key]
        format = thing[2]
        if format not in formater.formats_short:
            format = formater.default_format
        return {"title": thing[0],
            "content": thing[1],
            "content_html": formater.format(thing[1], format),
            "format_used": thing[2],
            "time": thing[4],
            "formats_long_to_short": formater.formats_long_to_short,
            "formats_long": formater.formats_long,
            "key": key}
    except KeyError:
        abort(404, "Key " + key + " not found")

@route("/:key/raw")
def view_paste_raw(key):
    try:
        thing = paste.thing[key]
        response.content_type = "text/plain"
        return thing[1]
    except KeyError:
        abort(404, "Key " + key + " not found")

@route('/static/:path#.+#')
def server_static(path):
    return static_file(path, root='./static')

if __name__ == '__main__':
    import bottle
    bottle.debug(True)
    run(host="localhost", port=8080, reloader=True)
else:
    import bottle
    bottle.TEMPLATE_PATH.insert(0,
        os.path.join(os.path.dirname(__file__), 'views/'))
    application = bottle.default_app()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- Mode: Python; py-indent-offset: 4 -*-
# vim: tabstop=4 shiftwidth=4 expandtab
#
# Copyright (C) 2010 Red Hat, Inc., John (J5) Palmieri <johnp@redhat.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
# USA

import os
import sys
import glob

from gi.repository import GLib, GObject, Gio, Pango, GdkPixbuf, Gtk

DEMOROOTDIR = os.path.abspath(os.path.dirname(__file__))
DEMOCODEDIR = os.path.join(DEMOROOTDIR, 'demos')
sys.path.insert(0, DEMOROOTDIR)

class Demo(GObject.GObject):
    __gtype_name__ = 'GtkDemo'

    def __init__(self, title, module, filename):
        super(Demo, self).__init__()

        self.title = title
        self.module = module
        self.filename = filename

    @classmethod
    def new_from_file(cls, path):
        relpath = os.path.relpath(path, DEMOROOTDIR)
        packagename = os.path.dirname(relpath).replace(os.sep, '.')
        modulename = os.path.basename(relpath)[0:-3]

        package = __import__(packagename, globals(), locals(), [modulename], -1)
        module = getattr(package, modulename)

        try:
            return cls(module.title, module, path)
        except AttributeError as e:
            raise AttributeError('(%s): %s' % (path, e.message))

class DemoTreeStore(Gtk.TreeStore):
    __gtype_name__ = 'GtkDemoTreeStore'

    def __init__(self, *args):
        #TODO: super does not seem to work here?
        #super(Gtk.TreeStore, self).__init__(str, Demo, Pango.Style)
        Gtk.TreeStore.__init__(self, str, Demo, Pango.Style)

        self._parent_nodes = {}

        for filename in self._list_dir(DEMOCODEDIR):
            fullpath = os.path.join(DEMOCODEDIR, filename)
            initfile = os.path.join(os.path.dirname(fullpath), '__init__.py')

            if fullpath != initfile \
            and os.path.isfile(initfile) \
            and fullpath.endswith('.py'):
                parentname = os.path.dirname(os.path.relpath(fullpath, DEMOCODEDIR))

                if parentname:
                    parent = self._get_parent_node(parentname)
                else:
                    parent = None

                demo = Demo.new_from_file(fullpath)
                self.append(parent, (demo.title, demo, Pango.Style.NORMAL))

    def _list_dir(self, path):
        demo_file_list = []

        for filename in os.listdir(path):
            fullpath = os.path.join(path, filename)

            if os.path.isdir(fullpath):
                demo_file_list.extend(self._list_dir(fullpath))
            elif os.path.isfile(fullpath):
                demo_file_list.append(fullpath)

        return sorted(demo_file_list, key=str.lower)

    def _get_parent_node(self, name):
        if not name in self._parent_nodes.keys():
            node = self.append(None, (name, None, Pango.Style.NORMAL))
            self._parent_nodes[name] = node

        return self._parent_nodes[name]

class GtkDemoWindow(Gtk.Window):
    __gtype_name__ = 'GtkDemoWindow'

    def __init__(self):
        super(GtkDemoWindow, self).__init__(type=Gtk.WindowType.TOPLEVEL)

        self.set_title('PyGI GTK+ Code Demos')
        self.set_default_size (600, 400)
        self.setup_default_icon()

        hbox = Gtk.HBox(homogeneous=False, spacing=0)
        self.add(hbox)

        tree = self.create_tree()
        hbox.pack_start(tree, False, False, 0)

        notebook = Gtk.Notebook()
        hbox.pack_start(notebook, True, True, 0)

        (text_widget, info_buffer) = self.create_text(False)
        notebook.append_page(text_widget, Gtk.Label.new_with_mnemonic('_Info'))
        self.info_buffer = info_buffer

        tag = info_buffer.create_tag('title', font = 'Sans 18')

        (text_widget, source_buffer) = self.create_text(True)
        notebook.append_page(text_widget, Gtk.Label.new_with_mnemonic('_Source'))

        self.source_buffer = source_buffer
        tag = source_buffer.create_tag('comment', foreground = 'DodgerBlue')
        tag = source_buffer.create_tag('type', foreground = 'ForestGreen')
        tag = source_buffer.create_tag('string',
                                       foreground = 'RosyBrown',
                                       weight = Pango.Weight.BOLD)
        tag = source_buffer.create_tag('control', foreground = 'purple')
        tag = source_buffer.create_tag('preprocessor',
                                       style = Pango.Style.OBLIQUE,
                                       foreground = 'burlywood4')
        tag = source_buffer.create_tag('function' ,
                                       weight = Pango.Weight.BOLD,
                                       foreground = 'DarkGoldenrod4')

        self.source_buffer = source_buffer
        self.show_all()

        self.selection_cb(self.tree_view.get_selection(),
                          self.tree_view.get_model())

    def find_file(self, base=''):
        dir = os.path.join(DEMOCODEDIR, 'data')
        logo_file = os.path.join(dir, 'gtk-logo-rgb.gif')
        base_file = os.path.join(dir, base)

        if (GLib.file_test(logo_file, GLib.FileTest.EXISTS) and
             GLib.file_test(base_file, GLib.FileTest.EXISTS)):
            return base_file
        else:
            filename = os.path.join(DEMOCODEDIR, base)

            if GLib.file_test(filename, GLib.FileTest.EXISTS):
                return filename

            # can't find the file
            raise IOError('Cannot find demo data file "%s"' % base)

    def setup_default_icon(self):
        filename = self.find_file ('gtk-logo-rgb.gif')
        pixbuf = GdkPixbuf.Pixbuf.new_from_file(filename)
        transparent = pixbuf.add_alpha(True, 0xff, 0xff, 0xff)
        list = []
        list.append(transparent)
        Gtk.Window.set_default_icon_list(list)

    def selection_cb(self, selection, model):
        sel = selection.get_selected()
        if sel == ():
            return

        treeiter = sel[1]
        title = model.get_value(treeiter, 0)
        demo = model.get_value(treeiter, 1)

        if demo is None:
            return

        description = demo.module.description
        code = GLib.file_get_contents(demo.filename)[1]

        # output and style the title
        (start, end) = self.info_buffer.get_bounds()
        self.info_buffer.delete(start, end)
        (start, end) = self.source_buffer.get_bounds()
        self.source_buffer.delete(start, end)

        start = self.info_buffer.get_iter_at_offset(0)
        end = start.copy()
        self.info_buffer.insert(end, title)
        start = end.copy()
        start.backward_chars(len(title))
        self.info_buffer.apply_tag_by_name('title', start, end)
        self.info_buffer.insert(end, '\n')

        # output the description
        self.info_buffer.insert(end, description)

        # output the code
        start = self.source_buffer.get_iter_at_offset(0)
        end = start.copy()
        self.source_buffer.insert(end, code)

    def row_activated_cb(self, view, path, col, store):
        treeiter = store.get_iter(path)
        demo = store.get_value(treeiter, 1)
        if demo is not None:
            demo.module.main(self)

    def create_tree(self):
        tree_store = DemoTreeStore()
        tree_view = Gtk.TreeView()
        self.tree_view = tree_view
        tree_view.set_model(tree_store)
        selection = tree_view.get_selection()
        selection.set_mode(Gtk.SelectionMode.BROWSE)
        tree_view.set_size_request(200, -1)

        cell = Gtk.CellRendererText()
        column = Gtk.TreeViewColumn(title = 'Widget (double click for demo)',
                                    cell_renderer = cell,
                                    text = 0,
                                    style = 2)

        first_iter = tree_store.get_iter_first()
        if first_iter is not None:
            selection.select_iter(first_iter)

        selection.connect('changed', self.selection_cb, tree_store)
        tree_view.connect('row_activated', self.row_activated_cb, tree_store)

        tree_view.append_column(column)

        tree_view.collapse_all()
        tree_view.set_headers_visible(False)
        scrolled_window = Gtk.ScrolledWindow(hadjustment = None,
                                             vadjustment = None)
        scrolled_window.set_policy(Gtk.PolicyType.NEVER,
                                   Gtk.PolicyType.AUTOMATIC)

        scrolled_window.add(tree_view)

        label = Gtk.Label(label = 'Widget (double click for demo)')

        box = Gtk.Notebook()
        box.append_page(scrolled_window, label)

        tree_view.grab_focus()

        return box

    def create_text(self, is_source):
        scrolled_window = Gtk.ScrolledWindow(hadjustment = None,
                                             vadjustment = None)
        scrolled_window.set_policy(Gtk.PolicyType.AUTOMATIC,
                                   Gtk.PolicyType.AUTOMATIC)
        scrolled_window.set_shadow_type(Gtk.ShadowType.IN)

        text_view = Gtk.TextView()
        buffer = Gtk.TextBuffer()

        text_view.set_buffer(buffer)
        text_view.set_editable(False)
        text_view.set_cursor_visible(False)

        scrolled_window.add(text_view)

        if is_source:
            font_desc = Pango.FontDescription('monospace')
            text_view.modify_font(font_desc)
            text_view.set_wrap_mode(Gtk.WrapMode.NONE)
        else:
            text_view.set_wrap_mode(Gtk.WrapMode.WORD)
            text_view.set_pixels_above_lines(2)
            text_view.set_pixels_below_lines(2)

        return(scrolled_window, buffer)

def main():
    mainloop = GLib.MainLoop()

    demowindow = GtkDemoWindow()
    demowindow.connect('delete-event', quit, mainloop)
    demowindow.show()

    mainloop.run()

def quit(widget, event, mainloop):
    mainloop.quit()

if __name__ == '__main__':
    main()
<commit_msg>[gtk-demo] add source colorizer loosely based on PyGTK's pygtk-demo and GTK+'s gtk-demo code<commit_after>#!/usr/bin/env python
# -*- Mode: Python; py-indent-offset: 4 -*-
# vim: tabstop=4 shiftwidth=4 expandtab
#
# Copyright (C) 2010 Red Hat, Inc., John (J5) Palmieri <johnp@redhat.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
# USA

import os
import sys
import glob
import tokenize
import keyword

from gi.repository import GLib, GObject, Gio, Pango, GdkPixbuf, Gtk

DEMOROOTDIR = os.path.abspath(os.path.dirname(__file__))
DEMOCODEDIR = os.path.join(DEMOROOTDIR, 'demos')
sys.path.insert(0, DEMOROOTDIR)

class InputStream(object):
    '''
    Simple Wrapper for File-like objects. [c]StringIO doesn't provide
    a readline function for use with generate_tokens.
    Using a iterator-like interface doesn't succeed, because the readline
    function isn't used in such a context. (see <python-lib>/tokenize.py)
    '''
    def __init__(self, data):
        self.__data = [ '%s\n' % x for x in data.splitlines() ]
        self.__lcount = 0

    def readline(self):
        try:
            line = self.__data[self.__lcount]
            self.__lcount += 1
        except IndexError:
            line = ''
            self.__lcount = 0

        return line

class Demo(GObject.GObject):
    __gtype_name__ = 'GtkDemo'

    def __init__(self, title, module, filename):
        super(Demo, self).__init__()

        self.title = title
        self.module = module
        self.filename = filename

    @classmethod
    def new_from_file(cls, path):
        relpath = os.path.relpath(path, DEMOROOTDIR)
        packagename = os.path.dirname(relpath).replace(os.sep, '.')
        modulename = os.path.basename(relpath)[0:-3]

        package = __import__(packagename, globals(), locals(), [modulename], -1)
        module = getattr(package, modulename)

        try:
            return cls(module.title, module, path)
        except AttributeError as e:
            raise AttributeError('(%s): %s' % (path, e.message))

class DemoTreeStore(Gtk.TreeStore):
    __gtype_name__ = 'GtkDemoTreeStore'

    def __init__(self, *args):
        #TODO: super does not seem to work here?
        #super(Gtk.TreeStore, self).__init__(str, Demo, Pango.Style)
        Gtk.TreeStore.__init__(self, str, Demo, Pango.Style)

        self._parent_nodes = {}

        for filename in self._list_dir(DEMOCODEDIR):
            fullpath = os.path.join(DEMOCODEDIR, filename)
            initfile = os.path.join(os.path.dirname(fullpath), '__init__.py')

            if fullpath != initfile \
            and os.path.isfile(initfile) \
            and fullpath.endswith('.py'):
                parentname = os.path.dirname(os.path.relpath(fullpath, DEMOCODEDIR))

                if parentname:
                    parent = self._get_parent_node(parentname)
                else:
                    parent = None

                demo = Demo.new_from_file(fullpath)
                self.append(parent, (demo.title, demo, Pango.Style.NORMAL))

    def _list_dir(self, path):
        demo_file_list = []

        for filename in os.listdir(path):
            fullpath = os.path.join(path, filename)

            if os.path.isdir(fullpath):
                demo_file_list.extend(self._list_dir(fullpath))
            elif os.path.isfile(fullpath):
                demo_file_list.append(fullpath)

        return sorted(demo_file_list, key=str.lower)

    def _get_parent_node(self, name):
        if not name in self._parent_nodes.keys():
            node = self.append(None, (name, None, Pango.Style.NORMAL))
            self._parent_nodes[name] = node

        return self._parent_nodes[name]

class GtkDemoWindow(Gtk.Window):
    __gtype_name__ = 'GtkDemoWindow'

    def __init__(self):
        super(GtkDemoWindow, self).__init__(type=Gtk.WindowType.TOPLEVEL)

        self.set_title('PyGI GTK+ Code Demos')
        self.set_default_size (600, 400)
        self.setup_default_icon()

        hbox = Gtk.HBox(homogeneous=False, spacing=0)
        self.add(hbox)

        tree = self.create_tree()
        hbox.pack_start(tree, False, False, 0)

        notebook = Gtk.Notebook()
        hbox.pack_start(notebook, True, True, 0)

        (text_widget, info_buffer) = self.create_text(False)
        notebook.append_page(text_widget, Gtk.Label.new_with_mnemonic('_Info'))

        self.info_buffer = info_buffer
        self.info_buffer.create_tag('title', font = 'Sans 18')

        (text_widget, source_buffer) = self.create_text(True)
        notebook.append_page(text_widget, Gtk.Label.new_with_mnemonic('_Source'))

        self.source_buffer = source_buffer
        self.source_buffer.create_tag('bold',
                                      weight=Pango.Weight.BOLD)
        self.source_buffer.create_tag('italic',
                                      style=Pango.Style.ITALIC)
        self.source_buffer.create_tag('comment',
                                      foreground='#c0c0c0')
        self.source_buffer.create_tag('decorator',
                                      foreground='#7d7d7d',
                                      style=Pango.Style.ITALIC)
        self.source_buffer.create_tag('keyword',
                                      foreground='#0000ff')
        self.source_buffer.create_tag('number',
                                      foreground='#800000')
        self.source_buffer.create_tag('string',
                                      foreground='#00aa00',
                                      style=Pango.Style.ITALIC)

        self.show_all()

        self.selection_cb(self.tree_view.get_selection(),
                          self.tree_view.get_model())

    def find_file(self, base=''):
        dir = os.path.join(DEMOCODEDIR, 'data')
        logo_file = os.path.join(dir, 'gtk-logo-rgb.gif')
        base_file = os.path.join(dir, base)

        if (GLib.file_test(logo_file, GLib.FileTest.EXISTS) and
             GLib.file_test(base_file, GLib.FileTest.EXISTS)):
            return base_file
        else:
            filename = os.path.join(DEMOCODEDIR, base)

            if GLib.file_test(filename, GLib.FileTest.EXISTS):
                return filename

            # can't find the file
            raise IOError('Cannot find demo data file "%s"' % base)

    def setup_default_icon(self):
        filename = self.find_file ('gtk-logo-rgb.gif')
        pixbuf = GdkPixbuf.Pixbuf.new_from_file(filename)
        transparent = pixbuf.add_alpha(True, 0xff, 0xff, 0xff)
        list = []
        list.append(transparent)
        Gtk.Window.set_default_icon_list(list)

    def selection_cb(self, selection, model):
        sel = selection.get_selected()
        if sel == ():
            return

        treeiter = sel[1]
        title = model.get_value(treeiter, 0)
        demo = model.get_value(treeiter, 1)

        if demo is None:
            return

        description = demo.module.description
        code = GLib.file_get_contents(demo.filename)[1]

        # output and style the title
        (start, end) = self.info_buffer.get_bounds()
        self.info_buffer.delete(start, end)
        (start, end) = self.source_buffer.get_bounds()
        self.source_buffer.delete(start, end)

        start = self.info_buffer.get_iter_at_offset(0)
        end = start.copy()
        self.info_buffer.insert(end, title)
        start = end.copy()
        start.backward_chars(len(title))
        self.info_buffer.apply_tag_by_name('title', start, end)
        self.info_buffer.insert(end, '\n')

        # output the description
        self.info_buffer.insert(end, description)

        # output the code
        start = self.source_buffer.get_iter_at_offset(0)
        end = start.copy()
        self.source_buffer.insert(end, code)
        self.fontify()

    def row_activated_cb(self, view, path, col, store):
        treeiter = store.get_iter(path)
        demo = store.get_value(treeiter, 1)
        if demo is not None:
            demo.module.main(self)

    def create_tree(self):
        tree_store = DemoTreeStore()
        tree_view = Gtk.TreeView()
        self.tree_view = tree_view
        tree_view.set_model(tree_store)
        selection = tree_view.get_selection()
        selection.set_mode(Gtk.SelectionMode.BROWSE)
        tree_view.set_size_request(200, -1)

        cell = Gtk.CellRendererText()
        column = Gtk.TreeViewColumn(title = 'Widget (double click for demo)',
                                    cell_renderer = cell,
                                    text = 0,
                                    style = 2)

        first_iter = tree_store.get_iter_first()
        if first_iter is not None:
            selection.select_iter(first_iter)

        selection.connect('changed', self.selection_cb, tree_store)
        tree_view.connect('row_activated', self.row_activated_cb, tree_store)

        tree_view.append_column(column)

        tree_view.collapse_all()
        tree_view.set_headers_visible(False)
        scrolled_window = Gtk.ScrolledWindow(hadjustment = None,
                                             vadjustment = None)
        scrolled_window.set_policy(Gtk.PolicyType.NEVER,
                                   Gtk.PolicyType.AUTOMATIC)

        scrolled_window.add(tree_view)

        label = Gtk.Label(label = 'Widget (double click for demo)')

        box = Gtk.Notebook()
        box.append_page(scrolled_window, label)

        tree_view.grab_focus()

        return box

    def create_text(self, is_source):
        scrolled_window = Gtk.ScrolledWindow(hadjustment = None,
                                             vadjustment = None)
        scrolled_window.set_policy(Gtk.PolicyType.AUTOMATIC,
                                   Gtk.PolicyType.AUTOMATIC)
        scrolled_window.set_shadow_type(Gtk.ShadowType.IN)

        text_view = Gtk.TextView()
        buffer = Gtk.TextBuffer()

        text_view.set_buffer(buffer)
        text_view.set_editable(False)
        text_view.set_cursor_visible(False)

        scrolled_window.add(text_view)

        if is_source:
            font_desc = Pango.FontDescription('monospace')
            text_view.modify_font(font_desc)
            text_view.set_wrap_mode(Gtk.WrapMode.NONE)
        else:
            text_view.set_wrap_mode(Gtk.WrapMode.WORD)
            text_view.set_pixels_above_lines(2)
            text_view.set_pixels_below_lines(2)

        return(scrolled_window, buffer)

    def fontify(self):
        start_iter = self.source_buffer.get_iter_at_offset(0)
        end_iter = self.source_buffer.get_iter_at_offset(0)
        data = self.source_buffer.get_text(self.source_buffer.get_start_iter(),
                                           self.source_buffer.get_end_iter(),
                                           False)

        builtin_constants = ['None', 'True', 'False']
        is_decorator = False
        is_func = False

        for x in tokenize.generate_tokens(InputStream(data).readline):
            # x has 5-tuples
            tok_type, tok_str = x[0], x[1]
            srow, scol = x[2]
            erow, ecol = x[3]

            start_iter.set_line(srow-1)
            start_iter.set_line_offset(scol)
            end_iter.set_line(erow-1)
            end_iter.set_line_offset(ecol)

            if tok_type == tokenize.BACKQUOTE:
                self.source_buffer.apply_tag_by_name('bold', start_iter, end_iter)
            elif tok_type == tokenize.COMMENT:
                self.source_buffer.apply_tag_by_name('comment', start_iter, end_iter)
            elif tok_type == tokenize.NAME:
                if tok_str in keyword.kwlist or tok_str in builtin_constants:
                    self.source_buffer.apply_tag_by_name('keyword', start_iter, end_iter)

                    if tok_str == 'def' or tok_str == 'class':
                        # Next token is going to be a function/method/class name
                        is_func = True
                        continue
                elif tok_str == 'self':
                    self.source_buffer.apply_tag_by_name('italic', start_iter, end_iter)
                else:
                    if is_func is True:
                        self.source_buffer.apply_tag_by_name('bold', start_iter, end_iter)
                    elif is_decorator is True:
                        self.source_buffer.apply_tag_by_name('decorator', start_iter, end_iter)
            elif tok_type == tokenize.STRING:
                self.source_buffer.apply_tag_by_name('string', start_iter, end_iter)
            elif tok_type == tokenize.NUMBER:
                self.source_buffer.apply_tag_by_name('number', start_iter, end_iter)
            elif tok_type == tokenize.OP:
                if tok_str == '@':
                    self.source_buffer.apply_tag_by_name('decorator', start_iter, end_iter)

                    # next token is going to be the decorator name
                    is_decorator = True
                    continue

            if is_func is True:
                is_func = False

            if is_decorator is True:
                is_decorator = False

def main():
    mainloop = GLib.MainLoop()

    demowindow = GtkDemoWindow()
    demowindow.connect('delete-event', quit, mainloop)
    demowindow.show()

    mainloop.run()

def quit(widget, event, mainloop):
    mainloop.quit()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
welcome = """
--- dENUMerator ---

by bl4de | bloorq@gmail.com | Twitter: @_bl4de | HackerOne: bl4de

Enumerates list of subdomains (output from tools like Sublist3r or subbrute)
and creates output file with servers responding on port 80/HTTP

This indicates (in most caes) working webserver

usage:
$ ./denumerator.py [domain_list_file]
"""

import requests
import sys

requests.packages.urllib3.disable_warnings()

HTTP_OK = 200
HTTP_UNAUTHORIZED = 401
HTTP_FORBIDDEN = 403
HTTP_NOT_MODIFIED = 304
HTTP_FOUND = 302
HTTP_INTERNAL_SERVER_ERROR = 500

allowed_http_responses = [
    HTTP_OK,
    HTTP_FORBIDDEN,
    HTTP_NOT_MODIFIED,
    HTTP_FOUND,
    HTTP_UNAUTHORIZED,
    HTTP_INTERNAL_SERVER_ERROR
]

def usage():
    print welcome

def send_request(proto, domain):
    protocols = {
        'http': 'http://',
        'https': 'https://'
    }
    resp = requests.get(protocols.get(proto.lower()) + domain,
                        timeout=5,
                        allow_redirects=False,
                        verify=False,
                        headers={
                            'Host': domain
    })

    if resp.status_code in allowed_http_responses:
        print '[+] domain {}:\t\t HTTP {}'.format(domain, resp.status_code)
        output_file.write('{}\n'.format(domain))
    return resp.status_code

def enumerate_domains(domains, output_file):
    i = len(domains)
    for d in domains:
        try:
            d = d.strip('\n').strip('\r')
            return_code = send_request('http', d)
            # if http not working, try https
            if (return_code not in allowed_http_responses):
                send_request('https', d)

        except requests.exceptions.InvalidURL:
            print '[-] {} is not a valid URL :/'.format(d)
        except requests.exceptions.ConnectTimeout:
            print '[-] {} :('.format(d)
            continue
        except requests.exceptions.ConnectionError:
            print '[-] connection to {} aborted :/'.format(d)
        except requests.exceptions.ReadTimeout:
            print '[-] {} read timeout :/'.format(d)
        except requests.exceptions.TooManyRedirects:
            print '[-] {} probably went into redirects loop :('.format(d)
        else:
            pass

if (len(sys.argv) < 2):
    print welcome
    exit(0)

domains = open(sys.argv[1].strip(), 'rw').readlines()
output_file = open('output.txt', 'w')

enumerate_domains(domains, output_file)

output_file.close()
<commit_msg>refactoring [denumerator]<commit_after>#!/usr/bin/env python
welcome = """
--- dENUMerator ---

by bl4de | bloorq@gmail.com | Twitter: @_bl4de | HackerOne: bl4de

Enumerates list of subdomains (output from tools like Sublist3r or subbrute)
and creates output file with servers responding on port 80/HTTP

This indicates (in most caes) working webserver

usage:
$ ./denumerator.py [domain_list_file]
"""

import requests
import sys

requests.packages.urllib3.disable_warnings()

allowed_http_responses = [200,302,304,401,403,500]

def usage():
    print welcome

def send_request(proto, domain):
    protocols = {
        'http': 'http://',
        'https': 'https://'
    }
    resp = requests.get(protocols.get(proto.lower()) + domain,
                        timeout=5,
                        allow_redirects=False,
                        verify=False,
                        headers={
                            'Host': domain
    })

    if resp.status_code in allowed_http_responses:
        print '[+] domain {}:\t\t HTTP {}'.format(domain, resp.status_code)
        output_file.write('{}\n'.format(domain))
    return resp.status_code

def enumerate_domains(domains, output_file):
    i = len(domains)
    for d in domains:
        try:
            d = d.strip('\n').strip('\r')
            return_code = send_request('http', d)
            # if http not working, try https
            if (return_code not in allowed_http_responses):
                send_request('https', d)

        except requests.exceptions.InvalidURL:
            print '[-] {} is not a valid URL :/'.format(d)
        except requests.exceptions.ConnectTimeout:
            print '[-] {} :('.format(d)
            continue
        except requests.exceptions.ConnectionError:
            print '[-] connection to {} aborted :/'.format(d)
        except requests.exceptions.ReadTimeout:
            print '[-] {} read timeout :/'.format(d)
        except requests.exceptions.TooManyRedirects:
            print '[-] {} probably went into redirects loop :('.format(d)
        else:
            pass

if (len(sys.argv) < 2):
    print welcome
    exit(0)

domains = open(sys.argv[1].strip(), 'rw').readlines()
output_file = open('output.txt', 'w')

enumerate_domains(domains, output_file)

output_file.close()
<|endoftext|>


<commit_before>import numpy as np
import pygame
from pygame import gfxdraw
import sys
from src.play import ConsoleView
from src.play.view import View
from src.play.view import Move
from src.play.model.Game import BLACK
from src.play.model.Game import WHITE

brown = (165, 42, 42)
black = (0, 0, 0)
white = (255, 255, 255)
yellow = (255, 255, 0)
orange = (255, 165, 0)
size = (500, 550)
board_size = 400
offset = 50
stone_radius = 20

class PygameGuiView(View):

    def __init__(self, game):
        View.__init__(self, game)
        self.console_view = ConsoleView(game)
        self.running = False
        self.cell_size = board_size / (self.game.size - 1)
        self.buttons = []

    def open(self, game_controller):
        self.game_controller = game_controller
        pygame.init()
        self.running = True
        self.screen = pygame.display.set_mode(size)
        pygame.display.set_caption('Go')
        self.buttons.append(Button(210, 480, 80, 40, 'Pass', self.screen, self.send_pass_move))
        self.console_view.print_board()
        self.render()

        while self.running:
            event = pygame.event.poll()
            if event.type == pygame.MOUSEBUTTONUP:
                x, y = event.pos
                col = int(round((x - offset) / self.cell_size))
                row = int(round((y - offset) / self.cell_size))
                if 0 <= col < self.game.size and 0 <= row < self.game.size:
                    self.game_controller.current_player.receive_next_move_from_gui(Move(col, row))
                for btn in self.buttons:
                    btn.check_mouse_released()
            if event.type == pygame.QUIT:
                self.running = False
            for btn in self.buttons:
                btn.is_mouse_over_btn()
            self.render()

        # this exiting mechanism doesn't work (on macOS at least), causes a freeze TODO
        pygame.quit()
        sys.exit(0)

    def show_player_turn_start(self, name):
        self.console_view.show_player_turn_start(name)

    def show_player_turn_end(self, name):
        self.console_view.show_player_turn_end(name)

    def send_pass_move(self):
        self.game_controller.current_player.receive_next_move_from_gui(Move(is_pass=True))

    def render(self):
        # board
        self.screen.fill(brown)
        for i in range(0, self.game.size):
            # horizontals
            pygame.draw.line(self.screen, black, [offset, offset + i * self.cell_size],
                             [offset + board_size, offset + i * self.cell_size], 1)
            # verticals
            pygame.draw.line(self.screen, black, [offset + i * self.cell_size, offset],
                             [offset + i * self.cell_size, offset + board_size], 1)
        # stones
        b = self.game.board.copy()  # is it necessary to copy it?
        b[b == BLACK] = 2
        b[b == WHITE] = 3
        black_rows, black_cols = np.where(b == 2)
        white_rows, white_cols = np.where(b == 3)
        for i in range(0, len(black_rows)):
            indices = black_cols[i], black_rows[i]
            self.draw_stone(indices, black)
        for i in range(0, len(white_rows)):
            indices = white_cols[i], white_rows[i]
            self.draw_stone(indices, white)
        for btn in self.buttons:
            btn.draw()
        pygame.display.flip()  # update the screen

    def draw_stone(self, indices, col):
        x = int(offset + self.cell_size * indices[0])
        y = int(offset + indices[1] * self.cell_size)
        # antialiasing via stackoverflow.com/a/26774279/2474159
        pygame.gfxdraw.aacircle(self.screen, x, y, stone_radius, col)
        pygame.gfxdraw.filled_circle(self.screen, x, y, stone_radius, col)

    def show_error(self, msg):
        self.console_view.show_error(msg)

class Button:  # adapted from gamedev.net/forums/topic/686666-pygame-buttons/?do=findComment&comment=5333411

    def __init__(self, x, y, w, h, text, screen, on_click):
        self.x = x
        self.y = y
        self.w = w
        self.h = h
        self.rect = (self.x, self.y, self.w, self.h)
        pygame.font.init()
        self.font = pygame.font.Font(None, 25)
        self.text = text
        self.mouse_is_over_btn = False
        self.screen = screen
        self.on_click = on_click

    def draw(self):
        if self.mouse_is_over_btn:
            bg = yellow
        else:
            bg = orange
        surf = self.font.render(self.text, True, black, bg)
        xo = self.x + (self.w - surf.get_width()) / 2
        yo = self.y + (self.h - surf.get_height()) / 2
        self.screen.fill(bg, self.rect)
        pygame.draw.rect(self.screen, black, self.rect, 1)
        self.screen.blit(surf, (xo, yo))

    def is_mouse_over_btn(self):
        pos = pygame.mouse.get_pos()
        self.mouse_is_over_btn = self.x <= pos[0] < self.x + self.w and self.y <= pos[1] < self.y + self.h

    def check_mouse_released(self):
        if self.mouse_is_over_btn:
            self.on_click()
<commit_msg>implement label showing whos turn it is<commit_after>import numpy as np
import pygame
from pygame import gfxdraw
import sys
from src.play import ConsoleView
from src.play.view import View
from src.play.view import Move
from src.play.model.Game import BLACK
from src.play.model.Game import WHITE

brown = (165, 42, 42)
black = (0, 0, 0)
white = (255, 255, 255)
yellow = (255, 255, 0)
orange = (255, 165, 0)
blue = (0, 0, 255)
size = (500, 600)
board_size = 400
x_offset = 50
y_offset = 100
stone_radius = 20

class PygameGuiView(View):

    def __init__(self, game):
        View.__init__(self, game)
        self.console_view = ConsoleView(game)
        self.running = False
        self.cell_size = board_size / (self.game.size - 1)
        self.buttons = []
        self.labels = []

    def open(self, game_controller):
        self.game_controller = game_controller
        pygame.init()
        self.running = True
        self.screen = pygame.display.set_mode(size)
        pygame.display.set_caption('Go')
        self.buttons.append(Button(210, 530, 80, 40, 'Pass', self.screen, self.send_pass_move))
        self.labels.append(Label(100, 30, 300, 40, self.get_turn_label_text, self.screen))
        self.console_view.print_board()
        self.render()

        while self.running:
            event = pygame.event.poll()
            if event.type == pygame.MOUSEBUTTONUP:
                x, y = event.pos
                col = int(round((x - x_offset) / self.cell_size))
                row = int(round((y - y_offset) / self.cell_size))
                if 0 <= col < self.game.size and 0 <= row < self.game.size:
                    self.game_controller.current_player.receive_next_move_from_gui(Move(col, row))
                for btn in self.buttons:
                    btn.check_mouse_released()
            if event.type == pygame.QUIT:
                self.running = False
            for btn in self.buttons:
                btn.is_mouse_over_btn()
            self.render()

        # this exiting mechanism doesn't work (on macOS at least), causes a freeze TODO
        pygame.quit()
        sys.exit(0)

    def show_player_turn_start(self, name):
        self.console_view.show_player_turn_start(name)

    def show_player_turn_end(self, name):
        self.console_view.show_player_turn_end(name)

    def send_pass_move(self):
        self.game_controller.current_player.receive_next_move_from_gui(Move(is_pass=True))

    def get_turn_label_text(self):
        return 'It\'s ' + self.game_controller.current_player.name + ' turn'

    def render(self):
        # board
        self.screen.fill(brown)
        for i in range(0, self.game.size):
            # horizontals
            pygame.draw.line(self.screen, black, [x_offset, y_offset + i * self.cell_size],
                             [x_offset + board_size, y_offset + i * self.cell_size], 1)
            # verticals
            pygame.draw.line(self.screen, black, [x_offset + i * self.cell_size, y_offset],
                             [x_offset + i * self.cell_size, y_offset + board_size], 1)
        # stones
        b = self.game.board.copy()  # is it necessary to copy it?
        b[b == BLACK] = 2
        b[b == WHITE] = 3
        black_rows, black_cols = np.where(b == 2)
        white_rows, white_cols = np.where(b == 3)
        for i in range(0, len(black_rows)):
            indices = black_cols[i], black_rows[i]
            self.draw_stone(indices, black)
        for i in range(0, len(white_rows)):
            indices = white_cols[i], white_rows[i]
            self.draw_stone(indices, white)
        for btn in self.buttons:
            btn.draw()
        for label in self.labels:
            label.draw()
        pygame.display.flip()  # update the screen

    def draw_stone(self, indices, col):
        x = int(x_offset + self.cell_size * indices[0])
        y = int(y_offset + indices[1] * self.cell_size)
        # antialiasing via stackoverflow.com/a/26774279/2474159
        pygame.gfxdraw.aacircle(self.screen, x, y, stone_radius, col)
        pygame.gfxdraw.filled_circle(self.screen, x, y, stone_radius, col)

    def show_error(self, msg):
        self.console_view.show_error(msg)

class Button:  # adapted from gamedev.net/forums/topic/686666-pygame-buttons/?do=findComment&comment=5333411

    def __init__(self, x, y, w, h, text, screen, on_click):
        self.x = x
        self.y = y
        self.w = w
        self.h = h
        self.rect = (self.x, self.y, self.w, self.h)
        pygame.font.init()
        self.font = pygame.font.Font(None, 25)  # compute font size from h?
        self.text = text
        self.mouse_is_over_btn = False
        self.screen = screen
        self.on_click = on_click

    def draw(self):
        if self.mouse_is_over_btn:
            bg = yellow
        else:
            bg = orange
        surf = self.font.render(self.text, True, black, bg)
        xo = self.x + (self.w - surf.get_width()) / 2
        yo = self.y + (self.h - surf.get_height()) / 2
        self.screen.fill(bg, self.rect)
        pygame.draw.rect(self.screen, black, self.rect, 1)
        self.screen.blit(surf, (xo, yo))

    def is_mouse_over_btn(self):
        pos = pygame.mouse.get_pos()
        self.mouse_is_over_btn = self.x <= pos[0] < self.x + self.w and self.y <= pos[1] < self.y + self.h

    def check_mouse_released(self):
        if self.mouse_is_over_btn:
            self.on_click()

class Label:  # make Button extend Label?

    def __init__(self, x, y, w, h, get_text, screen):
        self.x = x
        self.y = y
        self.w = w
        self.h = h
        self.rect = (self.x, self.y, self.w, self.h)
        pygame.font.init()
        self.font = pygame.font.Font(None, 30)
        self.get_text = get_text  # method, get's called in draw()
        self.screen = screen

    def draw(self):
        surf = self.font.render(self.get_text(), True, yellow, brown)
        xo = self.x + (self.w - surf.get_width()) / 2
        yo = self.y + (self.h - surf.get_height()) / 2
        self.screen.blit(surf, (xo, yo))
<|endoftext|>