<commit_before>"""
Demultiplex haloplex data including molecular tags.
"""
# TODO add pairwise alignment of indexes to correct for sequencing error (biopython's Bio.pairwise2)

from __future__ import print_function

import argparse
import Bio
#import collections
import itertools
import os
import re
import sys

#from scilifelab.illumina.hiseq import HiSeqSampleSheet
from scilifelab.utils.fastq_utils import FastQParser

# TODO these classes don't print the correct error message - maybe a problem with super? review docs
class NoIndexMatchException(Exception):
    def __init__(self, message, *args):
        if not message:
            message = "no match to supplied indexes."
        super(NoIndexMatchException, self).__init__(self, message)
        #Exception.__init__(self, message)
        self.args = args

class RevComMatchException(Exception):
    def __init__(self, message, *args):
        if not message:
            message = "no match to supplied indexes but match to reverse complement of supplied indexes. Double-check read direction."
        super(RevComMatchException, self).__init__(self, message)
        Exception.__init__(self, message)
        self.args = args

def main(read_one, read_two, read_index, data_directory, read_index_num, output_directory, halo_index_file, halo_index_length, molecular_tag_length=None):
#def main(**kwargs):

    if not output_directory:
        raise SyntaxError("Must specify output directory.")

    # Check for minimum required index information from user
    if halo_index_file:
        index_dict, index_revcom_dict = load_index_file(halo_index_file)
    elif halo_index_length:
        assert(type(halo_index_length) == int and halo_index_length > 0), "Haloplex index length must be a positive integer."
        if molecular_tag_length:
            assert(type(molecular_tag_length) == int and molecular_tag_length >= 0), "Molecular tag length must be a postive integer."
        index_dict, index_revcom_dict = None, None
    else:
        raise SyntaxError("Either an index file or the index length must be specified.")

    # Check for minimum required fastq location data from user and process
    if read_one and read_two and read_index:
        if data_directory or read_index_num:
            raise SyntaxError("Ambiguous: too many options specified. Specify either file paths or directory and read index number.")
        else:
            parse_readset(read_one, read_two, read_index, index_dict, index_revcom_dict, output_directory, halo_index_length, molecular_tag_length)
    elif data_directory and read_index_num:
        parse_directory(data_directory, read_index_num)
        #for readset in parse_directory(directory, read_index_num):
        # etc.
        # requires changing parse_directory to generator and changing parse_readset to accept a dict for the reads e.g. {1:x, 2:x, i:x}
        #parse_directory(directory, read_index_num, index_dict, index_revcom_dict, output_directory, halo_index_length, molecular_tag_length)
    else:
        raise SyntaxError("Either a directory and read index number or explicit paths to sequencing files must be specified.")
        print("End of function", file=sys.stderr)

# TODO turn this into a generator and use it in a for loop above
#def parse_directory(data_directory, read_index_num, index_dict, index_revcom_dict, halo_index_length, molecular_tag_length):
def parse_directory(data_directory, read_index_num):
    """
    Searches the directory for fastq file sets and calls parse_readset() on them.
    """
    raise NotImplementedError("I haven't implemented this yet, so don't go using it.")

def parse_readset(read_1_fq, read_2_fq, read_index_fq, index_dict, index_revcom_dict, halo_index_length, molecular_tag_length=None):
    """
    Parse input fastq files by index reads.
    """
    # TODO make a real directory, not this hack for testing
    output_dir = os.path.join(os.getcwd(), "processed_files")
    os.makedirs(output_dir)

    fqp_1, fqp_2, fqp_ind = map(FastQParser, (read_1_fq, read_2_fq, read_index_fq))

    match, non_match, revcom_match = 0, 0, 0

    for read_1, read_2, read_ind in itertools.izip(fqp_1, fqp_2, fqp_ind):
        # Get the haloplex index and the molecular tag from the index read's sequence data
        try:
            halo_index, molecular_tag = parse_index(read_ind[1], index_dict, index_revcom_dict,
                                                    halo_index_length, molecular_tag_length)
            match += 1
        except RevComMatchException as e:
            revcom_match += 1
            halo_index, molecular_tag = e[0], e[1]
        except NoIndexMatchException as e:
            non_match += 1
            halo_index, molecular_tag = e[0], e[1]
        # TODO modify headers of the reads and split by index either in dict, yield, or write to file
        # possibly use scilifelab.utils.fastq_utils.FastQWriter()
        if halo_index and molecular_tag:
            for read_num, read in enumerate([read_1, read_2]):
                read[0] = read[0] + "{}:{}:".format(halo_index, molecular_tag)
                file_name = os.path.join(output_dir, "{}_R{}.fastq".format(halo_index, read_num+1))
                with open(file_name, 'a') as f:
                    f.write("\n".join(read))
    import pdb; pdb.set_trace()

def parse_index(index_seq, index_dict=None, index_revcom_dict=None, halo_index_length=None, molecular_tag_length=None):
    """
    Split an index up into its haloplex index and the random molecular tag.
    Returns the halo index and the molecular tag as a tuple.
    """
    halo_index, molecular_tag = index_seq[:halo_index_length], index_seq[halo_index_length:molecular_tag_length]

    # TODO add check against known indexes (Bio.pairwise2) -- this will also determine default molecular tag length
    #from Bio import pairwise2 as pw2
    #    pw2.align.globalms(halo_index, key, 2, -1, -1, -1)
    if not halo_index in index_dict.keys():
        if halo_index in index_revcom_dict.keys():
            raise RevComMatchException(None, halo_index, molecular_tag)
        else:
            raise NoIndexMatchException(None, halo_index, molecular_tag)
    #    return None, None, None
    #index_name = index_dict[halo_index]
    #return halo_index, molecular_tag, index_name
    return halo_index, molecular_tag

def load_index_file(csv_file):
    """
    Load known indexes from a csv file.
    csv file should be in format:
        sequence[,index_name]
    where index_name is optional.
    Returns a dict of sequence->name pairs.
    """
    index_dict          = {}
    index_dict_revcom   = {}
    with open(csv_file, 'r') as f:
        for line in f:
            # could also use csv.sniffer to dynamically determine delimiter
            index = re.split(r'[\t,;]', line.strip())
            # include reverse complement
            rev_com_index = Bio.Seq.Seq(index[0]).reverse_complement().tostring()
            try:
                index_dict[index[0]]                = index[1]
                index_dict_revcom[rev_com_index]    = index[1]
            except IndexError:
                index_dict[index[0]]                = None
                index_dict_revcom[rev_com_index]    = None
    return index_dict, index_dict_revcom

def check_input(**kwa):
    pass

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument("-o", "--output-directory",
                                help="The directory to be used for storing output data.")

    # TODO I don't know about this whole subparsers thing. maybe._
    subparsers = parser.add_subparsers(title="subcommands")

    indexfromfile = subparsers.add_parser("indexfile",
                                help="Parse indexes from a file.")
    # TODO change to positional argument (only one argument)
    indexfromfile.add_argument( "-i", "--halo-index-file",
                                help="File containing haloplex indexes (one per line, optional name " \
                                     "in second column separated by tab, comma, or semicolon).")
    indexlength = subparsers.add_parser("indexlength", help="Parse indexes using a specified index length.")
    indexlength.add_argument( "-l", "--halo-index-length", type=int,
                                help="The length of the haloplex index. Required if indexes are not supplied (option -i).")
    indexlength.add_argument( "-m", "--molecular-tag-length", type=int,
                                help="The length of the (random) molecular tag. If not specified, " \
                                     "the remainder of the read after the halo index is used.")

    files_parser = subparsers.add_parser("fastqfiles", help="Specify the paths to the fastq files.")
    files_parser.add_argument("-1", "--read-one",
                                help="Read 1 fastq file.")
    files_parser.add_argument("-2", "--read-two",
                                help="Read 2 fastq file.")
    files_parser.add_argument("-r", "--read-index",
                                help="Index read fastq file.")

    directory_parser = subparsers.add_parser("fastqdir", help="Specify the directory containing the fastq files.")
    directory_parser.add_argument("-d", "--data-directory",
                                help="Directory containing fastq read data.")
    directory_parser.add_argument("-n", "--read-index-num", default=2, type=int,
                                help="Which read is the index (e.g. 1, 2, 3). Default 2.")
    # TODO parser.add_argument("-s", "--single-read", action="store_true", help="Specify that the data is single-read (not paired-end). Default false.")

    # TODO ensure exclusive subcommands for indexes, fastq data: check namespaces
    # TODO set unset variables to None? or check for presence of variables here
    arg_vars = vars(parser.parse_args())

    main(**arg_vars)
<commit_msg>Parsing by index length and manually specified files complete.<commit_after>"""
Demultiplex haloplex data including molecular tags.
"""
# TODO add pairwise alignment of indexes to correct for sequencing error (biopython's Bio.pairwise2)

from __future__ import print_function

import argparse
import Bio
#import collections
import itertools
import os
import re
import shutil
import sys

#from scilifelab.illumina.hiseq import HiSeqSampleSheet
from scilifelab.utils.fastq_utils import FastQParser

# TODO these classes don't print the correct error message - maybe a problem with super? review docs
class NoIndexMatchException(Exception):
    def __init__(self, message, *args):
        if not message:
            message = "no match to supplied indexes."
        super(NoIndexMatchException, self).__init__(self, message)
        #Exception.__init__(self, message)
        self.args = args

class RevComMatchException(Exception):
    def __init__(self, message, *args):
        if not message:
            message = "no match to supplied indexes but match to reverse complement of supplied indexes. Double-check read direction."
        super(RevComMatchException, self).__init__(self, message)
        Exception.__init__(self, message)
        self.args = args

def main(read_one, read_two, read_index, data_directory, read_index_num, output_directory, halo_index_file, halo_index_length, molecular_tag_length=None, force_overwrite=False):
#def main(**kwargs):

    if not output_directory:
        raise SyntaxError("Must specify output directory.")
    output_directory = create_output_dir(output_directory, force_overwrite)

    # Check for minimum required index information from user
    if halo_index_file:
        index_dict, index_revcom_dict = load_index_file(halo_index_file)
    elif halo_index_length:
        assert(type(halo_index_length) == int and halo_index_length > 0), "Haloplex index length must be a positive integer."
        if molecular_tag_length:
            assert(type(molecular_tag_length) == int and molecular_tag_length >= 0), "Molecular tag length must be a postive integer."
        index_dict, index_revcom_dict = None, None
    else:
        raise SyntaxError("Either an index file or the index length must be specified.")

    # Check for minimum required fastq location data from user and process
    if read_one and read_two and read_index:
        if data_directory or read_index_num:
            raise SyntaxError("Ambiguous: too many options specified. Specify either file paths or directory and read index number.")
        else:
            parse_readset(read_one, read_two, read_index, index_dict, index_revcom_dict, output_directory, halo_index_length, molecular_tag_length)
    elif data_directory and read_index_num:
        parse_directory(data_directory, read_index_num)
        #for readset in parse_directory(directory, read_index_num):
        # etc.
        # requires changing parse_directory to generator and changing parse_readset to accept a dict for the reads e.g. {1:x, 2:x, i:x}
        #parse_directory(directory, read_index_num, index_dict, index_revcom_dict, output_directory, halo_index_length, molecular_tag_length)
    else:
        raise SyntaxError("Either a directory and read index number or explicit paths to sequencing files must be specified.")
        print("End of function", file=sys.stderr)

def create_output_dir(output_directory, force_overwrite):
    """
    Create the output directory, forcing overwrite if the -f flag is passed; otherwise, fail if file exists."
    """
    output_directory = os.path.abspath(output_directory)
    if os.path.exists(output_directory):
        if force_overwrite:
            print("Warning: removing/overwriting output directory \"{}\"...".format(output_directory), file=sys.stderr, end="")
            shutil.rmtree(output_directory)
            print(" removed.", file=sys.stderr)
    # This can be outside the conditional because I don't want to raise my own exception in case of failure
    os.makedirs(output_directory)
    return output_directory

# TODO turn this into a generator and use it in a for loop above
#def parse_directory(data_directory, read_index_num, index_dict, index_revcom_dict, halo_index_length, molecular_tag_length):
def parse_directory(data_directory, read_index_num):
    """
    Searches the directory for fastq file sets and calls parse_readset() on them.
    """
    raise NotImplementedError("I haven't implemented this yet, so don't go using it.")

def parse_readset(read_1_fq, read_2_fq, read_index_fq, index_dict, index_revcom_dict, output_directory, halo_index_length, molecular_tag_length=None, force_append=False):
    """
    Parse input fastq files by index reads.
    """

    fqp_1, fqp_2, fqp_ind = map(FastQParser, (read_1_fq, read_2_fq, read_index_fq))

    match, non_match, revcom_match = 0, 0, 0

    for read_1, read_2, read_ind in itertools.izip(fqp_1, fqp_2, fqp_ind):
        # Get the haloplex index and the molecular tag from the index read's sequence data
        try:
            halo_index, molecular_tag = parse_index(read_ind[1], index_dict, index_revcom_dict,
                                                    halo_index_length, molecular_tag_length)
            match += 1
        except RevComMatchException as e:
            revcom_match += 1
            halo_index, molecular_tag = e[0], e[1]
        except NoIndexMatchException as e:
            non_match += 1
            halo_index, molecular_tag = e[0], e[1]
        # TODO modify headers of the reads and split by index either in dict, yield, or write to file
        # possibly use scilifelab.utils.fastq_utils.FastQWriter()
        if halo_index and molecular_tag:
            for read_num, read in enumerate([read_1, read_2]):
                read[0] = read[0] + "{}:{}:".format(halo_index, molecular_tag)
                file_name = os.path.join(output_directory, "{}_R{}.fastq".format(halo_index, read_num+1))
                with open(file_name, 'a') as f:
                    f.write("\n".join(read) + "\n")

def parse_index(index_seq, index_dict=None, index_revcom_dict=None, halo_index_length=None, molecular_tag_length=None):
    """
    Split an index up into its haloplex index and the random molecular tag.
    Returns the halo index and the molecular tag as a tuple.
    """
    halo_index, molecular_tag = index_seq[:halo_index_length], index_seq[halo_index_length:molecular_tag_length]

    # TODO add check against known indexes (Bio.pairwise2) -- this will also determine default molecular tag length
    #from Bio import pairwise2 as pw2
    #    pw2.align.globalms(halo_index, key, 2, -1, -1, -1)
    #if not halo_index in index_dict.keys():
    #    if halo_index in index_revcom_dict.keys():
    #        raise RevComMatchException(None, halo_index, molecular_tag)
    #    else:
    #        raise NoIndexMatchException(None, halo_index, molecular_tag)
    #    return None, None, None
    #index_name = index_dict[halo_index]
    #return halo_index, molecular_tag, index_name
    return halo_index, molecular_tag

def load_index_file(csv_file):
    """
    Load known indexes from a csv file.
    csv file should be in format:
        sequence[,index_name]
    where index_name is optional.
    Returns a dict of sequence->name pairs.
    """
    index_dict          = {}
    index_dict_revcom   = {}
    with open(csv_file, 'r') as f:
        for line in f:
            # could also use csv.sniffer to dynamically determine delimiter
            index = re.split(r'[\t,;]', line.strip())
            # include reverse complement
            rev_com_index = Bio.Seq.Seq(index[0]).reverse_complement().tostring()
            try:
                index_dict[index[0]]                = index[1]
                index_dict_revcom[rev_com_index]    = index[1]
            except IndexError:
                index_dict[index[0]]                = None
                index_dict_revcom[rev_com_index]    = None
    return index_dict, index_dict_revcom

def check_input(**kwa):
    pass

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument("-o", "--output-directory", required=True,
                                help="The directory to be used for storing output data.")

    parser.add_argument( "-i", "--halo-index-file",
                                help="File containing haloplex indexes (one per line, optional name " \
                                     "in second column separated by tab, comma, or semicolon).")
    parser.add_argument( "-l", "--halo-index-length", type=int,
                                help="The length of the haloplex index. Required if indexes are not supplied (option -i).")
    parser.add_argument( "-m", "--molecular-tag-length", type=int,
                                help="The length of the (random) molecular tag. If not specified, " \
                                     "the remainder of the read after the halo index is used.")

    parser.add_argument("-1", "--read-one",
                                help="Read 1 fastq file.")
    parser.add_argument("-2", "--read-two",
                                help="Read 2 fastq file.")
    parser.add_argument("-r", "--read-index",
                                help="Index read fastq file.")

    parser.add_argument("-d", "--data-directory",
                                help="Directory containing fastq read data.")
    parser.add_argument("-n", "--read-index-num", type=int,
                                help="Which read is the index (e.g. 1, 2, 3).")

    parser.add_argument("-f", "--force-overwrite", action="store_true",
                                help="Force overwrite to output directory.")

    # TODO parser.add_argument("-s", "--single-read", action="store_true", help="Specify that the data is single-read (not paired-end). Default false.")

    # TODO ensure exclusive subcommands for indexes, fastq data: check namespaces
    # TODO set unset variables to None? or check for presence of variables here
    arg_vars = vars(parser.parse_args())

    main(**arg_vars)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
'''Note that this script will only function for headless licenses at this time.'''
import argparse
import json
import os
import requests
import sys
import urllib

from pathlib import Path

url='https://master.binary.ninja/headless-download'

if sys.platform == "darwin":
	userpath = Path.home() / 'Library' / 'Application Support' / 'Binary Ninja'
elif sys.platform.startswith("linux"):
	userpath = Path.home() / '.binaryninja'
else:
	userpath = Path.home() / 'AppData' / 'Roaming' / 'Binary Ninja'

serial = ''
parser = argparse.ArgumentParser(description='Download a Binary Ninja installer with a headless license')
parser.add_argument("--serial", help="serial number")
parser.add_argument('--dev', dest='dev', default=False, action='store_true')
args = parser.parse_args()

if args.serial:
	serial = args.serial
elif userpath.is_dir():
	if (userpath / 'license.dat').is_file():
		with open(userpath / 'license.dat') as f:
			licenses = json.load(f)
			for license in licenses:
				if 'headless' in license['product']:
					serial = license['serial']

if serial == '':
	parser.print_help()
	sys.exit(-1)

params = {'serial': serial, 'dev': str(args.dev).lower()}

r = requests.get(url, params)
results = json.loads(r.text)
if results['ok']:
	url = results['url']
	r = requests.get(url, allow_redirects=True)
	filename = os.path.basename(urllib.parse.urlparse(url).path)
	if len(r.content) < 2**13:
		print(f'Error downloading from {url}')
	else:
		open(filename, 'wb').write(r.content)
else:
	print('Download failed.')
<commit_msg>Refactor download_headless.py - Support usage as a library - Optionally get license from a environment variable - Quiet output option<commit_after>#!/usr/bin/env python3
'''Note that this script will only function for headless licenses at this time.'''
import json
import os
import requests
import sys
import urllib

from pathlib import Path

class DownloadException(Exception):
    pass

url='https://master.binary.ninja/headless-download'
env_lic = 'HEADLESS_LICENSE'
min_download = 8192

def get_system_license_path() -> Path:
	"""Returns the default path to the Binary Ninja license.dat on the current platform

	:return: Path object to the license.dat file.
	:rtype: Path
	"""
	if sys.platform == "darwin":
		return Path.home() / 'Library' / 'Application Support' / 'Binary Ninja' / 'license.dat'
	elif sys.platform.startswith("linux"):
		return Path.home() / '.binaryninja' / 'license.dat'
	else:
		return Path.home() / 'AppData' / 'Roaming' / 'Binary Ninja' / 'license.dat'

def get_serial_from_license(path: str) -> str:
	"""Extracts the serial number from the license specified in path

	:param path: path to license file
	:type path: str
	:raises ValueError: When no headless license is found in the provided license.dat
	:return: returns the serial number of the headless license
	:rtype: str
	"""
	with open(path) as f:
		for license in [l for l in json.load(f) if 'Headless' in l['product']]:
			return license['serial']
		raise ValueError(f'No "computer" license in {path}')

def get_serial_from_environment(env: str='HEADLESS_LICENSE') -> str:
	"""Extracts the serial number from the license file referenced by the env

	:param env: environment variable containing headless license path, defaults to 'HEADLESS_LICENSE'
	:type env: str, optional
	:raises ValueError: When provided envronment variable doesn't exist or license doesn't contain a headless license.
	:return: the serial number
	:rtype: str
	"""
	if env not in os.environ:
		raise ValueError(f'Please add an environment variable {env} pointing to your headless Binary Ninja license')

	return get_serial_from_license(os.environ[env])

def download_headless(serial: str, output_path: str=None, dev: bool=False) -> str:
	"""Downloads the headless Binary Ninja installation binaries.

	:param serial: Serial number for verification
	:type serial: str
	:param output_path: path to write the binaries to, defaults to None
	:type output_path: str, optional
	:param dev: set to True if you wish to download 'dev' branch, defaults to False
	:type dev: bool, optional
	:raises DownloadException: On failure to download the requested binaries.
	"""
	r = requests.get(url, {'serial': serial, 'dev': str(dev).lower()})
	results = json.loads(r.text)
	if not results['ok']:
		raise DownloadException('Download failed.')

	req_url = results['url']
	content = requests.get(req_url, allow_redirects=True)
	if len(content.content) < min_download:
		raise DownloadException(f'Error downloading from {req_url}')

	if output_path is None:
		output_path = os.path.basename(urllib.parse.urlparse(req_url).path)
	with open(output_path, 'wb') as f:
		f.write(content.content)
	return output_path

if __name__ == '__main__':
	import argparse
	parser = argparse.ArgumentParser(description='Download a Binary Ninja installer given a headless license')
	parser.add_argument('--serial', default=None, help='serial number')
	parser.add_argument('--env', default=False, action='store_true', help='will extract serial from enviroment variable "HEADLESS_LICENSE" if not specfied')
	parser.add_argument('--dev', default=False, action='store_true', help='download the development branch')
	parser.add_argument('--output', default=None, help='path to write the file to (defaults to current directory)')
	parser.add_argument('-q', '--quiet', default=False, action='store_true', help='Don\'t show any output')
	args = parser.parse_args()

	try:
		if args.serial is None:
			if args.env:
				args.serial = get_serial_from_environment()
			else:
				args.serial = get_serial_from_license(get_system_license_path())

		download_path = download_headless(args.serial, args.output, args.dev)
		if not args.quiet:
			print(f"Successfully downloaded to: {download_path}")
		sys.exit(0)
	except Exception as e:
		if not args.quiet:
			print(e)
		sys.exit(1)
<|endoftext|>


<commit_before>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import os
import io
import sys
import re
import argparse
import textwrap
from argparse import RawTextHelpFormatter

def is_valid_file(x):
	if not os.path.exists(x):
		raise argparse.ArgumentTypeError("{0} does not exist".format(x))
	return x

def main(args):
	with open(args.infile) as fp:
		locus = None
		dna = ''
		in_pp = False
		in_features = False
		for line in fp:
			if line.startswith('LOCUS'):
				locus = line.split()[1]
				dna = ''
				in_pp = False
				in_features = False
			elif line.startswith('FEATURES'):
				in_features = True
				for pp in args.coords.get(locus, []):
					args.coords[locus][pp].file.write(line)
					args.coords[locus][pp].file.write('     source          1..')
					args.coords[locus][pp].file.write(str(1+args.coords[locus][pp].right-args.coords[locus][pp].left))
					args.coords[locus][pp].file.write('\n')
			elif line.startswith('ORIGIN'):
				in_features = False
				dna = '\n'
			elif line.startswith('//'):
				dna = dna.replace('\n', '')
				for pp in args.coords.get(locus, []):
						args.coords[locus][pp].file.write('ORIGIN\n')
						i = 0
						for block in textwrap.wrap(dna[ args.coords[locus][pp].left-1 : args.coords[locus][pp].right ], 10):
							if(i%60 == 0):
								args.coords[locus][pp].file.write('\n')
								args.coords[locus][pp].file.write(str(i+1).rjust(9))
								args.coords[locus][pp].file.write(' ')
								args.coords[locus][pp].file.write(block.lower())
							else:
								args.coords[locus][pp].file.write(' ')
								args.coords[locus][pp].file.write(block.lower())
							i += 10
						args.coords[locus][pp].file.write('\n')
						args.coords[locus][pp].file.write('//\n')
			elif in_features:
				if not line.startswith('      '):
					in_pp = False
					left = min(map(int, re.findall(r"\d+", line)))
					right = max(map(int, re.findall(r"\d+", line)))
					for pp in args.coords.get(locus, []):
						if args.coords[locus][pp].left <= left and left <= args.coords[locus][pp].right and right <= args.coords[locus][pp].right:
							offset = args.coords[locus][pp].left - 1
							for match in re.findall(r"\d+", line):
								line = line.replace(match, str(int(match)-offset))
							args.coords[locus][pp].file.write(line)
							in_pp = pp
				elif in_pp:
					args.coords[locus][in_pp].file.write(line)
			elif dna:
				line = line[10:].replace(' ','')
				dna += line.upper()
			if not in_features and not dna:
				for pp in args.coords.get(locus, []):
					if line.startswith('LOCUS'):
						match = re.findall(r"\d+ bp", line)[0]
						replacement = str( 1 + args.coords[locus][pp].right - args.coords[locus][pp].left ) + " bp"
						line = line.replace(match, replacement.rjust(len(match)))
						args.coords[locus][pp].file.write(line)
					else:
						args.coords[locus][pp].file.write(line)

if __name__ == '__main__':
	usage = 'extract_prophages.py [-opt1, [-opt2, ...]] -c coordinates infile'
	parser = argparse.ArgumentParser(description='', formatter_class=RawTextHelpFormatter, usage=usage)
	parser.add_argument('infile', type=is_valid_file, help='input file in genbank format')
	parser.add_argument('-c', '--coordinates', type=is_valid_file, required=True, help='prophage coordinates')
	#parser.add_argument('-o', '--outfile', action="store", default=sys.stdout, type=argparse.FileType('w'), help='where to write the output [stdout]')
	parser.add_argument('--ids', action="store", help=argparse.SUPPRESS)
	args = parser.parse_args()

	args.coords = dict()
	with open(args.coordinates) as fp:
		for line in fp:
			pp, locus, left, right, *_ = line.split('\t')
			obj = lambda: None
			obj.left = int(left)
			obj.right = int(right)
			obj.file = open(os.path.splitext(args.infile)[0] + "_" + pp + '.gb', 'w')
			args.coords.setdefault(locus,{})[pp] = obj
	main(args)

<commit_msg>Removed extra newline after ORIGIN tag<commit_after>#!/usr/bin/python3
# -*- coding: utf-8 -*-

import os
import io
import sys
import re
import argparse
import textwrap
from argparse import RawTextHelpFormatter

def is_valid_file(x):
	if not os.path.exists(x):
		raise argparse.ArgumentTypeError("{0} does not exist".format(x))
	return x

def main(args):
	with open(args.infile) as fp:
		locus = None
		dna = ''
		in_pp = False
		in_features = False
		for line in fp:
			if line.startswith('LOCUS'):
				locus = line.split()[1]
				dna = ''
				in_pp = False
				in_features = False
			elif line.startswith('FEATURES'):
				in_features = True
				for pp in args.coords.get(locus, []):
					args.coords[locus][pp].file.write(line)
					args.coords[locus][pp].file.write('     source          1..')
					args.coords[locus][pp].file.write(str(1+args.coords[locus][pp].right-args.coords[locus][pp].left))
					args.coords[locus][pp].file.write('\n')
			elif line.startswith('ORIGIN'):
				in_features = False
				dna = '\n'
			elif line.startswith('//'):
				dna = dna.replace('\n', '')
				for pp in args.coords.get(locus, []):
						args.coords[locus][pp].file.write('ORIGIN')
						i = 0
						for block in textwrap.wrap(dna[ args.coords[locus][pp].left-1 : args.coords[locus][pp].right ], 10):
							if(i%60 == 0):
								args.coords[locus][pp].file.write('\n')
								args.coords[locus][pp].file.write(str(i+1).rjust(9))
								args.coords[locus][pp].file.write(' ')
								args.coords[locus][pp].file.write(block.lower())
							else:
								args.coords[locus][pp].file.write(' ')
								args.coords[locus][pp].file.write(block.lower())
							i += 10
						args.coords[locus][pp].file.write('\n')
						args.coords[locus][pp].file.write('//\n')
			elif in_features:
				if not line.startswith('      '):
					in_pp = False
					left = min(map(int, re.findall(r"\d+", line)))
					right = max(map(int, re.findall(r"\d+", line)))
					for pp in args.coords.get(locus, []):
						if args.coords[locus][pp].left <= left and left <= args.coords[locus][pp].right and right <= args.coords[locus][pp].right:
							offset = args.coords[locus][pp].left - 1
							for match in re.findall(r"\d+", line):
								line = line.replace(match, str(int(match)-offset))
							args.coords[locus][pp].file.write(line)
							in_pp = pp
				elif in_pp:
					args.coords[locus][in_pp].file.write(line)
			elif dna:
				line = line[10:].replace(' ','')
				dna += line.upper()
			if not in_features and not dna:
				for pp in args.coords.get(locus, []):
					if line.startswith('LOCUS'):
						match = re.findall(r"\d+ bp", line)[0]
						replacement = str( 1 + args.coords[locus][pp].right - args.coords[locus][pp].left ) + " bp"
						line = line.replace(match, replacement.rjust(len(match)))
						args.coords[locus][pp].file.write(line)
					else:
						args.coords[locus][pp].file.write(line)

if __name__ == '__main__':
	usage = 'extract_prophages.py [-opt1, [-opt2, ...]] -c coordinates infile'
	parser = argparse.ArgumentParser(description='', formatter_class=RawTextHelpFormatter, usage=usage)
	parser.add_argument('infile', type=is_valid_file, help='input file in genbank format')
	parser.add_argument('-c', '--coordinates', type=is_valid_file, required=True, help='prophage coordinates')
	#parser.add_argument('-o', '--outfile', action="store", default=sys.stdout, type=argparse.FileType('w'), help='where to write the output [stdout]')
	parser.add_argument('--ids', action="store", help=argparse.SUPPRESS)
	args = parser.parse_args()

	args.coords = dict()
	with open(args.coordinates) as fp:
		for line in fp:
			pp, locus, left, right, *_ = line.split('\t')
			obj = lambda: None
			obj.left = int(left)
			obj.right = int(right)
			obj.file = open(os.path.splitext(args.infile)[0] + "_" + pp + '.gb', 'w')
			args.coords.setdefault(locus,{})[pp] = obj
	main(args)

<|endoftext|>


<commit_before>"""
Generates the release notes for the latest release, in Markdown.

Convert CHANGELOG.rst to Markdown, and extracts just the latest release.

Writes to ``scripts/latest-release-notes.md``, which can be
used with https://github.com/softprops/action-gh-release.
"""
from pathlib import Path

import pypandoc

this_dir = Path(__file__).parent
rst_text = (this_dir.parent / "CHANGELOG.rst").read_text(encoding="UTF-8")
md_text = pypandoc.convert_text(
    rst_text, "md", format="rst", extra_args=["--wrap=preserve"]
)

output_lines = []
first_heading_found = False
for line in md_text.splitlines():
    if line.startswith("# "):
        if first_heading_found:
            break
        first_heading_found = True
    else:
        output_lines.append(line)

output_fn = this_dir / "latest-release-notes.md"
output_fn.write_text("\n".join(output_lines), encoding="UTF-8")
print(output_fn, "generated.")
<commit_msg>Fix gen-release-notes script<commit_after>"""
Generates the release notes for the latest release, in Markdown.

Convert CHANGELOG.rst to Markdown, and extracts just the latest release.

Writes to ``scripts/latest-release-notes.md``, which can be
used with https://github.com/softprops/action-gh-release.
"""
from pathlib import Path

import pypandoc

this_dir = Path(__file__).parent
rst_text = (this_dir.parent / "CHANGELOG.rst").read_text(encoding="UTF-8")
md_text = pypandoc.convert_text(
    rst_text, "md", format="rst", extra_args=["--wrap=preserve"]
)

output_lines = []
first_heading_found = False
for line in md_text.splitlines():
    if line.startswith("# "):
        # Skip the first section (normally # Releases).
        pass
    elif line.startswith("## "):
        # First second-level section, note it and skip the text,
        # as we are only interested in the individual release items.
        if first_heading_found:
            break
        first_heading_found = True
    else:
        output_lines.append(line)

output_fn = this_dir / "latest-release-notes.md"
output_fn.write_text("\n".join(output_lines), encoding="UTF-8")
print(output_fn, "generated.")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#!/usr/bin/env python

"""
"""

import os
import csv
import logging
from datetime import datetime, timedelta

log_file = "./scripts/import_ev_reunion.log"
logging.basicConfig(filename=log_file,level=logging.DEBUG)

import calebasse.settings
import django.core.management

django.core.management.setup_environ(calebasse.settings)

from django.db import transaction

from calebasse.agenda.models import Event, EventType
from calebasse.personnes.models import Worker
from calebasse.ressources.models import Service

from scripts.import_rs import PERIOD_FAURE_NOUS

# Configuration
db_path = "./scripts/20121221-192258"

dbs = ["F_ST_ETIENNE_SESSAD_TED", "F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD"]
tables = ['rs', 'ev', 'details_rs', 'details_ev']

# Global mappers. This dicts are used to map a Faure id with a calebasse object.
tables_data = {}

def _to_datetime(str_date):
    if not str_date:
        return None
    return datetime.strptime(str_date[:19], "%Y-%m-%d %H:%M:%S")

def _to_date(str_date):
    dt = _to_datetime(str_date)
    return dt and dt.date()

def _to_time(str_date):
    dt = _to_datetime(str_date)
    return dt and dt.time()

def _to_duration(str_date):
    dt = _to_datetime(str_date)
    if dt is None:
        return timedelta(minutes=15)
    return dt - datetime(1900, 01, 01, 0, 0)

def _get_dict(cols, line):
    res = {}
    for i, data in enumerate(line):
        res[cols[i]] = data.decode('utf-8')
    return res

import utils

worker_indexes = utils.QuerysetIndex(Worker.objects.all(),
        'old_cmpp_id', 'old_camsp_id', 'old_sessad_dys_id', 'old_sessad_ted_id')

def _get_therapists(line, table_name, service):
    global worker_indexes
    p_ids = []
    for id, detail in tables_data['details_%s' % table_name].iteritems():
        if detail['%s_id' % table_name] == line['id']:
            p_ids.append(detail['thera_id'])
    participants = []
    for p_id in p_ids:
        try:
            if service.name == "CMPP":
                worker = worker_indexes.by_old_cmpp_id[p_id]
            elif service.name == "CAMSP":
                worker = worker_indexes.by_old_camsp_id[p_id]
            elif service.name == "SESSAD DYS":
                worker = worker_indexes.by_old_sessad_dys_id[p_id]
            elif service.name == "SESSAD TED":
                worker = worker_indexes.by_old_sessad_ted_id[p_id]
            participants.append(worker)
        except KeyError:
            logging.warning("%s %s_id %s thera_id %s" %\
                    (service.name, table_name, line['id'], p_id) +\
                    " therapeute non trouve")

    return participants

def _is_timetable(line):
    return line and line['libelle'].replace(u'é', u'e').upper() in ('ARRIVEE', 'DEPART')

def _get_ev(tables, line):
    ev_id = line['rr_ev_id']
    if not ev_id or not ev_id.startswith('ev_'):
        return None
    return tables['ev'].get(ev_id[3:])

ServicesThrough = Event.services.through
ParticipantsThrough = Event.participants.through

event_type_4 = EventType.objects.get(id=4)

def create_event(line, service, tables_data, rs_by_id, events_by_id, events,
        participants_by_id, exceptions_set):
    if line['id'] not in rs_by_id:
        # Manage exception
        exception_date = None
        exception_to = None
        exception_error = False
        if line['rr_ev_id']:
            assert line['rr_ev_id'].startswith('ev_')
            ev_id = int(line['rr_ev_id'][3:])
            exception_date = _to_date(line['date_rdv'])
            exception_to = events_by_id.get(str(ev_id))
            if exception_to:
                p = (exception_date, exception_to)
                if p in exceptions_set:
                    logging.info("rs_id %s is duplicated" % line["id"])
                    return
                exceptions_set.add(p)
            else:
                exception_error = True
        start_datetime = datetime.strptime(
                line['date_rdv'][:-13] + ' ' + line['heure'][11:-4],
                "%Y-%m-%d %H:%M:%S")
        end_datetime = start_datetime + timedelta(
                hours=int(line['duree'][11:-10]),
                minutes=int(line['duree'][14:-7]),
                )
        participants = _get_therapists(line, 'rs', service)
        if not participants:
            logging.error("%s rs_id %s exception aucun participant importable" % (service.name,
                line["id"]))
            return
        if not exception_error:
            event = Event(
                    title=line['libelle'][:60],
                    description=line['texte'],
                    event_type=event_type_4,
                    start_datetime=start_datetime,
                    end_datetime=end_datetime,
                    old_rs_id=line['id'],
                    exception_to = Event(id=exception_to),
                    exception_date = exception_date
                    )
            participants_by_id[line['id']] = participants
            events.append(event)
        else:
            logging.error("%s rs_id %s exception pas d'ev trouve %s" % (service.name,
                line["id"], line))

def create_recurrent_event(line, service, tables_data, events, participants_by_id):
    if not Event.objects.filter(old_ev_id=line['id'], services__in=[service]):
        start_date = _to_date(line['date_debut'])
        end_date = _to_date(line['date_fin'])
        if end_date and start_date > end_date:
            logging.warning('%s ev_id %s date_fin < date_debut' % \
                    (service.name, line['id']))
        time = _to_time(line['heure'])
        duration = _to_duration(line['duree'])
        start_datetime = datetime.combine(start_date, time)
        end_datetime = start_datetime + duration

        # connect rythme
        rythme = int(line['rythme'])
        if PERIOD_FAURE_NOUS.get(rythme):
            recurrence_periodicity = PERIOD_FAURE_NOUS[rythme]
        else:
            recurrence_periodicity = None
            logging.warning('%s ev_id %s rythme not found %s' % \
                    (service.name, line['id'], line['rythme']))

        participants = _get_therapists(line, 'ev', service)
        if not participants:
            logging.error("%s ev_id %s exception aucun participant importable" % (service.name,
                line["id"]))
            return
        try:
            event = Event(
                start_datetime=start_datetime,
                end_datetime=end_datetime,
                event_type=event_type_4,
                title=line['libelle'],
                old_ev_id=line['id'],
                description=line['note'],
                recurrence_periodicity=recurrence_periodicity,
                recurrence_end_date=end_date)
            participants_by_id[line['id']] = participants
            events.append(event)
        except django.core.exceptions.ValidationError, e:
            logging.error(service.name + ' ev recurrence non valide %s %s' % (line, e))

@transaction.commit_on_success
def main():
    for db in dbs:
        if "F_ST_ETIENNE_CMPP" == db:
            service = Service.objects.get(name="CMPP")
        elif "F_ST_ETIENNE_CAMSP" == db:
            service = Service.objects.get(name="CAMSP")
        elif "F_ST_ETIENNE_SESSAD_TED" == db:
            service = Service.objects.get(name="SESSAD TED")
        elif "F_ST_ETIENNE_SESSAD" == db:
            service = Service.objects.get(name="SESSAD DYS")

        for table in tables:
            tables_data[table] = None
            csvfile = open(os.path.join(db_path, db, '%s.csv' % table), 'rb')
            csvlines = csv.reader(csvfile, delimiter=';', quotechar='|')
            cols = csvlines.next()
            tables_data[table] = {}
            for line in csvlines:
                data = _get_dict(cols, line)
                tables_data[table][data['id']] = data
            csvfile.close()

        total = 0
        events = []
        participants = dict()
        for id, line in tables_data['ev'].iteritems():
            if not _is_timetable(line):
                total += 1
                create_recurrent_event(line, service, tables_data, events,
                        participants)

        # create events
        logging.info('%s creating %s events on %s',
                service.name, len(events), total)
        Event.objects.bulk_create(events)
        participants_through = []
        # get their ids
        events_by_id = dict(Event.objects \
                .exclude(event_type_id=1) \
                .filter(services__isnull=True, old_ev_id__isnull=False) \
                .values_list('old_ev_id', 'id'))
        for line_id, participants in participants.iteritems():
            for participant in participants:
                pt = ParticipantsThrough(event_id=events_by_id[line_id],
                        people_id=participant.people_ptr_id)
                participants_through.append(pt)
        ParticipantsThrough.objects.bulk_create(participants_through)
        ServicesThrough.objects.bulk_create([
            ServicesThrough(service_id=service.id, event_id=event_id) for event_id in events_by_id.values()])
        participants = {}

        total = 0
        events = []
        participants = dict()
        exceptions_set = set()
        # get their ids
        rs_by_id = dict(Event.objects \
                .exclude(event_type_id=1) \
                .filter(services=service, old_rs_id__isnull=False) \
                .values_list('old_rs_id', 'id'))
        for id, line in tables_data['rs'].iteritems():
            if (not line['enfant_id'] or not int(line['enfant_id'])) \
                    and not _is_timetable(line) \
                    and (not _is_timetable(_get_ev(tables_data, line))):
                total += 1
                create_event(line, service, tables_data, rs_by_id, events_by_id,
                        events, participants, exceptions_set)
        # create events
        logging.info('%s creating %s events on %s',
                service.name, len(events), total)
        Event.objects.bulk_create(events)
        participants_through = []
        # get their ids
        events_by_id = dict(Event.objects \
                .exclude(event_type_id=1) \
                .filter(services__isnull=True, old_rs_id__isnull=False) \
                .values_list('old_rs_id', 'id'))
        for line_id, participants in participants.iteritems():
            for participant in participants:
                pt = ParticipantsThrough(event_id=events_by_id[line_id],
                        people_id=participant.people_ptr_id)
                participants_through.append(pt)
        ParticipantsThrough.objects.bulk_create(participants_through)
        ServicesThrough.objects.bulk_create([
            ServicesThrough(service_id=service.id, event_id=event_id) for event_id in events_by_id.values()])
        participants = {}

if __name__ == "__main__":
    main()

<commit_msg>scripts: log already loaded lines in import_ev_reunion.py<commit_after># -*- coding: utf-8 -*-
#!/usr/bin/env python

"""
"""

import os
import csv
import logging
from datetime import datetime, timedelta

log_file = "./scripts/import_ev_reunion.log"
logging.basicConfig(filename=log_file,level=logging.DEBUG)

import calebasse.settings
import django.core.management

django.core.management.setup_environ(calebasse.settings)

from django.db import transaction

from calebasse.agenda.models import Event, EventType
from calebasse.personnes.models import Worker
from calebasse.ressources.models import Service

from scripts.import_rs import PERIOD_FAURE_NOUS

# Configuration
db_path = "./scripts/20121221-192258"

dbs = ["F_ST_ETIENNE_SESSAD_TED", "F_ST_ETIENNE_CMPP", "F_ST_ETIENNE_CAMSP", "F_ST_ETIENNE_SESSAD"]
tables = ['rs', 'ev', 'details_rs', 'details_ev']

# Global mappers. This dicts are used to map a Faure id with a calebasse object.
tables_data = {}

def _to_datetime(str_date):
    if not str_date:
        return None
    return datetime.strptime(str_date[:19], "%Y-%m-%d %H:%M:%S")

def _to_date(str_date):
    dt = _to_datetime(str_date)
    return dt and dt.date()

def _to_time(str_date):
    dt = _to_datetime(str_date)
    return dt and dt.time()

def _to_duration(str_date):
    dt = _to_datetime(str_date)
    if dt is None:
        return timedelta(minutes=15)
    return dt - datetime(1900, 01, 01, 0, 0)

def _get_dict(cols, line):
    res = {}
    for i, data in enumerate(line):
        res[cols[i]] = data.decode('utf-8')
    return res

import utils

worker_indexes = utils.QuerysetIndex(Worker.objects.all(),
        'old_cmpp_id', 'old_camsp_id', 'old_sessad_dys_id', 'old_sessad_ted_id')

def _get_therapists(line, table_name, service):
    global worker_indexes
    p_ids = []
    for id, detail in tables_data['details_%s' % table_name].iteritems():
        if detail['%s_id' % table_name] == line['id']:
            p_ids.append(detail['thera_id'])
    participants = []
    for p_id in p_ids:
        try:
            if service.name == "CMPP":
                worker = worker_indexes.by_old_cmpp_id[p_id]
            elif service.name == "CAMSP":
                worker = worker_indexes.by_old_camsp_id[p_id]
            elif service.name == "SESSAD DYS":
                worker = worker_indexes.by_old_sessad_dys_id[p_id]
            elif service.name == "SESSAD TED":
                worker = worker_indexes.by_old_sessad_ted_id[p_id]
            participants.append(worker)
        except KeyError:
            logging.warning("%s %s_id %s thera_id %s" %\
                    (service.name, table_name, line['id'], p_id) +\
                    " therapeute non trouve")

    return participants

def _is_timetable(line):
    return line and line['libelle'].replace(u'é', u'e').upper() in ('ARRIVEE', 'DEPART')

def _get_ev(tables, line):
    ev_id = line['rr_ev_id']
    if not ev_id or not ev_id.startswith('ev_'):
        return None
    return tables['ev'].get(ev_id[3:])

ServicesThrough = Event.services.through
ParticipantsThrough = Event.participants.through

event_type_4 = EventType.objects.get(id=4)

def create_event(line, service, tables_data, rs_by_id, events_by_id, events,
        participants_by_id, exceptions_set):
    if line['id'] in rs_by_id:
        logging.warning("%s rs %s already loaded", service.name, line['id'])
        return
    # Manage exception
    exception_date = None
    exception_to = None
    exception_error = False
    if line['rr_ev_id']:
        assert line['rr_ev_id'].startswith('ev_')
        ev_id = int(line['rr_ev_id'][3:])
        exception_date = _to_date(line['date_rdv'])
        exception_to = events_by_id.get(str(ev_id))
        if exception_to:
            p = (exception_date, exception_to)
            if p in exceptions_set:
                logging.info("rs_id %s is duplicated" % line["id"])
                return
            exceptions_set.add(p)
        else:
            exception_error = True
    start_datetime = datetime.strptime(
            line['date_rdv'][:-13] + ' ' + line['heure'][11:-4],
            "%Y-%m-%d %H:%M:%S")
    end_datetime = start_datetime + timedelta(
            hours=int(line['duree'][11:-10]),
            minutes=int(line['duree'][14:-7]),
            )
    participants = _get_therapists(line, 'rs', service)
    if not participants:
        logging.error("%s rs_id %s exception aucun participant importable" % (service.name,
            line["id"]))
        return
    if not exception_error:
        event = Event(
                title=line['libelle'][:60],
                description=line['texte'],
                event_type=event_type_4,
                start_datetime=start_datetime,
                end_datetime=end_datetime,
                old_rs_id=line['id'],
                exception_to = Event(id=exception_to),
                exception_date = exception_date
                )
        participants_by_id[line['id']] = participants
        events.append(event)
    else:
        logging.error("%s rs_id %s exception pas d'ev trouve %s" % (service.name,
            line["id"], line))

def create_recurrent_event(line, service, tables_data, events, participants_by_id):
    if Event.objects.filter(old_ev_id=line['id'], services__in=[service]):
        logging.warning("%s ev %s already loaded", service.name, line['id'])
        return
    start_date = _to_date(line['date_debut'])
    end_date = _to_date(line['date_fin'])
    if end_date and start_date > end_date:
        logging.warning('%s ev_id %s date_fin < date_debut' % \
                (service.name, line['id']))
    time = _to_time(line['heure'])
    duration = _to_duration(line['duree'])
    start_datetime = datetime.combine(start_date, time)
    end_datetime = start_datetime + duration

    # connect rythme
    rythme = int(line['rythme'])
    if PERIOD_FAURE_NOUS.get(rythme):
        recurrence_periodicity = PERIOD_FAURE_NOUS[rythme]
    else:
        recurrence_periodicity = None
        logging.warning('%s ev_id %s rythme not found %s' % \
                (service.name, line['id'], line['rythme']))

    participants = _get_therapists(line, 'ev', service)
    if not participants:
        logging.error("%s ev_id %s exception aucun participant importable" % (service.name,
            line["id"]))
        return
    try:
        event = Event(
            start_datetime=start_datetime,
            end_datetime=end_datetime,
            event_type=event_type_4,
            title=line['libelle'],
            old_ev_id=line['id'],
            description=line['note'],
            recurrence_periodicity=recurrence_periodicity,
            recurrence_end_date=end_date)
        participants_by_id[line['id']] = participants
        events.append(event)
    except django.core.exceptions.ValidationError, e:
        logging.error(service.name + ' ev recurrence non valide %s %s' % (line, e))

@transaction.commit_on_success
def main():
    for db in dbs:
        if "F_ST_ETIENNE_CMPP" == db:
            service = Service.objects.get(name="CMPP")
        elif "F_ST_ETIENNE_CAMSP" == db:
            service = Service.objects.get(name="CAMSP")
        elif "F_ST_ETIENNE_SESSAD_TED" == db:
            service = Service.objects.get(name="SESSAD TED")
        elif "F_ST_ETIENNE_SESSAD" == db:
            service = Service.objects.get(name="SESSAD DYS")

        for table in tables:
            tables_data[table] = None
            csvfile = open(os.path.join(db_path, db, '%s.csv' % table), 'rb')
            csvlines = csv.reader(csvfile, delimiter=';', quotechar='|')
            cols = csvlines.next()
            tables_data[table] = {}
            for line in csvlines:
                data = _get_dict(cols, line)
                tables_data[table][data['id']] = data
            csvfile.close()

        total = 0
        events = []
        participants = dict()
        for id, line in tables_data['ev'].iteritems():
            if not _is_timetable(line):
                total += 1
                create_recurrent_event(line, service, tables_data, events,
                        participants)

        # create events
        logging.info('%s creating %s events on %s',
                service.name, len(events), total)
        Event.objects.bulk_create(events)
        participants_through = []
        # get their ids
        events_by_id = dict(Event.objects \
                .exclude(event_type_id=1) \
                .filter(services__isnull=True, old_ev_id__isnull=False) \
                .values_list('old_ev_id', 'id'))
        for line_id, participants in participants.iteritems():
            for participant in participants:
                pt = ParticipantsThrough(event_id=events_by_id[line_id],
                        people_id=participant.people_ptr_id)
                participants_through.append(pt)
        ParticipantsThrough.objects.bulk_create(participants_through)
        ServicesThrough.objects.bulk_create([
            ServicesThrough(service_id=service.id, event_id=event_id) for event_id in events_by_id.values()])
        participants = {}

        total = 0
        events = []
        participants = dict()
        exceptions_set = set()
        # get their ids
        rs_by_id = dict(Event.objects \
                .exclude(event_type_id=1) \
                .filter(services=service, old_rs_id__isnull=False) \
                .values_list('old_rs_id', 'id'))
        for id, line in tables_data['rs'].iteritems():
            if (not line['enfant_id'] or not int(line['enfant_id'])) \
                    and not _is_timetable(line) \
                    and (not _is_timetable(_get_ev(tables_data, line))):
                total += 1
                create_event(line, service, tables_data, rs_by_id, events_by_id,
                        events, participants, exceptions_set)
        # create events
        logging.info('%s creating %s events on %s',
                service.name, len(events), total)
        Event.objects.bulk_create(events)
        participants_through = []
        # get their ids
        events_by_id = dict(Event.objects \
                .exclude(event_type_id=1) \
                .filter(services__isnull=True, old_rs_id__isnull=False) \
                .values_list('old_rs_id', 'id'))
        for line_id, participants in participants.iteritems():
            for participant in participants:
                pt = ParticipantsThrough(event_id=events_by_id[line_id],
                        people_id=participant.people_ptr_id)
                participants_through.append(pt)
        ParticipantsThrough.objects.bulk_create(participants_through)
        ServicesThrough.objects.bulk_create([
            ServicesThrough(service_id=service.id, event_id=event_id) for event_id in events_by_id.values()])
        participants = {}

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Look for events with the same event id in the datasets xzt and xzt_broken5 and plot them.
"""
import h5py
import numpy as np
import matplotlib.pyplot as plt

from get_dataset_info import get_dataset_info
from plotting.histogramm_3d_utils import make_3d_plots, reshape_3d_to_3d, get_title_arrays
from plotting.histogramm_3d_utils import get_event_no_from_file, get_title_arrays, get_some_hists_from_file, make_plots_from_array

datainfo_xzt = get_dataset_info("xztc")
datainfo_xzt_broken5 = get_dataset_info("xzt_broken5")

train_file_xzt = datainfo_xzt["train_file"]
train_file_xzt_broken5 = datainfo_xzt_broken5["train_file"]

hists, labels = get_some_hists_from_file(train_file_xzt, 1, 50)

title_array=get_title_arrays(labels)
title=title_array[0][0]+title_array[0][1]+title_array[0][2]

hists_b, labels_b = get_event_no_from_file(train_file_xzt_broken5, target_event_id=labels[0][0], event_track=None)

print(labels, labels_b)

#xztc hists haben shape (11,18,50,31)
hists=np.sum(hists[0], axis=-1)

fig = make_plots_from_array(hists, hists_b[0], suptitle=title, min_counts=0, titles=["Original","Manipulation"])
plt.show(fig)

<commit_msg>Added evelop mode<commit_after># -*- coding: utf-8 -*-
"""
Look for events with the same event id in the datasets xzt and xzt_broken5 and plot them.
"""
import h5py
import numpy as np
import matplotlib.pyplot as plt

from get_dataset_info import get_dataset_info
from plotting.histogramm_3d_utils import make_3d_plots, reshape_3d_to_3d, get_title_arrays
from plotting.histogramm_3d_utils import get_event_no_from_file, get_some_hists_from_file, make_plots_from_array

mode="develop"

datainfo_xzt = get_dataset_info("xztc")
train_file_xzt = datainfo_xzt["train_file"]

def make_broken5_manip(hists_temp, chance):
    #Input (X,11,18,50,31) xztc hists
    #Output: (X,11,18,50) manipulated xzt hists

    #all doms facing upwards AND having >0 counts have a 30% of getting reduced by one

    #1=upwards facing, taken from the paper
    up_mask=np.array([True,]*12 + [False,]*19)
    #chance for upward facing doms with >0 counts to have one count removed:
    #chance=0.3

    #the counts to subtract: upwards facing doms have a 30% chance of getting one count removed
    subt = np.multiply(up_mask, np.random.choice([0,1],size=hists_temp.shape, p=[1-chance,chance]))
    #only remove one count when there is one to begin with
    subt=np.multiply(subt,hists_temp>=1)
    #Ultimately, all doms facing upwards AND having >0 counts have a 30% of getting reduced by one
    hists_temp=hists_temp-subt
    #sum over channel axis to get X,11,18,50 xzt data
    hists_temp=np.sum(hists_temp, axis=-1)

    return hists_temp

if mode=="compare_datasets":
    #check if the generate script worked properly by plotting the same event from
    #xztc (summed over c) and xzt_broken5

    datainfo_xzt_broken5 = get_dataset_info("xzt_broken5")
    train_file_xzt_broken5 = datainfo_xzt_broken5["train_file"]

    hists, labels = get_some_hists_from_file(train_file_xzt, 1, 50)

    title_array=get_title_arrays(labels)
    title=title_array[0][0]+title_array[0][1]+title_array[0][2]

    hists_b, labels_b = get_event_no_from_file(train_file_xzt_broken5, target_event_id=labels[0][0], event_track=None)

    print(labels, labels_b)

    #xztc hists haben shape (11,18,50,31)
    hists=np.sum(hists[0], axis=-1)

    fig = make_plots_from_array(hists, hists_b[0], suptitle=title, min_counts=0, titles=["Original","Manipulation"])
    plt.show(fig)

elif mode=="develop":
    #used for finding a suitable manipulation of the data
    how_many=2
    energy_threshold=50
    chance=0.3

    hists, labels = get_some_hists_from_file(train_file_xzt, how_many, energy_threshold)
    title_array=get_title_arrays(labels)

    manip_hists = make_broken5_manip(hists, chance)

    for i in range(len(hists)):
        title=title_array[0][i]+title_array[1][i]+title_array[2][i]
        fig = make_plots_from_array(hists[i], manip_hists[i], suptitle=title, min_counts=0, titles=["Original","Manipulation"])
        plt.show(fig)
<|endoftext|>


<commit_before>"""HTML plotting."""

import os

from .pdf import BaseFigure
from .utils import to_data_frames

def speedup_saturation(speedup):
    diff = abs(1.0 - speedup)
    # scale up a 20% difference into maximum 50% saturation - lower
    # difference should be closer to 255 (white)
    saturation_ratio = min(0.5, diff / 0.2 * 0.5)
    return 255 - int(saturation_ratio * 255.0)

def speedup_colors(speedup):
    ret = []
    for s in speedup:
        saturation = speedup_saturation(s)
        if s < 1.0:
            # slowdown is red
            ret.append('rgb(255,{0},{0})'.format(saturation))
        else:
            # speedup is green
            ret.append('rgb({0},255,{0})'.format(saturation))
    return ret

def significance_colors(significance, threshold=0.05):
    ret = []
    for s in significance:
        if s < threshold:
            saturation = 128 + int(128 * (s / threshold))
            ret.append('rgb({0},{0},255)'.format(saturation))
        else:
            ret.append('white')
    return ret

def title_to_html_anchor(title):
    return title.replace(' ', '_')

def token_to_length(tokens):
    length = []
    for token in tokens:
        words = token.split("_")
        for idx in range(len(words)):
            if(words[idx] == "len"):
                lenidx = idx + 1
                thislength = []
                while lenidx < len(words) and words[lenidx].isnumeric():
                    thislength.append(int(words[lenidx]))
                    lenidx += 1
                length.append(thislength)
    return length

def token_to_batch(tokens):
    batch = []
    for token in tokens:
        words = token.split("_")
        for idx in range(len(words)):
            if(words[idx] == "batch"):
                batchidx = idx + 1
                thisbatch = []
                while batchidx < len(words) and words[batchidx].isnumeric():
                    thisbatch.append(int(words[batchidx]))
                    batchidx += 1
                batch.append(thisbatch)
    return batch

def token_to_elements(tokens):
    length = token_to_length(tokens)
    batch = token_to_batch(tokens)
    elements = []
    for i in range(len(length)):
        n = int(1)
        for j in range(len(length[i])):
            n *= int(length[i][j])
        for j in range(len(batch[i])):
            n *= int(batch[i][j])
        elements.append(n)
    return elements

class HTMLFigure(BaseFigure):

    def make(self):
        from plotly import graph_objs as go
        data_frames = to_data_frames(self.primary, self.secondary)
        logscale = False

        traces = []

        traces.append(go.Scatter(
            x=token_to_elements(data_frames[0].token),
            y=data_frames[0].median_sample,
            hovertext=token_to_length(data_frames[0].token),
            name=self.labels[0]
        ))

        for i in range(1,len(data_frames)):
            traces.append(go.Scatter(
                x= token_to_elements(data_frames[i].token),
                y=data_frames[i].median_sample,
                hovertext=data_frames[i].token,
                name=self.labels[i]
            ))
            traces.append(go.Scatter(
                x= token_to_elements(data_frames[i].token),
                y=data_frames[i].speedup,
                name='Speedup {} over {}'.format(self.labels[i], self.labels[0]),
                yaxis='y2',
                error_y = dict(
                    type='data',
                    symmetric=False,
                    array=data_frames[i].speedup_high - data_frames[i].speedup,
                    arrayminus=data_frames[i].speedup - data_frames[i].speedup_low,
                )
            ))

        if logscale:
            x_title = 'Problem size (elements, logarithmic)'
            axis_type = 'log'
            y_title = 'Time (ms, logarithmic)'
        else:
            x_title = 'Problem size (elements)'
            axis_type = 'linear'
            y_title = 'Time (ms)'

        layout = go.Layout(
            title=self.title,
            xaxis=dict(
                title=x_title,
                type=axis_type,
            ),
            yaxis=dict(
                title=y_title,
                type=axis_type,
                rangemode='tozero'
            ),
            yaxis2=dict(
                title='Speedup',
                overlaying='y',
                side='right',
                type='linear',
                rangemode='tozero'
            ),
            hovermode = 'x unified',
            width = 900,
            height = 600,
            legend = dict(
                yanchor="top",
                xanchor="right",
                x=1.2
            )
        )

        fig = go.Figure(data=traces, layout=layout)

        # Add speedup=1 reference line
        fig.add_shape(
            type='line',
            x0=min(token_to_elements(data_frames[0].token)),
            y0=1,
            x1=max(token_to_elements(data_frames[0].token)),
            y1=1,
            line=dict(color='grey', dash='dash'),
            yref='y2'
        )

        nrows = len(data_frames[0].index)
        headers = ['Problem size', 'Elements']
        values = [
            token_to_length(data_frames[0].token),
            token_to_elements(data_frames[0].token)
        ]
        fill_colors = [
            ['white'] * nrows,
            ['white'] * nrows,
        ]
        for i in range(len(data_frames)):
            headers.append(self.labels[i] + ' (median)')
            values.append(["{:.4f}".format(x) for x in data_frames[i].median_sample])
            fill_colors.append(['white'] * nrows)

            if i > 0:
                headers.append('Speedup {} over {}'.format(self.labels[i], self.labels[0]))
                values.append(["{:.4f}".format(x) for x in data_frames[i].speedup])
                fill_colors.append(speedup_colors(data_frames[i].speedup))

                headers.append('Significance of {} over {}'.format(self.labels[i], self.labels[0]))
                values.append(["{:.4f}".format(x) for x in data_frames[i].speedup_pval])
                fill_colors.append(significance_colors(data_frames[i].speedup_pval))

        table = go.Figure(
            data = [
                go.Table(header=dict(values=headers),
                         cells=dict(values=values, fill_color=fill_colors),
                         ),
            ],
            layout = go.Layout(
                title=self.title,
            )
        )
        # 900 seems to be enough for 2 dirs
        # widen table by 200px per extra dir
        table_width = 900
        if len(data_frames) > 2:
            table_width += (len(data_frames) - 2) * 200
        table.update_layout(width=table_width, height=600)

        self.plot = fig
        self.table = table

def make_html(figures, title, docdir, outdirs):
    # TODO: this needs to read the output from the post-processing;
    # graphing and post-processing should be separate.

    # # use first dir's dat files as a basis for what to graph.
    # # assumption is that other dirs have the same-named files.
    # dat_files = glob.glob(os.path.join(dirs[0], '*.dat'))
    # # sort files so diagrams show up in consistent order for each run
    # dat_files.sort()

    # construct the output file "figs.html"
    outfile = open(docdir / 'figs.html', 'w')
    outfile.write('''

    <title>{}</title>

'''.format(title))

    # make links to each figure at the top of the report
    outfile.write('''Quick links:
''')
    for figure in figures:
        plot, table = figure.plot, figure.table
        fig_title = plot.layout.title.text
        anchor = title_to_html_anchor(fig_title)
        outfile.write('''{}
'''.format(anchor, fig_title))

    # include specs in the report
    outfile.write('''

''')
    for d in outdirs:
        outfile.write('''{} specs:

'''.format(
            os.path.basename(os.path.normpath(d)))
        )
        specs_path = os.path.join(d, 'specs.txt')
        try:
            with open(specs_path) as specs:
              for line in specs:
                  outfile.write(line)
        except FileNotFoundError:
            outfile.write("N/A")

        outfile.write('''
	''')
    outfile.write('''

''')

    # only the first figure needs js included
    include_js = True
    for figure in figures:
        plot, table = figure.plot, figure.table
        fig_title = plot.layout.title.text
        anchor = title_to_html_anchor(fig_title)
        outfile.write('''
'''.format(anchor))

        outfile.write(plot.to_html(full_html=False, include_plotlyjs=include_js))
        include_js = False
        outfile.write('''	''')
        outfile.write(table.to_html(full_html=False, include_plotlyjs=False))
        outfile.write('''
''')

    outfile.write('''

    ''')
<commit_msg>rocfft-perf: html report fixes<commit_after>"""HTML plotting."""

import os

from .pdf import BaseFigure
from .utils import to_data_frames

def speedup_saturation(speedup):
    diff = abs(1.0 - speedup)
    # scale up a 20% difference into maximum 50% saturation - lower
    # difference should be closer to 255 (white)
    saturation_ratio = min(0.5, diff / 0.2 * 0.5)
    return 255 - int(saturation_ratio * 255.0)

def speedup_colors(speedup):
    ret = []
    for s in speedup:
        saturation = speedup_saturation(s)
        if s < 1.0:
            # slowdown is red
            ret.append('rgb(255,{0},{0})'.format(saturation))
        else:
            # speedup is green
            ret.append('rgb({0},255,{0})'.format(saturation))
    return ret

def significance_colors(significance, threshold=0.05):
    ret = []
    for s in significance:
        if s < threshold:
            saturation = 128 + int(128 * (s / threshold))
            ret.append('rgb({0},{0},255)'.format(saturation))
        else:
            ret.append('white')
    return ret

def title_to_html_anchor(title):
    return title.replace(' ', '_')

def token_to_length(tokens):
    length = []
    for token in tokens:
        words = token.split("_")
        for idx in range(len(words)):
            if(words[idx] == "len"):
                lenidx = idx + 1
                thislength = []
                while lenidx < len(words) and words[lenidx].isnumeric():
                    thislength.append(int(words[lenidx]))
                    lenidx += 1
                length.append(thislength)
    return length

def token_to_batch(tokens):
    batch = []
    for token in tokens:
        words = token.split("_")
        for idx in range(len(words)):
            if(words[idx] == "batch"):
                batchidx = idx + 1
                thisbatch = []
                while batchidx < len(words) and words[batchidx].isnumeric():
                    thisbatch.append(int(words[batchidx]))
                    batchidx += 1
                batch.append(thisbatch)
    return batch

def token_to_elements(tokens):
    length = token_to_length(tokens)
    batch = token_to_batch(tokens)
    elements = []
    for i in range(len(length)):
        n = int(1)
        for j in range(len(length[i])):
            n *= int(length[i][j])
        for j in range(len(batch[i])):
            n *= int(batch[i][j])
        elements.append(n)
    return elements

def token_to_size_description(tokens):
    length = token_to_length(tokens)
    batch = token_to_batch(tokens)
    descriptions = []
    for cur_len, cur_batch in zip(length, batch):
        def join_ints(ints):
            return 'x'.join([str(val) for val in ints])
        desc = join_ints(cur_len) + 'b' + join_ints(cur_batch)
        descriptions.append(desc)
    return descriptions

class HTMLFigure(BaseFigure):

    def make(self):
        from plotly import graph_objs as go
        data_frames = to_data_frames(self.primary, self.secondary)
        for df in data_frames:
            df['elements'] = token_to_elements(df.token)
            df.sort_values(by='elements', inplace=True)
        logscale = False

        traces = []

        traces.append(go.Scatter(
            x=data_frames[0].elements,
            y=data_frames[0].median_sample,
            hovertext=token_to_size_description(data_frames[0].token),
            name=self.labels[0]
        ))

        for i in range(1,len(data_frames)):
            traces.append(go.Scatter(
                x=data_frames[i].elements,
                y=data_frames[i].median_sample,
                hovertext=data_frames[i].token,
                name=self.labels[i]
            ))
            traces.append(go.Scatter(
                x=data_frames[i].elements,
                y=data_frames[i].speedup,
                name='Speedup {} over {}'.format(self.labels[i], self.labels[0]),
                yaxis='y2',
                error_y = dict(
                    type='data',
                    symmetric=False,
                    array=data_frames[i].speedup_high - data_frames[i].speedup,
                    arrayminus=data_frames[i].speedup - data_frames[i].speedup_low,
                )
            ))

        if logscale:
            x_title = 'Problem size (elements, logarithmic)'
            axis_type = 'log'
            y_title = 'Time (ms, logarithmic)'
        else:
            x_title = 'Problem size (elements)'
            axis_type = 'linear'
            y_title = 'Time (ms)'

        layout = go.Layout(
            title=self.title,
            xaxis=dict(
                title=x_title,
                type=axis_type,
            ),
            yaxis=dict(
                title=y_title,
                type=axis_type,
                rangemode='tozero'
            ),
            yaxis2=dict(
                title='Speedup',
                overlaying='y',
                side='right',
                type='linear',
                rangemode='tozero'
            ),
            hovermode = 'x unified',
            width = 900,
            height = 600,
            legend = dict(
                yanchor="top",
                xanchor="right",
                x=1.2
            )
        )

        fig = go.Figure(data=traces, layout=layout)

        # Add speedup=1 reference line
        fig.add_shape(
            type='line',
            x0=min(data_frames[0].elements),
            y0=1,
            x1=max(data_frames[0].elements),
            y1=1,
            line=dict(color='grey', dash='dash'),
            yref='y2'
        )

        nrows = len(data_frames[0].index)
        headers = ['Problem size', 'Elements']
        values = [
            token_to_size_description(data_frames[0].token),
            data_frames[0].elements
        ]
        fill_colors = [
            ['white'] * nrows,
            ['white'] * nrows,
        ]
        for i in range(len(data_frames)):
            headers.append(self.labels[i] + ' (median)')
            values.append(["{:.4f}".format(x) for x in data_frames[i].median_sample])
            fill_colors.append(['white'] * nrows)

            if i > 0:
                headers.append('Speedup {} over {}'.format(self.labels[i], self.labels[0]))
                values.append(["{:.4f}".format(x) for x in data_frames[i].speedup])
                fill_colors.append(speedup_colors(data_frames[i].speedup))

                headers.append('Significance of {} over {}'.format(self.labels[i], self.labels[0]))
                values.append(["{:.4f}".format(x) for x in data_frames[i].speedup_pval])
                fill_colors.append(significance_colors(data_frames[i].speedup_pval))

        table = go.Figure(
            data = [
                go.Table(header=dict(values=headers),
                         cells=dict(values=values, fill_color=fill_colors),
                         ),
            ],
            layout = go.Layout(
                title=self.title,
            )
        )
        # 900 seems to be enough for 2 dirs
        # widen table by 200px per extra dir
        table_width = 900
        if len(data_frames) > 2:
            table_width += (len(data_frames) - 2) * 200
        table.update_layout(width=table_width, height=600)

        self.plot = fig
        self.table = table

def make_html(figures, title, docdir, outdirs):
    # TODO: this needs to read the output from the post-processing;
    # graphing and post-processing should be separate.

    # # use first dir's dat files as a basis for what to graph.
    # # assumption is that other dirs have the same-named files.
    # dat_files = glob.glob(os.path.join(dirs[0], '*.dat'))
    # # sort files so diagrams show up in consistent order for each run
    # dat_files.sort()

    # construct the output file "figs.html"
    outfile = open(docdir / 'figs.html', 'w')
    outfile.write('''

    <title>{}</title>

'''.format(title))

    # make links to each figure at the top of the report
    outfile.write('''Quick links:
''')
    for figure in figures:
        plot, table = figure.plot, figure.table
        fig_title = plot.layout.title.text
        anchor = title_to_html_anchor(fig_title)
        outfile.write('''{}
'''.format(anchor, fig_title))

    # include specs in the report
    outfile.write('''

''')
    for d in outdirs:
        outfile.write('''{} specs:

'''.format(
            os.path.basename(os.path.normpath(d)))
        )
        specs_path = os.path.join(d, 'specs.txt')
        try:
            with open(specs_path) as specs:
              for line in specs:
                  outfile.write(line)
        except FileNotFoundError:
            outfile.write("N/A")

        outfile.write('''
	''')
    outfile.write('''

''')

    # only the first figure needs js included
    include_js = True
    for figure in figures:
        plot, table = figure.plot, figure.table
        fig_title = plot.layout.title.text
        anchor = title_to_html_anchor(fig_title)
        outfile.write('''
'''.format(anchor))

        outfile.write(plot.to_html(full_html=False, include_plotlyjs=include_js))
        include_js = False
        outfile.write('''	''')
        outfile.write(table.to_html(full_html=False, include_plotlyjs=False))
        outfile.write('''
''')

    outfile.write('''

    ''')
<|endoftext|>


<commit_before>import logging

from pylons import c
from ming.orm import ThreadLocalORMSession

from allura import model as M
from allura.lib.utils import chunked_find

log = logging.getLogger(__name__)

def main(options):
    q_project = {}
    if options.nbhd:
        nbhd = M.Neighborhood.query.get(url_prefix=options.nbhd)
        if not nbhd:
            return "Invalid neighborhood url prefix."
        q_project['neighborhood_id'] = nbhd._id
    if options.project:
        q_project['shortname'] = options.project
    elif options.project_regex:
        q_project['shortname'] = {'$regex': options.project_regex}

    log.info('Refreshing repositories')
    if options.clean:
        log.info('Removing all repository objects')
        M.repo.CommitDoc.m.remove({})
        M.repo.TreeDoc.m.remove({})
        M.repo.TreesDoc.m.remove({})
        M.repo.DiffInfoDoc.m.remove({})
        M.repo.CommitRunDoc.m.remove({})

    for chunk in chunked_find(M.Project, q_project):
        for p in chunk:
            log.info("Refreshing repos for project '%s'." % p.shortname)
            if options.dry_run:
                continue
            c.project = p
            if options.mount_point:
                mount_points = [options.mount_point]
            else:
                mount_points = [ac.options.mount_point for ac in
                                M.AppConfig.query.find(dict(project_id=p._id))]
            for app in (p.app_instance(mp) for mp in mount_points):
                c.app = app
                if not hasattr(app, 'repo'): continue
                try:
                    c.app.repo._impl._setup_hooks()
                except:
                    log.exception('Error setting up hooks for %r', c.app.repo)
                try:
                    if options.all:
                        log.info('Refreshing ALL commits in %r', c.app.repo)
                    else:
                        log.info('Refreshing NEW commits in %r', c.app.repo)
                    c.app.repo.refresh(options.all, notify=options.notify)
                except:
                    log.exception('Error refreshing %r', c.app.repo)
        ThreadLocalORMSession.flush_all()
        ThreadLocalORMSession.close_all()

def parse_options():
    import argparse
    parser = argparse.ArgumentParser(description='Scan repos on filesytem and '
            'update repo metadata in MongoDB. Run for all repos (no args), '
            'or restrict by neighborhood, project, or code tool mount point.')
    parser.add_argument('--nbhd', action='store', default='', dest='nbhd',
            help='Restrict update to a particular neighborhood, e.g. /p/.')
    parser.add_argument('--project', action='store', default='', dest='project',
            help='Restrict update to a particular project. To specify a '
            'subproject, use a slash: project/subproject.')
    parser.add_argument('--project-regex', action='store', default='',
            dest='project_regex',
            help='Restrict update to projects for which the shortname matches '
            'the provided regex.')
    parser.add_argument('--mount_point', default='', dest='mount_point',
            help='Restrict update to repos at the given tool mount point. ')
    parser.add_argument('--clean', action='store_true', dest='clean',
            default=False, help='Remove all repo-related mongo docs before '
            'refresh.')
    parser.add_argument('--all', action='store_true', dest='all', default=False,
            help='Refresh all commits (not just the ones that are new).')
    parser.add_argument('--notify', action='store_true', dest='notify',
            default=False, help='Send email notifications of new commits.')
    parser.add_argument('--dry-run', action='store_true', dest='dry_run',
            default=False, help='Log names of projects that would have their '
            'repos refreshed, but do not perform the actual refresh.')
    return parser.parse_args()

if __name__ == '__main__':
    import sys
    sys.exit(main(parse_options()))
<commit_msg>[#4142] Add faulthandler to repo refresh script.<commit_after>import logging

import faulthandler
from pylons import c
from ming.orm import ThreadLocalORMSession

from allura import model as M
from allura.lib.utils import chunked_find

log = logging.getLogger(__name__)

def main(options):
    q_project = {}
    if options.nbhd:
        nbhd = M.Neighborhood.query.get(url_prefix=options.nbhd)
        if not nbhd:
            return "Invalid neighborhood url prefix."
        q_project['neighborhood_id'] = nbhd._id
    if options.project:
        q_project['shortname'] = options.project
    elif options.project_regex:
        q_project['shortname'] = {'$regex': options.project_regex}

    log.info('Refreshing repositories')
    if options.clean:
        log.info('Removing all repository objects')
        M.repo.CommitDoc.m.remove({})
        M.repo.TreeDoc.m.remove({})
        M.repo.TreesDoc.m.remove({})
        M.repo.DiffInfoDoc.m.remove({})
        M.repo.CommitRunDoc.m.remove({})

    for chunk in chunked_find(M.Project, q_project):
        for p in chunk:
            log.info("Refreshing repos for project '%s'." % p.shortname)
            if options.dry_run:
                continue
            c.project = p
            if options.mount_point:
                mount_points = [options.mount_point]
            else:
                mount_points = [ac.options.mount_point for ac in
                                M.AppConfig.query.find(dict(project_id=p._id))]
            for app in (p.app_instance(mp) for mp in mount_points):
                c.app = app
                if not hasattr(app, 'repo'): continue
                try:
                    c.app.repo._impl._setup_hooks()
                except:
                    log.exception('Error setting up hooks for %r', c.app.repo)
                try:
                    if options.all:
                        log.info('Refreshing ALL commits in %r', c.app.repo)
                    else:
                        log.info('Refreshing NEW commits in %r', c.app.repo)
                    c.app.repo.refresh(options.all, notify=options.notify)
                except:
                    log.exception('Error refreshing %r', c.app.repo)
        ThreadLocalORMSession.flush_all()
        ThreadLocalORMSession.close_all()

def parse_options():
    import argparse
    parser = argparse.ArgumentParser(description='Scan repos on filesytem and '
            'update repo metadata in MongoDB. Run for all repos (no args), '
            'or restrict by neighborhood, project, or code tool mount point.')
    parser.add_argument('--nbhd', action='store', default='', dest='nbhd',
            help='Restrict update to a particular neighborhood, e.g. /p/.')
    parser.add_argument('--project', action='store', default='', dest='project',
            help='Restrict update to a particular project. To specify a '
            'subproject, use a slash: project/subproject.')
    parser.add_argument('--project-regex', action='store', default='',
            dest='project_regex',
            help='Restrict update to projects for which the shortname matches '
            'the provided regex.')
    parser.add_argument('--mount_point', default='', dest='mount_point',
            help='Restrict update to repos at the given tool mount point. ')
    parser.add_argument('--clean', action='store_true', dest='clean',
            default=False, help='Remove all repo-related mongo docs before '
            'refresh.')
    parser.add_argument('--all', action='store_true', dest='all', default=False,
            help='Refresh all commits (not just the ones that are new).')
    parser.add_argument('--notify', action='store_true', dest='notify',
            default=False, help='Send email notifications of new commits.')
    parser.add_argument('--dry-run', action='store_true', dest='dry_run',
            default=False, help='Log names of projects that would have their '
            'repos refreshed, but do not perform the actual refresh.')
    return parser.parse_args()

if __name__ == '__main__':
    import sys
    faulthandler.enable()
    sys.exit(main(parse_options()))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__version__ = '0.5.2'

<commit_msg>Updated version<commit_after># -*- coding: utf-8 -*-

__version__ = '0.6.0'

<|endoftext|>


<commit_before>from threading import Thread
import atexit

from annotypes import Anno, Array
from tornado.ioloop import IOLoop

from malcolm.core import Table

class IOLoopHelper(object):
    _loop = None  # type: IOLoop
    _thread = None  # type: Thread

    @classmethod
    def loop(cls):
        if cls._loop is None:
            loop = IOLoop.current()
            cls._loop = loop

            def run():
                loop.start()
                # loop.stop() called from somewhere else
                loop.close()

            cls._thread = Thread(target=run)
            cls._thread.setDaemon(True)
            cls._thread.start()
            atexit.register(cls.stop)
        return cls._loop

    @classmethod
    def call(cls, func, *args, **kwargs):
        cls.loop().add_callback(func, *args, **kwargs)

    @classmethod
    def stop(cls):
        if cls._loop is not None:
            # Remove cls._loop so all other stop() methods are no-ops
            loop = cls._loop
            cls._loop = None
            loop.add_callback(loop.stop)
            # Wait until done
            cls._thread.join()
            cls._thread = None

with Anno("The Malcolm Resource Identifier for the Block"):
    AMris = Array[str]
with Anno("A human readable label for the Block"):
    ALabels = Array[str]

class BlockTable(Table):
    def __init__(self, mri, label):
        # type: (AMris, ALabels) -> None
        self.mri = mri
        self.label = label
<commit_msg>Create event loop when running in python3<commit_after>from threading import Thread
import atexit

from annotypes import Anno, Array
from six import PY3
from tornado.ioloop import IOLoop

from malcolm.core import Table

class IOLoopHelper(object):
    _loop = None  # type: IOLoop
    _thread = None  # type: Thread

    @classmethod
    def loop(cls):
        if cls._loop is None:
            if PY3:
                # the event loop is not created automatically if we are not
                # the main thread
                import asyncio
                asyncio.set_event_loop(asyncio.new_event_loop())
            loop = IOLoop.current()
            cls._loop = loop

            def run():
                loop.start()
                # loop.stop() called from somewhere else
                loop.close()

            cls._thread = Thread(target=run)
            cls._thread.setDaemon(True)
            cls._thread.start()
            atexit.register(cls.stop)
        return cls._loop

    @classmethod
    def call(cls, func, *args, **kwargs):
        cls.loop().add_callback(func, *args, **kwargs)

    @classmethod
    def stop(cls):
        if cls._loop is not None:
            # Remove cls._loop so all other stop() methods are no-ops
            loop = cls._loop
            cls._loop = None
            loop.add_callback(loop.stop)
            # Wait until done
            cls._thread.join()
            cls._thread = None

with Anno("The Malcolm Resource Identifier for the Block"):
    AMris = Array[str]
with Anno("A human readable label for the Block"):
    ALabels = Array[str]

class BlockTable(Table):
    def __init__(self, mri, label):
        # type: (AMris, ALabels) -> None
        self.mri = mri
        self.label = label
<|endoftext|>