<commit_before>#!/usr/bin/env python
#########################################################################################
#
# Get or set orientation of nifti 3d or 4d data.
#
# ---------------------------------------------------------------------------------------
# Copyright (c) 2014 Polytechnique Montreal <www.neuro.polymtl.ca>
# Authors: Julien Cohen-Adad
# Modified: 2014-10-18
#
# About the license: see the file LICENSE.TXT
#########################################################################################

import sys
import os
import getopt
import commands
import sct_utils as sct
import time
from sct_convert import convert
from msct_image import Image

# DEFAULT PARAMETERS
class Param:
    ## The constructor
    def __init__(self):
        self.debug = 0
        self.fname_data = ''
        self.fname_out = ''
        self.orientation = ''
        self.list_of_correct_orientation = 'RIP LIP RSP LSP RIA LIA RSA LSA IRP ILP SRP SLP IRA ILA SRA SLA RPI LPI RAI LAI RPS LPS RAS LAS PRI PLI ARI ALI PRS PLS ARS ALS IPR SPR IAR SAR IPL SPL IAL SAL PIR PSR AIR ASR PIL PSL AIL ASL'
        self.change_header = ''
        self.verbose = 0
        self.remove_tmp_files = 1

# main
#=======================================================================================================================
def main():

    # Parameters for debug mode
    if param.debug:
        print '\n*** WARNING: DEBUG MODE ON ***\n'
        # get path of the testing data
        status, path_sct_data = commands.getstatusoutput('echo $SCT_TESTING_DATA_DIR')
        param.fname_data = path_sct_data+'/dmri/dwi_moco_mean.nii.gz'
        param.orientation = ''
        param.change_header = ''
        param.remove_tmp_files = 0
        param.verbose = 1
    else:
        # Check input parameters
        try:
            opts, args = getopt.getopt(sys.argv[1:], 'hi:o:r:s:a:v:')
        except getopt.GetoptError:
            usage()
        if not opts:
            usage()
        for opt, arg in opts:
            if opt == '-h':
                usage()
            elif opt in '-i':
                param.fname_data = arg
            elif opt in '-o':
                param.fname_out = arg
            elif opt in '-r':
                param.remove_tmp_files = int(arg)
            elif opt in '-s':
                param.orientation = arg
            elif opt in '-t':
                param.threshold = arg
            elif opt in '-a':
                param.change_header = arg
            elif opt in '-v':
                param.verbose = int(arg)

    # run main program
    get_or_set_orientation()

# get_or_set_orientation
#=======================================================================================================================
def get_or_set_orientation():

    fsloutput = 'export FSLOUTPUTTYPE=NIFTI; '  # for faster processing, all outputs are in NIFTI

    # display usage if a mandatory argument is not provided
    if param.fname_data == '':
        sct.printv('ERROR: All mandatory arguments are not provided. See usage.', 1, 'error')

    # check existence of input files
    sct.printv('\ncheck existence of input files...', param.verbose)
    sct.check_file_exist(param.fname_data, param.verbose)

    # find what to do
    if param.orientation == '' and param.change_header is '':
        todo = 'get_orientation'
    else:
        todo = 'set_orientation'
        # check if orientation is correct
        if check_orientation_input():
            sct.printv('\nERROR in '+os.path.basename(__file__)+': orientation is not recognized. Use one of the following orientation: '+param.list_of_correct_orientation+'\n', 1, 'error')
            sys.exit(2)

    # display input parameters
    sct.printv('\nInput parameters:', param.verbose)
    sct.printv('  data ..................'+param.fname_data, param.verbose)

    # Extract path/file/extension
    path_data, file_data, ext_data = sct.extract_fname(param.fname_data)
    if param.fname_out == '':
        # path_out, file_out, ext_out = '', file_data+'_'+param.orientation, ext_data
        fname_out = path_data+file_data+'_'+param.orientation+ext_data
    else:
        fname_out = param.fname_out

    # create temporary folder
    sct.printv('\nCreate temporary folder...', param.verbose)
    path_tmp = sct.slash_at_the_end('tmp.'+time.strftime("%y%m%d%H%M%S"), 1)
    sct.run('mkdir '+path_tmp, param.verbose)

    # Copying input data to tmp folder and convert to nii
    # NB: cannot use c3d here because c3d cannot convert 4D data.
    sct.printv('\nCopying input data to tmp folder and convert to nii...', param.verbose)
    sct.run('cp '+param.fname_data+' '+path_tmp+'data'+ext_data, param.verbose)

    # go to tmp folder
    os.chdir(path_tmp)

    # convert to nii format
    convert('data'+ext_data, 'data.nii')
    # sct.run('fslchfiletype NIFTI data', param.verbose)

    # Get dimensions of data
    sct.printv('\nGet dimensions of data...', param.verbose)
    nx, ny, nz, nt, px, py, pz, pt = Image('data.nii').dim
    sct.printv('  ' + str(nx) + ' x ' + str(ny) + ' x ' + str(nz)+ ' x ' + str(nt), param.verbose)

    # if 4d, loop across the data
    if nt == 1:
        if todo == 'set_orientation':
            # set orientation
            sct.printv('\nChange orientation...', param.verbose)
            if param.change_header is '':
                set_orientation('data.nii', param.orientation, 'data_orient.nii')
            else:
                set_orientation('data.nii', param.change_header, 'data_orient.nii', True)
        elif todo == 'get_orientation':
            # get orientation
            sct.printv('\nGet orientation...', param.verbose)
            sct.printv(get_orientation('data.nii'), 1)

    else:
        # split along T dimension
        sct.printv('\nSplit along T dimension...', param.verbose)
        sct.run(fsloutput+'fslsplit data data_T', param.verbose)

        if todo == 'set_orientation':
            # set orientation
            sct.printv('\nChange orientation...', param.verbose)
            for it in range(nt):
                file_data_split = 'data_T'+str(it).zfill(4)+'.nii'
                file_data_split_orient = 'data_orient_T'+str(it).zfill(4)+'.nii'
                set_orientation(file_data_split, param.orientation, file_data_split_orient)
            # Merge files back
            sct.printv('\nMerge file back...', param.verbose)
            cmd = fsloutput+'fslmerge -t data_orient'
            for it in range(nt):
                file_data_split_orient = 'data_orient_T'+str(it).zfill(4)+'.nii'
                cmd = cmd+' '+file_data_split_orient
            sct.run(cmd, param.verbose)

        elif todo == 'get_orientation':
            sct.printv('\nGet orientation...', param.verbose)
            sct.printv(get_orientation('data_T0000.nii'), 1)

    # come back to parent folder
    os.chdir('..')

    # Generate output files
    if todo == 'set_orientation':
        sct.printv('\nGenerate output files...', param.verbose)
        sct.generate_output_file(path_tmp+'data_orient.nii', fname_out)

    # Remove temporary files
    if param.remove_tmp_files == 1:
        sct.printv('\nRemove temporary files...', param.verbose)
        sct.run('rm -rf '+path_tmp, param.verbose)

    # to view results
    if todo == 'set_orientation':
        sct.printv('\nDone! To view results, type:', param.verbose)
        sct.printv('fslview '+fname_out+' &', param.verbose, 'code')
        print

# check_orientation_input
# ==========================================================================================
def check_orientation_input():
    """check if orientation input by user is correct"""

    if param.orientation in param.list_of_correct_orientation:
        return 0
    else:
        return -1

# get_orientation (uses FSL)
# ==========================================================================================
def get_orientation(fname):
    status, output = sct.run('isct_orientation3d -i '+fname+' -get ', 0)
    orientation = output[26:]
    return orientation

# set_orientation
# ==========================================================================================
def set_orientation(fname_in, orientation, fname_out, inversion=False):
    if not inversion:
        sct.run('isct_orientation3d -i '+fname_in+' -orientation '+orientation+' -o '+fname_out, 0)
    else:
        from msct_image import Image
        input_image = Image(fname_in)
        input_image.change_orientation(orientation, True)
        input_image.setFileName(fname_out)
        input_image.save()
    # return full path
    return os.path.abspath(fname_out)

# Print usage
# ==========================================================================================
def usage():
    print """
"""+os.path.basename(__file__)+"""
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Part of the Spinal Cord Toolbox <https://sourceforge.net/projects/spinalcordtoolbox>

DESCRIPTION
  Get or set orientation of 3D or 4D data. Available orientations are:
  """+param_default.list_of_correct_orientation+"""

USAGE
  Get orientation: """+os.path.basename(__file__)+""" -i <data>
  Set orientation: """+os.path.basename(__file__)+""" -i <data> -s <orient>

MANDATORY ARGUMENTS
  -i <file>        image to get or set orientation from. Can be 3D or 4D.

OPTIONAL ARGUMENTS
  -s <orient>      orientation. Default=None.
  -o <fname_out>   output file name. Default=<file>_<orient>.<ext>.
  -a <orient>      actual orientation of image data (for corrupted data). Change the data
                     orientation to match orientation in the header.
  -r {0,1}         remove temporary files. Default="""+str(param_default.remove_tmp_files)+"""
  -v {0,1}         verbose. Default="""+str(param_default.verbose)+"""
  -h               help. Show this message

EXAMPLE
  """+os.path.basename(__file__)+""" -i dwi.nii.gz -s RPI\n"""

    # exit program
    sys.exit(2)

#=======================================================================================================================
# Start program
#=======================================================================================================================
if __name__ == "__main__":
    # initialize parameters
    param = Param()
    param_default = Param()
    # call main function
    main()
<commit_msg>DOC: added TODO<commit_after>#!/usr/bin/env python
#########################################################################################
#
# Get or set orientation of nifti 3d or 4d data.
#
# ---------------------------------------------------------------------------------------
# Copyright (c) 2014 Polytechnique Montreal <www.neuro.polymtl.ca>
# Authors: Julien Cohen-Adad
# Modified: 2014-10-18
#
# About the license: see the file LICENSE.TXT
#########################################################################################

# TODO: add exception in case set_orientation does not output file.

import sys
import os
import getopt
import commands
import sct_utils as sct
import time
from sct_convert import convert
from msct_image import Image

# DEFAULT PARAMETERS
class Param:
    ## The constructor
    def __init__(self):
        self.debug = 0
        self.fname_data = ''
        self.fname_out = ''
        self.orientation = ''
        self.list_of_correct_orientation = 'RIP LIP RSP LSP RIA LIA RSA LSA IRP ILP SRP SLP IRA ILA SRA SLA RPI LPI RAI LAI RPS LPS RAS LAS PRI PLI ARI ALI PRS PLS ARS ALS IPR SPR IAR SAR IPL SPL IAL SAL PIR PSR AIR ASR PIL PSL AIL ASL'
        self.change_header = ''
        self.verbose = 0
        self.remove_tmp_files = 1

# main
#=======================================================================================================================
def main():

    # Parameters for debug mode
    if param.debug:
        print '\n*** WARNING: DEBUG MODE ON ***\n'
        # get path of the testing data
        status, path_sct_data = commands.getstatusoutput('echo $SCT_TESTING_DATA_DIR')
        param.fname_data = path_sct_data+'/dmri/dwi_moco_mean.nii.gz'
        param.orientation = ''
        param.change_header = ''
        param.remove_tmp_files = 0
        param.verbose = 1
    else:
        # Check input parameters
        try:
            opts, args = getopt.getopt(sys.argv[1:], 'hi:o:r:s:a:v:')
        except getopt.GetoptError:
            usage()
        if not opts:
            usage()
        for opt, arg in opts:
            if opt == '-h':
                usage()
            elif opt in '-i':
                param.fname_data = arg
            elif opt in '-o':
                param.fname_out = arg
            elif opt in '-r':
                param.remove_tmp_files = int(arg)
            elif opt in '-s':
                param.orientation = arg
            elif opt in '-t':
                param.threshold = arg
            elif opt in '-a':
                param.change_header = arg
            elif opt in '-v':
                param.verbose = int(arg)

    # run main program
    get_or_set_orientation()

# get_or_set_orientation
#=======================================================================================================================
def get_or_set_orientation():

    fsloutput = 'export FSLOUTPUTTYPE=NIFTI; '  # for faster processing, all outputs are in NIFTI

    # display usage if a mandatory argument is not provided
    if param.fname_data == '':
        sct.printv('ERROR: All mandatory arguments are not provided. See usage.', 1, 'error')

    # check existence of input files
    sct.printv('\ncheck existence of input files...', param.verbose)
    sct.check_file_exist(param.fname_data, param.verbose)

    # find what to do
    if param.orientation == '' and param.change_header is '':
        todo = 'get_orientation'
    else:
        todo = 'set_orientation'
        # check if orientation is correct
        if check_orientation_input():
            sct.printv('\nERROR in '+os.path.basename(__file__)+': orientation is not recognized. Use one of the following orientation: '+param.list_of_correct_orientation+'\n', 1, 'error')
            sys.exit(2)

    # display input parameters
    sct.printv('\nInput parameters:', param.verbose)
    sct.printv('  data ..................'+param.fname_data, param.verbose)

    # Extract path/file/extension
    path_data, file_data, ext_data = sct.extract_fname(param.fname_data)
    if param.fname_out == '':
        # path_out, file_out, ext_out = '', file_data+'_'+param.orientation, ext_data
        fname_out = path_data+file_data+'_'+param.orientation+ext_data
    else:
        fname_out = param.fname_out

    # create temporary folder
    sct.printv('\nCreate temporary folder...', param.verbose)
    path_tmp = sct.slash_at_the_end('tmp.'+time.strftime("%y%m%d%H%M%S"), 1)
    sct.run('mkdir '+path_tmp, param.verbose)

    # Copying input data to tmp folder and convert to nii
    # NB: cannot use c3d here because c3d cannot convert 4D data.
    sct.printv('\nCopying input data to tmp folder and convert to nii...', param.verbose)
    sct.run('cp '+param.fname_data+' '+path_tmp+'data'+ext_data, param.verbose)

    # go to tmp folder
    os.chdir(path_tmp)

    # convert to nii format
    convert('data'+ext_data, 'data.nii')
    # sct.run('fslchfiletype NIFTI data', param.verbose)

    # Get dimensions of data
    sct.printv('\nGet dimensions of data...', param.verbose)
    nx, ny, nz, nt, px, py, pz, pt = Image('data.nii').dim
    sct.printv('  ' + str(nx) + ' x ' + str(ny) + ' x ' + str(nz)+ ' x ' + str(nt), param.verbose)

    # if 4d, loop across the data
    if nt == 1:
        if todo == 'set_orientation':
            # set orientation
            sct.printv('\nChange orientation...', param.verbose)
            if param.change_header is '':
                set_orientation('data.nii', param.orientation, 'data_orient.nii')
            else:
                set_orientation('data.nii', param.change_header, 'data_orient.nii', True)
        elif todo == 'get_orientation':
            # get orientation
            sct.printv('\nGet orientation...', param.verbose)
            sct.printv(get_orientation('data.nii'), 1)

    else:
        # split along T dimension
        sct.printv('\nSplit along T dimension...', param.verbose)
        sct.run(fsloutput+'fslsplit data data_T', param.verbose)

        if todo == 'set_orientation':
            # set orientation
            sct.printv('\nChange orientation...', param.verbose)
            for it in range(nt):
                file_data_split = 'data_T'+str(it).zfill(4)+'.nii'
                file_data_split_orient = 'data_orient_T'+str(it).zfill(4)+'.nii'
                set_orientation(file_data_split, param.orientation, file_data_split_orient)
            # Merge files back
            sct.printv('\nMerge file back...', param.verbose)
            cmd = fsloutput+'fslmerge -t data_orient'
            for it in range(nt):
                file_data_split_orient = 'data_orient_T'+str(it).zfill(4)+'.nii'
                cmd = cmd+' '+file_data_split_orient
            sct.run(cmd, param.verbose)

        elif todo == 'get_orientation':
            sct.printv('\nGet orientation...', param.verbose)
            sct.printv(get_orientation('data_T0000.nii'), 1)

    # come back to parent folder
    os.chdir('..')

    # Generate output files
    if todo == 'set_orientation':
        sct.printv('\nGenerate output files...', param.verbose)
        sct.generate_output_file(path_tmp+'data_orient.nii', fname_out)

    # Remove temporary files
    if param.remove_tmp_files == 1:
        sct.printv('\nRemove temporary files...', param.verbose)
        sct.run('rm -rf '+path_tmp, param.verbose)

    # to view results
    if todo == 'set_orientation':
        sct.printv('\nDone! To view results, type:', param.verbose)
        sct.printv('fslview '+fname_out+' &', param.verbose, 'code')
        print

# check_orientation_input
# ==========================================================================================
def check_orientation_input():
    """check if orientation input by user is correct"""

    if param.orientation in param.list_of_correct_orientation:
        return 0
    else:
        return -1

# get_orientation (uses FSL)
# ==========================================================================================
def get_orientation(fname):
    status, output = sct.run('isct_orientation3d -i '+fname+' -get ', 0)
    orientation = output[26:]
    return orientation

# set_orientation
# ==========================================================================================
def set_orientation(fname_in, orientation, fname_out, inversion=False):
    if not inversion:
        sct.run('isct_orientation3d -i '+fname_in+' -orientation '+orientation+' -o '+fname_out, 0)
    else:
        from msct_image import Image
        input_image = Image(fname_in)
        input_image.change_orientation(orientation, True)
        input_image.setFileName(fname_out)
        input_image.save()
    # return full path
    return os.path.abspath(fname_out)

# Print usage
# ==========================================================================================
def usage():
    print """
"""+os.path.basename(__file__)+"""
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Part of the Spinal Cord Toolbox <https://sourceforge.net/projects/spinalcordtoolbox>

DESCRIPTION
  Get or set orientation of 3D or 4D data. Available orientations are:
  """+param_default.list_of_correct_orientation+"""

USAGE
  Get orientation: """+os.path.basename(__file__)+""" -i <data>
  Set orientation: """+os.path.basename(__file__)+""" -i <data> -s <orient>

MANDATORY ARGUMENTS
  -i <file>        image to get or set orientation from. Can be 3D or 4D.

OPTIONAL ARGUMENTS
  -s <orient>      orientation. Default=None.
  -o <fname_out>   output file name. Default=<file>_<orient>.<ext>.
  -a <orient>      actual orientation of image data (for corrupted data). Change the data
                     orientation to match orientation in the header.
  -r {0,1}         remove temporary files. Default="""+str(param_default.remove_tmp_files)+"""
  -v {0,1}         verbose. Default="""+str(param_default.verbose)+"""
  -h               help. Show this message

EXAMPLE
  """+os.path.basename(__file__)+""" -i dwi.nii.gz -s RPI\n"""

    # exit program
    sys.exit(2)

#=======================================================================================================================
# Start program
#=======================================================================================================================
if __name__ == "__main__":
    # initialize parameters
    param = Param()
    param_default = Param()
    # call main function
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
import subprocess
import sys
from bs4 import BeautifulSoup
from collections import defaultdict
from lxml import etree

FREELING_COMMAND = "analyze -f es.cfg --noloc --noner --nonumb --nodate --noquant".split()

corpus_file = sys.argv[1]
output_dir = sys.argv[2]

print >> sys.stderr, "Loading Corpus"
corpus = etree.parse(corpus_file)

print >> sys.stderr, "Parsing Corpus"

sentences = corpus.findall(".//sentence")
total = len(sentences)

verbs = defaultdict(int)
senses = defaultdict(int)

for idx, sentence in enumerate(sentences, start=1):
    print >> sys.stderr, "Parsing sentence {} of {}".format(idx, total)

    verb_words = sentence.findall(".//word[@verb='true']")

    if len(verb_words) == 0:
        continue

    lexical = sentence.find("lexical")

    verb = lexical.attrib["verb"]
    verbs[verb] += 1
    verb_id = "{:05d}".format(verbs[verb])

    sense = u"{}-{}".format(verb, lexical.attrib["sense"])
    senses[sense] += 1

    verb_positions = {(int(vw.attrib["id"]) - 1) for vw in verb_words}
    verb_forms = {vw.text for vw in verb_words}

    raw_sentence = re.sub(r'\s\s+', ' ',
                          BeautifulSoup(
                              etree.tostring(sentence.find("content"), encoding='UTF-8'), 'lxml'
                          ).get_text().replace('\n', ' ').strip())

    proc = subprocess.Popen(FREELING_COMMAND, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
    (pipe_out, pipe_err) = proc.communicate(input=raw_sentence.encode('UTF-8'))

    if pipe_err.strip() != '':
        print >> sys.stderr, pipe_err
        sys.exit(1)

    words = [w.strip().split()[:3] for w in pipe_out.split('\n') if w.strip() != '']

    sentence_shift = len(words) - len(raw_sentence.split())
    verb_found = False

    for widx, word in enumerate(words):
        if word[1] == verb and word[0] in verb_forms:
            if (idx+sentence_shift) in verb_positions:
                word.append("verb")
                verb_found = True
            else:
                print >> sys.stderr, u"Possible conflict in file {} - sentence {} - sense {}".format(
                    verb, verb_id, sense
                )

    if not verb_found:
        print >> sys.stderr, u"Verb not found in file {} - sentence {} - sense {}".format(verb, verb_id, sense)

    with open(os.path.join(output_dir, verb), "a") as fout:
        fout.write("#{} ".format(verb_id) + sense.encode('UTF-8') + '\n')

        for word in words:
            fout.write('\t'.join(word) + '\n')

        fout.write('\n')

print >> sys.stderr, "\nAll corpus parsed"
print >> sys.stderr, "Saving stats of senses"

with open("senses_stats.txt", "w") as fout:
    for sense in sorted(senses):
        fout.write(u"{} {}".format(sense, senses[sense]))
<commit_msg>Solved the potential problems<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import re
import subprocess
import sys
from bs4 import BeautifulSoup
from collections import defaultdict
from lxml import etree

FREELING_COMMAND = "analyze -f es.cfg --noloc --noner --nonumb --nodate --noquant".split()

corpus_file = sys.argv[1]
output_dir = sys.argv[2]

print >> sys.stderr, "Loading Corpus"
corpus = etree.parse(corpus_file)

print >> sys.stderr, "Parsing Corpus"

sentences = corpus.findall(".//sentence")
total = len(sentences)

verbs = defaultdict(int)
senses = defaultdict(int)

for idx, sentence in enumerate(sentences, start=1):
    print >> sys.stderr, "Parsing sentence {} of {}".format(idx, total)

    verb_words = sentence.findall(".//word[@verb='true']")

    if len(verb_words) == 0:
        continue

    lexical = sentence.find("lexical")

    verb = lexical.attrib["verb"]
    verbs[verb] += 1
    verb_id = "{:05d}".format(verbs[verb])

    sense = u"{}-{}".format(verb, lexical.attrib["sense"])
    senses[sense] += 1

    verb_positions = {(int(vw.attrib["id"]) - 1) for vw in verb_words}
    verb_forms = {vw.text for vw in verb_words}

    raw_sentence = re.sub(r'\s\s+', ' ',
                          BeautifulSoup(
                              etree.tostring(sentence.find("content"), encoding='UTF-8'), 'lxml'
                          ).get_text().replace('\n', ' ').strip())

    proc = subprocess.Popen(FREELING_COMMAND, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
    (pipe_out, pipe_err) = proc.communicate(input=raw_sentence.encode('UTF-8'))

    if pipe_err.strip() != '':
        print >> sys.stderr, pipe_err
        sys.exit(1)

    words = [w.decode('UTF-8').strip().split()[:3] for w in pipe_out.split('\n') if w.strip() != '']

    sentence_shift = len(words) - len(raw_sentence.split())
    verb_found = False

    for widx, word in enumerate(words):
        if word[1] == verb and word[0] in verb_forms:
            if (widx+sentence_shift) in verb_positions:
                word.append("verb")
                verb_found = True
            else:
                print >> sys.stderr, u"Possible conflict in file {} - sentence {} - sense {}".format(
                    verb, verb_id, sense
                )

    if not verb_found:
        print >> sys.stderr, u"Verb not found in file {} - sentence {} - sense {}".format(verb, verb_id, sense)

    with open(os.path.join(output_dir, verb), "a") as fout:
        fout.write("#{} ".format(verb_id) + sense.encode('UTF-8') + '\n')

        for word in words:
            fout.write('\t'.join(word).encode('UTF-8') + '\n')

        fout.write('\n')

print >> sys.stderr, "\nAll corpus parsed"
print >> sys.stderr, "Saving stats of senses"

with open("senses_stats.txt", "w") as fout:
    for sense in sorted(senses):
        fout.write(u"{} {}\n".format(sense, senses[sense]).encode('UTF-8'))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

r"""Tool for viewing masters, their hosts and their ports.

Has three modes:
  a) In normal mode, simply prints the list of all known masters, sorted by
     hostname, along with their associated ports, for the perusal of the user.
  b) In --audit mode, tests to make sure that no masters conflict/overlap on
     ports (even on different masters) and that no masters have unexpected
     ports (i.e. differences of more than 100 between master, slave, and alt).
     Audit mode returns non-zero error code if conflicts are found. In audit
     mode, --verbose causes it to print human-readable output as well.
  c) In --find mode, prints a set of available ports for the given master.

Ports are well-formed if they follow this spec:
XYYZZ
|| \__The last two digits identify the master, e.g. master.chromium
|\____The second and third digits identify the master host, e.g. master1.golo
\_____The first digit identifies the port type, e.g. master_port

In particular,
X==3: master_port (Web display)
X==4: slave_port (for slave TCP/RCP connections)
X==5: master_port_alt (Alt web display, with "force build" disabled)
The values X==1,2, and 6 are not used due to too few free ports in those ranges.

In all modes, --csv causes the output (if any) to be formatted as
comma-separated values.
"""

import json
import optparse
import os
import sys

# Should be <snip>/build/scripts/tools
SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
BASE_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir))
sys.path.insert(0, os.path.join(BASE_DIR, 'scripts'))
sys.path.insert(0, os.path.join(BASE_DIR, 'site_config'))

import config_bootstrap
from config_bootstrap import Master
from slave import bootstrap

# These are ports which are likely to be used by another service, or which have
# been officially reserved by IANA.
PORT_BLACKLIST = set([
    # We don't care about reserved ports below 30000, the lowest port we use.
    31457,  # TetriNET
    31620,  # LM-MON
    33434,  # traceroute
    34567,  # EDI service
    35357,  # OpenStack ID Service
    40000,  # SafetyNET p Real-time Industrial Ethernet protocol
    41794,  # Crestron Control Port
    41795,  # Crestron Control Port
    45824,  # Server for the DAI family of client-server products
    47001,  # WinRM
    47808,  # BACnet Building Automation and Control Networks
    48653,  # Robot Raconteur transport
    49151,  # Reserved
    # There are no reserved ports in the 50000-65535 range.
])

PORT_TYPE_MAP = {
  'port': Master.Base.MASTER_PORT,
  'slave_port': Master.Base.SLAVE_PORT,
  'alt_port': Master.Base.MASTER_PORT_ALT,
}

# A map of (full) master host to master class used in 'get_master_class'
# lookup.
MASTER_HOST_MAP = dict((m.master_host, m)
                       for m in Master.get_base_masters())

def get_args():
  """Process command-line arguments."""
  parser = optparse.OptionParser(
      description='Tool to list all masters along with their hosts and ports.')

  parser.add_option('-l', '--list', action='store_true', default=False,
                    help='Output a list of all ports in use by all masters. '
                         'Default behavior if no other options are given.')
  parser.add_option('--sort-by', action='store',
                    help='Define the primary key by which rows are sorted. '
                    'Possible values are: "port", "alt_port", "slave_port", '
                    '"host", and "name". Only one value is allowed (for now).')
  parser.add_option('--find', action='store', metavar='NAME',
                    help='Outputs three available ports for the given master.')
  parser.add_option('--audit', action='store_true', default=False,
                    help='Output conflict diagnostics and return an error '
                         'code if misconfigurations are found.')
  parser.add_option('--presubmit', action='store_true', default=False,
                    help='The same as --audit, but prints no output. '
                         'Overrides all other options.')

  parser.add_option('--csv', action='store_true', default=False,
                    help='Print output in comma-separated values format.')
  parser.add_option('--json', action='store_true', default=False,
                    help='Print output in JSON format. Overrides --csv.')
  parser.add_option('--full-host-names', action='store_true', default=False,
                    help='Refrain from truncating the master host names')

  opts, _ = parser.parse_args()

  opts.verbose = True

  if not (opts.find or opts.audit or opts.presubmit):
    opts.list = True

  if opts.presubmit:
    opts.list = False
    opts.audit = True
    opts.find = False
    opts.verbose = False

  return opts

def getint(string):
  """Try to parse an int (port number) from a string."""
  try:
    ret = int(string)
  except ValueError:
    ret = 0
  return ret

def human_print(lines, verbose):
  """Given a list of lists of tokens, pretty prints them in columns.

  Requires all lines to have the same number of tokens, as otherwise the desired
  behavior is not clearly defined (i.e. which columns should be left empty for
  shorter lines?).
  """

  for line in lines:
    assert len(line) == len(lines[0])

  num_cols = len(lines[0])
  format_string = ''
  for col in xrange(num_cols - 1):
    col_width = max(len(str(line[col])) for line in lines) + 1
    format_string += '%-' + str(col_width) + 's '
  format_string += '%s'

  if verbose:
    for line in lines:
      print format_string % tuple(line)

def csv_print(lines, verbose):
  """Given a list of lists of tokens, prints them as comma-separated values.

  Requires all lines to have the same number of tokens, as otherwise the desired
  behavior is not clearly defined (i.e. which columns should be left empty for
  shorter lines?).
  """

  for line in lines:
    assert len(line) == len(lines[0])

  if verbose:
    for line in lines:
      print ','.join(str(t) for t in line)
    print '\n'

def master_map(masters, output, opts):
  """Display a list of masters and their associated hosts and ports."""

  host_key = 'host' if not opts.full_host_names else 'fullhost'

  lines = [['Master', 'Config Dir', 'Host', 'Web port', 'Slave port',
            'Alt port', 'URL']]
  for master in masters:
    lines.append([
        master['name'], master['dirname'], master[host_key], master['port'],
        master['slave_port'], master['alt_port'], master['buildbot_url']])

  output(lines, opts.verbose)

def get_master_class(master):
  return MASTER_HOST_MAP.get(master['fullhost'])

def get_master_port(master):
  master_class = get_master_class(master)
  if not master_class:
    return None
  return '%02d' % (master_class.master_port_base,)

def master_audit(masters, output, opts):
  """Check for port conflicts and misconfigurations on masters.

  Outputs lists of masters whose ports conflict and who have misconfigured
  ports. If any misconfigurations are found, returns a non-zero error code.
  """
  # Return value. Will be set to 1 the first time we see an error.
  ret = 0

  # Look for masters using the wrong ports for their port types.
  lines = [['Masters with misconfigured ports based on port type:']]
  for master in masters:
    for port_type, port_digit in PORT_TYPE_MAP.iteritems():
      if not str(master[port_type]).startswith(str(port_digit)):
        ret = 1
        lines.append([master['name']])
        break
  output(lines, opts.verbose)
  print

  # Look for masters using the wrong ports for their port types.
  lines = [['Masters with misconfigured ports based on hostname:']]
  for master in masters:
    digits = get_master_port(master)
    if digits:
      for port in PORT_TYPE_MAP.iterkeys():
        if str(master[port])[1:3] != digits:
          ret = 1
          lines.append([master['name']])
          break
  output(lines, opts.verbose)
  print

  # Look for masters configured to use the same ports.
  web_ports = {}
  slave_ports = {}
  alt_ports = {}
  all_ports = {}
  for master in masters:
    web_ports.setdefault(master['port'], []).append(master)
    slave_ports.setdefault(master['slave_port'], []).append(master)
    alt_ports.setdefault(master['alt_port'], []).append(master)

    for port_type in ('port', 'slave_port', 'alt_port'):
      all_ports.setdefault(master[port_type], []).append(master)

  # Check for blacklisted ports.
  lines = [['Blacklisted port', 'Master', 'Host']]
  for port, lst in all_ports.iteritems():
    if port in PORT_BLACKLIST:
      ret = 1
      for m in lst:
        lines.append([port, m['name'], m['host']])
  output(lines, opts.verbose)
  print

  # Check for conflicting web ports.
  lines = [['Web port', 'Master', 'Host']]
  for port, lst in web_ports.iteritems():
    if len(lst) > 1:
      ret = 1
      for m in lst:
        lines.append([port, m['name'], m['host']])
  output(lines, opts.verbose)
  print

  # Check for conflicting slave ports.
  lines = [['Slave port', 'Master', 'Host']]
  for port, lst in slave_ports.iteritems():
    if len(lst) > 1:
      ret = 1
      for m in lst:
        lines.append([port, m['name'], m['host']])
  output(lines, opts.verbose)
  print

  # Check for conflicting alt ports.
  lines = [['Alt port', 'Master', 'Host']]
  for port, lst in alt_ports.iteritems():
    if len(lst) > 1:
      ret = 1
      for m in lst:
        lines.append([port, m['name'], m['host']])
  output(lines, opts.verbose)
  print

  # Look for masters whose port, slave_port, alt_port aren't separated by 10000.
  lines = [['Master', 'Host', 'Web port', 'Slave port', 'Alt port']]
  for master in masters:
    if (getint(master['slave_port']) - getint(master['port']) != 10000 or
        getint(master['alt_port']) - getint(master['slave_port']) != 10000):
      ret = 1
      lines.append([master['name'], master['host'],
                   master['port'], master['slave_port'], master['alt_port']])
  output(lines, opts.verbose)

  return ret

def build_port_str(master_class, port_type, digits):
  port_base = PORT_TYPE_MAP[port_type]
  port = '%d%02d%02d' % (port_base, master_class.master_port_base, digits)
  assert len(port) == 5, "Invalid port generated (%s)" % (port,)
  return port

def find_port(mastername, masters, output, opts):
  """Finds a triplet of free ports appropriate for the given master."""
  master = None
  for master in masters:
    if master['name'] == mastername:
      break
  else:
    lines = [['master %s not found' % mastername],
             ['use the list function to see all masters.']]
    output(lines, opts.verbose)
    return 1

  used_ports = set()
  for m in masters:
    for port in ('port', 'slave_port', 'alt_port'):
      used_ports.add(m.get(port, 0))
  used_ports = used_ports | PORT_BLACKLIST

  def _inner_loop():
    master_class = get_master_class(master)

    for digits in xrange(0, 100):
      port = build_port_str(master_class, 'port', digits)
      slave_port = build_port_str(master_class, 'slave_port', digits)
      alt_port = build_port_str(master_class, 'alt_port', digits)
      if all([
          int(port) not in used_ports,
          int(slave_port) not in used_ports,
          int(alt_port) not in used_ports]):
        return port, slave_port, alt_port
    return None, None, None
  port, slave_port, alt_port = _inner_loop()

  if not all([port, slave_port, alt_port]):
    lines = [['unable to find available ports on host %s' % master['fullhost']]]
    output(lines, opts.verbose)
    return 1

  lines = [['Web port', 'Slave port', 'Alt port']]
  lines.append([port, slave_port, alt_port])
  output(lines, opts.verbose)

def format_host_name(host):
  for suffix in ('.chromium.org', '.corp.google.com'):
    if host.endswith(suffix):
      return host[:-len(suffix)]
  return host

def extract_masters():
  """Extracts the data we want from a collection of possibly-masters."""
  good_masters = []
  for master in config_bootstrap.Master.get_all_masters():
    host = getattr(master, 'master_host', '')
    local_config_path = getattr(master, 'local_config_path', '')
    build_dir = os.path.basename(os.path.abspath(os.path.join(local_config_path,
                                                        os.pardir, os.pardir)))
    is_internal = build_dir == 'build_internal'
    good_masters.append({
        'name': master.__name__,
        'host': format_host_name(host),
        'fullhost': host,
        'port': getattr(master, 'master_port', 0),
        'slave_port': getattr(master, 'slave_port', 0),
        'alt_port': getattr(master, 'master_port_alt', 0),
        'buildbot_url': getattr(master, 'buildbot_url', ''),
        'dirname': os.path.basename(local_config_path),
        'internal': is_internal
    })
  return good_masters

def real_main(include_internal=False):
  opts = get_args()

  bootstrap.ImportMasterConfigs(include_internal=include_internal)

  masters = extract_masters()

  # Define sorting order
  sort_keys = ['host', 'port', 'alt_port', 'slave_port', 'name']
  # Move key specified on command-line to the front of the list
  if opts.sort_by is not None:
    try:
      index = sort_keys.index(opts.sort_by)
    except ValueError:
      pass
    else:
      sort_keys.insert(0, sort_keys.pop(index))

  for key in reversed(sort_keys):
    masters.sort(key=lambda m: m[key]) # pylint: disable=cell-var-from-loop

  if opts.csv:
    printer = csv_print
  else:
    printer = human_print

  if opts.list:
    if opts.json:
      print json.dumps(masters,
                       sort_keys=True, indent=2, separators=(',', ': '))
    else:
      master_map(masters, printer, opts)

  ret = 0
  if opts.audit or opts.presubmit:
    ret = master_audit(masters, printer, opts)

  if opts.find:
    find_port(opts.find, masters, printer, opts)

  return ret

def main():
  return real_main(include_internal=False)

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>Replaced optparse by argparse in mastermap<commit_after>#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

r"""Tool for viewing masters, their hosts and their ports.

Has three modes:
  a) In normal mode, simply prints the list of all known masters, sorted by
     hostname, along with their associated ports, for the perusal of the user.
  b) In --audit mode, tests to make sure that no masters conflict/overlap on
     ports (even on different masters) and that no masters have unexpected
     ports (i.e. differences of more than 100 between master, slave, and alt).
     Audit mode returns non-zero error code if conflicts are found. In audit
     mode, --verbose causes it to print human-readable output as well.
  c) In --find mode, prints a set of available ports for the given master.

Ports are well-formed if they follow this spec:
XYYZZ
|| \__The last two digits identify the master, e.g. master.chromium
|\____The second and third digits identify the master host, e.g. master1.golo
\_____The first digit identifies the port type, e.g. master_port

In particular,
X==3: master_port (Web display)
X==4: slave_port (for slave TCP/RCP connections)
X==5: master_port_alt (Alt web display, with "force build" disabled)
The values X==1,2, and 6 are not used due to too few free ports in those ranges.

In all modes, --csv causes the output (if any) to be formatted as
comma-separated values.
"""

import argparse
import json
import os
import sys

# Should be <snip>/build/scripts/tools
SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
BASE_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir))
sys.path.insert(0, os.path.join(BASE_DIR, 'scripts'))
sys.path.insert(0, os.path.join(BASE_DIR, 'site_config'))

import config_bootstrap
from config_bootstrap import Master
from slave import bootstrap

# These are ports which are likely to be used by another service, or which have
# been officially reserved by IANA.
PORT_BLACKLIST = set([
    # We don't care about reserved ports below 30000, the lowest port we use.
    31457,  # TetriNET
    31620,  # LM-MON
    33434,  # traceroute
    34567,  # EDI service
    35357,  # OpenStack ID Service
    40000,  # SafetyNET p Real-time Industrial Ethernet protocol
    41794,  # Crestron Control Port
    41795,  # Crestron Control Port
    45824,  # Server for the DAI family of client-server products
    47001,  # WinRM
    47808,  # BACnet Building Automation and Control Networks
    48653,  # Robot Raconteur transport
    49151,  # Reserved
    # There are no reserved ports in the 50000-65535 range.
])

PORT_TYPE_MAP = {
  'port': Master.Base.MASTER_PORT,
  'slave_port': Master.Base.SLAVE_PORT,
  'alt_port': Master.Base.MASTER_PORT_ALT,
}

# A map of (full) master host to master class used in 'get_master_class'
# lookup.
MASTER_HOST_MAP = dict((m.master_host, m)
                       for m in Master.get_base_masters())

def get_args():
  """Process command-line arguments."""
  parser = argparse.ArgumentParser(
      description='Tool to list all masters along with their hosts and ports.')

  parser.add_argument('-l', '--list', action='store_true', default=False,
                    help='Output a list of all ports in use by all masters. '
                         'Default behavior if no other options are given.')
  parser.add_argument('--sort-by', action='store',
                    help='Define the primary key by which rows are sorted. '
                    'Possible values are: "port", "alt_port", "slave_port", '
                    '"host", and "name". Only one value is allowed (for now).')
  parser.add_argument('--find', action='store', metavar='NAME',
                    help='Outputs three available ports for the given master.')
  parser.add_argument('--audit', action='store_true', default=False,
                    help='Output conflict diagnostics and return an error '
                         'code if misconfigurations are found.')
  parser.add_argument('--presubmit', action='store_true', default=False,
                    help='The same as --audit, but prints no output. '
                         'Overrides all other options.')

  parser.add_argument('--csv', action='store_true', default=False,
                    help='Print output in comma-separated values format.')
  parser.add_argument('--json', action='store_true', default=False,
                    help='Print output in JSON format. Overrides --csv.')
  parser.add_argument('--full-host-names', action='store_true', default=False,
                    help='Refrain from truncating the master host names')

  opts = parser.parse_args()

  opts.verbose = True

  if not (opts.find or opts.audit or opts.presubmit):
    opts.list = True

  if opts.presubmit:
    opts.list = False
    opts.audit = True
    opts.find = False
    opts.verbose = False

  return opts

def getint(string):
  """Try to parse an int (port number) from a string."""
  try:
    ret = int(string)
  except ValueError:
    ret = 0
  return ret

def human_print(lines, verbose):
  """Given a list of lists of tokens, pretty prints them in columns.

  Requires all lines to have the same number of tokens, as otherwise the desired
  behavior is not clearly defined (i.e. which columns should be left empty for
  shorter lines?).
  """

  for line in lines:
    assert len(line) == len(lines[0])

  num_cols = len(lines[0])
  format_string = ''
  for col in xrange(num_cols - 1):
    col_width = max(len(str(line[col])) for line in lines) + 1
    format_string += '%-' + str(col_width) + 's '
  format_string += '%s'

  if verbose:
    for line in lines:
      print format_string % tuple(line)

def csv_print(lines, verbose):
  """Given a list of lists of tokens, prints them as comma-separated values.

  Requires all lines to have the same number of tokens, as otherwise the desired
  behavior is not clearly defined (i.e. which columns should be left empty for
  shorter lines?).
  """

  for line in lines:
    assert len(line) == len(lines[0])

  if verbose:
    for line in lines:
      print ','.join(str(t) for t in line)
    print '\n'

def master_map(masters, output, opts):
  """Display a list of masters and their associated hosts and ports."""

  host_key = 'host' if not opts.full_host_names else 'fullhost'

  lines = [['Master', 'Config Dir', 'Host', 'Web port', 'Slave port',
            'Alt port', 'URL']]
  for master in masters:
    lines.append([
        master['name'], master['dirname'], master[host_key], master['port'],
        master['slave_port'], master['alt_port'], master['buildbot_url']])

  output(lines, opts.verbose)

def get_master_class(master):
  return MASTER_HOST_MAP.get(master['fullhost'])

def get_master_port(master):
  master_class = get_master_class(master)
  if not master_class:
    return None
  return '%02d' % (master_class.master_port_base,)

def master_audit(masters, output, opts):
  """Check for port conflicts and misconfigurations on masters.

  Outputs lists of masters whose ports conflict and who have misconfigured
  ports. If any misconfigurations are found, returns a non-zero error code.
  """
  # Return value. Will be set to 1 the first time we see an error.
  ret = 0

  # Look for masters using the wrong ports for their port types.
  lines = [['Masters with misconfigured ports based on port type:']]
  for master in masters:
    for port_type, port_digit in PORT_TYPE_MAP.iteritems():
      if not str(master[port_type]).startswith(str(port_digit)):
        ret = 1
        lines.append([master['name']])
        break
  output(lines, opts.verbose)
  print

  # Look for masters using the wrong ports for their port types.
  lines = [['Masters with misconfigured ports based on hostname:']]
  for master in masters:
    digits = get_master_port(master)
    if digits:
      for port in PORT_TYPE_MAP.iterkeys():
        if str(master[port])[1:3] != digits:
          ret = 1
          lines.append([master['name']])
          break
  output(lines, opts.verbose)
  print

  # Look for masters configured to use the same ports.
  web_ports = {}
  slave_ports = {}
  alt_ports = {}
  all_ports = {}
  for master in masters:
    web_ports.setdefault(master['port'], []).append(master)
    slave_ports.setdefault(master['slave_port'], []).append(master)
    alt_ports.setdefault(master['alt_port'], []).append(master)

    for port_type in ('port', 'slave_port', 'alt_port'):
      all_ports.setdefault(master[port_type], []).append(master)

  # Check for blacklisted ports.
  lines = [['Blacklisted port', 'Master', 'Host']]
  for port, lst in all_ports.iteritems():
    if port in PORT_BLACKLIST:
      ret = 1
      for m in lst:
        lines.append([port, m['name'], m['host']])
  output(lines, opts.verbose)
  print

  # Check for conflicting web ports.
  lines = [['Web port', 'Master', 'Host']]
  for port, lst in web_ports.iteritems():
    if len(lst) > 1:
      ret = 1
      for m in lst:
        lines.append([port, m['name'], m['host']])
  output(lines, opts.verbose)
  print

  # Check for conflicting slave ports.
  lines = [['Slave port', 'Master', 'Host']]
  for port, lst in slave_ports.iteritems():
    if len(lst) > 1:
      ret = 1
      for m in lst:
        lines.append([port, m['name'], m['host']])
  output(lines, opts.verbose)
  print

  # Check for conflicting alt ports.
  lines = [['Alt port', 'Master', 'Host']]
  for port, lst in alt_ports.iteritems():
    if len(lst) > 1:
      ret = 1
      for m in lst:
        lines.append([port, m['name'], m['host']])
  output(lines, opts.verbose)
  print

  # Look for masters whose port, slave_port, alt_port aren't separated by 10000.
  lines = [['Master', 'Host', 'Web port', 'Slave port', 'Alt port']]
  for master in masters:
    if (getint(master['slave_port']) - getint(master['port']) != 10000 or
        getint(master['alt_port']) - getint(master['slave_port']) != 10000):
      ret = 1
      lines.append([master['name'], master['host'],
                   master['port'], master['slave_port'], master['alt_port']])
  output(lines, opts.verbose)

  return ret

def build_port_str(master_class, port_type, digits):
  port_base = PORT_TYPE_MAP[port_type]
  port = '%d%02d%02d' % (port_base, master_class.master_port_base, digits)
  assert len(port) == 5, "Invalid port generated (%s)" % (port,)
  return port

def find_port(mastername, masters, output, opts):
  """Finds a triplet of free ports appropriate for the given master."""
  master = None
  for master in masters:
    if master['name'] == mastername:
      break
  else:
    lines = [['master %s not found' % mastername],
             ['use the list function to see all masters.']]
    output(lines, opts.verbose)
    return 1

  used_ports = set()
  for m in masters:
    for port in ('port', 'slave_port', 'alt_port'):
      used_ports.add(m.get(port, 0))
  used_ports = used_ports | PORT_BLACKLIST

  def _inner_loop():
    master_class = get_master_class(master)

    for digits in xrange(0, 100):
      port = build_port_str(master_class, 'port', digits)
      slave_port = build_port_str(master_class, 'slave_port', digits)
      alt_port = build_port_str(master_class, 'alt_port', digits)
      if all([
          int(port) not in used_ports,
          int(slave_port) not in used_ports,
          int(alt_port) not in used_ports]):
        return port, slave_port, alt_port
    return None, None, None
  port, slave_port, alt_port = _inner_loop()

  if not all([port, slave_port, alt_port]):
    lines = [['unable to find available ports on host %s' % master['fullhost']]]
    output(lines, opts.verbose)
    return 1

  lines = [['Web port', 'Slave port', 'Alt port']]
  lines.append([port, slave_port, alt_port])
  output(lines, opts.verbose)

def format_host_name(host):
  for suffix in ('.chromium.org', '.corp.google.com'):
    if host.endswith(suffix):
      return host[:-len(suffix)]
  return host

def extract_masters():
  """Extracts the data we want from a collection of possibly-masters."""
  good_masters = []
  for master in config_bootstrap.Master.get_all_masters():
    host = getattr(master, 'master_host', '')
    local_config_path = getattr(master, 'local_config_path', '')
    build_dir = os.path.basename(os.path.abspath(os.path.join(local_config_path,
                                                        os.pardir, os.pardir)))
    is_internal = build_dir == 'build_internal'
    good_masters.append({
        'name': master.__name__,
        'host': format_host_name(host),
        'fullhost': host,
        'port': getattr(master, 'master_port', 0),
        'slave_port': getattr(master, 'slave_port', 0),
        'alt_port': getattr(master, 'master_port_alt', 0),
        'buildbot_url': getattr(master, 'buildbot_url', ''),
        'dirname': os.path.basename(local_config_path),
        'internal': is_internal
    })
  return good_masters

def real_main(include_internal=False):
  opts = get_args()

  bootstrap.ImportMasterConfigs(include_internal=include_internal)

  masters = extract_masters()

  # Define sorting order
  sort_keys = ['host', 'port', 'alt_port', 'slave_port', 'name']
  # Move key specified on command-line to the front of the list
  if opts.sort_by is not None:
    try:
      index = sort_keys.index(opts.sort_by)
    except ValueError:
      pass
    else:
      sort_keys.insert(0, sort_keys.pop(index))

  for key in reversed(sort_keys):
    masters.sort(key=lambda m: m[key])  # pylint: disable=cell-var-from-loop

  if opts.csv:
    printer = csv_print
  else:
    printer = human_print

  if opts.list:
    if opts.json:
      print json.dumps(masters,
                       sort_keys=True, indent=2, separators=(',', ': '))
    else:
      master_map(masters, printer, opts)

  ret = 0
  if opts.audit or opts.presubmit:
    ret = master_audit(masters, printer, opts)

  if opts.find:
    find_port(opts.find, masters, printer, opts)

  return ret

def main():
  return real_main(include_internal=False)

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before>"""Upload i5 analysis to CyBox"""
import os
from pyiem import util
import datetime
import glob
import logging
import sys

if len(sys.argv) > 1 and sys.argv[1] == 'debug':
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    logging.debug("Setting logging to debug...")

ceiling = datetime.datetime.utcnow() - datetime.timedelta(hours=24)
os.chdir("/mesonet/share/pickup/ntrans")

local_filenames = {}
remote_filenames = {}
for fn in glob.glob("*.zip"):
    valid = datetime.datetime.strptime(fn[3:15], '%Y%m%d%H%M')
    if valid >= ceiling:
        continue
    rp = valid.strftime("/NTRANS/wxarchive/%Y/%m/%d/%H")
    if rp not in remote_filenames:
        local_filenames[rp] = []
        remote_filenames[rp] = []
    # Move file to temp fn so that we don't attempt to upload this twice
    fn2 = "%s.xfer" % (fn,)
    os.rename(fn, fn2)
    local_filenames[rp].append(fn2)
    remote_filenames[rp].append(fn)

fs = None
for rp in remote_filenames:
    fs = util.send2box(local_filenames[rp], rp,
                       remotenames=remote_filenames[rp], fs=fs)
    for fn in local_filenames[rp]:
        os.unlink(fn)
<commit_msg>upload fx files for NTRANS project<commit_after>"""Upload i5 analysis to CyBox"""
import os
from pyiem import util
import datetime
import glob
import logging
import sys

if len(sys.argv) > 1 and sys.argv[1] == 'debug':
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    logging.debug("Setting logging to debug...")

ceiling = datetime.datetime.utcnow() - datetime.timedelta(hours=24)
os.chdir("/mesonet/share/pickup/ntrans")

local_filenames = {}
remote_filenames = {}
for fn in glob.glob("*.zip"):
    valid = datetime.datetime.strptime(fn[3:15], '%Y%m%d%H%M')
    if valid >= ceiling:
        continue
    rp = valid.strftime("/NTRANS/wxarchive/%Y/%m/%d/%H")
    if fn.startswith("fx"):
        rp = valid.strftime("/NTRANS/wxarchive/forecast/%Y/%m")
    if rp not in remote_filenames:
        local_filenames[rp] = []
        remote_filenames[rp] = []
    # Move file to temp fn so that we don't attempt to upload this twice
    fn2 = "%s.xfer" % (fn,)
    os.rename(fn, fn2)
    local_filenames[rp].append(fn2)
    remote_filenames[rp].append(fn)

fs = None
for rp in remote_filenames:
    fs = util.send2box(local_filenames[rp], rp,
                       remotenames=remote_filenames[rp], fs=fs)
    for fn in local_filenames[rp]:
        os.unlink(fn)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# See LICENSE file for copyright and license details.
import sys
tg = "%.2f dB" % float(sys.argv[2])
tp = "%.8f"    % float(sys.argv[3])
ag = "%.2f dB" % float(sys.argv[4])
ap = "%.8f"    % float(sys.argv[5])

if sys.argv[1][-5:].find(".flac") == 0:
  from mutagen.flac import FLAC
  audio = FLAC(sys.argv[1])
  audio["replaygain_track_gain"] = tg
  audio["replaygain_track_peak"] = tp
  audio["replaygain_album_gain"] = ag
  audio["replaygain_album_peak"] = ap
  audio.save()
elif sys.argv[1][-4:].find(".ogg") == 0:
  from mutagen.oggvorbis import OggVorbis
  audio = OggVorbis(sys.argv[1])
  audio["replaygain_track_gain"] = tg
  audio["replaygain_track_peak"] = tp
  audio["replaygain_album_gain"] = ag
  audio["replaygain_album_peak"] = ap
  audio.save()
elif sys.argv[1][-4:].find(".mp3") == 0:
  import mutagen, mutagen.id3
  audio = mutagen.id3.ID3(sys.argv[1])
  frame = mutagen.id3.Frames["RVA2"](desc="track", channel=1, gain=float(sys.argv[2]), peak=float(sys.argv[3]))
  audio.add(frame)
  frame = mutagen.id3.Frames["RVA2"](desc="album", channel=1, gain=float(sys.argv[4]), peak=float(sys.argv[5]))
  audio.add(frame)
  frame = mutagen.id3.Frames["TXXX"](encoding=3, desc="replaygain_track_gain", text=tg)
  audio.add(frame)
  frame = mutagen.id3.Frames["TXXX"](encoding=3, desc="replaygain_track_peak", text=tp)
  audio.add(frame)
  frame = mutagen.id3.Frames["TXXX"](encoding=3, desc="replaygain_album_gain", text=ag)
  audio.add(frame)
  frame = mutagen.id3.Frames["TXXX"](encoding=3, desc="replaygain_album_peak", text=ap)
  audio.add(frame)
  audio.save()
<commit_msg>add musepack support to rgtag.py<commit_after>#!/usr/bin/env python
# See LICENSE file for copyright and license details.
import sys
tg = "%.2f dB" % float(sys.argv[2])
tp = "%.8f"    % float(sys.argv[3])
ag = "%.2f dB" % float(sys.argv[4])
ap = "%.8f"    % float(sys.argv[5])

if sys.argv[1][-5:].find(".flac") == 0:
  from mutagen.flac import FLAC
  audio = FLAC(sys.argv[1])
  audio["replaygain_track_gain"] = tg
  audio["replaygain_track_peak"] = tp
  audio["replaygain_album_gain"] = ag
  audio["replaygain_album_peak"] = ap
  audio.save()
elif sys.argv[1][-4:].find(".ogg") == 0 or sys.argv[1][-4:].find(".oga") == 0:
  from mutagen.oggvorbis import OggVorbis
  audio = OggVorbis(sys.argv[1])
  audio["replaygain_track_gain"] = tg
  audio["replaygain_track_peak"] = tp
  audio["replaygain_album_gain"] = ag
  audio["replaygain_album_peak"] = ap
  audio.save()
elif sys.argv[1][-4:].find(".mp3") == 0:
  import mutagen, mutagen.id3
  audio = mutagen.id3.ID3(sys.argv[1])
  frame = mutagen.id3.Frames["RVA2"](desc="track", channel=1, gain=float(sys.argv[2]), peak=float(sys.argv[3]))
  audio.add(frame)
  frame = mutagen.id3.Frames["RVA2"](desc="album", channel=1, gain=float(sys.argv[4]), peak=float(sys.argv[5]))
  audio.add(frame)
  frame = mutagen.id3.Frames["TXXX"](encoding=3, desc="replaygain_track_gain", text=tg)
  audio.add(frame)
  frame = mutagen.id3.Frames["TXXX"](encoding=3, desc="replaygain_track_peak", text=tp)
  audio.add(frame)
  frame = mutagen.id3.Frames["TXXX"](encoding=3, desc="replaygain_album_gain", text=ag)
  audio.add(frame)
  frame = mutagen.id3.Frames["TXXX"](encoding=3, desc="replaygain_album_peak", text=ap)
  audio.add(frame)
  audio.save()
elif sys.argv[1][-4:].find(".mpc") == 0:
  from mutagen.apev2 import APEv2, APENoHeaderError
  try:
    audio = APEv2(sys.argv[1])
  except APENoHeaderError:
    audio = APEv2()
  audio["replaygain_track_gain"] = tg
  audio["replaygain_track_peak"] = tp
  audio["replaygain_album_gain"] = ag
  audio["replaygain_album_peak"] = ap
  audio.save(sys.argv[1])
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
    This is the most lit foundation to build robot code on
"""

import wpilib
# from robotpy_ext.common_drivers import navx
# from robotpy_ext.common_drivers import xl_max_sonar_ez
# from networktables import NetworkTable
# import networktables

class MyRobot(wpilib.IterativeRobot):
    def robotInit(self):
        """
        This function is called upon program startup and
        is used for initialization code.
        """
        # joystick 1 on the driver station
        self.stick = wpilib.Joystick(0)

        # object that handles basic drive operations
        self.robot_drive = wpilib.RobotDrive(0, 1, 2, 3)
        self.robot_drive.setInvertedMotor(0, True)
        self.robot_drive.setInvertedMotor(1, True)
#         self.robot_drive.setSafetyEnabled(True)
#         self.robot_drive.setExpiration(0.1)

        # initialize motors
        self.loader = wpilib.Spark(4)
        self.shooter = wpilib.Spark(5)
        self.intake = wpilib.Spark(6)
        self.winch = wpilib.Spark(7)
        self.vibMotor = wpilib.Spark(8)

        # Initialize Gyro
#         self.gyro = wpilib.ADXRS450_Gyro(0)

        # Initialize Accelerometer
#         self.accelerometer = wpilib.ADXL362(8, 0)

        #initialize switch
#         self.gear_switch = wpilib.DigitalInput(0)

    def autonomousInit(self):
        """This function is run once each time the robot enters autonomous mode."""
        self.auto_loop_counter = 0

    def autonomousPeriodic(self):
        """This function is called periodically during autonomous."""
        try:
            # Check if we've completed 100 loops (approximately 2 seconds)
            if self.auto_loop_counter < 100:
                self.robot_drive.mecanumDrive_Cartesian(-0.1, 0, 0, 0) # Drive forwards
                self.auto_loop_counter += 1
            else:
                self.robot_drive.mecanumDrive_Cartesian(0, 0, 0, 0)    #Stop robot
        except:
            if not self.isFmsAttached():
                raise

#     def teleopInit(self):
#         """This function is run once each time the robot enters operator control mode."""
#         wpilib.IterativeRobot.teleopInit(self)
#
#         xAxis = 0
#         yAxis = 0
#         rotation = 0
#         gyroAngle = 0

    def teleopPeriodic(self):
        """This function is called periodically during operator control."""
        try:
            xAxis = self.stick.getRawAxis(1)*-0.5
            yAxis = self.stick.getRawAxis(0)*-0.5
            rotation = self.stick.getRawAxis(4)*0.25
#             gyroAngle = self.gyro.getAngle()
            left_trig = self.stick.getRawAxis(2)
            right_trig = self.stick.getRawAxis(3)
#             right_trig = right_trig * -1
            gyroAngle = 0
#             self.robot_drive.mecanumDrive_Cartesian(xAxis, yAxis, rotation, gyroAngle)

            self.winch.set(right_trig * -1)
            self.vibMotor.set(left_trig * -1)

            if self.stick.getRawButton(1) is True:
                self.loader.set(1)
            else:
                self.loader.set(0)

            if self.stick.getRawButton(2) is True:
                self.intake.set(0.65)
            else:
                self.intake.set(0)

#             if self.stick.getRawButton(4) is True:
#                 self.stick.setRumble(0, 1)
#                 self.stick.setRumble(1, 1)
#                 self.vibMotor.set(1)
#             else:
#                 self.stick.setRumble(0, 0)
#                 self.stick.setRumble(1, 0)
#                 self.vibMotor.set(0)

            if self.stick.getRawButton(5) is True:
                self.shooter.set(0.72387)
            else:
                self.shooter.set(0)

        except:
            if not self.isFmsAttached():
                raise
    def testPeriodic(self):
        """This function is called periodically during test mode."""
        wpilib.LiveWindow.run()

    def disabledInit(self):
        wpilib.IterativeRobot.disabledInit(self)

if __name__ == "__main__":
    wpilib.run(MyRobot)
<commit_msg>Lots of changes.<commit_after>#!/usr/bin/env python3
"""
    This is the most lit foundation to build robot code on
"""

import wpilib
# from robotpy_ext.common_drivers import navx
# from robotpy_ext.common_drivers import xl_max_sonar_ez
# from networktables import NetworkTable
# import networktables

class MyRobot(wpilib.IterativeRobot):
    def robotInit(self):
        """
        This function is called upon program startup and
        is used for initialization code.
        """
        # joystick 1 on the driver station
        self.stick = wpilib.Joystick(0)

        # object that handles basic drive operations
        self.robot_drive = wpilib.RobotDrive(0, 1, 2, 3)
        self.robot_drive.setInvertedMotor(0, True)
        self.robot_drive.setInvertedMotor(1, True)
#         self.robot_drive.setSafetyEnabled(True)
#         self.robot_drive.setExpiration(0.1)

        # initialize motors
        self.loader = wpilib.Spark(4)
        self.shooter = wpilib.Spark(5)
        self.intake = wpilib.Spark(6)
        self.winch = wpilib.Spark(7)
        self.vibMotor = wpilib.Spark(8)

        # Initialize Gyro
#         self.gyro = wpilib.ADXRS450_Gyro(0)

        # Initialize Accelerometer
#         self.accelerometer = wpilib.ADXL362(8, 0)

        #initialize switch
#         self.gear_switch = wpilib.DigitalInput(0)

    def autonomousInit(self):
        """This function is run once each time the robot enters autonomous mode."""
        self.auto_loop_counter = 0

    def autonomousPeriodic(self):
        """This function is called periodically during autonomous."""
        try:
            # Check if we've completed 100 loops (approximately 2 seconds)
            if self.auto_loop_counter < 100:
                self.robot_drive.mecanumDrive_Cartesian(-0.1, 0, 0, 0) # Drive forwards
                self.auto_loop_counter += 1
            else:
                self.robot_drive.mecanumDrive_Cartesian(0, 0, 0, 0)    #Stop robot
        except:
            if not self.isFmsAttached():
                raise

#     def teleopInit(self):
#         """This function is run once each time the robot enters operator control mode."""
#         wpilib.IterativeRobot.teleopInit(self)
#
#         xAxis = 0
#         yAxis = 0
#         rotation = 0
#         gyroAngle = 0

    def teleopPeriodic(self):
        """This function is called periodically during operator control."""
        try:
            xAxis = self.stick.getRawAxis(1)*-0.5
            yAxis = self.stick.getRawAxis(0)*-0.5
            rotation = self.stick.getRawAxis(4)*0.25
#             gyroAngle = self.gyro.getAngle()
            left_trig = self.stick.getRawAxis(2)
            right_trig = self.stick.getRawAxis(3)
#             right_trig = right_trig * -1
            gyroAngle = 0
#             self.robot_drive.mecanumDrive_Cartesian(xAxis, yAxis, rotation, gyroAngle)

            self.winch.set(right_trig * -1)
            self.vibMotor.set(left_trig * -1)

            if self.stick.getRawButton(1) is True:
                self.loader.set(1)
            else:
                self.loader.set(0)

            if self.stick.getRawButton(2) is True:
                self.intake.set(0.65)
            else:
                self.intake.set(0)

            if self.stick.getRawButton(4) is True:
                self.stick.setRumble(0, 1)
                self.stick.setRumble(1, 1)
            else:
                self.stick.setRumble(0, 0)
                self.stick.setRumble(1, 0)

            if self.stick.getRawButton(5) is True:
                self.shooter.set(0.72387)
            else:
                self.shooter.set(0)

        except:
            if not self.isFmsAttached():
                raise
    def testPeriodic(self):
        """This function is called periodically during test mode."""
        wpilib.LiveWindow.run()

    def disabledInit(self):
        wpilib.IterativeRobot.disabledInit(self)

if __name__ == "__main__":
    wpilib.run(MyRobot)
<|endoftext|>


<commit_before>import glob
import os, os.path
import sys
import csv
import dataserv
import matplotchart as mpc
import statistics
import sqlite3
from datetime import date, datetime
import time
from dateutil import parser

def filelinecount(filename):
    if type(filename) != str:
        filename = str(filename)

    with open(filename) as file:
        for i, l in enumerate(file):
            pass

        return i + 1

def linecount():
    numlines = []
    for file in list(os.walk('.'))[0][2]:
        if file.endswith('py'):
            try:
                numlines.append(filelinecount(file))
            except: pass

    return sum(numlines)

def plottest():
    import matplotchart as mpc
    import matplotlib.pyplot as plt
    import pandas as pd
    qdata = mpc.sqltopandas()
    dates, values = qdata['Order Date'], qdata['Order Total']
    mpc.plotdates(dates, values)

def datefromcsv():
    reader = csv.DictReader(fil, delimiter=';')         #read the csv file
    for row in reader:
        date = datetime.strptime(row['Order Date'], '%Y-%m-%d %H:%M:%S')     #datetime value in the right date format
        values[date.strftime('%Y-%m-%d')] += 1          #increment the date with a step of 1

    for date, value in sorted(values.items()):
        result = (value / 3927.2) * 100          #Sla calcul with the theoritic number of line
        print 'Date: {}'.format(date)        #SLA display
        print 'Result: {}'.format(result)

if __name__ == "__main__":
    x = linecount()
    print(x)
<commit_msg>fix datefromcsv<commit_after>import glob
import os, os.path
import sys
import csv
import dataserv
import matplotchart as mpc
import statistics
import sqlite3
from datetime import date, datetime
import time
from dateutil import parser

def filelinecount(filename):
    if type(filename) != str:
        filename = str(filename)

    with open(filename) as file:
        for i, l in enumerate(file):
            pass

        return i + 1

def linecount():
    numlines = []
    for file in list(os.walk('.'))[0][2]:
        if file.endswith('py'):
            try:
                numlines.append(filelinecount(file))
            except: pass

    return sum(numlines)

def plottest():
    import matplotchart as mpc
    import matplotlib.pyplot as plt
    import pandas as pd
    qdata = mpc.sqltopandas()
    dates, values = qdata['Order Date'], qdata['Order Total']
    mpc.plotdates(dates, values)

def datefromcsv(file):
    reader = csv.DictReader(file, delimiter=';')         #read the csv file
    for row in reader:
        date = datetime.strptime(row['Order Date'], '%Y-%m-%d %H:%M:%S')     #datetime value in the right date format
        values[date.strftime('%Y-%m-%d')] += 1          #increment the date with a step of 1

    for date, value in sorted(values.items()):
        result = (value / 3927.2) * 100          #Sla calcul with the theoritic number of line
        print('Date: {}'.format(date))        #SLA display
        print('Result: {}'.format(result))

if __name__ == "__main__":
    x = linecount()
    print(x)
<|endoftext|>


<commit_before>from setuptools import Extension, find_packages, setup

arpreq = Extension('arpreq', sources=['arpreq/arpreq.c'],
                   extra_compile_args=['-std=c11'])

setup(name='hades',
      version='0.1',
      description="Distributed AG DSN RADIUS MAC authentication. "
                  "Site node agent and captive portal",
      packages=find_packages(exclude=["*.tests"]),
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          "Flask",
          "Flask-Babel",
          "SQLAlchemy",
          "celery",
          "netaddr",
          "psycopg2",
          "pyroute2",
      ],
      ext_modules=[arpreq],
      scripts=['scripts/hades'],
      classifiers=[
          'Development Status :: 5 - Production/Stable',
          'License :: OSI Approved :: MIT License',
          'Environment :: No Input/Output (Daemon)',
          'Environment :: Web Environment',
          'Intended Audience :: System Administrators',
          'Operating System :: POSIX :: Linux',
          'Programming Language :: Python :: 3 :: Only',
          'Programming Language :: Python :: 3.4',
          'Programming Language :: Python :: 3.5',
          'Programming Language :: Python :: Implementation :: CPython',
          'Topic :: Internet :: WWW/HTTP :: WSGI :: Application',
          'Topic :: System :: Networking',
      ],
      )

<commit_msg>arpreq: Build with -std=c99<commit_after>from setuptools import Extension, find_packages, setup

arpreq = Extension('arpreq', sources=['arpreq/arpreq.c'],
                   extra_compile_args=['-std=c99'])

setup(name='hades',
      version='0.1',
      description="Distributed AG DSN RADIUS MAC authentication. "
                  "Site node agent and captive portal",
      packages=find_packages(exclude=["*.tests"]),
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          "Flask",
          "Flask-Babel",
          "SQLAlchemy",
          "celery",
          "netaddr",
          "psycopg2",
          "pyroute2",
      ],
      ext_modules=[arpreq],
      scripts=['scripts/hades'],
      classifiers=[
          'Development Status :: 5 - Production/Stable',
          'License :: OSI Approved :: MIT License',
          'Environment :: No Input/Output (Daemon)',
          'Environment :: Web Environment',
          'Intended Audience :: System Administrators',
          'Operating System :: POSIX :: Linux',
          'Programming Language :: Python :: 3 :: Only',
          'Programming Language :: Python :: 3.4',
          'Programming Language :: Python :: 3.5',
          'Programming Language :: Python :: Implementation :: CPython',
          'Topic :: Internet :: WWW/HTTP :: WSGI :: Application',
          'Topic :: System :: Networking',
      ],
      )

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from setuptools import setup, find_packages

setup(
    name           = 'service-deployment-tools',
    version        = '0.6.1',
    provides       = ["service_deployment_tools"],
    author         = 'Kyle Anderson',
    author_email   = 'kwa@yelp.com',
    description    = 'Tools for Yelps SOA infrastructure',
    packages       = find_packages(exclude=["tests", "scripts"]),
    setup_requires = ['setuptools'],
    include_package_data=True,
    install_requires = [
        'isodate >= 0.5.0',
        'service-configuration-lib >= 0.6.0',
        'marathon == 0.3.2',
        'argparse',
        'pycurl',
        'requests',
        'pysensu-yelp >= 0.1.5',
        'GitPython == 0.1.7',
        'kazoo == 1.3.1',
        'sensu-plugin == 0.1.0',
    ],
    scripts = [
        'service_deployment_tools/setup_marathon_job.py',
        'service_deployment_tools/setup_chronos_jobs.py',
        'service_deployment_tools/am_i_mesos_leader.py',
        'service_deployment_tools/synapse_srv_namespaces_fact.py',
        'service_deployment_tools/cleanup_marathon_jobs.py',
        'service_deployment_tools/check_marathon_services_frontends.py',
        'service_deployment_tools/list_marathon_service_instances.py',
        'service_deployment_tools/monitoring/check_synapse_replication.py',
        'service_deployment_tools/monitoring/check_classic_service_replication.py',
        'service_deployment_tools/deploy_marathon_services',
        'service_deployment_tools/generate_deployments_json.py',
        'service_deployment_tools/check_marathon_services_replication.py',
        'service_deployment_tools/generate_services_yaml.py'
    ]
)
<commit_msg>0.6.2 setup.py<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

from setuptools import setup, find_packages

setup(
    name           = 'service-deployment-tools',
    version        = '0.6.2',
    provides       = ["service_deployment_tools"],
    author         = 'Kyle Anderson',
    author_email   = 'kwa@yelp.com',
    description    = 'Tools for Yelps SOA infrastructure',
    packages       = find_packages(exclude=["tests", "scripts"]),
    setup_requires = ['setuptools'],
    include_package_data=True,
    install_requires = [
        'isodate >= 0.5.0',
        'service-configuration-lib >= 0.6.0',
        'marathon == 0.3.2',
        'argparse',
        'pycurl',
        'requests',
        'pysensu-yelp >= 0.1.5',
        'GitPython == 0.1.7',
        'kazoo == 1.3.1',
        'sensu-plugin == 0.1.0',
    ],
    scripts = [
        'service_deployment_tools/setup_marathon_job.py',
        'service_deployment_tools/setup_chronos_jobs.py',
        'service_deployment_tools/am_i_mesos_leader.py',
        'service_deployment_tools/synapse_srv_namespaces_fact.py',
        'service_deployment_tools/cleanup_marathon_jobs.py',
        'service_deployment_tools/check_marathon_services_frontends.py',
        'service_deployment_tools/list_marathon_service_instances.py',
        'service_deployment_tools/monitoring/check_synapse_replication.py',
        'service_deployment_tools/monitoring/check_classic_service_replication.py',
        'service_deployment_tools/deploy_marathon_services',
        'service_deployment_tools/generate_deployments_json.py',
        'service_deployment_tools/check_marathon_services_replication.py',
        'service_deployment_tools/generate_services_yaml.py'
    ]
)
<|endoftext|>


<commit_before>import os
from codecs import open
from distutils.command.build import build
from os import path

from setuptools import find_packages, setup

here = path.abspath(path.dirname(__file__))

# Get the long description from the relevant file
with open(path.join(here, '../README.md'), encoding='utf-8') as f:
    long_description = f.read()

class CustomBuild(build):
    def run(self):
        os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pretix.settings")
        import django
        django.setup()
        from django.conf import settings
        from django.core import management

        settings.COMPRESS_ENABLED = True
        settings.COMPRESS_OFFLINE = True

        management.call_command('collectstatic', verbosity=1, interactive=False)
        management.call_command('compress', verbosity=1, interactive=False)
        build.run(self)

cmdclass = {
    'build': CustomBuild
}

setup(
    name='pretix',
    version='0.0.0',
    description='Reinventing ticket presales',
    long_description=long_description,
    url='http://pretix.eu',
    author='Raphael Michel',
    author_email='mail@raphaelmichel.de',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'Intended Audience :: Other Audience',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Framework :: Django :: 1.8'
    ],

    keywords='tickets web shop ecommerce',
    install_requires=[
        'Django>=1.9,<1.10', 'python-dateutil>=2.4,<2.5',
        'pytz', 'django-bootstrap3>=6.2,<6.3', 'django-formset-js',
        'django-compressor==2.0', 'reportlab>=3.2,<3.3',
        'easy-thumbnails>=2.2,<3'
        'PyPDF2', 'BeautifulSoup4', 'html5lib',
        'slimit', 'lxml', 'static3==0.6.1', 'dj-static', 'chardet',
        'csscompressor'
    ],
    extras_require={
        'dev': ['django-debug-toolbar>=1.3.0,<2.0'],
        'test': ['pep8==1.5.7', 'pyflakes', 'pep8-naming', 'flake8', 'coverage',
                 'selenium', 'pytest', 'pytest-django'],
        'memcached': ['pylibmc'],
        'mysql': ['mysqlclient'],
        'paypal': ['paypalrestsdk>=1.9,<1.10,<2.0'],
        'postgres': ['psycopg2'],
        'redis': ['django-redis>=4.1,<4.2', 'redis>=2.10,<2.11'],
        'stripe': ['stripe>=1.22,<1.23']
    },

    packages=find_packages(exclude=['tests', 'tests.*']),
    include_package_data=True,
    cmdclass=cmdclass
)
<commit_msg>Added banktransfer requirements to setup.py<commit_after>import os
from codecs import open
from distutils.command.build import build
from os import path

from setuptools import find_packages, setup

here = path.abspath(path.dirname(__file__))

# Get the long description from the relevant file
with open(path.join(here, '../README.md'), encoding='utf-8') as f:
    long_description = f.read()

class CustomBuild(build):
    def run(self):
        os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pretix.settings")
        import django
        django.setup()
        from django.conf import settings
        from django.core import management

        settings.COMPRESS_ENABLED = True
        settings.COMPRESS_OFFLINE = True

        management.call_command('collectstatic', verbosity=1, interactive=False)
        management.call_command('compress', verbosity=1, interactive=False)
        build.run(self)

cmdclass = {
    'build': CustomBuild
}

setup(
    name='pretix',
    version='0.0.0',
    description='Reinventing ticket presales',
    long_description=long_description,
    url='http://pretix.eu',
    author='Raphael Michel',
    author_email='mail@raphaelmichel.de',
    license='Apache License 2.0',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Intended Audience :: Developers',
        'Intended Audience :: Other Audience',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Framework :: Django :: 1.8'
    ],

    keywords='tickets web shop ecommerce',
    install_requires=[
        'Django>=1.9,<1.10', 'python-dateutil>=2.4,<2.5',
        'pytz', 'django-bootstrap3>=6.2,<6.3', 'django-formset-js',
        'django-compressor==2.0', 'reportlab>=3.2,<3.3',
        'easy-thumbnails>=2.2,<3'
        'PyPDF2', 'BeautifulSoup4', 'html5lib',
        'slimit', 'lxml', 'static3==0.6.1', 'dj-static', 'chardet',
        'csscompressor', 'defusedxml', 'mt-940'
    ],
    extras_require={
        'dev': ['django-debug-toolbar>=1.3.0,<2.0'],
        'test': ['pep8==1.5.7', 'pyflakes', 'pep8-naming', 'flake8', 'coverage',
                 'selenium', 'pytest', 'pytest-django'],
        'memcached': ['pylibmc'],
        'mysql': ['mysqlclient'],
        'paypal': ['paypalrestsdk>=1.9,<1.10,<2.0'],
        'postgres': ['psycopg2'],
        'redis': ['django-redis>=4.1,<4.2', 'redis>=2.10,<2.11'],
        'stripe': ['stripe>=1.22,<1.23']
    },

    packages=find_packages(exclude=['tests', 'tests.*']),
    include_package_data=True,
    cmdclass=cmdclass
)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Miscellaneous statistics utilities."""

import math

from bayeslite.util import float_sum

def arithmetic_mean(array):
    """Arithmetic mean of elements of `array`.

    :param list<float> array: List of floats to compute arithmetic mean.

    :return: Arithmetic mean of `array`.
    :rtype: float
    """
    return float_sum(array) / len(array)

def pearsonr(a0, a1):
    """Pearson r correlation coefficient of two samples.

    For random variables X, Y:

                 cov(X, Y)
    r(X, Y) = ---------------,
              sigma_X sigma_Y

    where

      cov(X, Y) = E[(X - E[X]) (Y - E[Y])],
    {sigma_X}^2 = E[(X - E[X])^2], and
    {sigma_Y}^2 = E[(Y - E[Y])^2]

    For a sample a0, we take the mean of a0 instead of E[X], and the
    variance of a0 instead of E[(X - E[X])^2].

    https://en.wikipedia.org/wiki/Pearson_product-moment_correlation_coefficient

    :param list<float> a0: Observations of the first random variable.
    :param list<float> a1: Observations of the second random variable.

    :return: Pearson r correlation coefficient of samples `a0` and `a1`.
    :rtype: float
    """
    n = len(a0)
    assert n == len(a1)
    if n == 0:
        # No data, so no notion of correlation.
        return float('NaN')
    m0 = arithmetic_mean(a0)
    m1 = arithmetic_mean(a1)
    num = float_sum((x0 - m0)*(x1 - m1) for x0, x1 in zip(a0, a1))
    den0_sq = float_sum((x0 - m0)**2 for x0 in a0)
    den1_sq = float_sum((x1 - m1)**2 for x1 in a1)
    den = math.sqrt(den0_sq*den1_sq)
    if den == 0.0:
        # No variation in at least one column, so no notion of
        # correlation.
        return float('NaN')
    r = num / den
    # Clamp r in [-1, +1] in case of floating-point error.
    r = min(r, +1.0)
    r = max(r, -1.0)
    return r

def signum(x):
    """Sign of `x`.

    :param float x: Argument to signum.

    :return: Sign of `x`: ``-1 if x<0, 0 if x=0, +1 if x>0``.
    :rtype: int
    """
    if x < 0:
        return -1
    elif 0 < x:
        return +1
    else:
        return 0

def chi2_contingency(contingency, correction=None):
    """Pearson chi^2 statistic for test of independence on contingency table.

    https://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test#Test_of_independence

    :param list<list> contingency: Table counting values of two random
    variables X, Y in a population sample: for each pair of values
    x_i, y_j that X, Y can take, contingency[i][j] is the number of
    occurrences of that pair.

    :param boolean correction: If ``True``, move each observation
    count in the direction of the expectation by 1/2.

    :return: Observed Pearson chi^2 test statistic on `contingency`.
    :rtype: float
    """
    if correction is None:
        correction = False
    assert 0 < len(contingency)
    assert all(all(isinstance(v, int) for v in row) for row in contingency)
    n = float(sum(sum(row) for row in contingency))
    n0 = len(contingency)
    n1 = len(contingency[0])
    assert all(n1 == len(row) for row in contingency)
    p0 = [float_sum(contingency[i0][i1]/n for i1 in range(n1))
        for i0 in range(n0)]
    p1 = [float_sum(contingency[i0][i1]/n for i0 in range(n0))
        for i1 in range(n1)]
    def q(i0, i1):
        O = contingency[i0][i1]
        E = n*p0[i0]*p1[i1]
        if correction:
            O += 0.5*signum(E - O)
        return ((O - E)**2)/E
    return float_sum(q(i0, i1) for i0 in range(n0) for i1 in range(n1))

def f_oneway(groups):
    """F-test statistic for one-way analysis of variance (ANOVA).

    https://en.wikipedia.org/wiki/F-test#Multiple-comparison_ANOVA_problems

    :param list<list> groups: List of lists of the observed values of
        each group.  The outer list must length equal to the number of
        groups.

    :return: Observed F test statistic on `groups`.
    :rtype: float
    """
    K = len(groups)
    N = sum(len(group) for group in groups)
    means = [arithmetic_mean(group) for group in groups]
    overall_mean = float_sum(x for group in groups for x in group) / N
    bgv = float_sum(len(group) * (mean - overall_mean)**2 / (K - 1)
        for group, mean in zip(groups, means))
    wgv = float_sum(float((x - mean)**2)/float(N - K)
        for group, mean in zip(groups, means)
        for x in group)
    if wgv == 0.0:
        if bgv == 0.0:
            # No variation between or within groups, so we cannot
            # ascertain any correlation between them -- it is as if we
            # had no data about the groups: every value in every group
            # is the same.
            return float('NaN')
        else:
            # Within-group variability is zero, meaning for each
            # group, each value is the same; between-group variability
            # is nonzero, meaning there is variation between the
            # groups.  So if there were zero correlation we could not
            # possibly observe this, whereas all finite F statistics
            # could be observed with zero correlation.
            return float('+inf')
    return bgv / wgv

def t_cdf(x, df):
    """Approximate CDF for Student's t distribution.

    ``t_cdf(x,df) = P(T_df < x)``
    Values are tested to within 0.5% of values returned by the
    Cephes C library for numerical integration.

    :param float x: Argument to the survival function, must be positive.
    :param float df: Degrees of freedom of the chi^2 distribution.

    :return: Area from negative infinity to `x` under t distribution
        with degrees of freedom `df`.
    :rtype: float
    """
    import numpy

    if df <= 0:
        raise ValueError('Degrees of freedom must be positive.')
    if x == 0:
        return 0.5

    MONTE_CARLO_SAMPLES = 1e5
    random = numpy.random.RandomState(seed=0)
    T = random.standard_t(df, size=MONTE_CARLO_SAMPLES)
    return numpy.sum(T < x) / MONTE_CARLO_SAMPLES

def chi2_sf(x, df):
    """Approximate survival function for chi^2 distribution.

    ``chi2_sf(x, df) = P(CHI > x)``
    Values are tested to within 0.5% of values returned by the
    Cephes C library for numerical integration.

    :param float x: Argument to the survival function, must be positive.
    :param float df: Degrees of freedom of the chi^2 distribution.

    :return: Area from `x` to infinity under the chi^2 distribution
        with degrees of freedom `df`.
    :rtype: float
    """
    import numpy

    if df <= 0:
        raise ValueError('Degrees of freedom must be positive.')
    if x <= 0:
        return 1.0

    MONTE_CARLO_SAMPLES = 5e5
    random = numpy.random.RandomState(seed=0)
    CHI = random.chisquare(df, size=MONTE_CARLO_SAMPLES)
    return numpy.sum(CHI > x) / MONTE_CARLO_SAMPLES

def f_sf(x, df_num, df_den):
    """Approximate survival function for the F distribution.

    ``f_sf(x, df_num, df_den) = P(F > x)``
    Values are tested to within 1% of values returned by the
    Cephes C library for numerical integration.

    :param float x: Argument to the survival function, must be positive.
    :param float df_num: Degrees of freedom of the numerator.
    :param float df_den: Degrees of freedom of the denominator.

    :return: Area from negative infinity to `x` under F distribution
        with degrees of freedom `df`.
    :rtype: float
    """
    import numpy

    if df_num <= 0 or df_den <= 0:
        raise ValueError('Degrees of freedom must be positive.')
    if x <= 0:
        return 1.0

    MONTE_CARLO_SAMPLES = 1e5
    random = numpy.random.RandomState(seed=0)
    F = random.f(df_num, df_den, size=MONTE_CARLO_SAMPLES)
    return numpy.sum(F > x) / MONTE_CARLO_SAMPLES

def gauss_suff_stats(data):
    """Summarize an array of data as (count, mean, standard deviation).

    The algorithm is the "Online algorithm" presented in Knuth Volume
    2, 3rd ed, p. 232, originally credited to "Note on a Method for
    Calculating Corrected Sums of Squares and Products" B. P. Welford
    Technometrics Vol. 4, No. 3 (Aug., 1962), pp. 419-420.  This has
    the advantage over naively accumulating the sum and sum of squares
    that it is less subject to precision loss through massive
    cancellation.

    This version collected 8/31/15 from
    https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance

    """
    n = 0
    mean = 0.0
    M2 = 0.0 # n * sigma^2

    for x in data:
        n = n + 1
        delta = x - mean
        mean = mean + delta/n
        M2 = M2 + delta*(x - mean)

    if n < 1:
        return (n, mean, 0.0)
    else:
        return (n, mean, math.sqrt(M2 / float(n)))
<commit_msg>Omit needless verbiage in stats docstrings.<commit_after># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""Miscellaneous statistics utilities."""

import math

from bayeslite.util import float_sum

def arithmetic_mean(array):
    """Arithmetic mean of elements of `array`."""
    return float_sum(array) / len(array)

def pearsonr(a0, a1):
    """Pearson r, product-moment correlation coefficient, of two samples.

    Covariance divided by product of standard deviations.

    https://en.wikipedia.org/wiki/Pearson_product-moment_correlation_coefficient#For_a_sample
    """
    n = len(a0)
    assert n == len(a1)
    if n == 0:
        # No data, so no notion of correlation.
        return float('NaN')
    m0 = arithmetic_mean(a0)
    m1 = arithmetic_mean(a1)
    num = float_sum((x0 - m0)*(x1 - m1) for x0, x1 in zip(a0, a1))
    den0_sq = float_sum((x0 - m0)**2 for x0 in a0)
    den1_sq = float_sum((x1 - m1)**2 for x1 in a1)
    den = math.sqrt(den0_sq*den1_sq)
    if den == 0.0:
        # No variation in at least one column, so no notion of
        # correlation.
        return float('NaN')
    r = num / den
    # Clamp r in [-1, +1] in case of floating-point error.
    r = min(r, +1.0)
    r = max(r, -1.0)
    return r

def signum(x):
    """Sign of `x`: ``-1 if x<0, 0 if x=0, +1 if x>0``."""
    if x < 0:
        return -1
    elif 0 < x:
        return +1
    else:
        return 0

def chi2_contingency(contingency, correction=None):
    """Pearson chi^2 test of independence statistic on contingency table.

    https://en.wikipedia.org/wiki/Pearson%27s_chi-squared_test#Test_of_independence

    With ``correction=True``, apply Yates' correction: move each
    observation count in the direction of the expectation by 1/2.
    """
    if correction is None:
        correction = False
    assert 0 < len(contingency)
    assert all(all(isinstance(v, int) for v in row) for row in contingency)
    n = float(sum(sum(row) for row in contingency))
    n0 = len(contingency)
    n1 = len(contingency[0])
    assert all(n1 == len(row) for row in contingency)
    p0 = [float_sum(contingency[i0][i1]/n for i1 in range(n1))
        for i0 in range(n0)]
    p1 = [float_sum(contingency[i0][i1]/n for i0 in range(n0))
        for i1 in range(n1)]
    def q(i0, i1):
        O = contingency[i0][i1]
        E = n*p0[i0]*p1[i1]
        if correction:
            O += 0.5*signum(E - O)
        return ((O - E)**2)/E
    return float_sum(q(i0, i1) for i0 in range(n0) for i1 in range(n1))

def f_oneway(groups):
    """F-test statistic for one-way analysis of variance (ANOVA).

    https://en.wikipedia.org/wiki/F-test#Multiple-comparison_ANOVA_problems

    ``groups[i][j]`` is jth observation in ith group.
    """
    K = len(groups)
    N = sum(len(group) for group in groups)
    means = [arithmetic_mean(group) for group in groups]
    overall_mean = float_sum(x for group in groups for x in group) / N
    bgv = float_sum(len(group) * (mean - overall_mean)**2 / (K - 1)
        for group, mean in zip(groups, means))
    wgv = float_sum(float((x - mean)**2)/float(N - K)
        for group, mean in zip(groups, means)
        for x in group)
    if wgv == 0.0:
        if bgv == 0.0:
            # No variation between or within groups, so we cannot
            # ascertain any correlation between them -- it is as if we
            # had no data about the groups: every value in every group
            # is the same.
            return float('NaN')
        else:
            # Within-group variability is zero, meaning for each
            # group, each value is the same; between-group variability
            # is nonzero, meaning there is variation between the
            # groups.  So if there were zero correlation we could not
            # possibly observe this, whereas all finite F statistics
            # could be observed with zero correlation.
            return float('+inf')
    return bgv / wgv

def t_cdf(x, df):
    """Approximate CDF for Student's t distribution.

    ``t_cdf(x, df) = P(T_df < x)``
    """
    import numpy

    if df <= 0:
        raise ValueError('Degrees of freedom must be positive.')
    if x == 0:
        return 0.5

    MONTE_CARLO_SAMPLES = 1e5
    random = numpy.random.RandomState(seed=0)
    T = random.standard_t(df, size=MONTE_CARLO_SAMPLES)
    return numpy.sum(T < x) / MONTE_CARLO_SAMPLES

def chi2_sf(x, df):
    """Approximate survival function for chi^2 distribution.

    ``chi2_sf(x, df) = P(chi^2_df > x)``
    """
    import numpy

    if df <= 0:
        raise ValueError('Degrees of freedom must be positive.')
    if x <= 0:
        return 1.0

    MONTE_CARLO_SAMPLES = 5e5
    random = numpy.random.RandomState(seed=0)
    CHI = random.chisquare(df, size=MONTE_CARLO_SAMPLES)
    return numpy.sum(CHI > x) / MONTE_CARLO_SAMPLES

def f_sf(x, df_num, df_den):
    """Approximate survival function for the F distribution.

    ``f_sf(x, df_num, df_den) = P(F_{df_num, df_den} > x)``
    """
    import numpy

    if df_num <= 0 or df_den <= 0:
        raise ValueError('Degrees of freedom must be positive.')
    if x <= 0:
        return 1.0

    MONTE_CARLO_SAMPLES = 1e5
    random = numpy.random.RandomState(seed=0)
    F = random.f(df_num, df_den, size=MONTE_CARLO_SAMPLES)
    return numpy.sum(F > x) / MONTE_CARLO_SAMPLES

def gauss_suff_stats(data):
    """Summarize an array of data as (count, mean, standard deviation).

    The algorithm is the "Online algorithm" presented in Knuth Volume
    2, 3rd ed, p. 232, originally credited to "Note on a Method for
    Calculating Corrected Sums of Squares and Products" B. P. Welford
    Technometrics Vol. 4, No. 3 (Aug., 1962), pp. 419-420.  This has
    the advantage over naively accumulating the sum and sum of squares
    that it is less subject to precision loss through massive
    cancellation.

    This version collected 8/31/15 from
    https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
    """
    n = 0
    mean = 0.0
    M2 = 0.0 # n * sigma^2

    for x in data:
        n = n + 1
        delta = x - mean
        mean = mean + delta/n
        M2 = M2 + delta*(x - mean)

    if n < 1:
        return (n, mean, 0.0)
    else:
        return (n, mean, math.sqrt(M2 / float(n)))
<|endoftext|>