<commit_before>import re

__version__ = '0.0.1'
__author__ = 'Kasper Jacobsen'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014 Kasper Jacobsen'

class InvalidVersionException(Exception):
    pass

class SemVerPy():
    def __init__(self, version):
        self.pattern = r'(?P<major>\d+).(?P<minor>\d+).(?P<patch>\d+)(?:-(?P<build>\w*)|)'
        self.version = self._validate(version)

    def __str__(self):
        if self.version['build'] is not None:
            return '{major}.{minor}.{patch}-{build}'.format(**self.version)
        return '{major}.{minor}.{patch}'.format(**self.version)

    def __repr__(self):
        return '<{}({})>'.format(
            self.__class__.__name__, str(self.version)
        )

    def __getitem__(self, key):
        return self.version[key]

    def __setitem__(self, key, value):
        self.version[key] = value
        return self.version

    def _tuple(self):
        major = self.version['major']
        minor = self.version['minor']
        patch = self.version['patch']
        build = self.version['build']

        return major, minor, patch, build

    def __eq__(self, other):
        if not isinstance(other, SemVerPy):
            return False
        else:
            return self._tuple() == other._tuple()

    def __ne__(self, other):
        return not self == other

    def __lt__(self, other):
        if not isinstance(other, SemVerPy):
            return False
        for s, o in zip(self._tuple(), other._tuple()):
            if s < o:
                return True

    def __gt__(self, other):
        if not isinstance(other, SemVerPy):
            return False
        for s, o in zip(self._tuple(), other._tuple()):
            if s > o:
                return True

    def __le__(self, other):
        return self == other or self < other

    def __ge__(self, other):
        return self == other or self > other

    def _get_dict(self, string):
        return re.search(self.pattern, string).groupdict()

    @staticmethod
    def _parse_dict(version_dict):
        try:
            version_dict['major'] = int(version_dict['major'])
            version_dict['minor'] = int(version_dict['minor'])
            version_dict['patch'] = int(version_dict['patch'])
            return version_dict
        except ValueError:
            raise

    def _validate(self, string):
        try:
            version = self._get_dict(string)
            version_dict = self._parse_dict(version)
            return version_dict
        except AttributeError:
            raise InvalidVersionException('Not a valid version: {}'.format(string))

    def bump_major(self):
        self.version['major'] = self.version['major'] + 1
        self.version['minor'] = 0
        self.version['patch'] = 0
        return self.version

    def bump_minor(self):
        self.version['minor'] = self.version['minor'] + 1
        self.version['patch'] = 0
        return self.version

    def bump_patch(self):
        self.version['patch'] = self.version['patch'] + 1
        return self.version

    def set_build(self, build_string):
        self.version['build'] = build_string
        return self.version
<commit_msg>Added optional build name when bumping.<commit_after>import re

__version__ = '0.0.1'
__author__ = 'Kasper Jacobsen'
__license__ = 'MIT'
__copyright__ = 'Copyright 2014 Kasper Jacobsen'

class InvalidVersionException(Exception):
    pass

class SemVerPy():
    def __init__(self, version):
        self.pattern = r'(?P<major>\d+).(?P<minor>\d+).(?P<patch>\d+)(?:-(?P<build>\w*)|)'
        self.version = self._validate(version)

    def __str__(self):
        if self.version['build'] is not None:
            return '{major}.{minor}.{patch}-{build}'.format(**self.version)
        return '{major}.{minor}.{patch}'.format(**self.version)

    def __repr__(self):
        return '<{}({})>'.format(
            self.__class__.__name__, str(self.version)
        )

    def __getitem__(self, key):
        return self.version[key]

    def __setitem__(self, key, value):
        self.version[key] = value
        return self.version

    def _tuple(self):
        major = self.version['major']
        minor = self.version['minor']
        patch = self.version['patch']
        build = self.version['build']

        return major, minor, patch, build

    def __eq__(self, other):
        if not isinstance(other, SemVerPy):
            return False
        else:
            return self._tuple() == other._tuple()

    def __ne__(self, other):
        return not self == other

    def __lt__(self, other):
        if not isinstance(other, SemVerPy):
            return False
        for s, o in zip(self._tuple(), other._tuple()):
            if s < o:
                return True

    def __gt__(self, other):
        if not isinstance(other, SemVerPy):
            return False
        for s, o in zip(self._tuple(), other._tuple()):
            if s > o:
                return True

    def __le__(self, other):
        return self == other or self < other

    def __ge__(self, other):
        return self == other or self > other

    def _get_dict(self, string):
        return re.search(self.pattern, string).groupdict()

    @staticmethod
    def _parse_dict(version_dict):
        try:
            version_dict['major'] = int(version_dict['major'])
            version_dict['minor'] = int(version_dict['minor'])
            version_dict['patch'] = int(version_dict['patch'])
            return version_dict
        except ValueError:
            raise

    def _validate(self, string):
        try:
            version = self._get_dict(string)
            version_dict = self._parse_dict(version)
            return version_dict
        except AttributeError:
            raise InvalidVersionException('Not a valid version: {}'.format(string))

    def bump_major(self, build=None):
        self.version['major'] = self.version['major'] + 1
        self.version['minor'] = 0
        self.version['patch'] = 0
        self.version['build'] = build
        return self.version

    def bump_minor(self, build=None):
        self.version['minor'] = self.version['minor'] + 1
        self.version['patch'] = 0
        self.version['build'] = build
        return self.version

    def bump_patch(self, build=None):
        self.version['patch'] = self.version['patch'] + 1
        self.version['build'] = build
        return self.version

    def set_build(self, build_string):
        self.version['build'] = build_string
        return self.version
<|endoftext|>


<commit_before>#!/usr/bin/env python

import datetime
import json
import sys

import requests
import pytz
from twilio.rest import TwilioRestClient

import conf

def send(s):
    client.sms.messages.create(to=conf.TO, from_=conf.FROM, body=s)

# Use the first arg as the message to send, or use the default if not specified
default_message = "You haven't committed anything today!"
message = sys.argv[1] if len(sys.argv) > 1 else default_message

# Initialise twilio stuff
client = TwilioRestClient(conf.ACCOUNT_SID, conf.AUTH_TOKEN)

# Get Github contributions activity
url = 'https://github.com/users/%s/contributions_calendar_data' % conf.USERNAME
request = requests.get(url)
if request.ok:
    data = json.loads(request.text)

    # Set to epoch begin just in case this is a totally new account
    latest_contribution = datetime.datetime(1970, 1, 1, 0, 0)

    # Get data for the latest contribution
    for i in reversed(data):
        if i[1] > 0:
            latest_contribution = datetime.datetime.strptime(i[0], '%Y/%m/%d')
            break

    # Find out today's date in PST (since Github uses PST)
    today = datetime.datetime.now(pytz.timezone('US/Pacific'))

    # Haven't contributed anything today?
    if latest_contribution.date() < today.date():
      send(message)
else:
    send('There was a problem accessing the Github API :(')
<commit_msg>Improve logic in determining latest contribution<commit_after>#!/usr/bin/env python

import json
import sys

import requests
from twilio.rest import TwilioRestClient

import conf

def send(s):
    client.sms.messages.create(to=conf.TO, from_=conf.FROM, body=s)

# Use the first arg as the message to send, or use the default if not specified
default_message = "You haven't committed anything today!"
message = sys.argv[1] if len(sys.argv) > 1 else default_message

# Initialise twilio stuff
client = TwilioRestClient(conf.ACCOUNT_SID, conf.AUTH_TOKEN)

# Get Github contributions activity
url = 'https://github.com/users/%s/contributions_calendar_data' % conf.USERNAME
request = requests.get(url)
if request.ok:
    try:
        data = json.loads(request.text)

        # Get the number of commits made today
        commits_today = data[-1][1]

        if not commits_today:
          send(message)
    except:
        send('There was an error getting the number of commits today')
else:
    send('There was a problem accessing the Github API :(')
<|endoftext|>


<commit_before>#!/usr/bin/env python

import subprocess
import sys
import os
from glob import glob
from ififuncs import diff_textfiles
from ififuncs import make_manifest
import datetime
import time
import csv
from ififuncs import create_csv
from ififuncs import append_csv
from ififuncs import send_gmail

def set_environment(logfile):
    env_dict = os.environ.copy()
    # https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
    env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
    return env_dict

def make_framemd5(directory, log_filename_alteration):
    os.chdir(directory)

    tiff_check = glob('*.tiff')
    dpx_check = glob('*.dpx')
    if len(dpx_check) > 0:
        images = dpx_check

    elif len(tiff_check) > 0:
        images = tiff_check
    else:
        print 'no images found'
        return 'none'

    images.sort()
    if '864000' in images[0]:
        start_number = '864000'
    else:
        start_number = images[0].split("_")[-1].split(".")[0]
    global output_parent_directory
    container = images[0].split(".")[-1]
    print container

    output_parent_directory = config[1].rstrip()
    numberless_filename = images[0].split("_")[0:-1]
    ffmpeg_friendly_name = ''
    counter = 0
    while  counter <len(numberless_filename) :
        ffmpeg_friendly_name += numberless_filename[counter] + '_'
        counter += 1

    if start_number == '864000':
        output_dirname = output_parent_directory + '/' + os.path.basename(directory) + 'dpx_transcodes'
        basename = os.path.basename(directory)
    else:
        output_dirname = output_parent_directory + '/' + ffmpeg_friendly_name + 'dpx_transcodes'
        basename = ffmpeg_friendly_name
    try:
        os.makedirs(output_dirname)
        os.makedirs(output_dirname + '/logs')
        os.makedirs(output_dirname + '/md5')
        os.makedirs(output_dirname + '/video')
        os.makedirs(output_dirname + '/xml_files')

    except: OSError

    output = output_dirname + '/md5/%s.framemd5' % (basename)
    logfile = output_dirname + '/logs/%s%s.log' % (basename, log_filename_alteration)
    env_dict = set_environment(logfile)
    image_seq_without_container = ffmpeg_friendly_name
    ffmpeg_friendly_name += "%06d." + '%s' % container
    framemd5 = ['ffmpeg','-start_number', start_number, '-report','-f','image2','-framerate','24', '-i', ffmpeg_friendly_name,'-f','framemd5',output]
    print framemd5
    subprocess.call(framemd5, env=env_dict)
    info = [output_dirname, output, image_seq_without_container, start_number, container]
    return info

def remove_bad_files(root_dir):
    rm_these = ['.DS_Store', 'Thumbs.db', 'desktop.ini']
    for root, dirs, files in os.walk(root_dir):
        for name in files:
            path = os.path.join(root, name)
            for i in rm_these:
                if name == i:
                    print '***********************' + 'removing: ' + path
                    os.remove(path)
csv_report_filename = os.path.expanduser("~/Desktop/") + 'dpx_transcode_report' + time.strftime("_%Y_%m_%dT%H_%M_%S") + '.csv'
dpxconfig = os.path.expanduser("~/Desktop/") + 'make_dpx_config.txt'
with open(dpxconfig, 'r') as fo:
    config = fo.readlines()
emails = config[0].split(',')

source_directory = sys.argv[1]

create_csv(csv_report_filename, ('Sequence Name', 'Lossless?', 'Start time', 'Finish Time', 'Sequence Size', 'FFV1 Size', 'Compression Ratio'))
for root,dirnames,filenames in os.walk(source_directory):
        #if "tiff_scans"  in dirnames:
        source_directory = root# + '/tiff_scans'
        total_size = 0

        remove_bad_files(source_directory)
        source_parent_dir    = os.path.dirname(source_directory)
        normpath             = os.path.normpath(source_directory)
        relative_path        = normpath.split(os.sep)[-1]
        split_path           = os.path.split(os.path.basename(source_directory))[1]
        start = datetime.datetime.now()
        source_manifest = source_parent_dir + '/%s_manifest.md5' % relative_path
        #make_manifest(os.path.dirname(source_directory), os.path.basename(source_directory), source_manifest)
        info = make_framemd5(source_directory, 'dpx_framemd5')
        if info == 'none':
            continue
        for files in filenames:
            total_size += os.path.getsize(os.path.join(root,files))
        output_dirname              = info[0]
        source_textfile             = info[1]
        image_seq_without_container = info[2]
        start_number                = info[3]
        container                   = info[4]
        dpx_filename                = image_seq_without_container + "%06d." + container

        logfile = output_dirname + '/logs/%s_dpx_transcode.log' % os.path.basename(root)
        env_dict = set_environment(logfile)
        ffv12dpx = ['ffmpeg','-report','-f','image2','-framerate','24', '-start_number', start_number, '-i', os.path.abspath(dpx_filename) ,'-strict', '-2','-c:v','ffv1','-level', '3', '-pix_fmt', 'rgb48le',output_dirname +  '/video/' + os.path.basename(root) + '.mkv']
        print ffv12dpx

        subprocess.call(ffv12dpx,env=env_dict)
        parent_basename   =  os.path.basename(output_dirname)
        manifest_textfile = os.path.dirname(output_dirname) + '/' +  parent_basename + '_manifest.md5'
        ffv1_path         = output_dirname +  '/video/'  + os.path.basename(root) + '.mkv'
        ffv1_md5          = output_dirname +  '/md5/' + os.path.basename(root) + 'ffv1.framemd5'
        subprocess.call(['ffmpeg','-i', ffv1_path, '-pix_fmt', 'rgb48le','-f', 'framemd5', ffv1_md5])
        #other_textfile = other[1]
        judgement = diff_textfiles(source_textfile, ffv1_md5)
        make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile)
        finish = datetime.datetime.now()

        print total_size
        ffv1_size = os.path.getsize(ffv1_path)
        comp_ratio =  float(total_size) / float(os.path.getsize(ffv1_path))
        append_csv(csv_report_filename, (parent_basename,judgement, start, finish,total_size, fv1_size, comp_ratio))

#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())
<commit_msg>seq2ffv1 - fix broken variable<commit_after>#!/usr/bin/env python

import subprocess
import sys
import os
from glob import glob
from ififuncs import diff_textfiles
from ififuncs import make_manifest
import datetime
import time
import csv
from ififuncs import create_csv
from ififuncs import append_csv
from ififuncs import send_gmail

def set_environment(logfile):
    env_dict = os.environ.copy()
    # https://github.com/imdn/scripts/blob/0dd89a002d38d1ff6c938d6f70764e6dd8815fdd/ffmpy.py#L272
    env_dict['FFREPORT'] = 'file={}:level=48'.format(logfile)
    return env_dict

def make_framemd5(directory, log_filename_alteration):
    os.chdir(directory)

    tiff_check = glob('*.tiff')
    dpx_check = glob('*.dpx')
    if len(dpx_check) > 0:
        images = dpx_check

    elif len(tiff_check) > 0:
        images = tiff_check
    else:
        print 'no images found'
        return 'none'

    images.sort()
    if '864000' in images[0]:
        start_number = '864000'
    else:
        start_number = images[0].split("_")[-1].split(".")[0]
    global output_parent_directory
    container = images[0].split(".")[-1]
    print container

    output_parent_directory = config[1].rstrip()
    numberless_filename = images[0].split("_")[0:-1]
    ffmpeg_friendly_name = ''
    counter = 0
    while  counter <len(numberless_filename) :
        ffmpeg_friendly_name += numberless_filename[counter] + '_'
        counter += 1

    if start_number == '864000':
        output_dirname = output_parent_directory + '/' + os.path.basename(directory) + 'dpx_transcodes'
        basename = os.path.basename(directory)
    else:
        output_dirname = output_parent_directory + '/' + ffmpeg_friendly_name + 'dpx_transcodes'
        basename = ffmpeg_friendly_name
    try:
        os.makedirs(output_dirname)
        os.makedirs(output_dirname + '/logs')
        os.makedirs(output_dirname + '/md5')
        os.makedirs(output_dirname + '/video')
        os.makedirs(output_dirname + '/xml_files')

    except: OSError

    output = output_dirname + '/md5/%s.framemd5' % (basename)
    logfile = output_dirname + '/logs/%s%s.log' % (basename, log_filename_alteration)
    env_dict = set_environment(logfile)
    image_seq_without_container = ffmpeg_friendly_name
    ffmpeg_friendly_name += "%06d." + '%s' % container
    framemd5 = ['ffmpeg','-start_number', start_number, '-report','-f','image2','-framerate','24', '-i', ffmpeg_friendly_name,'-f','framemd5',output]
    print framemd5
    subprocess.call(framemd5, env=env_dict)
    info = [output_dirname, output, image_seq_without_container, start_number, container]
    return info

def remove_bad_files(root_dir):
    rm_these = ['.DS_Store', 'Thumbs.db', 'desktop.ini']
    for root, dirs, files in os.walk(root_dir):
        for name in files:
            path = os.path.join(root, name)
            for i in rm_these:
                if name == i:
                    print '***********************' + 'removing: ' + path
                    os.remove(path)
csv_report_filename = os.path.expanduser("~/Desktop/") + 'dpx_transcode_report' + time.strftime("_%Y_%m_%dT%H_%M_%S") + '.csv'
dpxconfig = os.path.expanduser("~/Desktop/") + 'make_dpx_config.txt'
with open(dpxconfig, 'r') as fo:
    config = fo.readlines()
emails = config[0].split(',')

source_directory = sys.argv[1]

create_csv(csv_report_filename, ('Sequence Name', 'Lossless?', 'Start time', 'Finish Time', 'Sequence Size', 'FFV1 Size', 'Compression Ratio'))
for root,dirnames,filenames in os.walk(source_directory):
        #if "tiff_scans"  in dirnames:
        source_directory = root# + '/tiff_scans'
        total_size = 0

        remove_bad_files(source_directory)
        source_parent_dir    = os.path.dirname(source_directory)
        normpath             = os.path.normpath(source_directory)
        relative_path        = normpath.split(os.sep)[-1]
        split_path           = os.path.split(os.path.basename(source_directory))[1]
        start = datetime.datetime.now()
        source_manifest = source_parent_dir + '/%s_manifest.md5' % relative_path
        #make_manifest(os.path.dirname(source_directory), os.path.basename(source_directory), source_manifest)
        info = make_framemd5(source_directory, 'dpx_framemd5')
        if info == 'none':
            continue
        for files in filenames:
            total_size += os.path.getsize(os.path.join(root,files))
        output_dirname              = info[0]
        source_textfile             = info[1]
        image_seq_without_container = info[2]
        start_number                = info[3]
        container                   = info[4]
        dpx_filename                = image_seq_without_container + "%06d." + container

        logfile = output_dirname + '/logs/%s_dpx_transcode.log' % os.path.basename(root)
        env_dict = set_environment(logfile)
        ffv12dpx = ['ffmpeg','-report','-f','image2','-framerate','24', '-start_number', start_number, '-i', os.path.abspath(dpx_filename) ,'-strict', '-2','-c:v','ffv1','-level', '3', '-pix_fmt', 'rgb48le',output_dirname +  '/video/' + os.path.basename(root) + '.mkv']
        print ffv12dpx

        subprocess.call(ffv12dpx,env=env_dict)
        parent_basename   =  os.path.basename(output_dirname)
        manifest_textfile = os.path.dirname(output_dirname) + '/' +  parent_basename + '_manifest.md5'
        ffv1_path         = output_dirname +  '/video/'  + os.path.basename(root) + '.mkv'
        ffv1_md5          = output_dirname +  '/md5/' + os.path.basename(root) + 'ffv1.framemd5'
        subprocess.call(['ffmpeg','-i', ffv1_path, '-pix_fmt', 'rgb48le','-f', 'framemd5', ffv1_md5])
        #other_textfile = other[1]
        judgement = diff_textfiles(source_textfile, ffv1_md5)
        make_manifest(output_parent_directory, os.path.basename(output_dirname), manifest_textfile)
        finish = datetime.datetime.now()

        print total_size
        ffv1_size = os.path.getsize(ffv1_path)
        comp_ratio =  float(total_size) / float(os.path.getsize(ffv1_path))
        append_csv(csv_report_filename, (parent_basename,judgement, start, finish,total_size, ffv1_size, comp_ratio))

#send_gmail(emails, csv_report_filename, 'makedpx completed', 'Hi,\n Please the attached log for details of the makedpx job, \nSincerely yours,\nIFIROBOT', config[2].rstrip(), config[3].rstrip())
<|endoftext|>


<commit_before>from tkinter import *
from point import point
from algorithmbezier import bezier, algorithmbezier
from drawing import draw

def main():
    print("Здравствуйте!")
    root = Tk()
    panel = Canvas(root, width = 500, height = 500)
    panel.pack()

    nums = [int(x) for x in input().split()]
    p = []
    # p = [point(10, 60), point(200, 20), point(90, 200), point(400, 500)]
    # p = [point(200, 100), point(400, 400), point(100, 400), point(300, 100)]
    # p = [point(100, 100), point(400, 100), point(400, 400), point(100, 400), point(100, 100)]
    for i in range(0, len(nums), 2):
        p.append(point(nums[i], nums[i + 1]))

    points = algorithmbezier(p)

    for i in range(len(points) - 1):
        panel.create_line(points[i].x, points[i].y, points[i + 1].x, points[i + 1].y, fill="red")
    draw(points, "test")

    root.mainloop()<commit_msg>commit<commit_after>from tkinter import *
from point import point
from algorithmbezier import bezier, algorithmbezier
from drawing import draw

def main(name, p):
    print("Здравствуйте!")
    # root = Tk()
    # panel = Canvas(root, width = 500, height = 500)
    # panel.pack()

    # nums = [int(x) for x in input().split()]
    # p = []
    # p = [point(10, 60), point(200, 20), point(90, 200), point(400, 500)]
    # p = [point(200, 100), point(400, 400), point(100, 400), point(300, 100)]
    # p = [point(100, 100), point(400, 100), point(400, 400), point(100, 400), point(100, 100)]
    # for i in range(0, len(nums), 2):
    #    p.append(point(nums[i], nums[i + 1]))

    points = algorithmbezier(p)

    # for i in range(len(points) - 1):
    #    panel.create_line(points[i].x, points[i].y, points[i + 1].x, points[i + 1].y, fill="red")
    draw(points, name, "bezier")

    # root.mainloop()<|endoftext|>


<commit_before>import string
import random

random.seed()

# ==============================
# = General sequence utilities =
# ==============================

def substitute(seq,pos,sub):
    return seq[:pos] + sub + seq[pos+1:]

complement_table = string.maketrans('ACGTRYSWKMBDHVN','TGCAYRSWMKVHDBN')

def reverse(seq):
    return seq[::-1]

def complement(seq):
    return seq.upper().translate(complement_table)

def reverse_complement(seq):
    """Compute reverse complement of sequence.

    Mindful of IUPAC ambiguities.
    Return all uppercase.
    """
    return reverse(complement(seq))
    # return seq.upper().translate(complement_table)[::-1]

def gc_content(seq):
    gc = seq.lower().count('g') + seq.lower().count('c')
    return float(gc) / len(seq)

def random_dna_seq(n):
    choice = random.choice
    return reduce(lambda cumul,garbage:cumul+choice('ACGT'),xrange(n),'')

# for generating 'safe' filenames from identifiers
cleanup_table = string.maketrans('/*|','___')
def cleanup_id(identifier):
    return identifier.translate(cleanup_table)

def seqhist(seqlist):
    seqdict = dict()
    for seq in seqlist:
        seqdict[seq] = seqdict.get(seq,0) + 1
    return seqdict

def seqmode(seqs):
    if isinstance(seqs,list):
        seqs = seqhist(seqs)
    return max(seqs.iterkeys(),key=lambda k: seqs[k])

# ==========================
# = Manual FASTA iteration =
# ==========================

# taken from biopython

identity = string.maketrans('','')
nonalpha = identity.translate(identity,string.ascii_letters)

def FastaIterator(handle,title2ids=lambda s: s):
    while True:
        line = handle.readline()
        if line == '' : return
        if line[0] == '>':
            break

    while True:
        if line[0] != '>':
            raise ValueError("Records in Fasta files should start with '>' character")
        descr = title2ids(line[1:].rstrip())
        fullline = ''
        line = handle.readline()
        while True:
            if not line : break
            if line[0] == '>': break
            fullline += line.translate(identity,nonalpha)
            line = handle.readline()

        yield (descr,fullline)

        if not line : return #StopIteration
    assert False, "Should not reach this line"

# ============================
# = biopython-specific tools =
# ============================

from Bio.Seq       import Seq
from Bio.SeqRecord import SeqRecord

def make_SeqRecord(name,seq):
    return SeqRecord(Seq(seq),id=name,name=name,description=name)

def get_string(seqobj):
    if isinstance(seqobj,SeqRecord):
        seq = seqobj.seq.tostring().upper()
    elif isinstance(seqobj,Seq):
        seq = seqobj.tostring().upper()
    elif isinstance(seqobj,str):
        seq = seqobj.upper()
    return seq

def get_features(feature_list,feature_type):
    target_features = []
    for feature in feature_list:
        if feature.type == feature_type:
            target_features.append(feature)
    return target_features

def advance_to_features(feature_iter,feature_types):
    # note, here feature_types is a list of possible stopping points
    for feature in feature_iter:
        if feature.type in feature_types:
            return feature
    raise ValueError, "didn't find %s in record" % feature_types

def advance_to_feature(feature_iter,feature_type):
    return advance_to_features(feature_iter,[feature_type])
<commit_msg>Added function to copy SeqFeatures<commit_after>import string
import random

from Bio.Seq        import Seq
from Bio.SeqRecord  import SeqRecord
from Bio.SeqFeature import SeqFeature, FeatureLocation

random.seed()

# ==============================
# = General sequence utilities =
# ==============================

def substitute(seq,pos,sub):
    return seq[:pos] + sub + seq[pos+1:]

complement_table = string.maketrans('ACGTRYSWKMBDHVN','TGCAYRSWMKVHDBN')

def reverse(seq):
    return seq[::-1]

def complement(seq):
    return seq.upper().translate(complement_table)

def reverse_complement(seq):
    """Compute reverse complement of sequence.

    Mindful of IUPAC ambiguities.
    Return all uppercase.
    """
    return reverse(complement(seq))
    # return seq.upper().translate(complement_table)[::-1]

def gc_content(seq):
    gc = seq.lower().count('g') + seq.lower().count('c')
    return float(gc) / len(seq)

def random_dna_seq(n):
    choice = random.choice
    return reduce(lambda cumul,garbage:cumul+choice('ACGT'),xrange(n),'')

# for generating 'safe' filenames from identifiers
cleanup_table = string.maketrans('/*|','___')
def cleanup_id(identifier):
    return identifier.translate(cleanup_table)

def seqhist(seqlist):
    seqdict = dict()
    for seq in seqlist:
        seqdict[seq] = seqdict.get(seq,0) + 1
    return seqdict

def seqmode(seqs):
    if isinstance(seqs,list):
        seqs = seqhist(seqs)
    return max(seqs.iterkeys(),key=lambda k: seqs[k])

# ==========================
# = Manual FASTA iteration =
# ==========================

# taken from biopython

identity = string.maketrans('','')
nonalpha = identity.translate(identity,string.ascii_letters)

def FastaIterator(handle,title2ids=lambda s: s):
    while True:
        line = handle.readline()
        if line == '' : return
        if line[0] == '>':
            break

    while True:
        if line[0] != '>':
            raise ValueError("Records in Fasta files should start with '>' character")
        descr = title2ids(line[1:].rstrip())
        fullline = ''
        line = handle.readline()
        while True:
            if not line : break
            if line[0] == '>': break
            fullline += line.translate(identity,nonalpha)
            line = handle.readline()

        yield (descr,fullline)

        if not line : return #StopIteration
    assert False, "Should not reach this line"

# ============================
# = biopython-specific tools =
# ============================

def make_SeqRecord(name,seq):
    return SeqRecord(Seq(seq),id=name,name=name,description=name)

def get_string(seqobj):
    if isinstance(seqobj,SeqRecord):
        seq = seqobj.seq.tostring().upper()
    elif isinstance(seqobj,Seq):
        seq = seqobj.tostring().upper()
    elif isinstance(seqobj,str):
        seq = seqobj.upper()
    return seq

def get_features(feature_list,feature_type):
    target_features = []
    for feature in feature_list:
        if feature.type == feature_type:
            target_features.append(feature)
    return target_features

def advance_to_features(feature_iter,feature_types):
    # note, here feature_types is a list of possible stopping points
    for feature in feature_iter:
        if feature.type in feature_types:
            return feature
    raise ValueError, "didn't find %s in record" % feature_types

def advance_to_feature(feature_iter,feature_type):
    return advance_to_features(feature_iter,[feature_type])

def copy_features( record_from, record_to, coord_mapping, offset=0 ):
    for feature in record_from.features:
        new_feature = copy.deepcopy(feature)
        new_start = coord_mapping[feature.location.start.position][-1] + offset
        new_end   = coord_mapping[feature.location.end.position][0] + offset
        new_location = FeatureLocation(new_start,new_end)
        new_feature.location = new_location
        record_to.features.append(new_feature)
<|endoftext|>


<commit_before>from __future__ import absolute_import

import os

from celery import Celery

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'remo.settings')

from django.conf import settings  # noqa

RUN_DAILY = 60 * 60 * 24
RUN_HOURLY = 60 * 60

app = Celery('remo')

app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()

@app.on_after_configure.connect
def setup_periodic_tasks(sender, **kwargs):
    from remo.base.tasks import celery_healthcheck
    from remo.events.tasks import notify_event_owners_to_input_metrics
    from remo.profiles.tasks import (check_mozillian_username, reset_rotm_nominees,
                                     send_rotm_nomination_reminder, set_unavailability_flag,
                                     resolve_nomination_action_items)
    from remo.remozilla.tasks import fetch_bugs
    from remo.reports.tasks import (send_first_report_notification, calculate_longest_streaks,
                                    send_second_report_notification, resolve_report_action_items)
    from remo.voting.tasks import extend_voting_period, resolve_action_items, create_rotm_poll

    sender.add_periodic_task(RUN_DAILY, notify_event_owners_to_input_metrics.s(),
                             name='notify-event-owners-to-input-metrics')

    sender.add_periodic_task(RUN_DAILY, check_mozillian_username.s(),
                             name='check-mozillian-username')

    sender.add_periodic_task(RUN_DAILY, reset_rotm_nominees.s(),
                             name='reset-rotm-nominees')

    sender.add_periodic_task(RUN_DAILY, send_rotm_nomination_reminder.s(),
                             name='send-rotm-nomination-reminder')

    sender.add_periodic_task(RUN_DAILY / 2, set_unavailability_flag.s(),
                             name='set-unavailability-flag')

    sender.add_periodic_task(RUN_DAILY, resolve_nomination_action_items.s(),
                             name='resolve-nomination-action-items')

    sender.add_periodic_task(RUN_HOURLY / 4, fetch_bugs.s(),
                             name='fetch-bugs')

    sender.add_periodic_task(RUN_DAILY, send_first_report_notification.s(),
                             name='send-first-report-notification')

    sender.add_periodic_task(RUN_DAILY, send_second_report_notification.s(),
                             name='send-second-report-notification')

    sender.add_periodic_task(RUN_DAILY, calculate_longest_streaks.s(),
                             name='calculate-longest-streaks')

    sender.add_periodic_task(RUN_DAILY, resolve_report_action_items.s(),
                             name='resolve-report-action-items')

    sender.add_periodic_task(RUN_DAILY / 3, extend_voting_period.s(),
                             name='extend-voting-period')

    sender.add_periodic_task(RUN_DAILY, resolve_action_items.s(),
                             name='resolve-action-items')

    sender.add_periodic_task(RUN_DAILY, create_rotm_poll.s(),
                             name='create-rotm-poll')

    sender.add_periodic_task(RUN_HOURLY, celery_healthcheck.s(),
                             name='celery-healthcheck')
<commit_msg>Configure celery to send errors to sentry.<commit_after>from __future__ import absolute_import

import os

from celery import Celery as BaseCelery
from raven.contrib.celery import register_signal, register_logger_signal
from raven.contrib.django.raven_compat.models import client as raven_client

# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'remo.settings')

from django.conf import settings  # noqa

RUN_DAILY = 60 * 60 * 24
RUN_HOURLY = 60 * 60

class Celery(BaseCelery):
    def on_configure(self):
        register_logger_signal(raven_client)
        register_signal(raven_client)

app = Celery('remo')

app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()

@app.on_after_configure.connect
def setup_periodic_tasks(sender, **kwargs):
    from remo.base.tasks import celery_healthcheck
    from remo.events.tasks import notify_event_owners_to_input_metrics
    from remo.profiles.tasks import (check_mozillian_username, reset_rotm_nominees,
                                     send_rotm_nomination_reminder, set_unavailability_flag,
                                     resolve_nomination_action_items)
    from remo.remozilla.tasks import fetch_bugs
    from remo.reports.tasks import (send_first_report_notification, calculate_longest_streaks,
                                    send_second_report_notification, resolve_report_action_items)
    from remo.voting.tasks import extend_voting_period, resolve_action_items, create_rotm_poll

    sender.add_periodic_task(RUN_DAILY, notify_event_owners_to_input_metrics.s(),
                             name='notify-event-owners-to-input-metrics')

    sender.add_periodic_task(RUN_DAILY, check_mozillian_username.s(),
                             name='check-mozillian-username')

    sender.add_periodic_task(RUN_DAILY, reset_rotm_nominees.s(),
                             name='reset-rotm-nominees')

    sender.add_periodic_task(RUN_DAILY, send_rotm_nomination_reminder.s(),
                             name='send-rotm-nomination-reminder')

    sender.add_periodic_task(RUN_DAILY / 2, set_unavailability_flag.s(),
                             name='set-unavailability-flag')

    sender.add_periodic_task(RUN_DAILY, resolve_nomination_action_items.s(),
                             name='resolve-nomination-action-items')

    sender.add_periodic_task(RUN_HOURLY / 4, fetch_bugs.s(),
                             name='fetch-bugs')

    sender.add_periodic_task(RUN_DAILY, send_first_report_notification.s(),
                             name='send-first-report-notification')

    sender.add_periodic_task(RUN_DAILY, send_second_report_notification.s(),
                             name='send-second-report-notification')

    sender.add_periodic_task(RUN_DAILY, calculate_longest_streaks.s(),
                             name='calculate-longest-streaks')

    sender.add_periodic_task(RUN_DAILY, resolve_report_action_items.s(),
                             name='resolve-report-action-items')

    sender.add_periodic_task(RUN_DAILY / 3, extend_voting_period.s(),
                             name='extend-voting-period')

    sender.add_periodic_task(RUN_DAILY, resolve_action_items.s(),
                             name='resolve-action-items')

    sender.add_periodic_task(RUN_DAILY, create_rotm_poll.s(),
                             name='create-rotm-poll')

    sender.add_periodic_task(RUN_HOURLY, celery_healthcheck.s(),
                             name='celery-healthcheck')
<|endoftext|>


<commit_before>import time
from collections import namedtuple

from cassandra import ConsistencyLevel
from cassandra.query import SimpleStatement

from dtest import Tester, debug
from tools import insert_c1c2, no_vnodes, query_c1c2, since

class TestRepair(Tester):

    def check_rows_on_node(self, node_to_check, rows, found=None, missings=None, restart=True):
        if found is None:
            found = []
        if missings is None:
            missings = []
        stopped_nodes = []

        for node in self.cluster.nodes.values():
            if node.is_running() and node is not node_to_check:
                stopped_nodes.append(node)
                node.stop(wait_other_notice=True)

        cursor = self.patient_cql_connection(node_to_check, 'ks')
        result = cursor.execute("SELECT * FROM cf LIMIT %d" % (rows * 2))
        assert len(result) == rows, len(result)

        for k in found:
            query_c1c2(cursor, k, ConsistencyLevel.ONE)

        for k in missings:
            query = SimpleStatement("SELECT c1, c2 FROM cf WHERE key='k%d'" % k, consistency_level=ConsistencyLevel.ONE)
            res = cursor.execute(query)
            assert len(filter(lambda x: len(x) != 0, res)) == 0, res

        if restart:
            for node in stopped_nodes:
                node.start(wait_other_notice=True)

    def simple_sequential_repair_test(self, ):
        self._simple_repair(sequential=True)

    def simple_parallel_repair_test(self, ):
        self._simple_repair(sequential=False)

    def empty_vs_gcable_sequential_repair_test(self):
        self._empty_vs_gcable_no_repair(sequential=True)

    def empty_vs_gcable_parallel_repair_test(self):
        self._empty_vs_gcable_no_repair(sequential=False)

    @no_vnodes()  # https://issues.apache.org/jira/browse/CASSANDRA-5220
    def simple_repair_order_preserving_test(self, ):
        self._simple_repair(order_preserving_partitioner=True)

    def _repair_options(self, ks='', cf=None, sequential=True):
        if cf is None:
            cf = []
        opts = []
        version = self.cluster.version()
        # since version 2.2, default is parallel, otherwise it's sequential
        if sequential:
            if version >= '2.2':
                opts += ['-seq']
        else:
            if version < '2.2':
                opts += ['-par']

        # test with full repair
        if version >= '2.2':
            opts += ['-full']
        if ks:
            opts += [ks]
        if cf:
            opts += cf
        return opts

    def _simple_repair(self, order_preserving_partitioner=False, sequential=True):
        cluster = self.cluster

        if order_preserving_partitioner:
            cluster.set_partitioner('org.apache.cassandra.dht.ByteOrderedPartitioner')

        # Disable hinted handoff and set batch commit log so this doesn't
        # interfer with the test (this must be after the populate)
        cluster.set_configuration_options(values={'hinted_handoff_enabled': False}, batch_commitlog=True)
        debug("Starting cluster..")
        cluster.populate(3).start()
        [node1, node2, node3] = cluster.nodelist()

        cursor = self.patient_cql_connection(node1)
        self.create_ks(cursor, 'ks', 3)
        self.create_cf(cursor, 'cf', read_repair=0.0, columns={'c1': 'text', 'c2': 'text'})

        # Insert 1000 keys, kill node 3, insert 1 key, restart node 3, insert 1000 more keys
        debug("Inserting data...")
        for i in xrange(0, 1000):
            insert_c1c2(cursor, i, ConsistencyLevel.ALL)
        node3.flush()
        node3.stop()
        insert_c1c2(cursor, 1000, ConsistencyLevel.TWO)
        node3.start(wait_other_notice=True)
        for i in xrange(1001, 2001):
            insert_c1c2(cursor, i, ConsistencyLevel.ALL)

        cluster.flush()

        # Verify that node3 has only 2000 keys
        debug("Checking data on node3...")
        self.check_rows_on_node(node3, 2000, missings=[1000])

        # Verify that node1 has 2001 keys
        debug("Checking data on node1...")
        self.check_rows_on_node(node1, 2001, found=[1000])

        # Verify that node2 has 2001 keys
        debug("Checking data on node2...")
        self.check_rows_on_node(node2, 2001, found=[1000])

        time.sleep(10)  # see CASSANDRA-4373
        # Run repair
        start = time.time()
        debug("starting repair...")
        node1.repair(self._repair_options(ks='ks', sequential=sequential))
        debug("Repair time: {end}".format(end=time.time() - start))

        # Validate that only one range was transfered
        l = node1.grep_log("/([0-9.]+) and /([0-9.]+) have ([0-9]+) range\(s\) out of sync")
        if cluster.version() > "1":
            assert len(l) == 2, "Lines matching: " + str([elt[0] for elt in l])
        else:
            # In pre-1.0, we should have only one line
            assert len(l) == 1, "Lines matching: " + str([elt[0] for elt in l])
        valid = [(node1.address(), node3.address()), (node3.address(), node1.address()),
                 (node2.address(), node3.address()), (node3.address(), node2.address())]
        for line, m in l:
            assert int(m.group(3)) == 1, "Expecting 1 range out of sync, got " + int(m.group(1))
            assert (m.group(1), m.group(2)) in valid, str((m.group(1), m.group(2)))
            valid.remove((m.group(1), m.group(2)))
            valid.remove((m.group(2), m.group(1)))

        # Check node3 now has the key
        self.check_rows_on_node(node3, 2001, found=[1000], restart=False)

    def _empty_vs_gcable_no_repair(self, sequential):
        """
        Repairing empty partition and tombstoned partition older than gc grace
        should be treated as the same and no repair is necessary.
        See CASSANDRA-8979.
        """
        cluster = self.cluster
        cluster.populate(2)
        cluster.set_configuration_options(values={'hinted_handoff_enabled': False}, batch_commitlog=True)
        cluster.start()
        [node1, node2] = cluster.nodelist()

        cursor = self.patient_cql_connection(node1)
        # create keyspace with RF=2 to be able to be repaired
        self.create_ks(cursor, 'ks', 2)
        # we create two tables, one has low gc grace seconds so that the data
        # can be dropped during test (but we don't actually drop them).
        # the other has default gc.
        # compaction is disabled not to purge data
        query = """
            CREATE TABLE cf1 (
                key text,
                c1 text,
                c2 text,
                PRIMARY KEY (key, c1)
            )
            WITH gc_grace_seconds=1
            AND compaction = {'class': 'SizeTieredCompactionStrategy', 'enabled': 'false'};
        """
        cursor.execute(query)
        time.sleep(.5)
        query = """
            CREATE TABLE cf2 (
                key text,
                c1 text,
                c2 text,
                PRIMARY KEY (key, c1)
            )
            WITH compaction = {'class': 'SizeTieredCompactionStrategy', 'enabled': 'false'};
        """
        cursor.execute(query)
        time.sleep(.5)

        # take down node2, so that only node1 has gc-able data
        node2.stop(wait_other_notice=True)
        for cf in ['cf1', 'cf2']:
            # insert some data
            for i in xrange(0, 10):
                for j in xrange(0, 1000):
                    query = SimpleStatement("INSERT INTO %s (key, c1, c2) VALUES ('k%d', 'v%d', 'value')" % (cf, i, j), consistency_level=ConsistencyLevel.ONE)
                    cursor.execute(query)
            node1.flush()
            # delete those data, half with row tombstone, and the rest with cell range tombstones
            for i in xrange(0, 5):
                query = SimpleStatement("DELETE FROM %s WHERE key='k%d'" % (cf, i), consistency_level=ConsistencyLevel.ONE)
                cursor.execute(query)
            node1.flush()
            for i in xrange(5, 10):
                for j in xrange(0, 1000):
                    query = SimpleStatement("DELETE FROM %s WHERE key='k%d' AND c1='v%d'" % (cf, i, j), consistency_level=ConsistencyLevel.ONE)
                    cursor.execute(query)
            node1.flush()

        # sleep until gc grace seconds pass so that cf1 can be dropped
        time.sleep(2)

        # bring up node2 and repair
        node2.start(wait_for_binary_proto=True, wait_other_notice=True)
        node2.repair(self._repair_options(ks='ks', sequential=sequential))

        # check no rows will be returned
        for cf in ['cf1', 'cf2']:
            for i in xrange(0, 10):
                query = SimpleStatement("SELECT c1, c2 FROM %s WHERE key='k%d'" % (cf, i), consistency_level=ConsistencyLevel.ALL)
                res = cursor.execute(query)
                assert len(filter(lambda x: len(x) != 0, res)) == 0, res

        # check log for no repair happened for gcable data
        l = node2.grep_log("/([0-9.]+) and /([0-9.]+) have ([0-9]+) range\(s\) out of sync for cf1")
        assert len(l) == 0, "GC-able data does not need to be repaired with empty data: " + str([elt[0] for elt in l])
        # check log for actual repair for non gcable data
        l = node2.grep_log("/([0-9.]+) and /([0-9.]+) have ([0-9]+) range\(s\) out of sync for cf2")
        assert len(l) > 0, "Non GC-able data should be repaired"

RepairTableContents = namedtuple('RepairTableContents',
                                 ['parent_repair_history', 'repair_history'])

@since('2.2')
class TestRepairDataSystemTable(Tester):
    """
    @jira_ticket CASSANDRA-5839

    Tests the `system_distributed.parent_repair_history` and
    `system_distributed.repair_history` tables by writing thousands of records
    to a cluster, then ensuring these tables are in valid states before and
    after running repair.
    """
    def setUp(self):
        """
        Prepares a cluster for tests of the repair history tables by starting
        a 5-node cluster, then inserting 5000 values with RF=3.
        """

        Tester.setUp(self)
        self.cluster.populate(5).start(wait_for_binary_proto=True)
        self.node1 = self.cluster.nodelist()[0]
        self.session = self.patient_cql_connection(self.node1)

        self.node1.stress(stress_options=['write', 'n=5000', 'cl=ONE', '-schema', 'replication(factor=3)'])

        self.cluster.flush()

    def repair_table_contents(self, node, include_system_keyspaces=True):
        """
        @param node the node to connect to and query
        @param include_system_keyspaces if truthy, return repair information about all keyspaces. If falsey, filter out keyspaces whose name contains 'system'

        Return a `RepairTableContents` `namedtuple` containing the rows in
        `node`'s `system_distributed.parent_repair_history` and
        `system_distributed.repair_history` tables. If `include_system_keyspaces`,
        include all results. If not `include_system_keyspaces`, filter out
        repair information about system keyspaces, or at least keyspaces with
        'system' in their names.
        """
        session = self.patient_cql_connection(node)

        def execute_with_all(stmt):
            return session.execute(SimpleStatement(stmt, consistency_level=ConsistencyLevel.ALL))

        parent_repair_history = execute_with_all('SELECT * FROM system_distributed.parent_repair_history;')
        repair_history = execute_with_all('SELECT * FROM system_distributed.repair_history;')

        if not include_system_keyspaces:
            parent_repair_history = [row for row in parent_repair_history
                                     if 'system' not in row.keyspace_name]
            repair_history = [row for row in repair_history if
                              'system' not in row.keyspace_name]
        return RepairTableContents(parent_repair_history=parent_repair_history,
                                   repair_history=repair_history)

    def initial_empty_repair_tables_test(self):
        debug('repair tables:')
        debug(self.repair_table_contents(node=self.node1, include_system_keyspaces=False))
        repair_tables_dict = self.repair_table_contents(node=self.node1, include_system_keyspaces=False)._asdict()
        for table_name, table_contents in repair_tables_dict.items():
            self.assertFalse(table_contents, '{} is non-empty'.format(table_name))

    def repair_history_template(self, node, parent):
        """
        @param repair_node calls repair and checks the contents of the repair history tables on this node
        @param parent whether to check the parent_repair_history or repair_history table

        A parameterized test of `system_distributed.parent_repair_history`
        and `system_distributed.parent_repair_history`. Tests them by:

        - running repair on `node` and
        - getting the contents of the `parent_repair_history` and `repair_history` tables on `node`.

        If `parent`, then this checks that there are a non-zero number of entries in `parent_repair_history`.
        If not `parent`, then this checks that there are a non-zero number of entries in `repair_history`.
        """
        # this error is from tripping an anticompaction guard, not catastrophic failure
        self.ignore_log_patterns = ['may not update a reader that has been obsoleted']
        node.repair()
        (parent_repair_history,
         repair_history) = self.repair_table_contents(node=node, include_system_keyspaces=False)

        self.assertTrue(len(parent_repair_history if parent else repair_history))

    def repair_parent_table_test(self):
        """
        Uses repair_history_template to test that `parent_repair_history` on a
        node is populated correctly after running repair.
        """
        self.repair_history_template(node=self.node1, parent=True)

    def repair_table_test(self):
        """
        Uses repair_history_template to test that `repair_history` on a node
        is populated correctly after running repair.
        """
        self.repair_history_template(node=self.node1, parent=False)
<commit_msg>add require for failing repair tests<commit_after>import time
from collections import namedtuple

from cassandra import ConsistencyLevel
from cassandra.query import SimpleStatement

from dtest import Tester, debug
from tools import insert_c1c2, no_vnodes, query_c1c2, since, require

class TestRepair(Tester):

    def check_rows_on_node(self, node_to_check, rows, found=None, missings=None, restart=True):
        if found is None:
            found = []
        if missings is None:
            missings = []
        stopped_nodes = []

        for node in self.cluster.nodes.values():
            if node.is_running() and node is not node_to_check:
                stopped_nodes.append(node)
                node.stop(wait_other_notice=True)

        cursor = self.patient_cql_connection(node_to_check, 'ks')
        result = cursor.execute("SELECT * FROM cf LIMIT %d" % (rows * 2))
        assert len(result) == rows, len(result)

        for k in found:
            query_c1c2(cursor, k, ConsistencyLevel.ONE)

        for k in missings:
            query = SimpleStatement("SELECT c1, c2 FROM cf WHERE key='k%d'" % k, consistency_level=ConsistencyLevel.ONE)
            res = cursor.execute(query)
            assert len(filter(lambda x: len(x) != 0, res)) == 0, res

        if restart:
            for node in stopped_nodes:
                node.start(wait_other_notice=True)

    def simple_sequential_repair_test(self, ):
        self._simple_repair(sequential=True)

    def simple_parallel_repair_test(self, ):
        self._simple_repair(sequential=False)

    def empty_vs_gcable_sequential_repair_test(self):
        self._empty_vs_gcable_no_repair(sequential=True)

    def empty_vs_gcable_parallel_repair_test(self):
        self._empty_vs_gcable_no_repair(sequential=False)

    @no_vnodes()  # https://issues.apache.org/jira/browse/CASSANDRA-5220
    def simple_repair_order_preserving_test(self, ):
        self._simple_repair(order_preserving_partitioner=True)

    def _repair_options(self, ks='', cf=None, sequential=True):
        if cf is None:
            cf = []
        opts = []
        version = self.cluster.version()
        # since version 2.2, default is parallel, otherwise it's sequential
        if sequential:
            if version >= '2.2':
                opts += ['-seq']
        else:
            if version < '2.2':
                opts += ['-par']

        # test with full repair
        if version >= '2.2':
            opts += ['-full']
        if ks:
            opts += [ks]
        if cf:
            opts += cf
        return opts

    def _simple_repair(self, order_preserving_partitioner=False, sequential=True):
        cluster = self.cluster

        if order_preserving_partitioner:
            cluster.set_partitioner('org.apache.cassandra.dht.ByteOrderedPartitioner')

        # Disable hinted handoff and set batch commit log so this doesn't
        # interfer with the test (this must be after the populate)
        cluster.set_configuration_options(values={'hinted_handoff_enabled': False}, batch_commitlog=True)
        debug("Starting cluster..")
        cluster.populate(3).start()
        [node1, node2, node3] = cluster.nodelist()

        cursor = self.patient_cql_connection(node1)
        self.create_ks(cursor, 'ks', 3)
        self.create_cf(cursor, 'cf', read_repair=0.0, columns={'c1': 'text', 'c2': 'text'})

        # Insert 1000 keys, kill node 3, insert 1 key, restart node 3, insert 1000 more keys
        debug("Inserting data...")
        for i in xrange(0, 1000):
            insert_c1c2(cursor, i, ConsistencyLevel.ALL)
        node3.flush()
        node3.stop()
        insert_c1c2(cursor, 1000, ConsistencyLevel.TWO)
        node3.start(wait_other_notice=True)
        for i in xrange(1001, 2001):
            insert_c1c2(cursor, i, ConsistencyLevel.ALL)

        cluster.flush()

        # Verify that node3 has only 2000 keys
        debug("Checking data on node3...")
        self.check_rows_on_node(node3, 2000, missings=[1000])

        # Verify that node1 has 2001 keys
        debug("Checking data on node1...")
        self.check_rows_on_node(node1, 2001, found=[1000])

        # Verify that node2 has 2001 keys
        debug("Checking data on node2...")
        self.check_rows_on_node(node2, 2001, found=[1000])

        time.sleep(10)  # see CASSANDRA-4373
        # Run repair
        start = time.time()
        debug("starting repair...")
        node1.repair(self._repair_options(ks='ks', sequential=sequential))
        debug("Repair time: {end}".format(end=time.time() - start))

        # Validate that only one range was transfered
        l = node1.grep_log("/([0-9.]+) and /([0-9.]+) have ([0-9]+) range\(s\) out of sync")
        if cluster.version() > "1":
            assert len(l) == 2, "Lines matching: " + str([elt[0] for elt in l])
        else:
            # In pre-1.0, we should have only one line
            assert len(l) == 1, "Lines matching: " + str([elt[0] for elt in l])
        valid = [(node1.address(), node3.address()), (node3.address(), node1.address()),
                 (node2.address(), node3.address()), (node3.address(), node2.address())]
        for line, m in l:
            assert int(m.group(3)) == 1, "Expecting 1 range out of sync, got " + int(m.group(1))
            assert (m.group(1), m.group(2)) in valid, str((m.group(1), m.group(2)))
            valid.remove((m.group(1), m.group(2)))
            valid.remove((m.group(2), m.group(1)))

        # Check node3 now has the key
        self.check_rows_on_node(node3, 2001, found=[1000], restart=False)

    def _empty_vs_gcable_no_repair(self, sequential):
        """
        Repairing empty partition and tombstoned partition older than gc grace
        should be treated as the same and no repair is necessary.
        See CASSANDRA-8979.
        """
        cluster = self.cluster
        cluster.populate(2)
        cluster.set_configuration_options(values={'hinted_handoff_enabled': False}, batch_commitlog=True)
        cluster.start()
        [node1, node2] = cluster.nodelist()

        cursor = self.patient_cql_connection(node1)
        # create keyspace with RF=2 to be able to be repaired
        self.create_ks(cursor, 'ks', 2)
        # we create two tables, one has low gc grace seconds so that the data
        # can be dropped during test (but we don't actually drop them).
        # the other has default gc.
        # compaction is disabled not to purge data
        query = """
            CREATE TABLE cf1 (
                key text,
                c1 text,
                c2 text,
                PRIMARY KEY (key, c1)
            )
            WITH gc_grace_seconds=1
            AND compaction = {'class': 'SizeTieredCompactionStrategy', 'enabled': 'false'};
        """
        cursor.execute(query)
        time.sleep(.5)
        query = """
            CREATE TABLE cf2 (
                key text,
                c1 text,
                c2 text,
                PRIMARY KEY (key, c1)
            )
            WITH compaction = {'class': 'SizeTieredCompactionStrategy', 'enabled': 'false'};
        """
        cursor.execute(query)
        time.sleep(.5)

        # take down node2, so that only node1 has gc-able data
        node2.stop(wait_other_notice=True)
        for cf in ['cf1', 'cf2']:
            # insert some data
            for i in xrange(0, 10):
                for j in xrange(0, 1000):
                    query = SimpleStatement("INSERT INTO %s (key, c1, c2) VALUES ('k%d', 'v%d', 'value')" % (cf, i, j), consistency_level=ConsistencyLevel.ONE)
                    cursor.execute(query)
            node1.flush()
            # delete those data, half with row tombstone, and the rest with cell range tombstones
            for i in xrange(0, 5):
                query = SimpleStatement("DELETE FROM %s WHERE key='k%d'" % (cf, i), consistency_level=ConsistencyLevel.ONE)
                cursor.execute(query)
            node1.flush()
            for i in xrange(5, 10):
                for j in xrange(0, 1000):
                    query = SimpleStatement("DELETE FROM %s WHERE key='k%d' AND c1='v%d'" % (cf, i, j), consistency_level=ConsistencyLevel.ONE)
                    cursor.execute(query)
            node1.flush()

        # sleep until gc grace seconds pass so that cf1 can be dropped
        time.sleep(2)

        # bring up node2 and repair
        node2.start(wait_for_binary_proto=True, wait_other_notice=True)
        node2.repair(self._repair_options(ks='ks', sequential=sequential))

        # check no rows will be returned
        for cf in ['cf1', 'cf2']:
            for i in xrange(0, 10):
                query = SimpleStatement("SELECT c1, c2 FROM %s WHERE key='k%d'" % (cf, i), consistency_level=ConsistencyLevel.ALL)
                res = cursor.execute(query)
                assert len(filter(lambda x: len(x) != 0, res)) == 0, res

        # check log for no repair happened for gcable data
        l = node2.grep_log("/([0-9.]+) and /([0-9.]+) have ([0-9]+) range\(s\) out of sync for cf1")
        assert len(l) == 0, "GC-able data does not need to be repaired with empty data: " + str([elt[0] for elt in l])
        # check log for actual repair for non gcable data
        l = node2.grep_log("/([0-9.]+) and /([0-9.]+) have ([0-9]+) range\(s\) out of sync for cf2")
        assert len(l) > 0, "Non GC-able data should be repaired"

RepairTableContents = namedtuple('RepairTableContents',
                                 ['parent_repair_history', 'repair_history'])

@since('2.2')
@require('CASSANDRA-9482')  # not a test for this ticket, but fails because of it
class TestRepairDataSystemTable(Tester):
    """
    @jira_ticket CASSANDRA-5839

    Tests the `system_distributed.parent_repair_history` and
    `system_distributed.repair_history` tables by writing thousands of records
    to a cluster, then ensuring these tables are in valid states before and
    after running repair.
    """
    def setUp(self):
        """
        Prepares a cluster for tests of the repair history tables by starting
        a 5-node cluster, then inserting 5000 values with RF=3.
        """

        Tester.setUp(self)
        self.cluster.populate(5).start(wait_for_binary_proto=True)
        self.node1 = self.cluster.nodelist()[0]
        self.session = self.patient_cql_connection(self.node1)

        self.node1.stress(stress_options=['write', 'n=5000', 'cl=ONE', '-schema', 'replication(factor=3)'])

        self.cluster.flush()

    def repair_table_contents(self, node, include_system_keyspaces=True):
        """
        @param node the node to connect to and query
        @param include_system_keyspaces if truthy, return repair information about all keyspaces. If falsey, filter out keyspaces whose name contains 'system'

        Return a `RepairTableContents` `namedtuple` containing the rows in
        `node`'s `system_distributed.parent_repair_history` and
        `system_distributed.repair_history` tables. If `include_system_keyspaces`,
        include all results. If not `include_system_keyspaces`, filter out
        repair information about system keyspaces, or at least keyspaces with
        'system' in their names.
        """
        session = self.patient_cql_connection(node)

        def execute_with_all(stmt):
            return session.execute(SimpleStatement(stmt, consistency_level=ConsistencyLevel.ALL))

        parent_repair_history = execute_with_all('SELECT * FROM system_distributed.parent_repair_history;')
        repair_history = execute_with_all('SELECT * FROM system_distributed.repair_history;')

        if not include_system_keyspaces:
            parent_repair_history = [row for row in parent_repair_history
                                     if 'system' not in row.keyspace_name]
            repair_history = [row for row in repair_history if
                              'system' not in row.keyspace_name]
        return RepairTableContents(parent_repair_history=parent_repair_history,
                                   repair_history=repair_history)

    def initial_empty_repair_tables_test(self):
        debug('repair tables:')
        debug(self.repair_table_contents(node=self.node1, include_system_keyspaces=False))
        repair_tables_dict = self.repair_table_contents(node=self.node1, include_system_keyspaces=False)._asdict()
        for table_name, table_contents in repair_tables_dict.items():
            self.assertFalse(table_contents, '{} is non-empty'.format(table_name))

    def repair_history_template(self, node, parent):
        """
        @param repair_node calls repair and checks the contents of the repair history tables on this node
        @param parent whether to check the parent_repair_history or repair_history table

        A parameterized test of `system_distributed.parent_repair_history`
        and `system_distributed.parent_repair_history`. Tests them by:

        - running repair on `node` and
        - getting the contents of the `parent_repair_history` and `repair_history` tables on `node`.

        If `parent`, then this checks that there are a non-zero number of entries in `parent_repair_history`.
        If not `parent`, then this checks that there are a non-zero number of entries in `repair_history`.
        """
        # this error is from tripping an anticompaction guard, not catastrophic failure
        self.ignore_log_patterns = ['may not update a reader that has been obsoleted']
        node.repair()
        (parent_repair_history,
         repair_history) = self.repair_table_contents(node=node, include_system_keyspaces=False)

        self.assertTrue(len(parent_repair_history if parent else repair_history))

    def repair_parent_table_test(self):
        """
        Uses repair_history_template to test that `parent_repair_history` on a
        node is populated correctly after running repair.
        """
        self.repair_history_template(node=self.node1, parent=True)

    def repair_table_test(self):
        """
        Uses repair_history_template to test that `repair_history` on a node
        is populated correctly after running repair.
        """
        self.repair_history_template(node=self.node1, parent=False)
<|endoftext|>


<commit_before>"""
    Module: Individual
    Description: Defines classes related to the creation and manipulation of
    and Individual for use in the genetic algorithm.
"""

from BitVector import BitVector
from bitarray import bitarray

class Individual:

    """ Encapsulates an Individual in the GA. """

    BIT_VECTOR = 0
    BIT_ARRAY = 1

    def __init__(self, length=0, method=None, defined=False, value=None):

        """ Creates a bit string of a certain length, using a certain underlying
        implementation.  """

        self.length = length

        if method not in [0, 1]:
            self.method = Individual.BIT_VECTOR
        else:
            self.method = method

        if self.method == Individual.BIT_VECTOR:
            self.data = BitVector(size=length)
            self.data = self.data.gen_random_bits(length)
            self.defined = BitVector(size=length)

            if defined:
                self.defined = None
            else:
                self.defined.reset(0)

            if value is not None:
                self.data = BitVector(bitlist = value)

        elif self.method == Individual.BIT_ARRAY:
            self.defined = bitarray(length)
            self.data = bitarray(length)

            if defined:
                self.defined = None
            else:
                self.defined.setall(False)

            if value is not None:
                self.data = [bool(x) for x in value]

    def __str__(self):

        """ Creates a consistent string method across implementations. """

        if self.method == Individual.BIT_VECTOR:
            return str(self.data)
        elif self.method == Individual.BIT_ARRAY:
            return self.data.to01()
        return ""

    def get(self, b):

        """ Returns the value at position b, that is either 1 or 0. """

        b -= 1
        if b >= self.length or b < 0:
            return

        if self.method == Individual.BIT_VECTOR:
            return self.data[b]
        elif self.method == Individual.BIT_ARRAY:
            return int(self.data[b])

    def set(self, b, v):

        """ Sets the bit at position b to value v. """

        b -= 1
        if b >= self.length or b < 0:
            return

        if self.method == Individual.BIT_VECTOR:
            self.data[b] = v
        elif self.method == Individual.BIT_ARRAY:
            self.data[b] = bool(v)

    def flip(self, b):

        """ Flips the bit at position b. """

        b -= 1
        if b >= self.length or b < 0:
            return

        if self.method == Individual.BIT_VECTOR:
            self.data[b] ^= 1
        elif self.method == Individual.BIT_ARRAY:
            self.data[b] = not self.data[b]

    def set_defined(self, b):

        """ Sets """

        b -= 1
        if b >= self.length or b < 0:
            return

        if self.method == Individual.BIT_VECTOR:
            self.defined[b] = 1
        elif self.method == Individual.BIT_ARRAY:
            self.defined[b] = not self.data[b]

    def allocate(self, first, second):

        """ Allocates uniformly from either first or second parent. """

        for i in range(self.length):
            self.set_defined(i)

class Factory:

    """ A factory class for creating individuals in bulk. """

    @staticmethod
    def create(length, method, amount):

        """ Creates an array of individuals. """

        array = [Individual(length, method, fa) for _ in range(amount)]
        return array,

# TESTS

if __name__ == "__main__":
    ind = Individual(9, Individual.BIT_VECTOR)
    num_fail = 0
    print()
    for x in range(2):
        if x == 0:
            print("Testing implementation : BitVector")
            ind.data = BitVector(bitlist=[0, 1, 1, 0, 1, 0, 1, 0, 0])
            ind.method = Individual.BIT_VECTOR
        else:
            print("Testing implementation : bitarray")
            ind.data = bitarray("011010100")
            ind.method = Individual.BIT_ARRAY
        print("With data: " + str(ind))
        print("  Testing get")

        if ind.get(5) == 1:
            print("    get(5) == 1 => pass")
        else:
            print("    get(5) == 1 => fail")
            num_fail += 1

        if ind.get(8) == 0:
            print("    get(8) == 0 => pass")
        else:
            print("    get(8) == 0 => fail")
            num_fail += 1

        if ind.get(10) is None:
            print("    get(10) == None => pass")
        else:
            print("    get(10) == None => fail")
            num_fail += 1
        print()
        print("  Testing set")
        ind.set(2, 1)
        if ind.get(2) == 1:
            print("    set(2,1) => pass")
        else:
            print("    set(2,1) => fail")
            num_fail += 1

        ind.set(7, 0) == 0
        if ind.get(7) == 0:
            print("    set(7,0) => pass")
        else:
            print("    set(7,0) => fail")
            num_fail += 1

        ind.set(6, 1)
        if ind.get(6) == 1:
            print("    set(6,1) => pass")
        else:
            print("    set(6,1) => fail")
            num_fail += 1

        print()
        print("  Testing flip")
        ind.flip(1)
        if ind.get(1) == 1:
            print("    flip(1) => pass")
        else:
            print("    flip(1) => fail")
            num_fail += 1

        ind.flip(8)
        if ind.get(8) == 1:
            print("    flip(8) => pass")
        else:
            print("    flip(8) => fail")
            num_fail += 1

        ind.flip(4)
        if ind.get(4) == 1:
            print("    flip(4) => pass")
        else:
            print("    flip(4) => fail")
            num_fail += 1
        print()
    print("----------------------")
    if num_fail == 0:
        print("Passed all tests.")
    else:
        print(str(num_fail) + " tests failed.")
<commit_msg>Added get_defined method to Individual<commit_after>"""
    Module: Individual
    Description: Defines classes related to the creation and manipulation of
    and Individual for use in the genetic algorithm.
"""

from BitVector import BitVector
from bitarray import bitarray

class Individual:

    """ Encapsulates an Individual in the GA. """

    BIT_VECTOR = 0
    BIT_ARRAY = 1

    def __init__(self, length=0, method=None, defined=False, value=None):

        """ Creates a bit string of a certain length, using a certain underlying
        implementation.  """

        self.length = length

        if method not in [0, 1]:
            self.method = Individual.BIT_VECTOR
        else:
            self.method = method

        if self.method == Individual.BIT_VECTOR:
            self.data = BitVector(size=length)
            self.data = self.data.gen_random_bits(length)
            self.defined = BitVector(size=length)

            if defined:
                self.defined = None
            else:
                self.defined.reset(0)

            if value is not None:
                self.data = BitVector(bitlist = value)

        elif self.method == Individual.BIT_ARRAY:
            self.defined = bitarray(length)
            self.data = bitarray(length)

            if defined:
                self.defined = None
            else:
                self.defined.setall(False)

            if value is not None:
                self.data = [bool(x) for x in value]

    def __str__(self):

        """ Creates a consistent string method across implementations. """

        if self.method == Individual.BIT_VECTOR:
            return str(self.data)
        elif self.method == Individual.BIT_ARRAY:
            return self.data.to01()
        return ""

    def get(self, b):

        """ Returns the value at position b, that is either 1 or 0. """

        b -= 1
        if b >= self.length or b < 0:
            return

        if self.method == Individual.BIT_VECTOR:
            return self.data[b]
        elif self.method == Individual.BIT_ARRAY:
            return int(self.data[b])

    def set(self, b, v):

        """ Sets the bit at position b to value v. """

        b -= 1
        if b >= self.length or b < 0:
            return

        if self.method == Individual.BIT_VECTOR:
            self.data[b] = v
        elif self.method == Individual.BIT_ARRAY:
            self.data[b] = bool(v)

    def flip(self, b):

        """ Flips the bit at position b. """

        b -= 1
        if b >= self.length or b < 0:
            return

        if self.method == Individual.BIT_VECTOR:
            self.data[b] ^= 1
        elif self.method == Individual.BIT_ARRAY:
            self.data[b] = not self.data[b]

    def set_defined(self, b):

        """ Sets a certain bit b as defined. """

        b -= 1
        if b >= self.length or b < 0:
            return

        if self.method == Individual.BIT_VECTOR:
            self.defined[b] = 1
        elif self.method == Individual.BIT_ARRAY:
            self.defined[b] = not self.data[b]

    def get_defined(self, b):

        """ Gets whether a certain bit is defined or not (boolean) """

        b -= 1
        if b >= self.length or b < 0:
            return

        if self.method == Individual.BIT_VECTOR:
            return True if self.defined[b] == 1 else False
        else:
            return self.defined[b]

    def allocate(self, first, second):

        """ Allocates uniformly from either first or second parent. """

        for i in range(self.length):
            self.set_defined(i)

class Factory:

    """ A factory class for creating individuals in bulk. """

    @staticmethod
    def create(length, method, amount):

        """ Creates an array of individuals. """

        array = [Individual(length, method, fa) for _ in range(amount)]
        return array,

# TESTS

if __name__ == "__main__":
    ind = Individual(9, Individual.BIT_VECTOR)
    num_fail = 0
    print()
    for x in range(2):
        if x == 0:
            print("Testing implementation : BitVector")
            ind.data = BitVector(bitlist=[0, 1, 1, 0, 1, 0, 1, 0, 0])
            ind.method = Individual.BIT_VECTOR
        else:
            print("Testing implementation : bitarray")
            ind.data = bitarray("011010100")
            ind.method = Individual.BIT_ARRAY
        print("With data: " + str(ind))
        print("  Testing get")

        if ind.get(5) == 1:
            print("    get(5) == 1 => pass")
        else:
            print("    get(5) == 1 => fail")
            num_fail += 1

        if ind.get(8) == 0:
            print("    get(8) == 0 => pass")
        else:
            print("    get(8) == 0 => fail")
            num_fail += 1

        if ind.get(10) is None:
            print("    get(10) == None => pass")
        else:
            print("    get(10) == None => fail")
            num_fail += 1
        print()
        print("  Testing set")
        ind.set(2, 1)
        if ind.get(2) == 1:
            print("    set(2,1) => pass")
        else:
            print("    set(2,1) => fail")
            num_fail += 1

        ind.set(7, 0) == 0
        if ind.get(7) == 0:
            print("    set(7,0) => pass")
        else:
            print("    set(7,0) => fail")
            num_fail += 1

        ind.set(6, 1)
        if ind.get(6) == 1:
            print("    set(6,1) => pass")
        else:
            print("    set(6,1) => fail")
            num_fail += 1

        print()
        print("  Testing flip")
        ind.flip(1)
        if ind.get(1) == 1:
            print("    flip(1) => pass")
        else:
            print("    flip(1) => fail")
            num_fail += 1

        ind.flip(8)
        if ind.get(8) == 1:
            print("    flip(8) => pass")
        else:
            print("    flip(8) => fail")
            num_fail += 1

        ind.flip(4)
        if ind.get(4) == 1:
            print("    flip(4) => pass")
        else:
            print("    flip(4) => fail")
            num_fail += 1
        print()
    print("----------------------")
    if num_fail == 0:
        print("Passed all tests.")
    else:
        print(str(num_fail) + " tests failed.")
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-

import os, sys, getopt
import csv
import time
import commands
import threading

# create output log folder

KBLAS_HOME='.'
TEST_LOGS_PATH='kblas-test-log'
cmd = ('cd '+KBLAS_HOME+'; mkdir -p '+TEST_LOGS_PATH)
print cmd
sys.stdout.flush()
os.system(cmd)

BIN_PATH='./testing/bin/'
if (not os.path.isdir(BIN_PATH)):
    print 'Unable to find executables folder! Exiting'
    exit()

#detect GPU devices
NGPUS=0
# first check using environment variable
if ( "CUDA_VISIBLE_DEVICES" in os.environ ):
    aux=os.environ["CUDA_VISIBLE_DEVICES"].strip(',')
    if ( len(aux) > 0):
      NGPUS=int( aux.count(',') ) + 1
if ( NGPUS == 0 ):
    # check using system
    cmd=('nvidia-smi -L | wc -l')
    NGPUS = int(commands.getstatusoutput(cmd)[1])

if (NGPUS < 1):
    print 'Unable to detect an NVIDIA GPU device to test on! Exiting'
    exit()

#set options
#check = ''
check = '-c'
defaultBatchCount = 1000
TEST_BATCH_SVD = 1
TEST_BATCH_QR = 1
TEST_BATCH_TRSM = 1
TEST_BATCH_TRMM = 1
TEST_BATCH_GEMM = 1
TEST_BATCH_SYRK = 1
TEST_BATCH_POTRF = 1

#--------------------------------
def task1(pVariants, pRanges, pExec, pOptions, pBatchCount, pDev, pOutfile):
    print 'running: '+pExec+' ... '
    os.system('echo running: '+pExec+' > '+pOutfile)
    for v in pVariants:
        for r in pRanges:
            cmd = (pExec+' '+ r + ' ' + pOptions + ' --dev ' + str(pDev) + ' ' + v + ' --batchCount ' + str(pBatchCount))
            os.system('echo >> '+pOutfile)
            os.system('echo '+cmd+' >> '+pOutfile)
            sys.stdout.flush()
            os.system(cmd+' >> '+pOutfile)
            time.sleep(1)
    print pExec+' done'

#--------------------------------
def launchQueue(*taskQueue):
    for t in taskQueue:
        t.start()
        t.join()

#--------------------------------
def parallelTaskLaunch(variants, programs, ranges, options, batchCount):
    deviceQueue = [[] for _ in xrange(NGPUS)]
    deviceThread = []

    dev = 0
    for p in programs:
        pp = BIN_PATH+p
        if (not os.path.isfile(pp)):
            print 'Unable to find '+pp+' executable! Skipping...'
        else:
            logFile = TEST_LOGS_PATH+'/'+p+'.txt'
            deviceQueue[dev].append( threading.Thread(target=task1, args=(variants, ranges, pp, options, batchCount, dev, logFile)) )
            dev = (dev+1)%NGPUS

    dev = 0
    while dev < NGPUS:
        q = deviceQueue[dev]
        #print q
        tq = threading.Thread( target=launchQueue, args=(q) )
        deviceThread.append(tq)
        tq.start()
        dev = dev + 1

    dev = 0
    while dev < NGPUS:
        tq = deviceThread[dev]
        tq.join()
        dev = dev + 1

############### BATCH_SVD
if (TEST_BATCH_SVD == 1):
    variants = ['']
    programs = ['test_dgesvj_batch', 'test_sgesvj_batch']
    ranges = ['--range 32:512:32']
    options = ''
    batchCount = 200;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_QR
if (TEST_BATCH_QR == 1):
    variants = ['']
    programs = ['test_dgeqrf_batch', 'test_sgeqrf_batch']
    ranges = ['--range 32:512:32']
    batchCount = defaultBatchCount;
    options = ''

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_GEMM
if (TEST_BATCH_GEMM == 1):
    variants = ['']
    programs = ['test_dgemm_batch', 'test_sgemm_batch', 'test_cgemm_batch', 'test_zgemm_batch']
    ranges = ['--range 32:512:32']
    options = '-c -t'
    batchCount = defaultBatchCount;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_TRMM
if (TEST_BATCH_TRMM == 1):
    variants = ['']
    programs = ['test_dtrmm_batch', 'test_strmm_batch', 'test_ctrmm_batch', 'test_ztrmm_batch']
    ranges = ['--range 32:512:32']
    options = '-c -t'
    batchCount = defaultBatchCount;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_TRSM
if (TEST_BATCH_TRSM == 1):
    variants = ['']
    programs = ['test_dtrsm_batch', 'test_strsm_batch', 'test_ctrsm_batch', 'test_ztrsm_batch']
    ranges = ['--range 32:512:32']
    options = '-c -t'
    batchCount = defaultBatchCount;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_SYRK
if (TEST_BATCH_SYRK == 1):
    variants = ['']
    programs = ['test_dsyrk_batch', 'test_ssyrk_batch', 'test_csyrk_batch', 'test_zsyrk_batch']
    ranges = ['--range 32:512:32']
    options = '-c -t'
    batchCount = defaultBatchCount;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_POTRF
if (TEST_BATCH_POTRF == 1):
    variants = ['']
    programs = ['test_dpotrf_batch', 'test_spotrf_batch', 'test_cpotrf_batch', 'test_zpotrf_batch']
    ranges = ['--range 32:512:32']
    options = '-c -t'
    batchCount = defaultBatchCount;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)
<commit_msg>update testing script<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-

import os, sys, getopt
import csv
import time
import commands
import threading

# create output log folder

KBLAS_HOME='.'
TEST_LOGS_PATH='kblas-test-log'
cmd = ('cd '+KBLAS_HOME+'; mkdir -p '+TEST_LOGS_PATH)
print cmd
sys.stdout.flush()
os.system(cmd)

BIN_PATH='./testing/bin/'
if (not os.path.isdir(BIN_PATH)):
    print 'Unable to find executables folder! Exiting'
    exit()

#detect GPU devices
NGPUS=0
# first check using environment variable
if ( "CUDA_VISIBLE_DEVICES" in os.environ ):
    aux=os.environ["CUDA_VISIBLE_DEVICES"].strip(',')
    if ( len(aux) > 0):
      NGPUS=int( aux.count(',') ) + 1
if ( NGPUS == 0 ):
    # check using system
    cmd=('nvidia-smi -L | wc -l')
    NGPUS = int(commands.getstatusoutput(cmd)[1])

if (NGPUS < 1):
    print 'Unable to detect an NVIDIA GPU device to test on! Exiting'
    exit()

print 'NGPUS: ' + str(NGPUS)

#set options
#check = ''
check = ' -c'
defaultBatchCount = 100
TEST_BATCH_SVD = 0
TEST_BATCH_QR = 0
TEST_BATCH_TRSM = 1
TEST_BATCH_TRMM = 1
TEST_BATCH_GEMM = 1
TEST_BATCH_SYRK = 1
TEST_BATCH_POTRF = 1

#--------------------------------
def task1(pVariants, pRanges, pExec, pOptions, pBatchCount, pDev, pOutfile):
    print 'running: '+pExec+' ... '
    os.system('echo running: '+pExec+' > '+pOutfile)
    for v in pVariants:
        for r in pRanges:
            cmd = (pExec+' '+ r + ' ' + pOptions + ' --dev ' + str(pDev) + ' ' + v + ' --batchCount ' + str(pBatchCount))
            os.system('echo >> '+pOutfile)
            os.system('echo '+cmd+' >> '+pOutfile)
            sys.stdout.flush()
            os.system(cmd+' >> '+pOutfile)
            time.sleep(1)
    print pExec+' done'

#--------------------------------
def launchQueue(*taskQueue):
    for t in taskQueue:
        t.start()
        t.join()

#--------------------------------
def parallelTaskLaunch(variants, programs, ranges, options, batchCount):
    deviceQueue = [[] for _ in xrange(NGPUS)]
    deviceThread = []

    dev = 0
    for p in programs:
        pp = BIN_PATH+p
        if (not os.path.isfile(pp)):
            print 'Unable to find '+pp+' executable! Skipping...'
        else:
            logFile = TEST_LOGS_PATH+'/'+p+'.txt'
            deviceQueue[dev].append( threading.Thread(target=task1, args=(variants, ranges, pp, options, batchCount, dev, logFile)) )
            dev = (dev+1)%NGPUS

    dev = 0
    while dev < NGPUS:
        q = deviceQueue[dev]
        #print q
        tq = threading.Thread( target=launchQueue, args=(q) )
        deviceThread.append(tq)
        tq.start()
        dev = dev + 1

    dev = 0
    while dev < NGPUS:
        tq = deviceThread[dev]
        tq.join()
        dev = dev + 1

############### BATCH_SVD
if (TEST_BATCH_SVD == 1):
    variants = ['']
    programs = ['test_dgesvj_batch', 'test_sgesvj_batch']
    ranges = ['--range 32:512:32']
    options = ''
    batchCount = 200;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_QR
if (TEST_BATCH_QR == 1):
    variants = ['']
    programs = ['test_dgeqrf_batch', 'test_sgeqrf_batch']
    ranges = ['--range 32:512:32']
    batchCount = defaultBatchCount;
    options = ''

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_GEMM
if (TEST_BATCH_GEMM == 1):
    variants = ['-NN',
                '-TN',
                '-NT',
                '-TT',
                ]
    programs = ['test_sgemm_batch',
                'test_dgemm_batch',
                'test_cgemm_batch',
                'test_zgemm_batch'
                ]
    ranges = ['--range 32:256:32']
    options = check
    batchCount = defaultBatchCount;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_TRMM
if (TEST_BATCH_TRMM == 1):
    variants = ['-L -NN -DN',
                '-L -TN -DN',
                # '-SL -U -NN -DN',
                # '-SL -U -TN -DN',
                # '-SR -L -NN -DN',
                # '-SR -L -TN -DN',
                # '-SR -U -NN -DN',
                # '-SR -U -TN -DN'
                ]

    programs = ['test_strmm_batch',
                'test_dtrmm_batch',
                'test_ctrmm_batch',
                'test_ztrmm_batch'
                ]
    ranges = ['--range 32:256:32']
    options = check
    batchCount = defaultBatchCount;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_TRSM
if (TEST_BATCH_TRSM == 1):
    variants = ['-SL -L -NN -DN',
                '-SL -L -TN -DN',
                # '-SL -U -NN -DN',
                # '-SL -U -TN -DN',
                '-SR -L -NN -DN',
                '-SR -L -TN -DN',
                # '-SR -U -NN -DN',
                # '-SR -U -TN -DN'
                ]
    programs = ['test_strsm_batch',
                'test_dtrsm_batch',
                # 'test_ctrsm_batch',
                # 'test_ztrsm_batch'
                ]
    ranges = ['--range 32:256:32']
    options = check
    batchCount = defaultBatchCount;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_SYRK
if (TEST_BATCH_SYRK == 1):
    variants = ['-L -NN',
                '-L -TN',
                # '-U -NN',
                # '-U -TN',
                ]
    programs = ['test_ssyrk_batch',
                'test_dsyrk_batch',
                'test_csyrk_batch',
                'test_zsyrk_batch'
                ]
    ranges = ['--range 32:256:32']
    options = check
    batchCount = defaultBatchCount;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)

############### BATCH_POTRF
if (TEST_BATCH_POTRF == 1):
    variants = ['-L',
                # '-U',
                ]
    programs = ['test_spotrf_batch',
                'test_dpotrf_batch',
                'test_cpotrf_batch',
                'test_zpotrf_batch'
                ]
    ranges = ['--range 32:256:32']
    options = check
    batchCount = defaultBatchCount;

    parallelTaskLaunch(variants, programs, ranges, options, batchCount)
<|endoftext|>