<commit_before>import os
import sys
import random
import compare_dirs
import pickle

data_base_path = os.getcwd()
back_base_path = os.getcwd()
backup_dst = back_base_path + '\\backup'
script_name = 'backup_script.txt'
script_path = data_base_path + '\\' + script_name

max_line_length = 120
min_line_length = max_line_length / 10

def to_native_slash(s):
	return s.replace('/', '\\')
	#return s

def to_memory_slash(s):
	return s.replace('\\', '/')
	#return s

def load_file(path):
	return [s.replace('\n', '') for s in open(path, 'r').readlines()]

def load_list():
	return load_file('..\\wordsEn.txt')

wordlist = load_list()

def pick_word():
	return random.choice(wordlist)

def generate_random_line(override_length = -1):
	ret = ''
	length = random.randint(0, max_line_length)
	if override_length >= 0:
		length = override_length
	if length >= min_line_length or override_length >= 0:
		while len(ret) < length:
			if len(ret) > 0:
				ret += ' '
			ret += pick_word()
	return ret

def generate_new_file(path):
	file = open(path, 'w')
	ret = ''
	lines = random.randint(20, 512)
	for i in range(lines):
		file.write(generate_random_line() + '\n')

def edit_line(line):
	words = line.split(' ')
	if '' in words:
		words.remove('')
	if len(words) > 1:
		start = random.randint(0, len(words) - 1)
		length = min(len(words) - start, random.randint(1, int(len(words) / 2)))
	else:
		start = 0
		length = len(words)
	for i in range(start, start + length):
		words[i] = pick_word()
	return ' '.join(words)

def edit_existing_file(path):
	lines = load_file(path)
	for i in range(random.randint(0, 5)):
		blockstart = random.randint(0, len(lines) - 1)
		r = random.randint(1, 100)
		count = min(len(lines) - blockstart, r)
		for j in range(blockstart, blockstart + count):
			#if j >= len(lines):
			#	print('%d, %d, %d, %d'%(len(lines), blockstart, r, count))
			lines[j] = edit_line(lines[j])

	if random.randint(0, 3) == 0:
		new_lines_count = random.randint(20, 512)
		for i in range(new_lines_count):
			lines.append(generate_random_line())

	file = open(path, 'w')
	for line in lines:
		file.write(line + '\n')

def exponential_distribution(k = 2):
	return (k ** random.uniform(1, 10)) / (k ** 10)

def generate_new_binaries_batch(base, addenda):
	for i in range(random.randint(1, 20)):
		path = 'binary.files/%08d.bin'%i
		addenda.append(path)
		file = open(base + '/' + path, 'wb')
		buffer = bytearray(os.urandom(int(exponential_distribution() * 1e+6)))
		file.write(buffer)

def generate_next_version(base, directories, files):
	addenda = []
	if len(files) == 0:
		for i in range(random.randint(0, 10)):
			path = pick_word() + '.txt'
			files.add(path)
			addenda.append(path)
			generate_new_file(base + '/' + path)
		os.mkdir('%s/binary.files'%base)
		addenda.append('binary.files')
		generate_new_binaries_batch(base, addenda)
	else:
		for i in range(min(random.randint(1, 100), len(files))):
			path = random.sample(files, 1)[0]
			edit_existing_file(base + '/' + path)

		if random.randint(1, 25) == 1:
			for i in range(random.randint(1, 5)):
				container = None
				if random.randint(0, len(directories)) == 0:
					container = ''
				else:
					container = random.sample(directories, 1)[0] + '/'
				path = container + pick_word()
				directories.add(path)
				addenda.append(path)
				os.mkdir(base + '/' + path)

		if random.randint(1, 5) == 1:
			for i in range(random.randint(1, 5)):
				container = None
				if random.randint(0, len(directories)) == 0:
					container = ''
				else:
					container = random.sample(directories, 1)[0] + '/'
				path = container + pick_word() + '.txt'
				files.add(path)
				addenda.append(path)
				generate_new_file(base + '/' + path)

		if random.randint(1, 10) == 1:
			generate_new_binaries_batch(base, addenda)

	return

def delete_directory(dir):
	os.system('rd /q /s "%s"'%(to_native_slash(dir)))

def isalpha(x):
	return x >= 'a' and x <= 'z' or x >= 'A' and x <= 'Z'

def simple_wd():
	path = os.getcwd()
	print(path)
	if len(path) > 2 and isalpha(path[0]) and path[1] == ':':
		path = path[2:]
	print(path)
	return to_memory_slash(path)

def generate_backup_script(base):
	cd = os.getcwd()
	script  = 'open %s\n' % backup_dst
	script += 'add %s\\%s\n' % (cd, base)
	script += 'exclude name dirs .svn\n'
	script += 'set change_criterium date\n'
	script += 'set use_snapshots false\n'
	script += 'backup\n'
	script += 'quit\n'
	open(script_path, 'w').write(script)

def initialize(base):
	delete_directory(base)
	os.mkdir(base)
	os.system('rd /q /s %s\\backup' % back_base_path)
	generate_backup_script(base)

def save_version(base):
	return compare_dirs.construct_tree(base)

def perform_backup():
	os.system('zekvok < %s > nul' % script_path)

def print_percent(i, width, n):
	percent = int(i * width / n)
	invpercent = width - percent
	sys.stdout.write('\r[%s%s] %d'%('#'*percent, ' '*invpercent, i))

def perform(base, version_count):
	initialize(base)
	directories = set()
	files = set()
	version_data = {}
	for i in range(version_count):
		print_percent(i, 80, version_count)
		generate_next_version(base, directories, files)
		version_data[i] = save_version(base)
		perform_backup()
	return version_data

def generate_data(version_count):
	filename = 'revision_data.pickle'
	try:
		file = open(filename, 'rb')
		return pickle.load(file)
	except:
		pass
	data = perform('test_repo', version_count)
	pickle.dump(data, open(filename, 'wb'))
	return data

def restore_backup(version):
	script  = 'open %s\\backup\n'
	script += 'select version %d\n'
	script += 'restore\n'
	script += 'quit\n'

	cd = os.getcwd()
	script = script % (data_base_path, version)
	open('restore_script.txt', 'w').write(script)
	os.system('zekvok < %s\\restore_script.txt > nul' % cd)

def perform_test(data, version_count):
	for i in range(version_count):
		print_percent(i, 80, version_count)
		restore_backup(i)
		if not compare_dirs.compare_tree_and_dir(data[i], 'test_repo'):
			return False
	return True

def test():
	os.system('copy /y ..\\..\\bin64\\zekvok.exe .')
	version_count = 100
	data = generate_data(version_count)
	result = perform_test(data, version_count)
	print('')
	if result:
		print('Hooray! No errors found!')
	else:
		print('Some error(s) found in the program.')
	return

test()
<commit_msg>Added test behaviors to test new fixed bugs.<commit_after>import os
import sys
import random
import compare_dirs
import pickle

data_base_path = os.getcwd()
back_base_path = os.getcwd()
backup_dst = back_base_path + '\\backup'
script_name = 'backup_script.txt'
script_path = data_base_path + '\\' + script_name

max_line_length = 120
min_line_length = max_line_length / 10

text_extensions = [
	'asm',
	'bat',
	'c',
	'cpp',
	'cs',
	'cxx',
	'f',
	'gcl',
	'h',
	'hpp',
	'hs',
	'htm',
	'html',
	'hxx',
	'java',
	'js',
	'log',
	'lua',
	'pas',
	'py',
	'qbk',
	's',
	'sh',
	'ss',
	'txt',
	'xml'
]

def to_native_slash(s):
	return s.replace('/', '\\')
	#return s

def to_memory_slash(s):
	return s.replace('\\', '/')
	#return s

def load_file(path):
	return [s.replace('\n', '') for s in open(path, 'r').readlines()]

def load_list():
	return load_file('..\\wordsEn.txt')

wordlist = load_list()

def pick_word():
	return random.choice(wordlist)

def generate_random_line(override_length = -1):
	ret = ''
	length = random.randint(0, max_line_length)
	if override_length >= 0:
		length = override_length
	if length >= min_line_length or override_length >= 0:
		while len(ret) < length:
			if len(ret) > 0:
				ret += ' '
			ret += pick_word()
	return ret

def generate_new_file(path):
	file = open(path, 'w')
	if random.randint(1, 3) == 1:
		return
	ret = ''
	lines = random.randint(20, 512)
	for i in range(lines):
		file.write(generate_random_line() + '\n')

def edit_line(line):
	words = line.split(' ')
	if '' in words:
		words.remove('')
	if len(words) > 1:
		start = random.randint(0, len(words) - 1)
		length = min(len(words) - start, random.randint(1, int(len(words) / 2)))
	else:
		start = 0
		length = len(words)
	for i in range(start, start + length):
		words[i] = pick_word()
	return ' '.join(words)

def edit_existing_file(path):
	lines = load_file(path)
	if len(lines) > 0:
		for i in range(random.randint(0, 5)):
			blockstart = random.randint(0, len(lines) - 1)
			r = random.randint(1, 100)
			count = min(len(lines) - blockstart, r)
			for j in range(blockstart, blockstart + count):
				#if j >= len(lines):
				#	print('%d, %d, %d, %d'%(len(lines), blockstart, r, count))
				lines[j] = edit_line(lines[j])

	if random.randint(0, 3) == 0 or len(lines) == 0:
		new_lines_count = random.randint(20, 512)
		for i in range(new_lines_count):
			lines.append(generate_random_line())

	file = open(path, 'w')
	for line in lines:
		file.write(line + '\n')

def exponential_distribution(k = 2):
	return (k ** random.uniform(1, 10)) / (k ** 10)

def generate_new_binaries_batch(base, addenda):
	for i in range(random.randint(1, 20)):
		path = 'binary.files/%08d.bin'%i
		addenda.append(path)
		file = open(base + '/' + path, 'wb')
		buffer = bytearray(os.urandom(int(exponential_distribution() * 1e+6)))
		file.write(buffer)

def random_filename():
	return pick_word() + '.' + random.choice(text_extensions)

def generate_next_version(base, directories, files):
	addenda = []
	if len(files) == 0:
		for i in range(random.randint(0, 10)):
			path = random_filename()
			files.add(path)
			addenda.append(path)
			generate_new_file(base + '/' + path)
		os.mkdir('%s/binary.files'%base)
		addenda.append('binary.files')
		generate_new_binaries_batch(base, addenda)
	else:
		for i in range(min(random.randint(1, 100), len(files))):
			path = random.sample(files, 1)[0]
			edit_existing_file(base + '/' + path)

		if random.randint(1, 25) == 1:
			for i in range(random.randint(1, 5)):
				container = None
				if random.randint(0, len(directories)) == 0:
					container = ''
				else:
					container = random.sample(directories, 1)[0] + '/'
				path = container + pick_word()
				directories.add(path)
				addenda.append(path)
				os.mkdir(base + '/' + path)

		if random.randint(1, 5) == 1:
			for i in range(random.randint(1, 5)):
				container = None
				if random.randint(0, len(directories)) == 0:
					container = ''
				else:
					container = random.sample(directories, 1)[0] + '/'
				path = container + random_filename()
				files.add(path)
				addenda.append(path)
				generate_new_file(base + '/' + path)

		if random.randint(1, 10) == 1:
			generate_new_binaries_batch(base, addenda)

	return

def delete_directory(dir):
	os.system('rd /q /s "%s"'%(to_native_slash(dir)))

def isalpha(x):
	return x >= 'a' and x <= 'z' or x >= 'A' and x <= 'Z'

def simple_wd():
	path = os.getcwd()
	print(path)
	if len(path) > 2 and isalpha(path[0]) and path[1] == ':':
		path = path[2:]
	print(path)
	return to_memory_slash(path)

def generate_backup_script(base):
	cd = os.getcwd()
	script  = 'open %s\n' % backup_dst
	script += 'add %s\\%s\n' % (cd, base)
	script += 'exclude name dirs .svn\n'
	script += 'set change_criterium date\n'
	script += 'set use_snapshots false\n'
	script += 'backup\n'
	script += 'quit\n'
	open(script_path, 'w').write(script)

def initialize(base):
	delete_directory(base)
	os.mkdir(base)
	os.system('rd /q /s %s\\backup' % back_base_path)
	generate_backup_script(base)

def save_version(base):
	return compare_dirs.construct_tree(base)

def perform_backup():
	os.system('zekvok < %s > nul' % script_path)

def print_percent(i, width, n):
	percent = int(i * width / n)
	invpercent = width - percent
	sys.stdout.write('\r[%s%s] %d'%('#'*percent, ' '*invpercent, i))

def perform(base, version_count):
	initialize(base)
	directories = set()
	files = set()
	version_data = {}
	for i in range(version_count):
		print_percent(i, 80, version_count)
		generate_next_version(base, directories, files)
		version_data[i] = save_version(base)
		perform_backup()
	return version_data

def generate_data(version_count):
	filename = 'revision_data.pickle'
	try:
		file = open(filename, 'rb')
		return pickle.load(file)
	except:
		pass
	data = perform('test_repo', version_count)
	pickle.dump(data, open(filename, 'wb'))
	return data

def restore_backup(version):
	script  = 'open %s\\backup\n'
	script += 'select version %d\n'
	script += 'restore\n'
	script += 'quit\n'

	cd = os.getcwd()
	script = script % (data_base_path, version)
	open('restore_script.txt', 'w').write(script)
	os.system('zekvok < %s\\restore_script.txt > nul' % cd)

def perform_test(data, version_count):
	for i in range(version_count):
		print_percent(i, 80, version_count)
		restore_backup(i)
		if not compare_dirs.compare_tree_and_dir(data[i], 'test_repo'):
			return False
	return True

def test():
	os.system('copy /y ..\\..\\bin64\\zekvok.exe .')
	version_count = 100
	data = generate_data(version_count)
	result = perform_test(data, version_count)
	print('')
	if result:
		print('Hooray! No errors found!')
	else:
		print('Some error(s) found in the program.')
	return

test()
<|endoftext|>


<commit_before>import functools
import re
import multiprocessing
from collections import Counter, defaultdict

from magichour.api.local.util.log import get_logger
from magichour.api.local.util.namedtuples import TimedTemplate, TimedEvent

logger = get_logger(__name__)

def process_line(templates, logline):
    for template in templates:
        if template.match.match(logline.text):
            return TimedTemplate(logline.ts, template.id, logline.id)
    # -1 = did not match any template
    return TimedTemplate(logline.ts, -1, logline.id)

re_type = re.compile(r'type=(\S+)')
def process_auditd_line(templates, logline):
    audit_msg_type = re_type.search(logline.text)
    if not audit_msg_type:
        raise ValueError('Does not match expected auditd format; missing type=TYPE: %s'%logline.text)

    audit_msg_type = audit_msg_type.group(1)
    if audit_msg_type not in templates:
        raise KeyError("type=%s not in dictionary"%audit_msg_type)

    return TimedTemplate(logline.ts, templates[audit_msg_type], logline.id)

def apply_templates(templates, loglines, mp=True, type_template_auditd=False, **kwargs):
    """
    Applies the templates on an iterable. This function creates a list of TimedTemplate named tuples.
    In effect this will produce a list of which templates occurred at which times.
    -1 is the template_id that is used for a logline which was unable to be matched to a template.

    The templates accepted by this function is exactly the output of functions in template.py
    This function has the option of running in either multiprocessing mode (mp=True by default) or not.

    Args:
        templates: iterable Templates to apply
        loglines: loglines which will be examined

    Kwargs:
        mp: whether or not to run in multiprocessing mode (default: True)
        type_template_auditd: whether or not to use specialized auditd type template processing (default: False)

    Returns:
        timed_templates: a list of TimedTemplate named tuples that represent which templates occurred at which times in the log file.

    """
    # Change processing mode for auditd data by Templating based on type=TYPE
    if type_template_auditd:
        process_function = process_auditd_line
    else:
        process_function = process_line

    if mp:
        # Use multiprocessing.Pool to use multiple CPUs
        pool = multiprocessing.Pool(multiprocessing.cpu_count())
        f = functools.partial(process_function, templates)

        timed_templates = pool.map(func=f, iterable=loglines)
    else:
        # Do this the naive way with one CPU
        timed_templates = []
        for logline in loglines:
            timed_templates.append(process_function(templates, logline))
    return timed_templates

#####

def count_templates(window):
    c = Counter()
    for timed_template in window.timed_templates:
        c[timed_template.template_id] += 1
    return c

def counter_issubset(counter1, counter2):
    return not counter1 - counter2

def apply_events_old(events, windows, mp=False):
    event_counters = {event.id : Counter(event.template_ids) for event in events}
    timed_events = []
    for window in windows:
        template_counts = count_templates(window)
        for event in events:
            num_occurrences = 0
            is_subset = True
            cur_counts = None
            while is_subset:
                if cur_counts:
                    is_subset = counter_issubset(event_counters[event.id], cur_counts)
                    cur_counts = cur_counts - event_counters[event.id]
                else:
                    is_subset = counter_issubset(event_counters[event.id], template_counts)
                    cur_counts = template_counts - event_counters[event.id]
                if is_subset:
                    num_occurrences += 1
            for occurrence in xrange(0, num_occurrences):
                timed_event = TimedEvent(window.start_time, window.end_time, event.id)
                timed_events.append(timed_event)
    return timed_events

#####

from collections import defaultdict

def count_templates(timed_templates):
    counts = defaultdict(int)
    for timed_template in timed_templates:
        counts[timed_template.template_id] += 1
    return counts

def get_least_common_template(template_counts, template_ids):
    least_common_template = None
    for template_id in template_ids:
        if not least_common_template or template_counts[template_id] < template_counts[least_common_template]:
            least_common_template = template_id
    return least_common_template

def jaccard(s1, s2):
    return len(s1 & s2) / float(len(s1 | s2))

def get_inner_list_vals(d):
    d_vals = []
    for d_val_list in d.itervalues():
        for d_val in d_val_list:
            d_vals.append(d_val)
    return d_vals

def jaccard_dicts(d1, d2, key_weight=0.0):
    jaccard_keys = jaccard(d1.viewkeys(), d2.viewkeys())
    d1_vals = get_inner_list_vals(d1)
    d2_vals = get_inner_list_vals(d2)
    jaccard_vals = jaccard(frozenset(d1_vals), frozenset(d2_vals))
    return (jaccard_keys * key_weight) + (jaccard_vals * (1-key_weight))

def calc_overlap(candidate_timed_template, orig_timed_template, logline_dict):
    candidate = logline_dict[candidate_timed_template.logline_id].replacements
    orig = logline_dict[orig_timed_template.logline_id].replacements
    return jaccard_dicts(candidate, orig) if candidate and orig else 0

def find_left_idx(idx, timed_templates, start_time):
    while idx > 0:
        if timed_templates[idx].ts < start_time:
            return idx
        idx -= 1
    return idx

def find_right_idx(idx, timed_templates, end_time):

    while idx < len(timed_templates):
        if timed_templates[idx].ts > end_time:
            return idx
        idx += 1
    return idx

def create_window(idx, timed_templates, window_size):
    timed_template = timed_templates[idx]
    start_time = timed_template.ts - window_size
    end_time = timed_template.ts + window_size
    left_idx = find_left_idx(idx, timed_templates, start_time)
    right_idx = find_right_idx(idx, timed_templates, end_time)
    return (left_idx, right_idx)

def search_window(idx, event, timed_templates, logline_dict, window_size):
    timed_template = timed_templates[idx]
    left_idx, right_idx = create_window(idx, timed_templates, window_size)
    window = timed_templates[left_idx:right_idx]
    results = {timed_template.template_id : timed_template}
    for template_id in event.template_ids:
        if template_id != timed_template.template_id:
            relevant = [tt for tt in window if tt.template_id == template_id]
            if not relevant:
                # We are missing one of the required template ID in this window
                return []
            relevant = sorted(relevant, reverse=True, key=lambda tt: calc_overlap(tt, timed_template, logline_dict))
            results[template_id] = relevant[0]
    return results.values()

# assume timed_templates are ordered
def apply_events(events, timed_templates, loglines, window_size=60, mp=False):
    template_counts = count_templates(timed_templates)
    logline_dict = {logline.id : logline for logline in loglines}

    # Create lookup table to speed up matching
    timed_template_dict = defaultdict(list)
    for idx, tt in enumerate(timed_templates):
        timed_template_dict[tt.template_id].append(idx)

    timed_events = []
    for event in events:
        lct_id = get_least_common_template(template_counts, event.template_ids)
        for idx in timed_template_dict[lct_id]:
            results = search_window(idx, event, timed_templates, logline_dict, window_size)
            if results:
                s = sorted(results, key=lambda result: result.ts)
                timed_event = TimedEvent(event.id, timed_templates=s)
                timed_events.append(timed_event)
    return timed_events<commit_msg>comment out old/unused apply_events_old()<commit_after>import functools
import re
import multiprocessing
from collections import Counter, defaultdict

from magichour.api.local.util.log import get_logger
from magichour.api.local.util.namedtuples import TimedTemplate, TimedEvent

logger = get_logger(__name__)

def process_line(templates, logline):
    for template in templates:
        if template.match.match(logline.text):
            return TimedTemplate(logline.ts, template.id, logline.id)
    # -1 = did not match any template
    return TimedTemplate(logline.ts, -1, logline.id)

re_type = re.compile(r'type=(\S+)')
def process_auditd_line(templates, logline):
    audit_msg_type = re_type.search(logline.text)
    if not audit_msg_type:
        raise ValueError('Does not match expected auditd format; missing type=TYPE: %s'%logline.text)

    audit_msg_type = audit_msg_type.group(1)
    if audit_msg_type not in templates:
        raise KeyError("type=%s not in dictionary"%audit_msg_type)

    return TimedTemplate(logline.ts, templates[audit_msg_type], logline.id)

def apply_templates(templates, loglines, mp=True, type_template_auditd=False, **kwargs):
    """
    Applies the templates on an iterable. This function creates a list of TimedTemplate named tuples.
    In effect this will produce a list of which templates occurred at which times.
    -1 is the template_id that is used for a logline which was unable to be matched to a template.

    The templates accepted by this function is exactly the output of functions in template.py
    This function has the option of running in either multiprocessing mode (mp=True by default) or not.

    Args:
        templates: iterable Templates to apply
        loglines: loglines which will be examined

    Kwargs:
        mp: whether or not to run in multiprocessing mode (default: True)
        type_template_auditd: whether or not to use specialized auditd type template processing (default: False)

    Returns:
        timed_templates: a list of TimedTemplate named tuples that represent which templates occurred at which times in the log file.

    """
    # Change processing mode for auditd data by Templating based on type=TYPE
    if type_template_auditd:
        process_function = process_auditd_line
    else:
        process_function = process_line

    if mp:
        # Use multiprocessing.Pool to use multiple CPUs
        pool = multiprocessing.Pool(multiprocessing.cpu_count())
        f = functools.partial(process_function, templates)

        timed_templates = pool.map(func=f, iterable=loglines)
    else:
        # Do this the naive way with one CPU
        timed_templates = []
        for logline in loglines:
            timed_templates.append(process_function(templates, logline))
    return timed_templates

#####

"""
def count_templates(window):
    c = Counter()
    for timed_template in window.timed_templates:
        c[timed_template.template_id] += 1
    return c

def counter_issubset(counter1, counter2):
    return not counter1 - counter2

def apply_events_old(events, windows, mp=False):
    event_counters = {event.id : Counter(event.template_ids) for event in events}
    timed_events = []
    for window in windows:
        template_counts = count_templates(window)
        for event in events:
            num_occurrences = 0
            is_subset = True
            cur_counts = None
            while is_subset:
                if cur_counts:
                    is_subset = counter_issubset(event_counters[event.id], cur_counts)
                    cur_counts = cur_counts - event_counters[event.id]
                else:
                    is_subset = counter_issubset(event_counters[event.id], template_counts)
                    cur_counts = template_counts - event_counters[event.id]
                if is_subset:
                    num_occurrences += 1
            for occurrence in xrange(0, num_occurrences):
                timed_event = TimedEvent(window.start_time, window.end_time, event.id)
                timed_events.append(timed_event)
    return timed_events

"""
#####

from collections import defaultdict

def count_templates(timed_templates):
    counts = defaultdict(int)
    for timed_template in timed_templates:
        counts[timed_template.template_id] += 1
    return counts

def get_least_common_template(template_counts, template_ids):
    least_common_template = None
    for template_id in template_ids:
        if not least_common_template or template_counts[template_id] < template_counts[least_common_template]:
            least_common_template = template_id
    return least_common_template

def jaccard(s1, s2):
    return len(s1 & s2) / float(len(s1 | s2))

def get_inner_list_vals(d):
    d_vals = []
    for d_val_list in d.itervalues():
        for d_val in d_val_list:
            d_vals.append(d_val)
    return d_vals

def jaccard_dicts(d1, d2, key_weight=0.0):
    jaccard_keys = jaccard(d1.viewkeys(), d2.viewkeys())
    d1_vals = get_inner_list_vals(d1)
    d2_vals = get_inner_list_vals(d2)
    jaccard_vals = jaccard(frozenset(d1_vals), frozenset(d2_vals))
    return (jaccard_keys * key_weight) + (jaccard_vals * (1-key_weight))

def calc_overlap(candidate_timed_template, orig_timed_template, logline_dict):
    candidate = logline_dict[candidate_timed_template.logline_id].replacements
    orig = logline_dict[orig_timed_template.logline_id].replacements
    return jaccard_dicts(candidate, orig) if candidate and orig else 0

def find_left_idx(idx, timed_templates, start_time):
    while idx > 0:
        if timed_templates[idx].ts < start_time:
            return idx
        idx -= 1
    return idx

def find_right_idx(idx, timed_templates, end_time):

    while idx < len(timed_templates):
        if timed_templates[idx].ts > end_time:
            return idx
        idx += 1
    return idx

def create_window(idx, timed_templates, window_size):
    timed_template = timed_templates[idx]
    start_time = timed_template.ts - window_size
    end_time = timed_template.ts + window_size
    left_idx = find_left_idx(idx, timed_templates, start_time)
    right_idx = find_right_idx(idx, timed_templates, end_time)
    return (left_idx, right_idx)

def search_window(idx, event, timed_templates, logline_dict, window_size):
    timed_template = timed_templates[idx]
    left_idx, right_idx = create_window(idx, timed_templates, window_size)
    window = timed_templates[left_idx:right_idx]
    results = {timed_template.template_id : timed_template}
    for template_id in event.template_ids:
        if template_id != timed_template.template_id:
            relevant = [tt for tt in window if tt.template_id == template_id]
            if not relevant:
                # We are missing one of the required template ID in this window
                return []
            relevant = sorted(relevant, reverse=True, key=lambda tt: calc_overlap(tt, timed_template, logline_dict))
            results[template_id] = relevant[0]
    return results.values()

# assume timed_templates are ordered
def apply_events(events, timed_templates, loglines, window_size=60, mp=False):
    template_counts = count_templates(timed_templates)
    logline_dict = {logline.id : logline for logline in loglines}

    # Create lookup table to speed up matching
    timed_template_dict = defaultdict(list)
    for idx, tt in enumerate(timed_templates):
        timed_template_dict[tt.template_id].append(idx)

    timed_events = []
    for event in events:
        lct_id = get_least_common_template(template_counts, event.template_ids)
        for idx in timed_template_dict[lct_id]:
            results = search_window(idx, event, timed_templates, logline_dict, window_size)
            if results:
                s = sorted(results, key=lambda result: result.ts)
                timed_event = TimedEvent(event.id, timed_templates=s)
                timed_events.append(timed_event)
    return timed_events<|endoftext|>


<commit_before>import json
import os
import sys

import defusedxml.ElementTree as ET
from defusedxml import minidom

class API:
    GeoPointQuery = None
    PlatformQuery = None
    LocationQuery = None
    AddressQuery = None
    AddressableQuery = None
    StopQuery = None
    POIQuery = None
    TripQuery = None

    def __init__(self, name):
        if self.__class__ == API:
            raise TypeError('Only API subclasses can be initialized.')
        self.name = name

    @property
    def geopoints(self):
        if self.GeoPointQuery is None:
            raise NotImplementedError('Querying geopoints is not supported by this network.')
        return self.GeoPointQuery(self)

    @property
    def platforms(self):
        if self.PlatformQuery is None:
            raise NotImplementedError('Querying platforms is not supported by this network.')
        return self.GeoPointQuery(self)

    @property
    def locations(self):
        if self.LocationQuery is None:
            raise NotImplementedError('Querying locations is not supported by this network.')
        return self.LocationQuery(self)

    @property
    def addresses(self):
        if self.AddressQuery is None:
            raise NotImplementedError('Querying addresses is not supported by this network.')
        return self.AddressQuery(self)

    @property
    def addressables(self):
        if self.AddressableQuery is None:
            raise NotImplementedError('Querying addressables is not supported by this network.')
        return self.AddressableQuery(self)

    @property
    def stops(self):
        if self.StopQuery is None:
            raise NotImplementedError('Querying stops is not supported by this network.')
        return self.StopQuery(self)

    @property
    def pois(self):
        if self.POIQuery is None:
            raise NotImplementedError('Querying POIs is not supported by this network.')
        return self.POIQuery(self)

    @property
    def trips(self):
        if self.TripQuery is None:
            raise NotImplementedError('Querying trips is not supported by this network.')
        return self.TripQuery(self)

class ParserError(Exception):
    def __init__(self, parser, message):
        self.parser = parser
        self.message = message
        self.pretty_data = self.parser.printable_data()

        if os.environ.get('CHOO_DEBUG'):
            message += '\n'+self.pretty_data

        super().__init__(message)
        self.message = self.actual

class Parser:
    def __init__(self, parent, data, *args, **kwargs):
        self.network = parent.network
        self.time = parent.time
        self.data = data
        self._args = args
        self._kwargs = kwargs

    def printable_data(self, pretty=True):
        raise NotImplementedError

class XMLParser(Parser):
    def printable_data(self, pretty=True):
        string = ET.tostring(self.data, 'utf-8')
        if pretty:
            string = minidom.parseString(string).toprettyxml(indent='  ')
        return string

class JSONParser(Parser):
    def printable_data(self, pretty=True):
        return json.dumps(self.data, indent=2 if pretty else None)

class parser_property(object):
    def __init__(self, func, name=None):
        self.func = func
        self.name = name or func.__name__

    def __get__(self, obj, cls):
        if obj is None:
            return self
        field = obj.Model._fields[self.name]
        try:
            value = obj.__dict__[self.name] = self.func(obj, obj.data, *obj._args, **obj._kwargs)
        except Exception as e:
            raise type(e)(str(e) +
                          '\n\n### CHOO DEBUG INFO:\n%s' % obj.printable_data()).with_traceback(sys.exc_info()[2])

        if not field.validate(value):
            raise TypeError('Invalid type for attribute %s.' % self.name)

        return value

    def __set__(self, obj, value):
        raise AttributeError("can't set a parser property")

    def __delete__(self, obj):
        raise AttributeError("can't delete a parser property")

def cached_property(func):
    def wrapped_func(self):
        value = func(self, self.data, *self._args, **self._kwargs)
        self.__dict__[func.__name__] = value
        return value
    return property(wrapped_func)
<commit_msg>API.platforms should return new PlatformQuery<commit_after>import json
import os
import sys

import defusedxml.ElementTree as ET
from defusedxml import minidom

class API:
    GeoPointQuery = None
    PlatformQuery = None
    LocationQuery = None
    AddressQuery = None
    AddressableQuery = None
    StopQuery = None
    POIQuery = None
    TripQuery = None

    def __init__(self, name):
        if self.__class__ == API:
            raise TypeError('Only API subclasses can be initialized.')
        self.name = name

    @property
    def geopoints(self):
        if self.GeoPointQuery is None:
            raise NotImplementedError('Querying geopoints is not supported by this network.')
        return self.GeoPointQuery(self)

    @property
    def platforms(self):
        if self.PlatformQuery is None:
            raise NotImplementedError('Querying platforms is not supported by this network.')
        return self.PlatformQuery(self)

    @property
    def locations(self):
        if self.LocationQuery is None:
            raise NotImplementedError('Querying locations is not supported by this network.')
        return self.LocationQuery(self)

    @property
    def addresses(self):
        if self.AddressQuery is None:
            raise NotImplementedError('Querying addresses is not supported by this network.')
        return self.AddressQuery(self)

    @property
    def addressables(self):
        if self.AddressableQuery is None:
            raise NotImplementedError('Querying addressables is not supported by this network.')
        return self.AddressableQuery(self)

    @property
    def stops(self):
        if self.StopQuery is None:
            raise NotImplementedError('Querying stops is not supported by this network.')
        return self.StopQuery(self)

    @property
    def pois(self):
        if self.POIQuery is None:
            raise NotImplementedError('Querying POIs is not supported by this network.')
        return self.POIQuery(self)

    @property
    def trips(self):
        if self.TripQuery is None:
            raise NotImplementedError('Querying trips is not supported by this network.')
        return self.TripQuery(self)

class ParserError(Exception):
    def __init__(self, parser, message):
        self.parser = parser
        self.message = message
        self.pretty_data = self.parser.printable_data()

        if os.environ.get('CHOO_DEBUG'):
            message += '\n'+self.pretty_data

        super().__init__(message)
        self.message = self.actual

class Parser:
    def __init__(self, parent, data, *args, **kwargs):
        self.network = parent.network
        self.time = parent.time
        self.data = data
        self._args = args
        self._kwargs = kwargs

    def printable_data(self, pretty=True):
        raise NotImplementedError

class XMLParser(Parser):
    def printable_data(self, pretty=True):
        string = ET.tostring(self.data, 'utf-8')
        if pretty:
            string = minidom.parseString(string).toprettyxml(indent='  ')
        return string

class JSONParser(Parser):
    def printable_data(self, pretty=True):
        return json.dumps(self.data, indent=2 if pretty else None)

class parser_property(object):
    def __init__(self, func, name=None):
        self.func = func
        self.name = name or func.__name__

    def __get__(self, obj, cls):
        if obj is None:
            return self
        field = obj.Model._fields[self.name]
        try:
            value = obj.__dict__[self.name] = self.func(obj, obj.data, *obj._args, **obj._kwargs)
        except Exception as e:
            raise type(e)(str(e) +
                          '\n\n### CHOO DEBUG INFO:\n%s' % obj.printable_data()).with_traceback(sys.exc_info()[2])

        if not field.validate(value):
            raise TypeError('Invalid type for attribute %s.' % self.name)

        return value

    def __set__(self, obj, value):
        raise AttributeError("can't set a parser property")

    def __delete__(self, obj):
        raise AttributeError("can't delete a parser property")

def cached_property(func):
    def wrapped_func(self):
        value = func(self, self.data, *self._args, **self._kwargs)
        self.__dict__[func.__name__] = value
        return value
    return property(wrapped_func)
<|endoftext|>


<commit_before>import tempfile
import json

import fabric

import vcloud_plugin_common

def configure(vcloud_config):
    _copy_vsphere_configuration_to_manager(vcloud_config)
    _update_vm()

def _copy_vsphere_configuration_to_manager(vcloud_config):
    tmp = tempfile.mktemp()
    with open(tmp, 'w') as f:
        json.dump(vcloud_config, f)
    fabric.api.put(tmp,
                   vcloud_plugin_common.Config.VCLOUD_CONFIG_PATH_DEFAULT)

def _get_distro():
    """ detect current distro """
    return fabric.api.run(
        'python -c "import platform; print platform.dist()[0]"')

def _update_vm():
    """ install some packeges for future deployments creation """
    distro = _get_distro()
    if 'Ubuntu' in distro:
        # update system to last version
        fabric.api.run("sudo apt-get update -q -y 2>&1")
        fabric.api.run("sudo apt-get dist-upgrade -q -y 2>&1")
        # install:
        # * zram-config for minimize out-of-memory cases with zswap
        # * other packages for create deployments from source
        fabric.api.run("sudo apt-get install zram-config -q -y 2>&1")
        _install_docker()

def _install_docker():
    kernel_version = fabric.api.run(
        'python -c "import platform; print platform.release()"')
    if kernel_version.startswith("3.13"):
        fabric.api.run("wget -qO- https://get.docker.com/ | sudo sh")
<commit_msg>CFY-2483: drop apt-get update for host<commit_after>import tempfile
import json

import fabric

import vcloud_plugin_common

def configure(vcloud_config):
    _copy_vsphere_configuration_to_manager(vcloud_config)
    _install_docker()

def _copy_vsphere_configuration_to_manager(vcloud_config):
    tmp = tempfile.mktemp()
    with open(tmp, 'w') as f:
        json.dump(vcloud_config, f)
    fabric.api.put(tmp,
                   vcloud_plugin_common.Config.VCLOUD_CONFIG_PATH_DEFAULT)

def _install_docker():
    distro = fabric.api.run(
        'python -c "import platform; print platform.dist()[0]"')
    kernel_version = fabric.api.run(
        'python -c "import platform; print platform.release()"')
    if kernel_version.startswith("3.13") and 'Ubuntu' in distro:
        fabric.api.run("wget -qO- https://get.docker.com/ | sudo sh")
<|endoftext|>


<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Set of utilities to add commands to a buildbot factory.

This is based on commands.py and adds skia-specific commands."""

from buildbot.process.properties import WithProperties
from buildbot.steps import shell

from master.factory import commands

class SkiaCommands(commands.FactoryCommands):

  def __init__(self, factory, configuration, workdir, target_arch,
               default_timeout, target_platform, environment_variables):
    """Instantiates subclass of FactoryCommands appropriate for Skia.

    factory: a BaseFactory
    configuration: 'Debug' or 'Release'
    workdir: string indicating path within slave directory
    target_arch: string such as 'x64'
    default_timeout: default timeout for each command, in seconds
    target_platform: a string such as skia_factory.TARGET_PLATFORM_LINUX,
        to be passed into FactoryCommands.__init__()
    environment_variables: dictionary of environment variables that should
        be passed to all commands
    """
    commands.FactoryCommands.__init__(
        self, factory=factory, target=configuration,
        build_dir='', target_platform=target_platform)
    # Store some parameters that the subclass may want to use later.
    self.default_timeout = default_timeout
    self.environment_variables = environment_variables
    self.factory = factory
    self.target_arch = target_arch
    self.workdir = workdir
    # TODO(epoger): It would be better for this path to be specified by
    # an environment variable or some such, so that it is not dependent on the
    # path from CWD to slave/skia_slave_scripts... but for now, this will do.
    self._local_slave_script_dir = self.PathJoin(
        '..', '..', '..', '..', '..', '..', 'slave', 'skia_slave_scripts')

  def AddClean(self, build_target='clean', description='Clean', timeout=None):
    """Does a 'make clean'"""
    cmd = 'make %s' % build_target
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddBuild(self, build_target, description='Build', timeout=None):
    """Adds a compile step to the build."""
    cmd = 'make %s' % build_target
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddUploadToBucket(self, source_filepath,
                        dest_gsbase='gs://chromium-skia-gm',
                        description='Upload', timeout=None):
    """Adds a step that uploads a file to a Google Storage Bucket."""
    path_to_upload_script = self.PathJoin(
        self._local_slave_script_dir, 'upload_to_bucket.py')
    cmd = 'python %s --source_filepath=%s --dest_gsbase=%s' % (
        path_to_upload_script, source_filepath, dest_gsbase)
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddUploadGMResults(self, gm_image_subdir, builder_name,
                         description='UploadGMResults', timeout=None):
    """Adds a step that calls upload_gm_results.py on the slave."""
    path_to_script = self.PathJoin(
        self._local_slave_script_dir, 'upload_gm_results.py')
    cmd = ['python', path_to_script,
           '--gm_image_subdir', gm_image_subdir,
           '--builder_name', builder_name,
           '--got_revision', WithProperties('%(got_revision)s'),
           ]
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddAndroidRunGM(self, device, description='RunGM', arguments='',
                      timeout=None):
    """ Runs GM on the device and pulls the output images to the host. """
    path_to_script = self.PathJoin(
        self._local_slave_script_dir, 'android_run_gm.py')
    cmd = ['python', path_to_script,
           '--device', device,
           '--args', arguments]
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddInstallAndroid(self, device, description='InstallAPK', timeout=None):
    """ Installs the Skia APK to the requested device. """
    path_to_script = self.PathJoin(
        self._local_slave_script_dir, 'install_android.py')
    cmd = ['python', path_to_script,
           '--device', device]
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddRunAndroid(self, device, binary_name, description='RunAndroid',
                    args='', timeout=None):
    """Runs any of the Skia binaries, provided that they have already been
    built and the APK has been installed."""
    path_to_script = self.PathJoin(
        self._local_slave_script_dir, 'run_android.py')
    cmd = ['python', path_to_script,
           '--binary_name', binary_name,
           '--device', device,
           '--args', args]
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddRunCommand(self, command, description='Run', timeout=None):
    """Runs an arbitrary command, perhaps a binary we built."""
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=command,
                         workdir=self.workdir, env=self.environment_variables)

  def AddRunCommandList(self, command_list, description='Run', timeout=None):
    """Runs a list of arbitrary commands."""
    # TODO(epoger): Change this so that build-step output shows each command
    # in the list separately--that will be a lot easier to follow.
    #
    # TODO(epoger): For now, this wraps the total command with WithProperties()
    # because *some* callers need it, and we can't use the string.join() command
    # to concatenate strings that have already been wrapped with
    # WithProperties().  Once I figure out how to make the build-step output
    # show each command separately, maybe I can remove this wrapper.
    self.AddRunCommand(command=WithProperties(' && '.join(command_list)),
                       description=description, timeout=timeout)
<commit_msg>Restore AddMergeIntoSvn() method I removed in https://code.google.com/p/skia/source/detail?r=4701 It's still needed by the housekeeping bot.<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Set of utilities to add commands to a buildbot factory.

This is based on commands.py and adds skia-specific commands."""

from buildbot.process.properties import WithProperties
from buildbot.steps import shell

from master.factory import commands

class SkiaCommands(commands.FactoryCommands):

  def __init__(self, factory, configuration, workdir, target_arch,
               default_timeout, target_platform, environment_variables):
    """Instantiates subclass of FactoryCommands appropriate for Skia.

    factory: a BaseFactory
    configuration: 'Debug' or 'Release'
    workdir: string indicating path within slave directory
    target_arch: string such as 'x64'
    default_timeout: default timeout for each command, in seconds
    target_platform: a string such as skia_factory.TARGET_PLATFORM_LINUX,
        to be passed into FactoryCommands.__init__()
    environment_variables: dictionary of environment variables that should
        be passed to all commands
    """
    commands.FactoryCommands.__init__(
        self, factory=factory, target=configuration,
        build_dir='', target_platform=target_platform)
    # Store some parameters that the subclass may want to use later.
    self.default_timeout = default_timeout
    self.environment_variables = environment_variables
    self.factory = factory
    self.target_arch = target_arch
    self.workdir = workdir
    # TODO(epoger): It would be better for this path to be specified by
    # an environment variable or some such, so that it is not dependent on the
    # path from CWD to slave/skia_slave_scripts... but for now, this will do.
    self._local_slave_script_dir = self.PathJoin(
        '..', '..', '..', '..', '..', '..', 'slave', 'skia_slave_scripts')

  def AddClean(self, build_target='clean', description='Clean', timeout=None):
    """Does a 'make clean'"""
    cmd = 'make %s' % build_target
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddBuild(self, build_target, description='Build', timeout=None):
    """Adds a compile step to the build."""
    cmd = 'make %s' % build_target
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddUploadToBucket(self, source_filepath,
                        dest_gsbase='gs://chromium-skia-gm',
                        description='Upload', timeout=None):
    """Adds a step that uploads a file to a Google Storage Bucket."""
    path_to_upload_script = self.PathJoin(
        self._local_slave_script_dir, 'upload_to_bucket.py')
    cmd = 'python %s --source_filepath=%s --dest_gsbase=%s' % (
        path_to_upload_script, source_filepath, dest_gsbase)
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddMergeIntoSvn(self, source_dir_path, dest_svn_url, merge_dir_path,
                      svn_username_file, svn_password_file,
                      commit_message=None, description='MergeIntoSvn',
                      timeout=None):
    """Adds a step that commits all files within a directory to a special
    SVN repository."""
    if not commit_message:
      commit_message = 'automated svn commit of %s step' % description

    path_to_upload_script = self.PathJoin(
        self._local_slave_script_dir, 'merge_into_svn.py')
    cmd = ['python', path_to_upload_script,
           '--commit_message', commit_message,
           '--dest_svn_url', dest_svn_url,
           '--merge_dir_path', merge_dir_path,
           '--source_dir_path', source_dir_path,
           '--svn_password_file', svn_password_file,
           '--svn_username_file', svn_username_file,
           ]
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddUploadGMResults(self, gm_image_subdir, builder_name,
                         description='UploadGMResults', timeout=None):
    """Adds a step that calls upload_gm_results.py on the slave."""
    path_to_script = self.PathJoin(
        self._local_slave_script_dir, 'upload_gm_results.py')
    cmd = ['python', path_to_script,
           '--gm_image_subdir', gm_image_subdir,
           '--builder_name', builder_name,
           '--got_revision', WithProperties('%(got_revision)s'),
           ]
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddAndroidRunGM(self, device, description='RunGM', arguments='',
                      timeout=None):
    """ Runs GM on the device and pulls the output images to the host. """
    path_to_script = self.PathJoin(
        self._local_slave_script_dir, 'android_run_gm.py')
    cmd = ['python', path_to_script,
           '--device', device,
           '--args', arguments]
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddInstallAndroid(self, device, description='InstallAPK', timeout=None):
    """ Installs the Skia APK to the requested device. """
    path_to_script = self.PathJoin(
        self._local_slave_script_dir, 'install_android.py')
    cmd = ['python', path_to_script,
           '--device', device]
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddRunAndroid(self, device, binary_name, description='RunAndroid',
                    args='', timeout=None):
    """Runs any of the Skia binaries, provided that they have already been
    built and the APK has been installed."""
    path_to_script = self.PathJoin(
        self._local_slave_script_dir, 'run_android.py')
    cmd = ['python', path_to_script,
           '--binary_name', binary_name,
           '--device', device,
           '--args', args]
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=cmd, workdir=self.workdir,
                         env=self.environment_variables)

  def AddRunCommand(self, command, description='Run', timeout=None):
    """Runs an arbitrary command, perhaps a binary we built."""
    if not timeout:
      timeout = self.default_timeout
    self.factory.addStep(shell.ShellCommand, description=description,
                         timeout=timeout, command=command,
                         workdir=self.workdir, env=self.environment_variables)

  def AddRunCommandList(self, command_list, description='Run', timeout=None):
    """Runs a list of arbitrary commands."""
    # TODO(epoger): Change this so that build-step output shows each command
    # in the list separately--that will be a lot easier to follow.
    #
    # TODO(epoger): For now, this wraps the total command with WithProperties()
    # because *some* callers need it, and we can't use the string.join() command
    # to concatenate strings that have already been wrapped with
    # WithProperties().  Once I figure out how to make the build-step output
    # show each command separately, maybe I can remove this wrapper.
    self.AddRunCommand(command=WithProperties(' && '.join(command_list)),
                       description=description, timeout=timeout)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2015, Google Inc. All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# be found in the LICENSE file.
"""End-to-end test for horizontal resharding automation."""

# "unittest" is used indirectly by importing "worker", but pylint does
# not grasp this.
# Import it explicitly to make pylint happy and stop it complaining about
# setUpModule, tearDownModule and the missing module docstring.
import unittest  # pylint: disable=unused-import

import environment
import utils
import worker

def setUpModule():
  worker.setUpModule()
  utils.Vtctld().start()

def tearDownModule():
  worker.tearDownModule()

class TestAutomationHorizontalResharding(worker.TestBaseSplitClone):
  """End-to-end test for horizontal resharding automation.

  This test reuses worker.py because worker.py also covers the happy path
  of the horizontal resharding code. Instead of running the different resharding
  steps "manually" as part of the test, they will be run by the automation
  cluster operation.
  """

  KEYSPACE = 'test_keyspace'

  def test_regular_operation(self):
    # Use a dedicated worker to run all vtworker commands.
    worker_proc, _, worker_rpc_port = utils.run_vtworker_bg(
        ['--cell', 'test_nj'],
        auto_log=True)
    vtworker_endpoint = 'localhost:' + str(worker_rpc_port)

    automation_server_proc, automation_server_port = (
        utils.run_automation_server())

    source_shard_list = '0'
    dest_shard_list = '-80,80-'
    _, vtctld_endpoint = utils.vtctld.rpc_endpoint()
    utils.run(
        environment.binary_argstr('automation_client') +
        ' --server localhost:' + str(automation_server_port) +
        ' --task HorizontalReshardingTask' +
        ' --param keyspace=' + self.KEYSPACE +
        ' --param source_shard_list=' + source_shard_list +
        ' --param dest_shard_list=' + dest_shard_list +
        ' --param vtctld_endpoint=' + vtctld_endpoint +
        ' --param vtworker_endpoint=' + vtworker_endpoint +
        ' --param min_healthy_rdonly_tablets=1')

    self.verify()

    utils.kill_sub_process(automation_server_proc, soft=True)
    utils.kill_sub_process(worker_proc, soft=True)

  def verify(self):
    self.assert_shard_data_equal(0, worker.shard_master,
                                 worker.shard_0_tablets.replica)
    self.assert_shard_data_equal(1, worker.shard_master,
                                 worker.shard_1_tablets.replica)

    # Verify effect of MigrateServedTypes. Dest shards are serving now.
    utils.check_srv_keyspace(
        'test_nj', self.KEYSPACE,
        'Partitions(master): -80 80-\n'
        'Partitions(rdonly): -80 80-\n'
        'Partitions(replica): -80 80-\n')

    # Check that query service is disabled (source shard) or enabled (dest).

    # source shard: query service must be disabled after MigrateServedTypes.
    utils.check_tablet_query_service(
        self, worker.shard_rdonly1,
        serving=False, tablet_control_disabled=True)
    utils.check_tablet_query_service(
        self, worker.shard_replica,
        serving=False, tablet_control_disabled=True)
    utils.check_tablet_query_service(
        self, worker.shard_master,
        serving=False, tablet_control_disabled=True)

    # dest shard -80: query service must be disabled after MigrateServedTypes.
    utils.check_tablet_query_service(
        self, worker.shard_0_rdonly1,
        serving=True, tablet_control_disabled=False)
    utils.check_tablet_query_service(
        self, worker.shard_0_replica,
        serving=True, tablet_control_disabled=False)
    utils.check_tablet_query_service(
        self, worker.shard_0_master,
        serving=True, tablet_control_disabled=False)

    # dest shard 80-: query service must be disabled after MigrateServedTypes.
    utils.check_tablet_query_service(
        self, worker.shard_1_rdonly1,
        serving=True, tablet_control_disabled=False)
    utils.check_tablet_query_service(
        self, worker.shard_1_replica,
        serving=True, tablet_control_disabled=False)
    utils.check_tablet_query_service(
        self, worker.shard_1_master,
        serving=True, tablet_control_disabled=False)

if __name__ == '__main__':
  utils.main(test_options=worker.add_test_options)
<commit_msg>test: Fix copy and paste mistake in comment.<commit_after>#!/usr/bin/env python
#
# Copyright 2015, Google Inc. All rights reserved.
# Use of this source code is governed by a BSD-style license that can
# be found in the LICENSE file.
"""End-to-end test for horizontal resharding automation."""

# "unittest" is used indirectly by importing "worker", but pylint does
# not grasp this.
# Import it explicitly to make pylint happy and stop it complaining about
# setUpModule, tearDownModule and the missing module docstring.
import unittest  # pylint: disable=unused-import

import environment
import utils
import worker

def setUpModule():
  worker.setUpModule()
  utils.Vtctld().start()

def tearDownModule():
  worker.tearDownModule()

class TestAutomationHorizontalResharding(worker.TestBaseSplitClone):
  """End-to-end test for horizontal resharding automation.

  This test reuses worker.py because worker.py also covers the happy path
  of the horizontal resharding code. Instead of running the different resharding
  steps "manually" as part of the test, they will be run by the automation
  cluster operation.
  """

  KEYSPACE = 'test_keyspace'

  def test_regular_operation(self):
    # Use a dedicated worker to run all vtworker commands.
    worker_proc, _, worker_rpc_port = utils.run_vtworker_bg(
        ['--cell', 'test_nj'],
        auto_log=True)
    vtworker_endpoint = 'localhost:' + str(worker_rpc_port)

    automation_server_proc, automation_server_port = (
        utils.run_automation_server())

    source_shard_list = '0'
    dest_shard_list = '-80,80-'
    _, vtctld_endpoint = utils.vtctld.rpc_endpoint()
    utils.run(
        environment.binary_argstr('automation_client') +
        ' --server localhost:' + str(automation_server_port) +
        ' --task HorizontalReshardingTask' +
        ' --param keyspace=' + self.KEYSPACE +
        ' --param source_shard_list=' + source_shard_list +
        ' --param dest_shard_list=' + dest_shard_list +
        ' --param vtctld_endpoint=' + vtctld_endpoint +
        ' --param vtworker_endpoint=' + vtworker_endpoint +
        ' --param min_healthy_rdonly_tablets=1')

    self.verify()

    utils.kill_sub_process(automation_server_proc, soft=True)
    utils.kill_sub_process(worker_proc, soft=True)

  def verify(self):
    self.assert_shard_data_equal(0, worker.shard_master,
                                 worker.shard_0_tablets.replica)
    self.assert_shard_data_equal(1, worker.shard_master,
                                 worker.shard_1_tablets.replica)

    # Verify effect of MigrateServedTypes. Dest shards are serving now.
    utils.check_srv_keyspace(
        'test_nj', self.KEYSPACE,
        'Partitions(master): -80 80-\n'
        'Partitions(rdonly): -80 80-\n'
        'Partitions(replica): -80 80-\n')

    # Check that query service is disabled (source shard) or enabled (dest).

    # source shard: query service must be disabled after MigrateServedTypes.
    utils.check_tablet_query_service(
        self, worker.shard_rdonly1,
        serving=False, tablet_control_disabled=True)
    utils.check_tablet_query_service(
        self, worker.shard_replica,
        serving=False, tablet_control_disabled=True)
    utils.check_tablet_query_service(
        self, worker.shard_master,
        serving=False, tablet_control_disabled=True)

    # dest shard -80: query service must be enabled after MigrateServedTypes.
    utils.check_tablet_query_service(
        self, worker.shard_0_rdonly1,
        serving=True, tablet_control_disabled=False)
    utils.check_tablet_query_service(
        self, worker.shard_0_replica,
        serving=True, tablet_control_disabled=False)
    utils.check_tablet_query_service(
        self, worker.shard_0_master,
        serving=True, tablet_control_disabled=False)

    # dest shard 80-: query service must be enabled after MigrateServedTypes.
    utils.check_tablet_query_service(
        self, worker.shard_1_rdonly1,
        serving=True, tablet_control_disabled=False)
    utils.check_tablet_query_service(
        self, worker.shard_1_replica,
        serving=True, tablet_control_disabled=False)
    utils.check_tablet_query_service(
        self, worker.shard_1_master,
        serving=True, tablet_control_disabled=False)

if __name__ == '__main__':
  utils.main(test_options=worker.add_test_options)
<|endoftext|>


<commit_before>##############################################################################
# MDTraj: A Python Library for Loading, Saving, and Manipulating
#         Molecular Dynamics Trajectories.
# Copyright 2012-2013 Stanford University and the Authors
#
# Authors: Jason Swails
# Contributors:
#
# MDTraj is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation, either version 2.1
# of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with MDTraj. If not, see <http://www.gnu.org/licenses/>.
#
# Portions of this code originate from the OpenMM molecular simulation
# toolkit, copyright (c) 2012 Stanford University and Peter Eastman. Those
# portions are distributed under the following terms:
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS, CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
# USE OR OTHER DEALINGS IN THE SOFTWARE.
##############################################################################
"""Load an md.Topology from CHARMM/XPLOR PSF files
"""

# Written by Jason Swails <jason.swails@gmail.com> 9/8/2014
# This code was mostly stolen and stripped down from ParmEd

##############################################################################
# Imports
##############################################################################

from __future__ import print_function, division

from mdtraj.core import topology, element as elem
from mdtraj.formats import pdb
from mdtraj.utils.unit import unit_definitions as u

__all__ = ['load_psf']

##############################################################################
# Functions
##############################################################################

class PSFError(Exception):
    """ Raised upon problems parsing a PSF file """
    pass

class _PSFEOF(Exception):
    """ Raised when EOF is hit on the parsed PSF """
    pass

def _convert(string, type, message):
    """Converts a string to the desired data type, making sure to raise PSFError
    with the given message in the case of a failure.

    Parameters
    ----------
    string : str
        String to convert
    type : simple data type
        Either int, float, or str
    message : str
        Message to assign to the PSFError if the conversion fails

    Returns
    -------
    The converted string to the desired datatype
    """
    try:
        return type(string)
    except ValueError:
        raise PSFError('Could not convert %s [%s]' % (message, string))

def _parse_psf_section(psf):
    """Parses a section of the PSF file, returning the data as a list of integer
    pointers

    Parameters
    ----------
    psf : open file object
        The open PSF file whose pointer is at the beginning of the section to be
        parsed

    Returns
    -------
    (title, pointers, data)

    title : str
        The label of the PSF section we are parsing
    pointers : (int/tuple of ints)
        If one pointer is set, pointers is simply the integer that is the value
        of that pointer. Otherwise it is a tuple with every pointer value
        defined in the first line
    data : list of integers
        A list of all data in the parsed section as integers

    Raises
    ------
    PSFError upon any parsing errors and _PSFEOF on end-of-file
    """
    line = psf.readline()
    while not line.strip():
        if not line:
            raise _PSFEOF('Unexpected EOF in PSF file')
        else:
            line = psf.readline()
    if '!' in line:
        words = line[:line.index('!')].split()
        title = line[line.index('!')+1:].strip().upper()
        # Strip out the description
        if ':' in title:
            title = title[:title.index(':')]
    else:
        raise PSFError('Could not determine section title')
    if len(words) == 1:
        pointers = _convert(words[0], int, 'pointer')
    else:
        pointers = tuple([_convert(w, int, 'pointer') for w in words])
    line = psf.readline().strip()
    if not line and title.startswith('NNB'):
        # This will correctly handle the NNB section (which has a spurious blank
        # line)
        line = psf.readline().strip()
    data = []
    if title == 'NATOM' or title == 'NTITLE':
        # Store these two sections as strings (ATOM section we will parse
        # later). The rest of the sections are integer pointers
        while line:
            data.append(line)
            line = psf.readline().strip()
    else:
        while line:
            words = line.split()
            data.extend([_convert(w, int, 'PSF data') for w in words])
            line = psf.readline().strip()
    return title, pointers, data

def load_psf(fname):
    """Load a CHARMM or XPLOR PSF file from disk

    Parameters
    ----------
    fname : str
        Path to the PSF file on disk

    Returns
    -------
    top : md.Topology
        The resulting topology as an md.Topology object

    Notes
    -----
    Only the bond and atom sections are read in, and all atoms are added to the
    same chain in the topology

    Raises
    ------
    PSFError if any parsing errors occur

    Examples
    --------
    >>> topology = md.load_psf('mysystem.psf')
    >>> # or
    >>> trajectory = md.load('trajectory.dcd', top='system.psf')
    """

    top = topology.Topology()

    with open(fname, 'r') as f:
        line = f.readline()
        if not line.startswith('PSF'):
            raise PSFError('Unrecognized PSF file.')

        # Store all of the sections and store them in a dict
        f.readline()
        psfsections = dict()
        while True:
            try:
                sec, ptr, data = _parse_psf_section(f)
            except _PSFEOF:
                break
            psfsections[sec] = (ptr, data)
            # We only have to parse up to the NBOND section
            if sec == 'NBOND': break

    c = top.add_chain()
    prev_residue = (None, None)

    pdb.PDBTrajectoryFile._loadNameReplacementTables()

    natom = _convert(psfsections['NATOM'][0], int, 'natom')
    for i in range(natom):
        words = psfsections['NATOM'][1][i].split()
        atid = _convert(words[0], int, 'atom index')
        if atid != i + 1:
            raise PSFError('Nonsequential atom indices detected!')
#       system = words[1]
        resid = _convert(words[2], int, 'residue number')
        rname = words[3]
        name = words[4]
#       attype = words[5]
#       charge = _convert(words[6], float, 'partial atomic charge')
        mass = _convert(words[7], float, 'atomic mass')
        curr_residue = (resid, rname)
        if prev_residue != curr_residue:
            prev_residue = curr_residue
            try:
                rname = pdb.PDBTrajectoryFile._residueNameReplacements[rname]
            except KeyError:
                pass
            r = top.add_residue(rname, c)

        try:
            name = pdb.PDBTrajectoryFile._atomNameReplacements[rname][name]
        except KeyError:
            pass

        # Try to guess the element from the atom name for some of the common
        # ions using the names that CHARMM assigns to ions. If it's not one of
        # these 'weird' ion names, look up the element by mass. If the mass is
        # 0, assume a lone pair
        upper = name.upper()
        if upper.startswith('CLA'):
            element = elem.chlorine
        elif upper.startswith('SOD'):
            element = elem.sodium
        elif upper.startswith('POT'):
            element = elem.potassium
        elif upper == 'CAL':
            element = elem.calcium
        elif mass == 0:
            element = None
        else:
            element = elem.Element.getByMass(mass*u.dalton)
        top.add_atom(name, element, r)

    # Add bonds to the topology
    atoms = list(top.atoms)
    bond_data = psfsections['NBOND'][1]
    nbond = _convert(psfsections['NBOND'][0], int, 'number of bonds')
    if len(bond_data) != nbond * 2:
        raise PSFError('Got %d indexes for %d bonds' % (len(bond_data), nbond))

    for i in range(nbond):
        i2 = i * 2
        top.add_bond(atoms[bond_data[i2]-1], atoms[bond_data[i2+1]-1])

    return top
<commit_msg>Update attribution.<commit_after>##############################################################################
# MDTraj: A Python Library for Loading, Saving, and Manipulating
#         Molecular Dynamics Trajectories.
# Copyright 2012-2013 Stanford University and the Authors
#
# Authors: Jason Swails
# Contributors:
#
# MDTraj is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation, either version 2.1
# of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with MDTraj. If not, see <http://www.gnu.org/licenses/>.
#
# Portions of this module originate from the ParmEd program, copyright (c) 2014
# Jason Swails, which is also distributed under the GNU Lesser General Public
# License
#
# Other portions of this code originate from the OpenMM molecular simulation
# toolkit, copyright (c) 2012 Stanford University and Peter Eastman. Those
# portions are distributed under the following terms:
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS, CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
# USE OR OTHER DEALINGS IN THE SOFTWARE.
##############################################################################
"""Load an md.Topology from CHARMM/XPLOR PSF files
"""

# Written by Jason Swails <jason.swails@gmail.com> 9/8/2014
# This code was mostly stolen and stripped down from ParmEd

##############################################################################
# Imports
##############################################################################

from __future__ import print_function, division

from mdtraj.core import topology, element as elem
from mdtraj.formats import pdb
from mdtraj.utils.unit import unit_definitions as u

__all__ = ['load_psf']

##############################################################################
# Functions
##############################################################################

class PSFError(Exception):
    """ Raised upon problems parsing a PSF file """
    pass

class _PSFEOF(Exception):
    """ Raised when EOF is hit on the parsed PSF """
    pass

def _convert(string, type, message):
    """Converts a string to the desired data type, making sure to raise PSFError
    with the given message in the case of a failure.

    Parameters
    ----------
    string : str
        String to convert
    type : simple data type
        Either int, float, or str
    message : str
        Message to assign to the PSFError if the conversion fails

    Returns
    -------
    The converted string to the desired datatype
    """
    try:
        return type(string)
    except ValueError:
        raise PSFError('Could not convert %s [%s]' % (message, string))

def _parse_psf_section(psf):
    """Parses a section of the PSF file, returning the data as a list of integer
    pointers

    Parameters
    ----------
    psf : open file object
        The open PSF file whose pointer is at the beginning of the section to be
        parsed

    Returns
    -------
    (title, pointers, data)

    title : str
        The label of the PSF section we are parsing
    pointers : (int/tuple of ints)
        If one pointer is set, pointers is simply the integer that is the value
        of that pointer. Otherwise it is a tuple with every pointer value
        defined in the first line
    data : list of integers
        A list of all data in the parsed section as integers

    Raises
    ------
    PSFError upon any parsing errors and _PSFEOF on end-of-file
    """
    line = psf.readline()
    while not line.strip():
        if not line:
            raise _PSFEOF('Unexpected EOF in PSF file')
        else:
            line = psf.readline()
    if '!' in line:
        words = line[:line.index('!')].split()
        title = line[line.index('!')+1:].strip().upper()
        # Strip out the description
        if ':' in title:
            title = title[:title.index(':')]
    else:
        raise PSFError('Could not determine section title')
    if len(words) == 1:
        pointers = _convert(words[0], int, 'pointer')
    else:
        pointers = tuple([_convert(w, int, 'pointer') for w in words])
    line = psf.readline().strip()
    if not line and title.startswith('NNB'):
        # This will correctly handle the NNB section (which has a spurious blank
        # line)
        line = psf.readline().strip()
    data = []
    if title == 'NATOM' or title == 'NTITLE':
        # Store these two sections as strings (ATOM section we will parse
        # later). The rest of the sections are integer pointers
        while line:
            data.append(line)
            line = psf.readline().strip()
    else:
        while line:
            words = line.split()
            data.extend([_convert(w, int, 'PSF data') for w in words])
            line = psf.readline().strip()
    return title, pointers, data

def load_psf(fname):
    """Load a CHARMM or XPLOR PSF file from disk

    Parameters
    ----------
    fname : str
        Path to the PSF file on disk

    Returns
    -------
    top : md.Topology
        The resulting topology as an md.Topology object

    Notes
    -----
    Only the bond and atom sections are read in, and all atoms are added to the
    same chain in the topology

    Raises
    ------
    PSFError if any parsing errors occur

    Examples
    --------
    >>> topology = md.load_psf('mysystem.psf')
    >>> # or
    >>> trajectory = md.load('trajectory.dcd', top='system.psf')
    """

    top = topology.Topology()

    with open(fname, 'r') as f:
        line = f.readline()
        if not line.startswith('PSF'):
            raise PSFError('Unrecognized PSF file.')

        # Store all of the sections and store them in a dict
        f.readline()
        psfsections = dict()
        while True:
            try:
                sec, ptr, data = _parse_psf_section(f)
            except _PSFEOF:
                break
            psfsections[sec] = (ptr, data)
            # We only have to parse up to the NBOND section
            if sec == 'NBOND': break

    c = top.add_chain()
    prev_residue = (None, None)

    pdb.PDBTrajectoryFile._loadNameReplacementTables()

    natom = _convert(psfsections['NATOM'][0], int, 'natom')
    for i in range(natom):
        words = psfsections['NATOM'][1][i].split()
        atid = _convert(words[0], int, 'atom index')
        if atid != i + 1:
            raise PSFError('Nonsequential atom indices detected!')
#       system = words[1]
        resid = _convert(words[2], int, 'residue number')
        rname = words[3]
        name = words[4]
#       attype = words[5]
#       charge = _convert(words[6], float, 'partial atomic charge')
        mass = _convert(words[7], float, 'atomic mass')
        curr_residue = (resid, rname)
        if prev_residue != curr_residue:
            prev_residue = curr_residue
            try:
                rname = pdb.PDBTrajectoryFile._residueNameReplacements[rname]
            except KeyError:
                pass
            r = top.add_residue(rname, c)

        try:
            name = pdb.PDBTrajectoryFile._atomNameReplacements[rname][name]
        except KeyError:
            pass

        # Try to guess the element from the atom name for some of the common
        # ions using the names that CHARMM assigns to ions. If it's not one of
        # these 'weird' ion names, look up the element by mass. If the mass is
        # 0, assume a lone pair
        upper = name.upper()
        if upper.startswith('CLA'):
            element = elem.chlorine
        elif upper.startswith('SOD'):
            element = elem.sodium
        elif upper.startswith('POT'):
            element = elem.potassium
        elif upper == 'CAL':
            element = elem.calcium
        elif mass == 0:
            element = None
        else:
            element = elem.Element.getByMass(mass*u.dalton)
        top.add_atom(name, element, r)

    # Add bonds to the topology
    atoms = list(top.atoms)
    bond_data = psfsections['NBOND'][1]
    nbond = _convert(psfsections['NBOND'][0], int, 'number of bonds')
    if len(bond_data) != nbond * 2:
        raise PSFError('Got %d indexes for %d bonds' % (len(bond_data), nbond))

    for i in range(nbond):
        i2 = i * 2
        top.add_bond(atoms[bond_data[i2]-1], atoms[bond_data[i2+1]-1])

    return top
<|endoftext|>


<commit_before>"""WikiQA data loader."""

from pathlib import Path

import keras
import pandas as pd

import matchzoo

_url = "https://download.microsoft.com/download/E/5/F/" \
       "E5FCFCEE-7005-4814-853D-DAA7C66507E0/WikiQACorpus.zip"

def load_data(stage='train', task='ranking'):
    """
    Load WikiQA data.

    :param stage: One of `train`, `dev`, and `test`.
    :param task: Could be one of `ranking`, `classification` or a
        :class:`matchzoo.engine.BaseTask` instance.
    :return: A DataPack if `ranking`, a tuple of (DataPack, classes) if
        `classification`.
    """
    if stage not in ('train', 'dev', 'test'):
        raise ValueError(f"{stage} is not a valid stage."
                         f"Must be one of `train`, `dev`, and `test`.")

    data_root = _download_data()
    file_path = data_root.joinpath(f'WikiQA-{stage}.tsv')
    data_pack = _read_data(file_path)

    if task == 'ranking':
        task = matchzoo.tasks.Ranking()
    if task == 'classification':
        task = matchzoo.tasks.Classification()

    if isinstance(task, matchzoo.tasks.Ranking):
        return data_pack
    elif isinstance(task, matchzoo.tasks.Classification):
        data_pack.one_hot_encode_label(task.num_classes, inplace=True)
        return data_pack, [False, True]
    else:
        raise ValueError(f"{task} is not a valid task.")

def _download_data():
    ref_path = keras.utils.data_utils.get_file(
        'wikiqa', _url, extract=True,
        cache_dir=matchzoo.USER_DATA_DIR,
        cache_subdir='wiki_qa'
    )
    return Path(ref_path).parent.joinpath('WikiQACorpus')

def _read_data(path):
    table = pd.read_table(path)
    df = pd.DataFrame({
        'text_left': table['Question'],
        'text_right': table['Sentence'],
        'id_left': table['DocumentID'],
        'id_right': table['SentenceID'],
        'label': table['Label']
    })
    return matchzoo.pack(df)
<commit_msg>Fix wiki qa id bug.<commit_after>"""WikiQA data loader."""

from pathlib import Path

import keras
import pandas as pd

import matchzoo

_url = "https://download.microsoft.com/download/E/5/F/" \
       "E5FCFCEE-7005-4814-853D-DAA7C66507E0/WikiQACorpus.zip"

def load_data(stage='train', task='ranking'):
    """
    Load WikiQA data.

    :param stage: One of `train`, `dev`, and `test`.
    :param task: Could be one of `ranking`, `classification` or a
        :class:`matchzoo.engine.BaseTask` instance.
    :return: A DataPack if `ranking`, a tuple of (DataPack, classes) if
        `classification`.
    """
    if stage not in ('train', 'dev', 'test'):
        raise ValueError(f"{stage} is not a valid stage."
                         f"Must be one of `train`, `dev`, and `test`.")

    data_root = _download_data()
    file_path = data_root.joinpath(f'WikiQA-{stage}.tsv')
    data_pack = _read_data(file_path)

    if task == 'ranking':
        task = matchzoo.tasks.Ranking()
    if task == 'classification':
        task = matchzoo.tasks.Classification()

    if isinstance(task, matchzoo.tasks.Ranking):
        return data_pack
    elif isinstance(task, matchzoo.tasks.Classification):
        data_pack.one_hot_encode_label(task.num_classes, inplace=True)
        return data_pack, [False, True]
    else:
        raise ValueError(f"{task} is not a valid task.")

def _download_data():
    ref_path = keras.utils.data_utils.get_file(
        'wikiqa', _url, extract=True,
        cache_dir=matchzoo.USER_DATA_DIR,
        cache_subdir='wiki_qa'
    )
    return Path(ref_path).parent.joinpath('WikiQACorpus')

def _read_data(path):
    table = pd.read_table(path)
    df = pd.DataFrame({
        'text_left': table['Question'],
        'text_right': table['Sentence'],
        'id_left': table['QuestionID'],
        'id_right': table['SentenceID'],
        'label': table['Label']
    })
    return matchzoo.pack(df)
<|endoftext|>


<commit_before># Test List module.
# Draw a window with all the files in the current folder.
# double-clicking will change folder.
#
# This test expects Win, Evt and FrameWork (and anything used by those)
# to work.

from FrameWork import *
import Win
import Qd
import List
import os

class TestList(Application):
	def __init__(self):
		os.chdir('Moes:')
		self.makemenubar()
		self.makewindow()

	def makewindow(self):
		r = (40, 40, 400, 300)
		w = Win.NewWindow(r, "List test", 1, 0, -1, 1, 0x55555555)
		r2 = (0, 0, 345, 245)
		self.list = List.LNew(r2, (0, 0, 1, 1), (0,0), 0, w, 0, 1, 1, 1)
		self.filllist()
		w.DrawGrowIcon()
		self.win = w

	def makeusermenus(self):
		self.filemenu = m = Menu(self.menubar, "File")
		self.quititem = MenuItem(m, "Quit", "Q", self.quit)

	def quit(self, *args):
		raise self

	def do_about(self, id, item, window, event):
		EasyDialogs.Message("""Test the List Manager interface.
		Double-click on a folder to change directory""")

	def do_activateEvt(self, *args):
		self.list.LActivate(1)	# XXXX Wrong...

	def do_update(self, *args):
		print 'LUPDATE'
		self.list.LUpdate()

	def do_inContent(self, partcode, window, event):
		(what, message, when, where, modifiers) = event
		Qd.SetPort(window)
		local = Qd.GlobalToLocal(where)
		print 'CLICK', where, '->', local
		dclick = self.list.LClick(local, modifiers)
		if dclick:
			h, v = self.list.LLastClick()
			file = self.list.LGetCell(1000, (h, v))
			os.chdir(file)
			self.filllist()

	def filllist(self):
		"""Fill the list with the contents of the current directory"""
		l = self.list
		l.LSetDrawingMode(0)
		l.LDelRow(0, 0)
		contents = os.listdir(':')
		print contents
		l.LAddRow(len(contents), 0)
		for i in range(len(contents)):
			l.LSetCell(contents[i], (0, i))
		l.LSetDrawingMode(1)
		l.LUpdate()

def main():
	App = TestList()
	App.mainloop()

if __name__ == '__main__':
	main()

<commit_msg>Uses new FrameWork, has multiple windows<commit_after># Test List module.
# Draw a window with all the files in the current folder.
# double-clicking will change folder.
#
# This test expects Win, Evt and FrameWork (and anything used by those)
# to work.
#
# Actually, it is more a test of FrameWork by now....

from FrameWork import *
import Win
import Qd
import List
import os

class ListWindow(Window):
	def open(self, name, where):
		self.where = where
		r = (40, 40, 400, 300)
		w = Win.NewWindow(r, name, 1, 0, -1, 1, 0x55555555)
		r2 = (0, 0, 345, 245)
		self.list = List.LNew(r2, (0, 0, 1, 1), (0,0), 0, w, 0, 1, 1, 1)
		self.filllist()
		w.DrawGrowIcon()
		self.wid = w
		self.do_postopen()

	def do_activate(self, onoff, evt):
		self.list.LActivate(onoff)

	def do_rawupdate(self, window, event):
		window.BeginUpdate()
		self.do_update(window, event)
		window.EndUpdate()

	def do_update(self, *args):
		self.list.LUpdate()

	def do_contentclick(self, local, modifiers, evt):
		dclick = self.list.LClick(local, modifiers)
		if dclick:
			h, v = self.list.LLastClick()
			file = self.list.LGetCell(1000, (h, v))
			self.where = os.path.join(self.where, file)
			self.filllist()

	def filllist(self):
		"""Fill the list with the contents of the current directory"""
		l = self.list
		l.LSetDrawingMode(0)
		l.LDelRow(0, 0)
		contents = os.listdir(self.where)
		l.LAddRow(len(contents), 0)
		for i in range(len(contents)):
			l.LSetCell(contents[i], (0, i))
		l.LSetDrawingMode(1)
		l.LUpdate()

class TestList(Application):
	def __init__(self):
		Application.__init__(self)
		self.num = 0
		self.listoflists = []

	def makeusermenus(self):
		self.filemenu = m = Menu(self.menubar, "File")
		self.newitem = MenuItem(m, "New window...", "O", self.open)
		self.quititem = MenuItem(m, "Quit", "Q", self.quit)

	def open(self, *args):
		import macfs
		fss, ok = macfs.GetDirectory()
		if not ok:
			return
		w = ListWindow(self)
		w.open('Window %d'%self.num, fss.as_pathname())
		self.num = self.num + 1
		self.listoflists.append(w)

	def quit(self, *args):
		raise self

	def do_about(self, id, item, window, event):
		EasyDialogs.Message("""Test the List Manager interface.
		Simple inward-only folder browser""")

def main():
	App = TestList()
	App.mainloop()

if __name__ == '__main__':
	main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
<commit_msg>Delete unnecessary file<commit_after><|endoftext|>


<commit_before>import os, unittest, sqlalchemy
from sqlalchemy import create_engine
from igf_data.utils.dbutils import read_dbconf_json
from igf_data.igfdb.baseadaptor import BaseAdaptor
from igf_data.igfdb.projectadaptor import ProjectAdaptor
from igf_data.igfdb.sampleadaptor import SampleAdaptor
from igf_data.igfdb.platformadaptor import PlatformAdaptor
from igf_data.igfdb.seqrunadaptor import SeqrunAdaptor
from igf_data.igfdb.experimentadaptor import ExperimentAdaptor
from igf_data.igfdb.igfTables import Base

class ExperimentAdaptor_test1(unittest.TestCase):
  def setUp(self):
    self.dbconfig = 'data/dbconfig.json'
    dbparam=read_dbconf_json(self.dbconfig)
    base = BaseAdaptor(**dbparam)
    self.engine = base.engine
    self.dbname=dbparam['dbname']
    Base.metadata.drop_all(self.engine)
    if os.path.exists(self.dbname):
      os.remove(self.dbname)
    Base.metadata.create_all(self.engine)
    self.session_class=base.get_session_class()

    base = BaseAdaptor(**{'session_class':self.session_class})
    base.start_session()
    platform_data=[{ "platform_igf_id" : "M001",
                     "model_name" : "MISEQ" ,
                     "vendor_name" : "ILLUMINA" ,
                     "software_name" : "RTA",
                     "software_version" : "RTA1.18.54"}]                        # platform data
    flowcell_rule_data=[{"platform_igf_id":"M001",
                         "flowcell_type":"MISEQ",
                         "index_1":"NO_CHANGE",
                         "index_2":"NO_CHANGE"}]                                # flowcell rule data
    pl=PlatformAdaptor(**{'session':base.session})
    pl.store_platform_data(data=platform_data)                                  # loading platform data
    pl.store_flowcell_barcode_rule(data=flowcell_rule_data)                     # loading flowcell rules data
    project_data=[{'project_igf_id':'ProjectA'}]                                # project data
    pa=ProjectAdaptor(**{'session':base.session})
    pa.store_project_and_attribute_data(data=project_data)                      # load project data
    sample_data=[{'sample_igf_id':'SampleA',
                  'project_igf_id':'ProjectA'}]                                 # sample data
    sa=SampleAdaptor(**{'session':base.session})
    sa.store_sample_and_attribute_data(data=sample_data)                        # store sample data
    seqrun_data=[{'seqrun_igf_id':'SeqrunA',
                  'flowcell_id':'000000000-D0YLK',
                  'platform_igf_id':'M001',
                  'flowcell':'MISEQ'}]                                          # seqrun data
    sra=SeqrunAdaptor(**{'session':base.session})
    sra.store_seqrun_and_attribute_data(data=seqrun_data)                       # load seqrun data
    experiment_data=[{'experiment_igf_id':'ExperimentA',
                      'sample_igf_id':'SampleA',
                      'library_name':'SampleA',
                      'platform_name':'MISEQ',
                      'project_igf_id':'ProjectA'}]                             # experiment data
    ea=ExperimentAdaptor(**{'session':base.session})
    ea.store_project_and_attribute_data(data=experiment_data)                   # load experiment data
    base.commit_session()
    base.close_session()

  def tearDown(self):
    Base.metadata.drop_all(self.engine)
    os.remove(self.dbname)

  def test_check_experiment_records_id(self):
    ea = ExperimentAdaptor(**{'session_class':self.session_class})
    ea.start_session()
    self.assertTrue(ea.check_experiment_records_id(experiment_igf_id='ExperimentA'))
    self.assertFalse(ea.check_experiment_records_id(experiment_igf_id='ExperimentB'))
    ea.close_session()

  def test_fetch_project_and_sample_for_experiment(self):
    ea = ExperimentAdaptor(**{'session_class':self.session_class})
    ea.start_session()
    project_id,sample_id=ea.fetch_project_and_sample_for_experiment(experiment_igf_id='ExperimentA')
    self.assertEqual(project_id,'ProjectA')
    self.assertEqual(sample_id,'SampleA')
    ea.close_session()
<commit_msg>bugfix for experiment adaptor test<commit_after>import os, unittest, sqlalchemy
from sqlalchemy import create_engine
from igf_data.utils.dbutils import read_dbconf_json
from igf_data.igfdb.baseadaptor import BaseAdaptor
from igf_data.igfdb.projectadaptor import ProjectAdaptor
from igf_data.igfdb.sampleadaptor import SampleAdaptor
from igf_data.igfdb.platformadaptor import PlatformAdaptor
from igf_data.igfdb.seqrunadaptor import SeqrunAdaptor
from igf_data.igfdb.experimentadaptor import ExperimentAdaptor
from igf_data.igfdb.igfTables import Base

class ExperimentAdaptor_test1(unittest.TestCase):
  def setUp(self):
    self.dbconfig = 'data/dbconfig.json'
    dbparam=read_dbconf_json(self.dbconfig)
    base = BaseAdaptor(**dbparam)
    self.engine = base.engine
    self.dbname=dbparam['dbname']
    Base.metadata.drop_all(self.engine)
    if os.path.exists(self.dbname):
      os.remove(self.dbname)
    Base.metadata.create_all(self.engine)
    self.session_class=base.get_session_class()

    base = BaseAdaptor(**{'session_class':self.session_class})
    base.start_session()
    platform_data=[{ "platform_igf_id" : "M001",
                     "model_name" : "MISEQ" ,
                     "vendor_name" : "ILLUMINA" ,
                     "software_name" : "RTA",
                     "software_version" : "RTA1.18.54"}]                        # platform data
    flowcell_rule_data=[{"platform_igf_id":"M001",
                         "flowcell_type":"MISEQ",
                         "index_1":"NO_CHANGE",
                         "index_2":"NO_CHANGE"}]                                # flowcell rule data
    pl=PlatformAdaptor(**{'session':base.session})
    pl.store_platform_data(data=platform_data)                                  # loading platform data
    pl.store_flowcell_barcode_rule(data=flowcell_rule_data)                     # loading flowcell rules data
    project_data=[{'project_igf_id':'ProjectA'}]                                # project data
    pa=ProjectAdaptor(**{'session':base.session})
    pa.store_project_and_attribute_data(data=project_data)                      # load project data
    sample_data=[{'sample_igf_id':'SampleA',
                  'project_igf_id':'ProjectA'}]                                 # sample data
    sa=SampleAdaptor(**{'session':base.session})
    sa.store_sample_and_attribute_data(data=sample_data)                        # store sample data
    seqrun_data=[{'seqrun_igf_id':'SeqrunA',
                  'flowcell_id':'000000000-D0YLK',
                  'platform_igf_id':'M001',
                  'flowcell':'MISEQ'}]                                          # seqrun data
    sra=SeqrunAdaptor(**{'session':base.session})
    sra.store_seqrun_and_attribute_data(data=seqrun_data)                       # load seqrun data
    experiment_data=[{'experiment_igf_id':'ExperimentA',
                      'sample_igf_id':'SampleA',
                      'library_name':'SampleA',
                      'platform_name':'MISEQ',
                      'project_igf_id':'ProjectA'}]                             # experiment data
    ea=ExperimentAdaptor(**{'session':base.session})
    ea.store_project_and_attribute_data(data=experiment_data)                   # load experiment data
    base.commit_session()
    base.close_session()

  def tearDown(self):
    Base.metadata.drop_all(self.engine)
    os.remove(self.dbname)

  def test_check_experiment_records_id(self):
    ea = ExperimentAdaptor(**{'session_class':self.session_class})
    ea.start_session()
    self.assertTrue(ea.check_experiment_records_id(experiment_igf_id='ExperimentA'))
    self.assertFalse(ea.check_experiment_records_id(experiment_igf_id='ExperimentB'))
    ea.close_session()

  def test_fetch_project_and_sample_for_experiment(self):
    ea = ExperimentAdaptor(**{'session_class':self.session_class})
    ea.start_session()
    project_id,sample_id=ea.fetch_project_and_sample_for_experiment(experiment_igf_id='ExperimentA')
    self.assertEqual(project_id,'ProjectA')
    self.assertEqual(sample_id,'SampleA')
    ea.close_session()

if __name__=='__main__':
  unittest.main()<|endoftext|>