<commit_before>#!/usr/bin/python
import datetime
import csv
import sys
import os
import json

from optparse import OptionParser

import requests
from BeautifulSoup import BeautifulSoup

from pytools.asynx import scheduled
from threading import Thread

import pandas as pd
import matplotlib.pylab as plt
import statsmodels.api as sm

import flask
from flask import Flask
from flask import request

__author__ = 'sekely'

requests.packages.urllib3.disable_warnings()
DEFAULT_CSV = '/tmp/toll_road.csv'
DEST = '32.051070, 34.785303'
ORIGIN = '32.000955, 34.845297'

app = Flask(__name__)

def parse_arg(arg_name, args, arg_type=None, default_val=None):
    arg = args.get(arg_name, default_val)
    if isinstance(arg, list):
        arg = arg[0]
    if arg_type:
        return arg_type(arg)
    return arg

class ServerStopped(Exception):
    pass

class Singleton(type):
    _instances = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
        return cls._instances[cls]

class Collector(object):
    __metaclass__ = Singleton
    headers = ['timestamp', 'price', 'traffic']
    started = False
    last_update = None

    _csv = None
    csv_file = property(lambda self: self._csv)
    api_key = None

    @csv_file.setter
    def csv_file(self, csv_file):
        self._csv = csv_file
        if not os.path.isfile(self._csv):
            with open(self.csv_file, 'w') as f:
                writer = csv.DictWriter(f, self.headers)
                writer.writeheader()

    def save_to_csv(self, data=None):
        if not data:
            data = {'timestamp': str(datetime.datetime.now()),
                    'key': None,
                    'values': None}
        with open(self.csv_file, 'a') as f:
            writer = csv.DictWriter(f, self.headers)
            writer.writerow(data)

    def get_price(self):
        r = requests.get('https://www.fastlane.co.il/mobile.aspx')
        parsed_html = BeautifulSoup(r.content)
        price = parsed_html.find('span', attrs={'id': 'lblPrice'}).text
        return int(price)

    def get_traffic(self):
        payload = {
            'key': self.api_key,
            'departure_time': 'now',
            'origins': ORIGIN,
            'destinations': DEST
        }
        base_url = 'https://maps.googleapis.com/maps/api/distancematrix/json'
        r = requests.get(base_url, params=payload)
        resp = json.loads(r.text)
        value = resp['rows'][0]['elements'][0]['duration_in_traffic']['value']
        return int(value)

    @scheduled(60)
    def start_sampling(self):
        if not self.started:
            raise ServerStopped()
        ts = str(datetime.datetime.now())
        price = self.get_price()
        if self.api_key:
            traffic = self.get_traffic()
        else:
            traffic = None
        data = {'timestamp': ts,
                'price': price,
                'traffic': traffic}
        self.save_to_csv(data)
        self.last_update = datetime.datetime.now()

    def do_start(self, *args, **kwargs):
        if not self.started:
            self.started = True
            t = Thread(target=self.start_sampling)
            t.start()
        return 'collector started\n'

    def do_stop(self, *args, **kwargs):
        self.started = False
        return 'stopping collector\n'

    def do_config(self, *args, **kwargs):
        for key, val in kwargs.iteritems():
            setattr(self, key, val[0])
        return 'finished collector config\n'

    def do_describe(self, *args, **kwargs):
        ret = {'csv_file': self.csv_file,
               'started': self.started,
               'last_update': self.last_update,
               'commands': [x.split('do_')[-1].replace('_', '-') for x in dir(self) if 'do_' in x]}
        return json.dumps(ret)

class Analyzer(object):
    _csv = None
    csv_file = property(lambda self: self._csv)
    last_update = None
    df = pd.DataFrame()

    @csv_file.setter
    def csv_file(self, val):
        self._csv = val
        self.extract_data()

    def extract_data(self, drop_na=False):
        if not self.csv_file:
            return
        dateparse = lambda dates: pd.datetime.strptime(dates, '%Y-%m-%d %H:%M:%S.%f')
        df = pd.read_csv(self.csv_file, parse_dates='timestamp', index_col='timestamp', date_parser=dateparse)
        df['traffic'] = df['traffic'].apply(lambda x: x / 60.)
        self.df = df.resample('T').mean()
        if drop_na:
            self.df.dropna(inplace=True)
        else:
            self.df.interpolate(inplace=True)
        self.last_update = datetime.datetime.now()

    def rolling_mean(self, window=10):
        means = self.df.rolling(window=window).mean()
        ewm_means = self.df.ewm(halflife=window).mean()
        means.columns = ['mean-%s' % col for col in means.columns]
        ewm_means.columns = ['ewm-%s' % col for col in ewm_means.columns]
        ts = pd.concat([means, ewm_means], axis=1)
        return ts

    def filter(self, lamb=1e5):
        cycle, trend = sm.tsa.filters.hpfilter(self.df, lamb=lamb)
        cycle.columns = ['%s-cycle' % col for col in cycle.columns]
        trend.columns = ['%s-trend' % col for col in trend.columns]
        ts = pd.concat([cycle, trend], axis=1)
        return ts

    @staticmethod
    def plot(ts):
        fig, ax = plt.subplots()
        lined = dict()

        ax.set_title('Click on legend line to toggle line on/off')
        lines = [ax.plot(ts[col], label=col) for col in ts.columns]
        leg = ax.legend(loc='best')

        for legline, origline in zip(leg.get_lines(), lines):
            legline.set_picker(5)  # 5 pts tolerance
            lined[legline] = origline[0]

        def onpick(event):
            # on the pick event, find the orig line corresponding to the
            # legend proxy line, and toggle the visibility
            legline = event.artist
            origline = lined[legline]
            vis = not origline.get_visible()
            origline.set_visible(vis)
            # Change the alpha on the line in the legend so we can see what lines
            # have been toggled
            if vis:
                legline.set_alpha(1.0)
            else:
                legline.set_alpha(0.2)
            fig.canvas.draw()

        fig.canvas.mpl_connect('pick_event', onpick)
        plt.show(False)

    def do_summary(self, *args, **kwargs):
        return flask.jsonify(self.df.describe().to_dict())

    def do_filter(self, *args, **kwargs):
        as_json = parse_arg('as_json', kwargs, default_val=False)
        lamb = parse_arg('lamb', kwargs, float, 1e5)
        orient = parse_arg('orient', kwargs, default_val='columns')
        j = self.filter(lamb).interpolate().\
            to_json(date_format='iso', orient=orient, double_precision=2, date_unit='s')
        if as_json:
            return j
        return flask.jsonify(json.loads(j))

    def do_raw_data(self, *args, **kwargs):
        as_json = parse_arg('as_json', kwargs, default_val=False)
        orient = parse_arg('orient', kwargs, default_val='columns')
        j = self.df.to_json(date_format='iso', orient=orient, double_precision=2, date_unit='s')
        if as_json:
            return j
        return flask.jsonify(json.loads(j))

    def do_rolling_mean(self, *args, **kwargs):
        as_json = parse_arg('as_json', kwargs, default_val=False)
        window = parse_arg('window', kwargs, int, 10)
        orient = parse_arg('orient', kwargs, default_val='columns')
        j = self.rolling_mean(window=window).interpolate().\
            to_json(date_format='iso', orient=orient, double_precision=2, date_unit='s')
        if as_json:
            return j
        return flask.jsonify(json.loads(j))

    def do_refresh(self, *args, **kwargs):
        self.extract_data()
        return "analyzer refreshed\n"

    def do_config(self, *args, **kwargs):
        for key in kwargs.iterkeys():
            val = parse_arg(key, kwargs)
            setattr(self, key, val)
        return 'finished analyzer config\n'

    def do_describe(self, *args, **kwargs):
        ret = {'csv_file': self.csv_file,
               'last_update': self.last_update,
               'samples': len(self.df),
               'commands': [x.split('do_')[-1].replace('_', '-') for x in dir(self) if 'do_' in x]}
        return flask.jsonify(ret)

_ar = Analyzer()
_cr = Collector()

def get_parser():
    parser = OptionParser()
    parser.add_option('-a', '--address', dest='host', default='127.0.0.1', help='server hostname')
    parser.add_option('-p', '--port', dest='port', default=5000, help='server port')
    return parser.parse_args()

@app.route('/collector/<command>')
def collector(command):
    command = command.replace('-', '_')
    cmd = getattr(_cr, 'do_%s' % command)
    return cmd(**request.args)

@app.route('/analyzer/<command>')
def analyzer(command):
    command = command.replace('-', '_')
    cmd = getattr(_ar, 'do_%s' % command)
    return cmd(**request.args)

if __name__ == '__main__':
    print "starting toll road server"
    options, _ = get_parser()
    try:
        app.run(host=options.host, port=options.port)
    except KeyboardInterrupt as e:
        print "shutting down sampling"
        sys.exit(0)
    sys.exit(1)
<commit_msg>minor: collector describe returns json<commit_after>#!/usr/bin/python
import datetime
import csv
import sys
import os
import json

from optparse import OptionParser

import requests
from BeautifulSoup import BeautifulSoup

from pytools.asynx import scheduled
from threading import Thread

import pandas as pd
import matplotlib.pylab as plt
import statsmodels.api as sm

import flask
from flask import Flask
from flask import request

__author__ = 'sekely'

requests.packages.urllib3.disable_warnings()
DEFAULT_CSV = '/tmp/toll_road.csv'
DEST = '32.051070, 34.785303'
ORIGIN = '32.000955, 34.845297'

app = Flask(__name__)

def parse_arg(arg_name, args, arg_type=None, default_val=None):
    arg = args.get(arg_name, default_val)
    if isinstance(arg, list):
        arg = arg[0]
    if arg_type:
        return arg_type(arg)
    return arg

class ServerStopped(Exception):
    pass

class Singleton(type):
    _instances = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
        return cls._instances[cls]

class Collector(object):
    __metaclass__ = Singleton
    headers = ['timestamp', 'price', 'traffic']
    started = False
    last_update = None

    _csv = None
    csv_file = property(lambda self: self._csv)
    api_key = None

    @csv_file.setter
    def csv_file(self, csv_file):
        self._csv = csv_file
        if not os.path.isfile(self._csv):
            with open(self.csv_file, 'w') as f:
                writer = csv.DictWriter(f, self.headers)
                writer.writeheader()

    def save_to_csv(self, data=None):
        if not data:
            data = {'timestamp': str(datetime.datetime.now()),
                    'key': None,
                    'values': None}
        with open(self.csv_file, 'a') as f:
            writer = csv.DictWriter(f, self.headers)
            writer.writerow(data)

    def get_price(self):
        r = requests.get('https://www.fastlane.co.il/mobile.aspx')
        parsed_html = BeautifulSoup(r.content)
        price = parsed_html.find('span', attrs={'id': 'lblPrice'}).text
        return int(price)

    def get_traffic(self):
        payload = {
            'key': self.api_key,
            'departure_time': 'now',
            'origins': ORIGIN,
            'destinations': DEST
        }
        base_url = 'https://maps.googleapis.com/maps/api/distancematrix/json'
        r = requests.get(base_url, params=payload)
        resp = json.loads(r.text)
        value = resp['rows'][0]['elements'][0]['duration_in_traffic']['value']
        return int(value)

    @scheduled(60)
    def start_sampling(self):
        if not self.started:
            raise ServerStopped()
        ts = str(datetime.datetime.now())
        price = self.get_price()
        if self.api_key:
            traffic = self.get_traffic()
        else:
            traffic = None
        data = {'timestamp': ts,
                'price': price,
                'traffic': traffic}
        self.save_to_csv(data)
        self.last_update = datetime.datetime.now()

    def do_start(self, *args, **kwargs):
        if not self.started:
            self.started = True
            t = Thread(target=self.start_sampling)
            t.start()
        return 'collector started\n'

    def do_stop(self, *args, **kwargs):
        self.started = False
        return 'stopping collector\n'

    def do_config(self, *args, **kwargs):
        for key, val in kwargs.iteritems():
            setattr(self, key, val[0])
        return 'finished collector config\n'

    def do_describe(self, *args, **kwargs):
        ret = {'csv_file': self.csv_file,
               'started': self.started,
               'last_update': self.last_update,
               'commands': [x.split('do_')[-1].replace('_', '-') for x in dir(self) if 'do_' in x]}
        return flask.jsonify(ret)

class Analyzer(object):
    _csv = None
    csv_file = property(lambda self: self._csv)
    last_update = None
    df = pd.DataFrame()

    @csv_file.setter
    def csv_file(self, val):
        self._csv = val
        self.extract_data()

    def extract_data(self, drop_na=False):
        if not self.csv_file:
            return
        dateparse = lambda dates: pd.datetime.strptime(dates, '%Y-%m-%d %H:%M:%S.%f')
        df = pd.read_csv(self.csv_file, parse_dates='timestamp', index_col='timestamp', date_parser=dateparse)
        df['traffic'] = df['traffic'].apply(lambda x: x / 60.)
        self.df = df.resample('T').mean()
        if drop_na:
            self.df.dropna(inplace=True)
        else:
            self.df.interpolate(inplace=True)
        self.last_update = datetime.datetime.now()

    def rolling_mean(self, window=10):
        means = self.df.rolling(window=window).mean()
        ewm_means = self.df.ewm(halflife=window).mean()
        means.columns = ['mean-%s' % col for col in means.columns]
        ewm_means.columns = ['ewm-%s' % col for col in ewm_means.columns]
        ts = pd.concat([means, ewm_means], axis=1)
        return ts

    def filter(self, lamb=1e5):
        cycle, trend = sm.tsa.filters.hpfilter(self.df, lamb=lamb)
        cycle.columns = ['%s-cycle' % col for col in cycle.columns]
        trend.columns = ['%s-trend' % col for col in trend.columns]
        ts = pd.concat([cycle, trend], axis=1)
        return ts

    @staticmethod
    def plot(ts):
        fig, ax = plt.subplots()
        lined = dict()

        ax.set_title('Click on legend line to toggle line on/off')
        lines = [ax.plot(ts[col], label=col) for col in ts.columns]
        leg = ax.legend(loc='best')

        for legline, origline in zip(leg.get_lines(), lines):
            legline.set_picker(5)  # 5 pts tolerance
            lined[legline] = origline[0]

        def onpick(event):
            # on the pick event, find the orig line corresponding to the
            # legend proxy line, and toggle the visibility
            legline = event.artist
            origline = lined[legline]
            vis = not origline.get_visible()
            origline.set_visible(vis)
            # Change the alpha on the line in the legend so we can see what lines
            # have been toggled
            if vis:
                legline.set_alpha(1.0)
            else:
                legline.set_alpha(0.2)
            fig.canvas.draw()

        fig.canvas.mpl_connect('pick_event', onpick)
        plt.show(False)

    def do_summary(self, *args, **kwargs):
        return flask.jsonify(self.df.describe().to_dict())

    def do_filter(self, *args, **kwargs):
        as_json = parse_arg('as_json', kwargs, default_val=False)
        lamb = parse_arg('lamb', kwargs, float, 1e5)
        orient = parse_arg('orient', kwargs, default_val='columns')
        j = self.filter(lamb).interpolate().\
            to_json(date_format='iso', orient=orient, double_precision=2, date_unit='s')
        if as_json:
            return j
        return flask.jsonify(json.loads(j))

    def do_raw_data(self, *args, **kwargs):
        as_json = parse_arg('as_json', kwargs, default_val=False)
        orient = parse_arg('orient', kwargs, default_val='columns')
        j = self.df.to_json(date_format='iso', orient=orient, double_precision=2, date_unit='s')
        if as_json:
            return j
        return flask.jsonify(json.loads(j))

    def do_rolling_mean(self, *args, **kwargs):
        as_json = parse_arg('as_json', kwargs, default_val=False)
        window = parse_arg('window', kwargs, int, 10)
        orient = parse_arg('orient', kwargs, default_val='columns')
        j = self.rolling_mean(window=window).interpolate().\
            to_json(date_format='iso', orient=orient, double_precision=2, date_unit='s')
        if as_json:
            return j
        return flask.jsonify(json.loads(j))

    def do_refresh(self, *args, **kwargs):
        self.extract_data()
        return "analyzer refreshed\n"

    def do_config(self, *args, **kwargs):
        for key in kwargs.iterkeys():
            val = parse_arg(key, kwargs)
            setattr(self, key, val)
        return 'finished analyzer config\n'

    def do_describe(self, *args, **kwargs):
        ret = {'csv_file': self.csv_file,
               'last_update': self.last_update,
               'samples': len(self.df),
               'commands': [x.split('do_')[-1].replace('_', '-') for x in dir(self) if 'do_' in x]}
        return flask.jsonify(ret)

_ar = Analyzer()
_cr = Collector()

def get_parser():
    parser = OptionParser()
    parser.add_option('-a', '--address', dest='host', default='127.0.0.1', help='server hostname')
    parser.add_option('-p', '--port', dest='port', default=5000, help='server port')
    return parser.parse_args()

@app.route('/collector/<command>')
def collector(command):
    command = command.replace('-', '_')
    cmd = getattr(_cr, 'do_%s' % command)
    return cmd(**request.args)

@app.route('/analyzer/<command>')
def analyzer(command):
    command = command.replace('-', '_')
    cmd = getattr(_ar, 'do_%s' % command)
    return cmd(**request.args)

if __name__ == '__main__':
    print "starting toll road server"
    options, _ = get_parser()
    try:
        app.run(host=options.host, port=options.port)
    except KeyboardInterrupt as e:
        print "shutting down sampling"
        sys.exit(0)
    sys.exit(1)
<|endoftext|>


<commit_before>from os.path import join

COURSE_DATA = join('..', 'course-data')
details_source = join(COURSE_DATA, 'details')
xml_source = join(COURSE_DATA, 'raw_xml')
course_dest = join(COURSE_DATA, 'courses')
info_path = join(COURSE_DATA, 'info.json')
term_dest = join(COURSE_DATA, 'terms')
mappings_path = join(COURSE_DATA, 'data-lists')
handmade_path = join(COURSE_DATA, 'data-mappings')
term_clbid_mapping_path = join(COURSE_DATA, 'courses', '_index')

def make_clbid_map_path(term):
    return join(term_clbid_mapping_path, str(term) + '.json')

def find_details_subdir(clbid):
    str_clbid = str(clbid).zfill(10)

    n_thousand = int(int(clbid) / 1000)
    thousands_subdir = (n_thousand * 1000)

    return join(str(thousands_subdir).zfill(5), str_clbid)

def make_course_path(clbid):
    clbid = str(clbid).zfill(10)
    return join(course_dest, find_details_subdir(clbid) + '.json')

def make_html_path(clbid):
    clbid = str(clbid).zfill(10)
    return join(details_source, find_details_subdir(clbid) + '.html')

def make_xml_term_path(term):
    return join(xml_source, str(term) + '.xml')

def make_built_term_path(term, kind, folder=term_dest):
    return join(folder, '%d.%s' % (term, kind))
<commit_msg>clean up some stuff in paths.py<commit_after>from os.path import join

COURSE_DATA = join('..', 'course-data')
details_source = join(COURSE_DATA, 'details')
xml_source = join(COURSE_DATA, 'raw_xml')
course_dest = join(COURSE_DATA, 'courses')
info_path = join(COURSE_DATA, 'info.json')
term_dest = join(COURSE_DATA, 'terms')
mappings_path = join(COURSE_DATA, 'data-lists')
handmade_path = join(COURSE_DATA, 'data-mappings')
term_clbid_mapping_path = join(COURSE_DATA, 'courses', '_index')

def make_clbid_map_path(term):
    return join(term_clbid_mapping_path, str(term) + '.json')

def find_details_subdir(clbid):
    str_clbid = str(clbid).zfill(10)

    n_thousand = int(clbid) // 1000
    thousands_subdir = str(n_thousand * 1000).zfill(5)

    return join(thousands_subdir, str_clbid)

def make_course_path(clbid):
    return join(course_dest, find_details_subdir(clbid) + '.json')

def make_html_path(clbid):
    return join(details_source, find_details_subdir(clbid) + '.html')

def make_xml_term_path(term):
    return join(xml_source, str(term) + '.xml')

def make_built_term_path(term, kind, folder=term_dest):
    return join(folder, '%d.%s' % (term, kind))
<|endoftext|>


<commit_before>#!/usr/bin/python3
# ^ Note "python3", not "python", because Cygwin currently installs both. You may not need to do this--run "python -V"; if your version is 3+, you can just set the location to /python
# Ideal command-line syntax? "glt *.txt -backup -rf"

import fileinput, re, sys, os, os.path, glob

#Set initial variables
backupmode = ""
filestotag = []
recursivefiles = []
recursion = False

# Interpret command-line parameters
def getparameters(mode):
	global backupmode
	global recursion
	global removeglt
	if mode == "-rf":
		recursion = True
	if mode == "-backup":
		backupmode = ".backup"
	if mode == "-strip":
		removeglt = True

allfilesintree = [os.path.join(root, name)
    for root, dirs, filestotag in os.walk(".")
	for name in filestotag]

# Begin script
for word in sys.argv:
	word = word.replace("\"\"", "")
	if word != __file__: # Don't include the script's own name
		filestotag.append(word)
for file in filestotag:
	if file[0] == "-": # Remove and interpret parameters preceded by a hyphen
		filestotag.remove(file)
		getparameters(file)
if recursion == True:
	for file in filestotag:
		for expandedfile in glob.glob(file):
			for word in allfilesintree:
				if word.endswith(expandedfile):
					recursivefiles.append(word)
	filestotag = recursivefiles
for file in filestotag:
	if file.find("*") != -1: # Expand wilcards
		filestotag.remove(file)
		for expandedfile in glob.glob(file):
			filestotag.append(expandedfile) # Shoot expanded files into the list
#print("Recursive!", recursivefiles)
for file in filestotag:
	if os.path.isfile(file) is not True:
		print("LOL WUT \""+ file + "\" is not a file")
		filestotag.remove(file)
	if file.endswith(".backup"):
		filestotag.remove(file)
if len(filestotag) is 0:
	print("No files to tag!")
else:
	# source = input("Enter source: ")
	# medium = input("Enter medium: ")
	# content = input("Enter content: ")
	# campaign = input("Enter campaign: ")
	# glt = "utm_source=" + source + "&utm_medium=" + medium + "&utm_content=" + content + "&utm_campaign=" + campaign
	# print(glt) # For now
	glt = "temp" # Remove
	for file in filestotag:
		print(file)
	for line in fileinput.input(filestotag, inplace=1, backup=backupmode):
	    line = re.sub('<a([^>]*)href="([^"\#]*)(\#[^"]*)(\?[^"]*)"','<a\\1href="\\2\\4\\3"', line.rstrip()) # Put section IDs at the end
	    line = re.sub('<a([^>]*)href="([^"]*http[^#?"]*?)"','<a\\1href="\\2?' + glt + '"', line.rstrip()) # Tag links without any section ID
	    line = re.sub('<a([^>]*)href="([^"]*http[^#?"]*?)#([^\"]*?)"','<a\\1href="\\2?' + glt + '#\\3"', line.rstrip()) # Tag links with section ID, before the ID
	    line = re.sub('<a([^>]*)href="([^"]*http[^#?"]*?)(\?(?![^#"]*?utm_)[^#"]*?)"','<a\\1href="\\2\\3&' + glt + '"', line.rstrip()) # Tag links with other parameters (but no GLT) and no section ID
	    line = re.sub('<a([^>]*)href="([^"]*http[^#?"]*?)(\?(?!.*?utm_).*?)(#*[^#"]*)"','<a\\1href="\\2\\3&' + glt + '\\4"', line.rstrip()) # Tag links with other parameters (but no GLT) and a section ID at the end
	    line = re.sub('<a([^>]*)href="([^"]*http[^"]*?[^"]*)">','<a\\1href="\\2" target="_blank">', line.rstrip()) # Append target="_blank"
	    print(line) # Writes directly to the file
	print("Done!")
# print("Testing")
# allfiles =[]
# for root, dirs, files in os.walk("."):
# 	for name in files:
# 		allfiles.append(name)
# print(allfiles)
# Use scenarios for recursive tagging:
# tag -rf index.html (every index.html in every directory)
# tag -rf *.* (every file in every directory)
# recursivefiles = [os.path.join(root, name)
#     for root, dirs, files in os.walk(".")
#         for name in files
#             if name.endswith((".html"))]
# for name in recursivefiles:
# 	print(name)
#print(os.path.basename(__file__))
#person = input('Enter your name: ')
#print('Hello', person)
# type = sys.argv[1]
# backupMode = ".backup"
# allfilesintree = [os.path.join(root, name)
#     for root, dirs, filestotag in os.walk(".")
#         for name in filestotag
#             if name.endswith(("." + type))]
# #print allfilestotag
# for name in allfilestotag:
# 	print name

<commit_msg>Working version with recursion!<commit_after>#!/usr/bin/python3
# ^ Note "python3", not "python", because Cygwin currently installs both. You may not need to do this--run "python -V"; if your version is 3+, you can just set the location to /python

import fileinput, re, sys, os, os.path, glob, fnmatch

#Set initial variables
allfilesintree = [os.path.join(root, name)
    for root, dirs, files in os.walk(".")
	for name in files]
backupmode = "" # none
recursion = False
removeglt = False
filestotag = []
matches = []

# Interpret parameters
def getparameters(mode):
	global backupmode
	global recursion
	global removeglt
	if mode == "-rf":
		recursion = True
	if mode == "-backup":
		backupmode = ".backup"
	if mode == "-strip":
		removeglt = True

# Begin script
for word in sys.argv:
	word = word.replace("\"\"", "")
	if word != __file__: # Don't include the script's own name
		filestotag.append(word)
for file in filestotag:
	if file[0] == "-": # Interpret parameters preceded by a hyphen, and...
		getparameters(file)
for file in filestotag:
	if file[0] == "-": # ...remove those parameters in a separate loop: otherwise, removing them shifts the count one over and allows the next parameter to be overlooked
		filestotag.remove(file)
if recursion == True:
	for file in filestotag:
		for root, dirnames, filenames in os.walk('.'):
			for filename in fnmatch.filter(filenames, file):
				matches.append(os.path.join(root, filename))
	filestotag = matches
else:
	for file in filestotag:
		if file.find("*") != -1: # Expand wilcards; this won't fire with -rf, though, because there'll be no asterisks left to expand
			filestotag.remove(file)
			for expandedfile in glob.glob(file):
				filestotag.append(expandedfile) # Shoot expanded files into the list
for file in filestotag:
	if os.path.isfile(file) is not True:
		print("LOL WUT \""+ file + "\" is not a file")
		filestotag.remove(file)
	if file.endswith(".backup"):
		filestotag.remove(file)
if len(filestotag) is 0:
	print("No files to tag!")
else:
	# source = input("Enter source: ")
	# medium = input("Enter medium: ")
	# content = input("Enter content: ")
	# campaign = input("Enter campaign: ")
	# glt = "utm_source=" + source + "&utm_medium=" + medium + "&utm_content=" + content + "&utm_campaign=" + campaign
	# print(glt) # For now
	glt = "temp" # Remove
	for file in filestotag:
		print(file)
	for line in fileinput.input(filestotag, inplace=1, backup=backupmode):
	    line = re.sub('<a([^>]*)href="([^"\#]*)(\#[^"]*)(\?[^"]*)"','<a\\1href="\\2\\4\\3"', line.rstrip()) # Put section IDs at the end
	    line = re.sub('<a([^>]*)href="([^"]*http[^#?"]*?)"','<a\\1href="\\2?' + glt + '"', line.rstrip()) # Tag links without any section ID
	    line = re.sub('<a([^>]*)href="([^"]*http[^#?"]*?)#([^\"]*?)"','<a\\1href="\\2?' + glt + '#\\3"', line.rstrip()) # Tag links with section ID, before the ID
	    line = re.sub('<a([^>]*)href="([^"]*http[^#?"]*?)(\?(?![^#"]*?utm_)[^#"]*?)"','<a\\1href="\\2\\3&' + glt + '"', line.rstrip()) # Tag links with other parameters (but no GLT) and no section ID
	    line = re.sub('<a([^>]*)href="([^"]*http[^#?"]*?)(\?(?!.*?utm_).*?)(#*[^#"]*)"','<a\\1href="\\2\\3&' + glt + '\\4"', line.rstrip()) # Tag links with other parameters (but no GLT) and a section ID at the end
	    line = re.sub('<a([^>]*)href="([^"]*http[^"]*?[^"]*)">','<a\\1href="\\2" target="_blank">', line.rstrip()) # Append target="_blank"
	    print(line) # Writes directly to the file
	print("Done!")
# print("Testing")
# allfiles =[]
# for root, dirs, files in os.walk("."):
# 	for name in files:
# 		allfiles.append(name)
# print(allfiles)
# Use scenarios for recursive tagging:
# tag -rf index.html (every index.html in every directory)
# tag -rf *.* (every file in every directory)
# recursivefiles = [os.path.join(root, name)
#     for root, dirs, files in os.walk(".")
#         for name in files
#             if name.endswith((".html"))]
# for name in recursivefiles:
# 	print(name)
#print(os.path.basename(__file__))
#person = input('Enter your name: ')
#print('Hello', person)
# type = sys.argv[1]
# backupMode = ".backup"
# allfilesintree = [os.path.join(root, name)
#     for root, dirs, filestotag in os.walk(".")
#         for name in filestotag
#             if name.endswith(("." + type))]
# #print allfilestotag
# for name in allfilestotag:
# 	print name

<|endoftext|>


<commit_before>print('hellow world and i like chicken') #here i add some useless comments

from math import pi
import numpy as np

def circunference(r):
    from math import pi
    return 2*r*pi

def areac(r):
    print(pi*r**2)
    return pi*r**2
<commit_msg>added plot circle<commit_after>print('hellow world and i like chicken') #here i add some useless comments

print('blablalalalalala')

import matplotlib.pyplot as plt

circle1 = plt.Circle((0, 0), 0.2, color='r')

from math import pi
import numpy as np

def circunference(r):
    from math import pi
    return 2*r*pi

def areac(r):
    return pi*r**2
<|endoftext|>


<commit_before>print('fine!')
<commit_msg>tests for remove<commit_after><|endoftext|>


<commit_before>import os
import time

from termcolor import colored

from tota.world import World
from tota.things import Ancient, Hero, Creep, Tower
from tota.utils import closes_empty_position, distance
from tota import settings

def get_hero_function(name):
    module = __import__('tota.heroes.' + name, fromlist=['create', ])
    create_function = getattr(module, 'create')

    return create_function()

class Game:
    """An instance of game controls the flow of the game.

       This includes player and creeps spawning, game main loop, deciding when
       to stop, importing map data, drawing each update, etc.
    """
    def __init__(self, radiant_heroes, dire_heroes, map_file_path, world_size,
                 debug=False, use_basic_icons=False):
        self.radiant_heroes = radiant_heroes
        self.dire_heroes = dire_heroes
        self.map_file_path = map_file_path
        self.debug = debug
        self.use_basic_icons = use_basic_icons

        self.heroes = []
        self.ancients = {}

        self.world = World(world_size, debug=debug)

        self.initialize_world_map()
        self.cache_ancients()
        self.initialize_heroes()

    def initialize_world_map(self):
        with open(self.map_file_path, encoding='utf-8') as map_file:
            map_text = map_file.read()
            self.world.import_map(map_text)

    def cache_ancients(self):
        def get_ancient(team):
            ancients = [thing for thing in self.world.things.values()
                        if isinstance(thing, Ancient) and thing.team == team]
            if not ancients:
                message = "Can't find the ancient for the {} team".format(team)
                raise Exception(message)
            elif len(ancients) > 1:
                message = "Team {} has 2 ancients".format(team)
                raise Exception(message)
            else:
                return ancients[0]

        for team in (settings.TEAM_DIRE, settings.TEAM_RADIANT):
            self.ancients[team] = get_ancient(team)

    def initialize_heroes(self):
        teams = {
            settings.TEAM_DIRE: self.dire_heroes,
            settings.TEAM_RADIANT: self.radiant_heroes,
        }
        for team, heroes in teams.items():
            for hero_name in heroes:
                hero = Hero(name=hero_name,
                            team=team,
                            act_function=get_hero_function(hero_name))
                self.heroes.append(hero)
                self.spawn_near_ancient(hero)

    def spawn_near_ancient(self, thing):
        """Spawn players or creeps near their ancient."""
        # start searching from the ancient position, outwards, until an empty
        # space is found, using breadth first graph search
        ancient = self.ancients[thing.team]
        spawn_at = closes_empty_position(ancient, self.world)
        if spawn_at:
            self.world.spawn(thing, spawn_at)
        else:
            message = "Can't spawn {} near its ancient".format(thing.name)
            raise Exception(message)

    def position_draw(self, position):
        """Get the string to draw for a given position of the world."""
        # decorations first, then things over them
        thing = self.world.things.get(position)

        if thing is not None:
            if self.use_basic_icons:
                icon = thing.ICON_BASIC
            else:
                icon = thing.ICON
            return colored(icon, settings.TEAM_COLORS[thing.team])
        else:
            return ' '

    def play(self, frames_per_second=2.0):
        """Game main loop, ending in a game result with description."""
        while True:
            # spawn creep wave
            if self.world.t % settings.CREEP_WAVE_COOLDOWN == 0:
                for team in (settings.TEAM_RADIANT, settings.TEAM_DIRE):
                    for i in range(settings.CREEP_WAVE_SIZE):
                        creep = Creep(team)
                        self.spawn_near_ancient(creep)

            self.world.step()
            self.update_experience()
            self.clean_deads()

            # maintain the flow of zombies if necessary
            self.draw()

            if self.debug:
                input()
            else:
                time.sleep(1.0 / frames_per_second)

            if self.game_ended():
                description = self.game_result()
                print('')
                print(description)

                return description

    def update_experience(self):
        for thing in list(self.world.things.values()):
            if thing.life <= 0:
                for hero in self.heroes:
                    if distance(hero, thing) < settings.XP_DISTANCE:
                        if isinstance(thing, Creep):
                            hero.xp += settings.XP_CREEP_DEAD
                        elif isinstance(thing, Hero):
                            hero.xp += settings.XP_HERO_DEAD
                        elif isinstance(thing, Tower):
                            hero.xp += settings.XP_TOWER_DEAD

                self.world.destroy(thing)

    def clean_deads(self):
        """Remove dead things from the world."""
        for thing in list(self.world.things.values()):
            if thing.life <= 0:
                self.world.destroy(thing)

    def draw(self):
        """Draw the world."""
        screen = ''

        # print the world
        screen += '\n'.join(u''.join(self.position_draw((x, y)) + ' '
                                     for x in range(self.world.size[0]))
                            for y in range(self.world.size[1]))

        # game stats
        screen += '\nticks:{}'.format(self.world.t)

        # print hero stats
        for hero in sorted(self.heroes, key=lambda x: x.name):
            if hero.life > 0:
                # a small "health bar" with unicode chars, from 0 to 10 chars
                life_chars_count = int((10.0 / hero.max_life) * hero.life)
                life_chars = life_chars_count * '\u2588'
                no_life_chars = (10 - life_chars_count) * '\u2591'
                life_bar = '\u2665 {}{}'.format(life_chars, no_life_chars)
            else:
                life_bar = '\u2620 [dead]'

            hero_template =  '{bar}({life}) {name} ({level})'
            hero_stats = hero_template.format(bar=life_bar,
                                              name=hero.name,
                                              life=hero.life,
                                              level=hero.level)

            screen += '\n' + colored(hero_stats,
                                     settings.TEAM_COLORS[hero.team])

        # print events (of last step) for debugging
        if self.debug:
            screen += u'\n'
            screen += u'\n'.join([colored('{}: {}'.format(thing.name, event),
                                          settings.TEAM_COLORS[thing.team])
                                  for t, thing, event in self.world.events
                                  if t == self.world.t])
        os.system('clear')
        print(screen)

    def destroyed_ancients(self):
        """Which ancients have been destroyed?"""
        return [ancient for ancient in self.ancients.values()
                if ancient.life <= 0]

    def game_ended(self):
        """Has the game ended?"""
        return len(self.destroyed_ancients()) > 0

    def game_result(self):
        """Was the game won?"""
        return '\n'.join('Team {} lost!'.format(ancient.team)
                         for ancient in self.destroyed_ancients())
<commit_msg>Show level, fix show life decimals<commit_after>import os
import time

from termcolor import colored

from tota.world import World
from tota.things import Ancient, Hero, Creep, Tower
from tota.utils import closes_empty_position, distance
from tota import settings

def get_hero_function(name):
    module = __import__('tota.heroes.' + name, fromlist=['create', ])
    create_function = getattr(module, 'create')

    return create_function()

class Game:
    """An instance of game controls the flow of the game.

       This includes player and creeps spawning, game main loop, deciding when
       to stop, importing map data, drawing each update, etc.
    """
    def __init__(self, radiant_heroes, dire_heroes, map_file_path, world_size,
                 debug=False, use_basic_icons=False):
        self.radiant_heroes = radiant_heroes
        self.dire_heroes = dire_heroes
        self.map_file_path = map_file_path
        self.debug = debug
        self.use_basic_icons = use_basic_icons

        self.heroes = []
        self.ancients = {}

        self.world = World(world_size, debug=debug)

        self.initialize_world_map()
        self.cache_ancients()
        self.initialize_heroes()

    def initialize_world_map(self):
        with open(self.map_file_path, encoding='utf-8') as map_file:
            map_text = map_file.read()
            self.world.import_map(map_text)

    def cache_ancients(self):
        def get_ancient(team):
            ancients = [thing for thing in self.world.things.values()
                        if isinstance(thing, Ancient) and thing.team == team]
            if not ancients:
                message = "Can't find the ancient for the {} team".format(team)
                raise Exception(message)
            elif len(ancients) > 1:
                message = "Team {} has 2 ancients".format(team)
                raise Exception(message)
            else:
                return ancients[0]

        for team in (settings.TEAM_DIRE, settings.TEAM_RADIANT):
            self.ancients[team] = get_ancient(team)

    def initialize_heroes(self):
        teams = {
            settings.TEAM_DIRE: self.dire_heroes,
            settings.TEAM_RADIANT: self.radiant_heroes,
        }
        for team, heroes in teams.items():
            for hero_name in heroes:
                hero = Hero(name=hero_name,
                            team=team,
                            act_function=get_hero_function(hero_name))
                self.heroes.append(hero)
                self.spawn_near_ancient(hero)

    def spawn_near_ancient(self, thing):
        """Spawn players or creeps near their ancient."""
        # start searching from the ancient position, outwards, until an empty
        # space is found, using breadth first graph search
        ancient = self.ancients[thing.team]
        spawn_at = closes_empty_position(ancient, self.world)
        if spawn_at:
            self.world.spawn(thing, spawn_at)
        else:
            message = "Can't spawn {} near its ancient".format(thing.name)
            raise Exception(message)

    def position_draw(self, position):
        """Get the string to draw for a given position of the world."""
        # decorations first, then things over them
        thing = self.world.things.get(position)

        if thing is not None:
            if self.use_basic_icons:
                icon = thing.ICON_BASIC
            else:
                icon = thing.ICON
            return colored(icon, settings.TEAM_COLORS[thing.team])
        else:
            return ' '

    def play(self, frames_per_second=2.0):
        """Game main loop, ending in a game result with description."""
        while True:
            # spawn creep wave
            if self.world.t % settings.CREEP_WAVE_COOLDOWN == 0:
                for team in (settings.TEAM_RADIANT, settings.TEAM_DIRE):
                    for i in range(settings.CREEP_WAVE_SIZE):
                        creep = Creep(team)
                        self.spawn_near_ancient(creep)

            self.world.step()
            self.update_experience()
            self.clean_deads()

            # maintain the flow of zombies if necessary
            self.draw()

            if self.debug:
                input()
            else:
                time.sleep(1.0 / frames_per_second)

            if self.game_ended():
                description = self.game_result()
                print('')
                print(description)

                return description

    def update_experience(self):
        for thing in list(self.world.things.values()):
            if thing.life <= 0:
                for hero in self.heroes:
                    if distance(hero, thing) < settings.XP_DISTANCE:
                        if isinstance(thing, Creep):
                            hero.xp += settings.XP_CREEP_DEAD
                        elif isinstance(thing, Hero):
                            hero.xp += settings.XP_HERO_DEAD
                        elif isinstance(thing, Tower):
                            hero.xp += settings.XP_TOWER_DEAD

                self.world.destroy(thing)

    def clean_deads(self):
        """Remove dead things from the world."""
        for thing in list(self.world.things.values()):
            if thing.life <= 0:
                self.world.destroy(thing)

    def draw(self):
        """Draw the world."""
        screen = ''

        # print the world
        screen += '\n'.join(u''.join(self.position_draw((x, y)) + ' '
                                     for x in range(self.world.size[0]))
                            for y in range(self.world.size[1]))

        # game stats
        screen += '\nticks:{}'.format(self.world.t)

        # print hero stats
        for hero in sorted(self.heroes, key=lambda x: x.name):
            if hero.life > 0:
                # a small "health bar" with unicode chars, from 0 to 10 chars
                life_chars_count = int((10.0 / hero.max_life) * hero.life)
                life_chars = life_chars_count * '\u2588'
                no_life_chars = (10 - life_chars_count) * '\u2591'
                life_bar = '\u2665 {}{}'.format(life_chars, no_life_chars)
            else:
                life_bar = '\u2620 [dead]'

            hero_template = '{bar}({life}) {name} ({level})'
            hero_stats = hero_template.format(bar=life_bar,
                                              name=hero.name,
                                              life=int(hero.life),
                                              level=hero.level)

            screen += '\n' + colored(hero_stats,
                                     settings.TEAM_COLORS[hero.team])

        # print events (of last step) for debugging
        if self.debug:
            screen += u'\n'
            screen += u'\n'.join([colored('{}: {}'.format(thing.name, event),
                                          settings.TEAM_COLORS[thing.team])
                                  for t, thing, event in self.world.events
                                  if t == self.world.t])
        os.system('clear')
        print(screen)

    def destroyed_ancients(self):
        """Which ancients have been destroyed?"""
        return [ancient for ancient in self.ancients.values()
                if ancient.life <= 0]

    def game_ended(self):
        """Has the game ended?"""
        return len(self.destroyed_ancients()) > 0

    def game_result(self):
        """Was the game won?"""
        return '\n'.join('Team {} lost!'.format(ancient.team)
                         for ancient in self.destroyed_ancients())
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Standard packages
import os
import sys
import argparse
import configuration
import subprocess

from toil.job import Job
from toil.job import JobException

def spawn_batch_jobs(job):
    """
    This is simply a placeholder root job for the workflow
    """

    job.fileStore.logToMaster("Initializing workflow\n")

def create_file(job, fpath):
    job.fileStore.logToMaster("Creating file: {}\n".format(fpath))
    subprocess.call(['touch', fpath])
    # job.fileStore.writeGlobalFile(fpath)

def run_and_log_command(command, logfile):
    """This function uses the python subprocess method to run the specified command and writes all error to the
    specified logfile

    :param command: The command-line command to execute.
    :type name: str.
    :param logfile: The logfile to output error messages to.
    :type logfile: str.
    :returns:  Nothing
    :raises: RuntimeError

    """

    with open(logfile, "wb") as err:
        sys.stdout.write("Executing {} and writing to logfile {}\n".format(command, logfile))
        err.write("Command: {}\n".format(command))
        p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=err, shell=True)
        output = p.communicate()
        code = p.returncode
        if code:
            raise RuntimeError("An error occurred when executing the commandline: {}. "
                               "Please check the logfile {} for details\n".format(command, logfile))

def run_bwa_mem(job, config, name, samples):
    """Run GATK's DiagnoseTargets against the supplied region

    :param config: The configuration dictionary.
    :type config: dict.
    :param sample: sample name.
    :type sample: str.
    :param fastq1: Input FastQ File.
    :type fastq1: str.
    :param fastq2: Input FastQ File.
    :type fastq2: str.
    :returns:  str -- Aligned and sorted BAM file name.

    """

    job.fileStore.logToMaster("Running BWA for sample {}\n".format(name))

    output_bam = "{}.bwa.sorted.bam".format(name)
    temp = "{}.bwa.sort.temp".format(name)
    logfile = "{}.bwa-align.log".format(name)

    cmd = ["{}".format(config['bwa']['bin']),
           "mem",
           "-t",
           "24",
           "-M",
           "-v",
           "2",
           "{}".format(config['reference']),
           "{}".format(samples[name]['fastq1']),
           "{}".format(samples[name]['fastq2']),
           "|",
           "{}".format(config['samtools']['bin']),
           "view",
           "-u",
           "-",
           "{}".format(config['samtools']['bin']),
           "sort",
           "-@",
           "24",
           "-O",
           "bam",
           "-o",
           "{}".format(output_bam),
           "-T",
           "{}".format(temp),
           "-"]

    command = " ".join(cmd)
    job.fileStore.logToMaster("BWA Command: {}\n".format(command))
    p = subprocess.Popen(command, shell=True)
    output = p.communicate()
    code = p.returncode

    return output_bam

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--samples_file', help="Input configuration file for samples")
    parser.add_argument('-c', '--configuration', help="Configuration file for various settings")
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    args.logLevel = "INFO"

    cwd = os.getcwd()

    sys.stdout.write("Parsing configuration data\n")
    config = configuration.configure_runtime(args.configuration)

    sys.stdout.write("Parsing sample data\n")
    samples = configuration.configure_samples(args.samples_file, config)

    root_job = Job.wrapJobFn(spawn_batch_jobs, cores=1)

    for sample in samples:
        fname = "{}/{}.txt".format(cwd, sample)
        align_job = Job.wrapJobFn(run_bwa_mem, config, sample, samples, cores=24,
                                  memory="112G".format(config['bwa']['max_mem']))

        root_job.addChild(align_job)

    # Start workflow execution
    Job.Runner.startToil(root_job, args)
<commit_msg>Testing<commit_after>#!/usr/bin/env python

# Standard packages
import os
import sys
import argparse
import configuration
import subprocess

from toil.job import Job
from toil.job import JobException

def spawn_batch_jobs(job):
    """
    This is simply a placeholder root job for the workflow
    """

    job.fileStore.logToMaster("Initializing workflow\n")

def create_file(job, fpath):
    job.fileStore.logToMaster("Creating file: {}\n".format(fpath))
    subprocess.call(['touch', fpath])
    # job.fileStore.writeGlobalFile(fpath)

def run_and_log_command(command, logfile):
    """This function uses the python subprocess method to run the specified command and writes all error to the
    specified logfile

    :param command: The command-line command to execute.
    :type name: str.
    :param logfile: The logfile to output error messages to.
    :type logfile: str.
    :returns:  Nothing
    :raises: RuntimeError

    """

    with open(logfile, "wb") as err:
        sys.stdout.write("Executing {} and writing to logfile {}\n".format(command, logfile))
        err.write("Command: {}\n".format(command))
        p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=err, shell=True)
        output = p.communicate()
        code = p.returncode
        if code:
            raise RuntimeError("An error occurred when executing the commandline: {}. "
                               "Please check the logfile {} for details\n".format(command, logfile))

def run_bwa_mem(job, config, name, samples):
    """Run GATK's DiagnoseTargets against the supplied region

    :param config: The configuration dictionary.
    :type config: dict.
    :param sample: sample name.
    :type sample: str.
    :param fastq1: Input FastQ File.
    :type fastq1: str.
    :param fastq2: Input FastQ File.
    :type fastq2: str.
    :returns:  str -- Aligned and sorted BAM file name.

    """

    job.fileStore.logToMaster("Running BWA for sample {}\n".format(name))

    output_bam = "{}.bwa.sorted.bam".format(name)
    temp = "{}.bwa.sort.temp".format(name)
    logfile = "{}.bwa-align.log".format(name)

    cmd = ["{}".format(config['bwa']['bin']),
           "mem",
           "-t",
           "24",
           "-M",
           "-v",
           "2",
           "{}".format(config['reference']),
           "{}".format(samples[name]['fastq1']),
           "{}".format(samples[name]['fastq2']),
           "|",
           "{}".format(config['samtools']['bin']),
           "view",
           "-u",
           "-",
           "{}".format(config['samtools']['bin']),
           "sort",
           "-@",
           "24",
           "-O",
           "bam",
           "-o",
           "{}".format(output_bam),
           "-T",
           "{}".format(temp),
           "-"]

    command = " ".join(cmd)
    job.fileStore.logToMaster("BWA Command: {}\n".format(command))
    p = subprocess.call(command, shell=True)

    return output_bam

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-s', '--samples_file', help="Input configuration file for samples")
    parser.add_argument('-c', '--configuration', help="Configuration file for various settings")
    Job.Runner.addToilOptions(parser)
    args = parser.parse_args()
    args.logLevel = "INFO"

    cwd = os.getcwd()

    sys.stdout.write("Parsing configuration data\n")
    config = configuration.configure_runtime(args.configuration)

    sys.stdout.write("Parsing sample data\n")
    samples = configuration.configure_samples(args.samples_file, config)

    root_job = Job.wrapJobFn(spawn_batch_jobs, cores=1)

    for sample in samples:
        fname = "{}/{}.txt".format(cwd, sample)
        align_job = Job.wrapJobFn(run_bwa_mem, config, sample, samples, cores=24,
                                  memory="112G".format(config['bwa']['max_mem']))

        root_job.addChild(align_job)

    # Start workflow execution
    Job.Runner.startToil(root_job, args)
<|endoftext|>


<commit_before>#!/usr/bin/python

# http://www.stolaf.edu/people/hansonr/sudoku/top95-analysis.htm
# http://www.sudokuwiki.org/sudoku.htm
# http://www.sudokuwiki.org/Strategy_Families
# http://www.sudokuwiki.org/Weekly_Sudoku.asp
# http://www.sadmansoftware.com/sudoku/solvingtechniques.php
# http://www.thonky.com/sudoku/

from __future__ import print_function

import sudoku

def test_solve_methods(solver, verbose=False):
	if solver.solved():
		return (False, (0, 'nothing'))
	if solver.solve_strip_naked_singletons(verbose):
		return (True, (1, 'naked singletons'))
	if solver.solve_hidden_n_tuples(1, verbose):
		return (True, (1.5, 'hidden singletons'))
	for n in range(2, 5):
		if solver.solve_naked_n_tuples(n, verbose):
			return (True, (n, 'naked %ss' % sudoku.n_tuple_name(n)))
		if solver.solve_hidden_n_tuples(n, verbose):
			return (True, (n+0.5, 'hidden %ss' % sudoku.n_tuple_name(n)))
	for unit_type in sudoku.Sudoku.UNIT_TYPES:
		if solver.solve_unit_intersection_removals(unit_type, verbose):
			intersection_plurals = {
				'row': "rows' pointing pairs/triples",
				'column': "columns' pointing pairs/triples",
				'block': 'block+line reductions'}
			intersection_plural = intersection_plurals[unit_type]
			return (True, (5, intersection_plural))
	for n in range(2, 5):
		if solver.solve_n_fish(n, verbose):
			n_fish_name = {2: 'X-wing', 3: 'swordfish', 4: 'jellyfish'}[n]
			return (True, (n+4, n_fish_name))
	if solver.solve_3d_medusas(verbose):
		return (True, (9, '3D Medusas'))
	for n in range(2, 4):
		if solver.solve_n_cell_subset_exclusion(n, verbose):
			return (True, (n+8, '%d-cell subset exclusion' % n))
	return (False, (-1, 'unsolved'))

def test_solve(solver, verbose=False):
	if solver.solved():
		if verbose:
			print('Already solved!')
		return (True, 'nothing')
	if verbose:
		print('Solving %r' % solver)
	num_solved = solver.num_solved()
	hardest = (-1, 'unsolved')
	while True:
		worked, method = test_solve_methods(solver, verbose)
		if not worked:
			break
		if method[0] > hardest[0]:
			hardest = method
	if verbose:
		if solver.solved():
			print('Completely solved!')
		else:
			print('...Cannot solve further (solved %d cells)' %
				(solver.num_solved() - num_solved))
	return (solver.solved(), hardest[1])

def test_cases():
	s = Sudoku('000000010400000000020000000000050407008000300001090000300400200050100000000806000') # hidden singleton
	s = Sudoku('000000016000900080500000000405000300000100500000800000600040200000030070010000000') # naked pair
	s = Sudoku('000000012008030000000000040120500000000004700060000000507000300000620000000100000') # hidden pair
	s = Sudoku('000000054700030000000000000000400016380070000020000000000500800105000000006000300') # naked triple
	s = Sudoku('000000031080000070000920000401000000000200800300000000090000250000080600000001000') # hidden triple
	s = Sudoku('000100030720000000000000008300000007000640000000000200061000050000070600000008400') # hidden quad
	s = Sudoku('000000013400200000600000000000460500010000007200500000000031000000000420080000000') # row's pointing pair/triple
	s = Sudoku('000000021300050000000000000500630000010000080000000900704000600600200000000108000') # column's pointing pair/triple
	s = Sudoku('000002040030000600050070000800000020000600010000350000200000900000900300100000000') # block+line reduction
	s = Sudoku('000070900020600000500000300040000021700030000000000000000201080900000700000400000') # X-wing
	s = Sudoku('050400000000030800000000001300080700060000050000200000000506040108000300000000000') # swordfish
	s = Sudoku('000000013040000080200060000609000400000800000000300000030100500000040706000000000') # 3D Medusa
	s = Sudoku('300050001000010000000000060002600300000204000000000005850000700000900020100000000') # 2-cell subset exclusion
	s = Sudoku('010000020000800600000003000000004300002010000800000090400070503000200000000000400') # 3-cell subset exclusion
	s = Sudoku('000000012400090000000000050070200000600000400000108000018000000000030700502000000') # solves 4
	s = Sudoku('000000608900002000000000300500060070000800000000030000020007500038100000000000040') # solves 0
	s = Sudoku('000060000020904060400070009030000092009080500100000030500000001090206040000050000') # unsolvable (solves 10)
	s = Sudoku('100007090030020008009600500005300900010080002600004000300000010041000007007000300') # "Escargot" (unsolvable)
	s.solve(verbose=True)
	s.verify()
	print(s)

def test_boards():
	print('#', 'solved?', 'board', 'method', sep='\t')
	with open('boards.txt', 'r') as boards:
		for line in boards:
			if not line or line.startswith('#'):
				continue
			s = sudoku.Sudoku(line)
			n = s.num_solved()
			solved, hardest = test_solve(s)
			try:
				s.verify()
			except:
				print('error:', line)
				print(s)
				break
			print(s.num_solved() - n, 'TRUE' if s.solved() else 'FALSE', line, hardest, sep='\t')

if __name__ == '__main__':
	test_boards()
<commit_msg>Strip lines so as to not print extra newlines.<commit_after>#!/usr/bin/python

# http://www.stolaf.edu/people/hansonr/sudoku/top95-analysis.htm
# http://www.sudokuwiki.org/sudoku.htm
# http://www.sudokuwiki.org/Strategy_Families
# http://www.sudokuwiki.org/Weekly_Sudoku.asp
# http://www.sadmansoftware.com/sudoku/solvingtechniques.php
# http://www.thonky.com/sudoku/

from __future__ import print_function

import sudoku

def test_solve_methods(solver, verbose=False):
	if solver.solved():
		return (False, (0, 'nothing'))
	if solver.solve_strip_naked_singletons(verbose):
		return (True, (1, 'naked singletons'))
	if solver.solve_hidden_n_tuples(1, verbose):
		return (True, (1.5, 'hidden singletons'))
	for n in range(2, 5):
		if solver.solve_naked_n_tuples(n, verbose):
			return (True, (n, 'naked %ss' % sudoku.n_tuple_name(n)))
		if solver.solve_hidden_n_tuples(n, verbose):
			return (True, (n+0.5, 'hidden %ss' % sudoku.n_tuple_name(n)))
	for unit_type in sudoku.Sudoku.UNIT_TYPES:
		if solver.solve_unit_intersection_removals(unit_type, verbose):
			intersection_plurals = {
				'row': "rows' pointing pairs/triples",
				'column': "columns' pointing pairs/triples",
				'block': 'block+line reductions'}
			intersection_plural = intersection_plurals[unit_type]
			return (True, (5, intersection_plural))
	for n in range(2, 5):
		if solver.solve_n_fish(n, verbose):
			n_fish_name = {2: 'X-wing', 3: 'swordfish', 4: 'jellyfish'}[n]
			return (True, (n+4, n_fish_name))
	if solver.solve_3d_medusas(verbose):
		return (True, (9, '3D Medusas'))
	for n in range(2, 4):
		if solver.solve_n_cell_subset_exclusion(n, verbose):
			return (True, (n+8, '%d-cell subset exclusion' % n))
	return (False, (-1, 'unsolved'))

def test_solve(solver, verbose=False):
	if solver.solved():
		if verbose:
			print('Already solved!')
		return (True, 'nothing')
	if verbose:
		print('Solving %r' % solver)
	num_solved = solver.num_solved()
	hardest = (-1, 'unsolved')
	while True:
		worked, method = test_solve_methods(solver, verbose)
		if not worked:
			break
		if method[0] > hardest[0]:
			hardest = method
	if verbose:
		if solver.solved():
			print('Completely solved!')
		else:
			print('...Cannot solve further (solved %d cells)' %
				(solver.num_solved() - num_solved))
	return (solver.solved(), hardest[1])

def test_cases():
	s = Sudoku('000000010400000000020000000000050407008000300001090000300400200050100000000806000') # hidden singleton
	s = Sudoku('000000016000900080500000000405000300000100500000800000600040200000030070010000000') # naked pair
	s = Sudoku('000000012008030000000000040120500000000004700060000000507000300000620000000100000') # hidden pair
	s = Sudoku('000000054700030000000000000000400016380070000020000000000500800105000000006000300') # naked triple
	s = Sudoku('000000031080000070000920000401000000000200800300000000090000250000080600000001000') # hidden triple
	s = Sudoku('000100030720000000000000008300000007000640000000000200061000050000070600000008400') # hidden quad
	s = Sudoku('000000013400200000600000000000460500010000007200500000000031000000000420080000000') # row's pointing pair/triple
	s = Sudoku('000000021300050000000000000500630000010000080000000900704000600600200000000108000') # column's pointing pair/triple
	s = Sudoku('000002040030000600050070000800000020000600010000350000200000900000900300100000000') # block+line reduction
	s = Sudoku('000070900020600000500000300040000021700030000000000000000201080900000700000400000') # X-wing
	s = Sudoku('050400000000030800000000001300080700060000050000200000000506040108000300000000000') # swordfish
	s = Sudoku('000000013040000080200060000609000400000800000000300000030100500000040706000000000') # 3D Medusa
	s = Sudoku('300050001000010000000000060002600300000204000000000005850000700000900020100000000') # 2-cell subset exclusion
	s = Sudoku('010000020000800600000003000000004300002010000800000090400070503000200000000000400') # 3-cell subset exclusion
	s = Sudoku('000000012400090000000000050070200000600000400000108000018000000000030700502000000') # solves 4
	s = Sudoku('000000608900002000000000300500060070000800000000030000020007500038100000000000040') # solves 0
	s = Sudoku('000060000020904060400070009030000092009080500100000030500000001090206040000050000') # unsolvable (solves 10)
	s = Sudoku('100007090030020008009600500005300900010080002600004000300000010041000007007000300') # "Escargot" (unsolvable)
	s.solve(verbose=True)
	s.verify()
	print(s)

def test_boards():
	print('#', 'solved?', 'board', 'method', sep='\t')
	with open('boards.txt', 'r') as boards:
		for line in boards:
			line = line.strip()
			if not line or line.startswith('#'):
				continue
			s = sudoku.Sudoku(line)
			n = s.num_solved()
			solved, hardest = test_solve(s)
			try:
				s.verify()
			except:
				print('error:', line)
				print(s)
				break
			print(s.num_solved() - n, 'TRUE' if s.solved() else 'FALSE', line, hardest, sep='\t')

if __name__ == '__main__':
	test_boards()
<|endoftext|>


<commit_before>import pyximport; pyximport.install()
import sparse_interaction
from scipy.sparse import csr_matrix, rand
from time import time
from sklearn.preprocessing import PolynomialFeatures
from code import interact
from numpy import  array_equal
interaction = sparse_interaction.SparseInteractionFeatures()

X = rand(10, 10000, 0.2).tocsr()#csr_matrix([[1,1,0,0],[0,0,1,1],[1,0,1,0],[0,1,0,1], [1,1,1,0], [0,1,1,0]])#
raw_input('made matrix')
#X = csr_matrix([[1,1,0,0],[0,0,1,1],[1,0,1,0],[0,1,0,1], [1,1,1,0], [0,1,1,0]])##rand(1000, 4, 0.5).tocsr()#

theirs = PolynomialFeatures(interaction_only=True)
t0 = time()
B = interaction.fit_transform(X)

t1 = time()
raw_input('transformed matrix')
print 'Mine:', t1-t0, B.shape

t0 = time()
C = theirs.fit_transform(X.toarray())
t1 = time()
print 'Theirs:', t1-t0, C.shape

#Get rid of the dummy feature sklearn adds
C = C[:,1:]

print B.toarray()
print C

print (B==C).sum() == B.shape[0]*B.shape[1] and B.shape==C.shape
<commit_msg>Delete test.py<commit_after><|endoftext|>


<commit_before>import codegen, jinja2, spidermonkey, sys
import simplejson as json

TESTS = [
	('{{ test }}', {'test': 'crap'}),
	('{% if a %}x{% endif %}', {'a': True}),
	('{% if a %}c{% endif %}b', {'a': False}),
	('{{ 1 if a else 2 }}', {'a': True}),
	('{{ 1 if a else 2 }}', {'a': False}),
	('{% if a %}d{% else %}e{% endif %}', {'a': False}),
	('{% if a %}f{% elif b %}g{% endif %}', {'b': True}),
	("{{ '%4.2f'|format(x) }}", {'x': 17.0}),
	('{{ d[:7] }}', {'d': '2011-05-27'}),
	('{{ a.x }}', {'a': {'x': 'z'}}),
	('{{ "%.6f"|format(a / b) }}', {'a': 5.0, 'b': 3}),
	('{{ "%.1f"|format(a.x / b.y * 100) }}', {'a': {'x': 20}, 'b': {'y': 5}}),
	('{% macro x(y) %}{{ y / 2 }}{% endmacro %}{{ x(z) }}', {'z': 512}),
	(
		'{% macro x(y, z) %}{{ y + z }}{% endmacro %}{{ x(y, z) }}',
		{'z': 512, 'y': 3},
	),
	('{{ x is none }}', {'x': None}),
	('{{ "%.2f%%"|format(a) }}', {'a': 5}),
]

def loader(i):
	return jinja2.DictLoader({'index': TESTS[i][0]})

def jstest(env, data):
	run = spidermonkey.Runtime()
	ctx = run.new_context()
	js = codegen.generate(env)
	jsobj = json.dumps(data)
	code = js + '\ntemplates.index.render(%s);' % jsobj
	return ctx.execute(code)

def pytest(env, data):
	tmpl = env.get_template('index')
	return tmpl.render(data)

def run(i, quiet=True):

	src, data = TESTS[i]
	env = jinja2.Environment(loader=loader(i))
	ast = codegen.compile(env, src)

	if not quiet:
		print ast
		print codegen.generate(env)

	js = jstest(env, data)
	py = pytest(env, data)

	if not quiet:
		print 'js:', repr(js)
		print 'py:', repr(py)

	if js.isdigit():
		return float(js) == float(py)
	if {'true': 'True', 'false': 'False'}.get(js, js) == py:
		return True
	return js == py

def test():
	for i, t in enumerate(TESTS):
		res = run(i)
		sys.stdout.write('.' if res else 'F')
	sys.stdout.write('\n')

if __name__ == '__main__':
	args = sys.argv[1:]
	if args:
		run(int(args[0]), False)
	else:
		test()
<commit_msg>Don't make the check die.<commit_after>import codegen, jinja2, spidermonkey, sys
import simplejson as json

TESTS = [
	('{{ test }}', {'test': 'crap'}),
	('{% if a %}x{% endif %}', {'a': True}),
	('{% if a %}c{% endif %}b', {'a': False}),
	('{{ 1 if a else 2 }}', {'a': True}),
	('{{ 1 if a else 2 }}', {'a': False}),
	('{% if a %}d{% else %}e{% endif %}', {'a': False}),
	('{% if a %}f{% elif b %}g{% endif %}', {'b': True}),
	("{{ '%4.2f'|format(x) }}", {'x': 17.0}),
	('{{ d[:7] }}', {'d': '2011-05-27'}),
	('{{ a.x }}', {'a': {'x': 'z'}}),
	('{{ "%.6f"|format(a / b) }}', {'a': 5.0, 'b': 3}),
	('{{ "%.1f"|format(a.x / b.y * 100) }}', {'a': {'x': 20}, 'b': {'y': 5}}),
	('{% macro x(y) %}{{ y / 2 }}{% endmacro %}{{ x(z) }}', {'z': 512}),
	(
		'{% macro x(y, z) %}{{ y + z }}{% endmacro %}{{ x(y, z) }}',
		{'z': 512, 'y': 3},
	),
	('{{ x is none }}', {'x': None}),
	('{{ "%.2f%%"|format(a) }}', {'a': 5}),
]

def loader(i):
	return jinja2.DictLoader({'index': TESTS[i][0]})

def jstest(env, data):
	run = spidermonkey.Runtime()
	ctx = run.new_context()
	js = codegen.generate(env)
	jsobj = json.dumps(data)
	code = js + '\ntemplates.index.render(%s);' % jsobj
	return ctx.execute(code)

def pytest(env, data):
	tmpl = env.get_template('index')
	return tmpl.render(data)

def run(i, quiet=True):

	src, data = TESTS[i]
	env = jinja2.Environment(loader=loader(i))
	ast = codegen.compile(env, src)

	if not quiet:
		print ast
		print codegen.generate(env)

	js = jstest(env, data)
	py = pytest(env, data)

	if not quiet:
		print 'js:', repr(js)
		print 'py:', repr(py)

	if isinstance(js, str) and js.isdigit():
		return float(js) == float(py)
	if {'true': 'True', 'false': 'False'}.get(js, js) == py:
		return True
	return js == py

def test():
	for i, t in enumerate(TESTS):
		res = run(i)
		sys.stdout.write('.' if res else 'F')
	sys.stdout.write('\n')

if __name__ == '__main__':
	args = sys.argv[1:]
	if args:
		run(int(args[0]), False)
	else:
		test()
<|endoftext|>


<commit_before>#! /usr/bin/env python3

import unittest
import random
from pyroaring import BitMap

class BaseTest(unittest.TestCase):

    def compare_with_set(self, bitmap, expected_set, universe):
        self.assertEqual(len(bitmap), len(expected_set))
        self.assertEqual(set(bitmap), expected_set)
        self.assertEqual(sorted(list(bitmap)), sorted(list(expected_set)))
        self.assertEqual(BitMap(expected_set), bitmap)
        for value in universe:
            if value in expected_set:
                self.assertIn(value, bitmap)
            else:
                self.assertNotIn(value, bitmap)

    def test_basic(self):
        bitmap = BitMap()
        expected_set = set()
        universe = range(100)
        self.compare_with_set(bitmap, expected_set, universe)
        values = list(universe)
        random.shuffle(values)
        for value in values:
            bitmap.add(value)
            expected_set.add(value)
            self.compare_with_set(bitmap, expected_set, universe)

    def test_bitmap_equality(self):
        bitmap1 = BitMap()
        bitmap2 = BitMap()
        self.assertEqual(bitmap1, bitmap2)
        bitmap1.add(42)
        self.assertNotEqual(bitmap1, bitmap2)
        bitmap2.add(27)
        self.assertNotEqual(bitmap1, bitmap2)
        bitmap1.add(27)
        self.assertNotEqual(bitmap1, bitmap2)
        bitmap2.add(42)
        self.assertEqual(bitmap1, bitmap2)

    def test_constructor_values(self):
        values = range(10,80,3)
        bitmap = BitMap(values)
        expected_set = set(values)
        universe = range(100)
        self.compare_with_set(bitmap, expected_set, universe)

    def test_constructor_copy(self):
        bitmap1 = BitMap(range(10))
        bitmap2 = BitMap(bitmap1)
        self.assertEqual(bitmap1, bitmap2)
        bitmap1.add(42)
        self.assertNotEqual(bitmap1, bitmap2)

class OperationsTest(BaseTest):

    def setUp(self):
        self.universe = range(100)
        self.set1 = self.get_random_set(self.universe)
        self.set2 = self.get_random_set(self.universe)
        self.bitmap1 = BitMap(self.set1)
        self.bitmap2 = BitMap(self.set2)

    @staticmethod
    def get_random_set(universe):
        size = random.randint(len(universe)/10, 4*len(universe)/10)
        return set(random.sample(universe, size))

    def do_test_binary_op(self, op):
        old_bitmap1 = BitMap(self.bitmap1)
        old_bitmap2 = BitMap(self.bitmap2)
        result_set = op(self.set1, self.set2)
        result_bitmap = op(self.bitmap1, self.bitmap2)
        self.assertEqual(self.bitmap1, old_bitmap1)
        self.assertEqual(self.bitmap2, old_bitmap2)
        self.compare_with_set(result_bitmap, result_set, self.universe)

    def test_or(self):
        for _ in range(10):
            self.do_test_binary_op(lambda x,y : x|y)

    def test_and(self):
        for _ in range(10):
            self.do_test_binary_op(lambda x,y : x&y)

    def do_test_binary_op_inplace(self, op):
        old_bitmap2 = BitMap(self.bitmap2)
        op(self.set1, self.set2)
        op(self.bitmap1, self.bitmap2)
        self.assertEqual(self.bitmap2, old_bitmap2)
        self.compare_with_set(self.bitmap1, self.set1, self.universe)

    def test_or_inplace(self):
        for _ in range(10):
            self.do_test_binary_op_inplace(lambda x,y : x.__ior__(y))

    def test_and_inplace(self):
        for _ in range(10):
            self.do_test_binary_op_inplace(lambda x,y : x.__iand__(y))

if __name__ == "__main__":
    unittest.main()
<commit_msg>Add unit tests for wrong types.<commit_after>#! /usr/bin/env python3

import unittest
import random
from pyroaring import BitMap

class BaseTest(unittest.TestCase):

    def compare_with_set(self, bitmap, expected_set, universe):
        self.assertEqual(len(bitmap), len(expected_set))
        self.assertEqual(set(bitmap), expected_set)
        self.assertEqual(sorted(list(bitmap)), sorted(list(expected_set)))
        self.assertEqual(BitMap(expected_set), bitmap)
        for value in universe:
            if value in expected_set:
                self.assertIn(value, bitmap)
            else:
                self.assertNotIn(value, bitmap)

class BasicTest(BaseTest):

    def test_basic(self):
        bitmap = BitMap()
        expected_set = set()
        universe = range(100)
        self.compare_with_set(bitmap, expected_set, universe)
        values = list(universe)
        random.shuffle(values)
        for value in values:
            bitmap.add(value)
            expected_set.add(value)
            self.compare_with_set(bitmap, expected_set, universe)

    def test_bitmap_equality(self):
        bitmap1 = BitMap()
        bitmap2 = BitMap()
        self.assertEqual(bitmap1, bitmap2)
        bitmap1.add(42)
        self.assertNotEqual(bitmap1, bitmap2)
        bitmap2.add(27)
        self.assertNotEqual(bitmap1, bitmap2)
        bitmap1.add(27)
        self.assertNotEqual(bitmap1, bitmap2)
        bitmap2.add(42)
        self.assertEqual(bitmap1, bitmap2)

    def test_constructor_values(self):
        values = range(10,80,3)
        bitmap = BitMap(values)
        expected_set = set(values)
        universe = range(100)
        self.compare_with_set(bitmap, expected_set, universe)

    def test_constructor_copy(self):
        bitmap1 = BitMap(range(10))
        bitmap2 = BitMap(bitmap1)
        self.assertEqual(bitmap1, bitmap2)
        bitmap1.add(42)
        self.assertNotEqual(bitmap1, bitmap2)

    def wrong_op(self, op):
        bitmap = BitMap()
        with self.assertRaises(ValueError):
            op(bitmap, -3)
        with self.assertRaises(ValueError):
            op(bitmap, 2**33)
        with self.assertRaises(ValueError):
            op(bitmap, 'bla')

    def test_wrong_add(self):
        self.wrong_op(lambda bitmap, value: bitmap.add(value))

    def test_wrong_contain(self):
        self.wrong_op(lambda bitmap, value: bitmap.__contains__(value))

    def test_wrong_constructor_values(self):
        with self.assertRaises(ValueError):
            bitmap = BitMap([3, 1, -3, 42])
        with self.assertRaises(ValueError):
            bitmap = BitMap([3, 2**33, 3, 42])
        with self.assertRaises(ValueError):
            bitmap = BitMap([3, 'bla', 3, 42])

class OperationsTest(BaseTest):

    def setUp(self):
        self.universe = range(100)
        self.set1 = self.get_random_set(self.universe)
        self.set2 = self.get_random_set(self.universe)
        self.bitmap1 = BitMap(self.set1)
        self.bitmap2 = BitMap(self.set2)

    @staticmethod
    def get_random_set(universe):
        size = random.randint(len(universe)/10, 4*len(universe)/10)
        return set(random.sample(universe, size))

    def do_test_binary_op(self, op):
        old_bitmap1 = BitMap(self.bitmap1)
        old_bitmap2 = BitMap(self.bitmap2)
        result_set = op(self.set1, self.set2)
        result_bitmap = op(self.bitmap1, self.bitmap2)
        self.assertEqual(self.bitmap1, old_bitmap1)
        self.assertEqual(self.bitmap2, old_bitmap2)
        self.compare_with_set(result_bitmap, result_set, self.universe)

    def test_or(self):
        for _ in range(10):
            self.do_test_binary_op(lambda x,y : x|y)

    def test_and(self):
        for _ in range(10):
            self.do_test_binary_op(lambda x,y : x&y)

    def do_test_binary_op_inplace(self, op):
        old_bitmap2 = BitMap(self.bitmap2)
        op(self.set1, self.set2)
        op(self.bitmap1, self.bitmap2)
        self.assertEqual(self.bitmap2, old_bitmap2)
        self.compare_with_set(self.bitmap1, self.set1, self.universe)

    def test_or_inplace(self):
        for _ in range(10):
            self.do_test_binary_op_inplace(lambda x,y : x.__ior__(y))

    def test_and_inplace(self):
        for _ in range(10):
            self.do_test_binary_op_inplace(lambda x,y : x.__iand__(y))

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2003-2009 Edgewall Software
# Copyright (C) 2003-2004 Jonas Borgström <jonas@edgewall.com>
# Copyright (C) 2004-2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Jonas Borgström <jonas@edgewall.com>
#         Christopher Lenz <cmlenz@gmx.de>

__all__ = ['Component', 'ExtensionPoint', 'implements', 'Interface',
           'TracError']

class TracError(Exception):
    """Exception base class for errors in Trac."""

    title = 'Trac Error'

    def __init__(self, message, title=None, show_traceback=False):
        """If message is a genshi.builder.tag object, everything up to the
        first <p> will be displayed in the red box, and everything after will
        be displayed below the red box.
        If title is given, it will be displayed as the large header above the
        error message.
        """
        Exception.__init__(self, message)
        self.message = message
        if title:
            self.title = title
        self.show_traceback = show_traceback

    def __unicode__(self):
        return unicode(self.message)

class Interface(object):
    """Marker base class for extension point interfaces."""

class ExtensionPoint(property):
    """Marker class for extension points in components."""

    def __init__(self, interface):
        """Create the extension point.

        @param interface: the `Interface` subclass that defines the protocol
            for the extension point
        """
        property.__init__(self, self.extensions)
        self.interface = interface
        self.__doc__ = 'List of components that implement `%s`' % \
                       self.interface.__name__

    def extensions(self, component):
        """Return a list of components that declare to implement the extension
        point interface.
        """
        extensions = ComponentMeta._registry.get(self.interface, [])
        return filter(None, [component.compmgr[cls] for cls in extensions])

    def __repr__(self):
        """Return a textual representation of the extension point."""
        return '<ExtensionPoint %s>' % self.interface.__name__

class ComponentMeta(type):
    """Meta class for components.

    Takes care of component and extension point registration.
    """
    _components = []
    _registry = {}

    def __new__(cls, name, bases, d):
        """Create the component class."""

        new_class = type.__new__(cls, name, bases, d)
        if name == 'Component':
            # Don't put the Component base class in the registry
            return new_class

        # Only override __init__ for Components not inheriting ComponentManager
        if True not in [issubclass(x, ComponentManager) for x in bases]:
            # Allow components to have a no-argument initializer so that
            # they don't need to worry about accepting the component manager
            # as argument and invoking the super-class initializer
            init = d.get('__init__')
            if not init:
                # Because we're replacing the initializer, we need to make sure
                # that any inherited initializers are also called.
                for init in [b.__init__._original for b in new_class.mro()
                             if issubclass(b, Component)
                             and '__init__' in b.__dict__]:
                    break
            def maybe_init(self, compmgr, init=init, cls=new_class):
                if cls not in compmgr.components:
                    compmgr.components[cls] = self
                    if init:
                        try:
                            init(self)
                        except:
                            del compmgr.components[cls]
                            raise
            maybe_init._original = init
            new_class.__init__ = maybe_init

        if d.get('abstract'):
            # Don't put abstract component classes in the registry
            return new_class

        ComponentMeta._components.append(new_class)
        registry = ComponentMeta._registry
        for interface in d.get('_implements', []):
            registry.setdefault(interface, []).append(new_class)
        for base in [base for base in bases if hasattr(base, '_implements')]:
            for interface in base._implements:
                registry.setdefault(interface, []).append(new_class)

        return new_class

class Component(object):
    """Base class for components.

    Every component can declare what extension points it provides, as well as
    what extension points of other components it extends.
    """
    __metaclass__ = ComponentMeta

    def __new__(cls, *args, **kwargs):
        """Return an existing instance of the component if it has already been
        activated, otherwise create a new instance.
        """
        # If this component is also the component manager, just invoke that
        if issubclass(cls, ComponentManager):
            self = super(Component, cls).__new__(cls)
            self.compmgr = self
            return self

        # The normal case where the component is not also the component manager
        compmgr = args[0]
        self = compmgr.components.get(cls)
        if self is None:
            self = super(Component, cls).__new__(cls)
            self.compmgr = compmgr
            compmgr.component_activated(self)
        return self

    @staticmethod
    def implements(*interfaces):
        """Can be used in the class definiton of `Component` subclasses to
        declare the extension points that are extended.
        """
        import sys

        frame = sys._getframe(1)
        locals_ = frame.f_locals

        # Some sanity checks
        assert locals_ is not frame.f_globals and '__module__' in locals_, \
               'implements() can only be used in a class definition'

        locals_.setdefault('_implements', []).extend(interfaces)

implements = Component.implements

class ComponentManager(object):
    """The component manager keeps a pool of active components."""

    def __init__(self):
        """Initialize the component manager."""
        self.components = {}
        self.enabled = {}
        if isinstance(self, Component):
            self.components[self.__class__] = self

    def __contains__(self, cls):
        """Return wether the given class is in the list of active components."""
        return cls in self.components

    def __getitem__(self, cls):
        """Activate the component instance for the given class, or return the
        existing the instance if the component has already been activated.
        """
        if cls not in self.enabled:
            self.enabled[cls] = self.is_component_enabled(cls)
        if not self.enabled[cls]:
            return None
        component = self.components.get(cls)
        if not component:
            if cls not in ComponentMeta._components:
                raise TracError('Component "%s" not registered' % cls.__name__)
            try:
                component = cls(self)
            except TypeError, e:
                raise TracError('Unable to instantiate component %r (%s)' %
                                (cls, e))
        return component

    def disable_component(self, component):
        """Force a component to be disabled.

        The argument `component` can be a class or an instance.
        """
        if not isinstance(component, type):
            component = component.__class__
        self.enabled[component] = False
        self.components[component] = None

    def component_activated(self, component):
        """Can be overridden by sub-classes so that special initialization for
        components can be provided.
        """

    def is_component_enabled(self, cls):
        """Can be overridden by sub-classes to veto the activation of a
        component.

        If this method returns False, the component with the given class will
        not be available.
        """
        return True
<commit_msg>0.12dev: Merged [8333] from 0.11-stable (#8160 - Setting `BaseException.message` is deprecated in Python 2.6).<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2003-2009 Edgewall Software
# Copyright (C) 2003-2004 Jonas Borgström <jonas@edgewall.com>
# Copyright (C) 2004-2005 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Jonas Borgström <jonas@edgewall.com>
#         Christopher Lenz <cmlenz@gmx.de>

__all__ = ['Component', 'ExtensionPoint', 'implements', 'Interface',
           'TracError']

class TracError(Exception):
    """Exception base class for errors in Trac."""

    title = 'Trac Error'

    def __init__(self, message, title=None, show_traceback=False):
        """If message is a genshi.builder.tag object, everything up to the
        first <p> will be displayed in the red box, and everything after will
        be displayed below the red box.
        If title is given, it will be displayed as the large header above the
        error message.
        """
        Exception.__init__(self, message)
        self._message = message
        if title:
            self.title = title
        self.show_traceback = show_traceback

    message = property(lambda self: self._message,
                       lambda self, v: setattr(self, '_message', v))

    def __unicode__(self):
        return unicode(self.message)

class Interface(object):
    """Marker base class for extension point interfaces."""

class ExtensionPoint(property):
    """Marker class for extension points in components."""

    def __init__(self, interface):
        """Create the extension point.

        @param interface: the `Interface` subclass that defines the protocol
            for the extension point
        """
        property.__init__(self, self.extensions)
        self.interface = interface
        self.__doc__ = 'List of components that implement `%s`' % \
                       self.interface.__name__

    def extensions(self, component):
        """Return a list of components that declare to implement the extension
        point interface.
        """
        extensions = ComponentMeta._registry.get(self.interface, [])
        return filter(None, [component.compmgr[cls] for cls in extensions])

    def __repr__(self):
        """Return a textual representation of the extension point."""
        return '<ExtensionPoint %s>' % self.interface.__name__

class ComponentMeta(type):
    """Meta class for components.

    Takes care of component and extension point registration.
    """
    _components = []
    _registry = {}

    def __new__(cls, name, bases, d):
        """Create the component class."""

        new_class = type.__new__(cls, name, bases, d)
        if name == 'Component':
            # Don't put the Component base class in the registry
            return new_class

        # Only override __init__ for Components not inheriting ComponentManager
        if True not in [issubclass(x, ComponentManager) for x in bases]:
            # Allow components to have a no-argument initializer so that
            # they don't need to worry about accepting the component manager
            # as argument and invoking the super-class initializer
            init = d.get('__init__')
            if not init:
                # Because we're replacing the initializer, we need to make sure
                # that any inherited initializers are also called.
                for init in [b.__init__._original for b in new_class.mro()
                             if issubclass(b, Component)
                             and '__init__' in b.__dict__]:
                    break
            def maybe_init(self, compmgr, init=init, cls=new_class):
                if cls not in compmgr.components:
                    compmgr.components[cls] = self
                    if init:
                        try:
                            init(self)
                        except:
                            del compmgr.components[cls]
                            raise
            maybe_init._original = init
            new_class.__init__ = maybe_init

        if d.get('abstract'):
            # Don't put abstract component classes in the registry
            return new_class

        ComponentMeta._components.append(new_class)
        registry = ComponentMeta._registry
        for interface in d.get('_implements', []):
            registry.setdefault(interface, []).append(new_class)
        for base in [base for base in bases if hasattr(base, '_implements')]:
            for interface in base._implements:
                registry.setdefault(interface, []).append(new_class)

        return new_class

class Component(object):
    """Base class for components.

    Every component can declare what extension points it provides, as well as
    what extension points of other components it extends.
    """
    __metaclass__ = ComponentMeta

    def __new__(cls, *args, **kwargs):
        """Return an existing instance of the component if it has already been
        activated, otherwise create a new instance.
        """
        # If this component is also the component manager, just invoke that
        if issubclass(cls, ComponentManager):
            self = super(Component, cls).__new__(cls)
            self.compmgr = self
            return self

        # The normal case where the component is not also the component manager
        compmgr = args[0]
        self = compmgr.components.get(cls)
        if self is None:
            self = super(Component, cls).__new__(cls)
            self.compmgr = compmgr
            compmgr.component_activated(self)
        return self

    @staticmethod
    def implements(*interfaces):
        """Can be used in the class definiton of `Component` subclasses to
        declare the extension points that are extended.
        """
        import sys

        frame = sys._getframe(1)
        locals_ = frame.f_locals

        # Some sanity checks
        assert locals_ is not frame.f_globals and '__module__' in locals_, \
               'implements() can only be used in a class definition'

        locals_.setdefault('_implements', []).extend(interfaces)

implements = Component.implements

class ComponentManager(object):
    """The component manager keeps a pool of active components."""

    def __init__(self):
        """Initialize the component manager."""
        self.components = {}
        self.enabled = {}
        if isinstance(self, Component):
            self.components[self.__class__] = self

    def __contains__(self, cls):
        """Return wether the given class is in the list of active components."""
        return cls in self.components

    def __getitem__(self, cls):
        """Activate the component instance for the given class, or return the
        existing the instance if the component has already been activated.
        """
        if cls not in self.enabled:
            self.enabled[cls] = self.is_component_enabled(cls)
        if not self.enabled[cls]:
            return None
        component = self.components.get(cls)
        if not component:
            if cls not in ComponentMeta._components:
                raise TracError('Component "%s" not registered' % cls.__name__)
            try:
                component = cls(self)
            except TypeError, e:
                raise TracError('Unable to instantiate component %r (%s)' %
                                (cls, e))
        return component

    def disable_component(self, component):
        """Force a component to be disabled.

        The argument `component` can be a class or an instance.
        """
        if not isinstance(component, type):
            component = component.__class__
        self.enabled[component] = False
        self.components[component] = None

    def component_activated(self, component):
        """Can be overridden by sub-classes so that special initialization for
        components can be provided.
        """

    def is_component_enabled(self, cls):
        """Can be overridden by sub-classes to veto the activation of a
        component.

        If this method returns False, the component with the given class will
        not be available.
        """
        return True
<|endoftext|>


<commit_before>from selenium import webdriver
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
import unittest
import os

URL = os.environ['VIEWER_URL']

class BasicWorkflowTest(unittest.TestCase):

    def setUp(self):
        self.browser = webdriver.Firefox()
        self.browser.implicitly_wait(2)

    def test_can_load(self):
        """ Test if page loads up to the point
        when we can access title
        """
        self.browser.get(URL)
        self.assertEqual(
            'NASB Microtask Viewer',
            self.browser.title,
            'We should see page title'
        )

    def test_can_load_chdir(self):
        """ Test if we can access chdir tab
        and read help messages
        """
        self.browser.get(URL)
        (
            self.browser
                .find_element_by_id('workflow_panel')
                .find_element_by_link_text('Characteristic Direction Analysis')
                .click()
        )

        wait = WebDriverWait(self.browser, 10)

        self.assertTrue(
            'Upload your dataset' in wait.until(
                expected_conditions.visibility_of_element_located((
                    By.ID,
                    'run_chdir_help'
                ))).text,
            'We should see chdir run message'
        )

        self.assertTrue(
            'No data' in wait.until(
                expected_conditions.visibility_of_element_located((
                    By.ID,
                    'chdir_downloads_message'
                ))).text,
            'We should see chdir downloads message'
        )

    def tearDown(self):
        self.browser.quit()

if __name__ == '__main__':
    unittest.main()
<commit_msg>Remove explicit wait<commit_after>from selenium import webdriver
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
import unittest
import os

URL = os.environ['VIEWER_URL']

class BasicWorkflowTest(unittest.TestCase):

    def setUp(self):
        self.browser = webdriver.Firefox()

    def test_can_load(self):
        """ Test if page loads up to the point
        when we can access title
        """
        self.browser.get(URL)
        self.assertEqual(
            'NASB Microtask Viewer',
            self.browser.title,
            'We should see page title'
        )

    def test_can_load_chdir(self):
        """ Test if we can access chdir tab
        and read help messages
        """
        self.browser.get(URL)
        (
            self.browser
                .find_element_by_id('workflow_panel')
                .find_element_by_link_text('Characteristic Direction Analysis')
                .click()
        )

        wait = WebDriverWait(self.browser, 10)

        self.assertTrue(
            'Upload your dataset' in wait.until(
                expected_conditions.visibility_of_element_located((
                    By.ID,
                    'run_chdir_help'
                ))).text,
            'We should see chdir run message'
        )

        self.assertTrue(
            'No data' in wait.until(
                expected_conditions.visibility_of_element_located((
                    By.ID,
                    'chdir_downloads_message'
                ))).text,
            'We should see chdir downloads message'
        )

    def tearDown(self):
        self.browser.quit()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>