<commit_before>'''
(c) 2011, 2012 Georgia Tech Research Corporation
This source code is released under the New BSD license.  Please see
http://wiki.quantsoftware.org/index.php?title=QSTK_License
for license details.

Created on October 5, 2014 (fork of Examples/Basic/tutorial1.py)

@author: Bradley Wogsland
@contact: bradley@wogsland.org
@summary: Solution to Homework 1
'''

# QSTK Imports
import QSTK.qstkutil.qsdateutil as du
import QSTK.qstkutil.tsutil as tsu
import QSTK.qstkutil.DataAccess as da

# Third Party Imports
import datetime as dt
import matplotlib.pyplot as plt
import pandas as pd

# Example call of function:
# vol, daily_ret, sharpe, cum_ret = simulate(startdate, enddate, ['GOOG','AAPL','GLD','XOM'], [0.2,0.3,0.4,0.1])
# The function should return:
#   Standard deviation of daily returns of the total portfolio
#   Average daily return of the total portfolio
#   Sharpe ratio (Always assume you have 252 trading days in an year. And risk free rate = 0) of the total portfolio
#   Cumulative return of the total portfolio
def simulate(startdate, endate, symbols, allocations):
    ''' Main Function'''
    print "In the simulate function.\n"

    # We need closing prices so the timestamp should be hours=16.
    timeofday = dt.timedelta(hours=16)

    # Get a list of trading days between the start and the end.
    ldt_timestamps = du.getNYSEdays(startdate, endate, timeofday)

    # Creating an object of the dataaccess class with Yahoo as the source.
    c_dataobj = da.DataAccess('Yahoo')

    # Keys to be read from the data, it is good to read everything in one go.
    ls_keys = ['open', 'high', 'low', 'close', 'volume', 'actual_close']

    # Reading the data, now d_data is a dictionary with the keys above.
    # Timestamps and symbols are the ones that were specified before.
    ldf_data = c_dataobj.get_data(ldt_timestamps, ls_symbols, ls_keys)
    d_data = dict(zip(ls_keys, ldf_data))

    # Filling the data for NAN
    for s_key in ls_keys:
        d_data[s_key] = d_data[s_key].fillna(method='ffill')
        d_data[s_key] = d_data[s_key].fillna(method='bfill')
        d_data[s_key] = d_data[s_key].fillna(1.0)

    # Getting the numpy ndarray of close prices.
    na_price = d_data['close'].values

    # Normalizing the prices to start at 1 and see relative returns
    na_normalized_price = na_price / na_price[0, :]

    # Copy the normalized prices to a new ndarry to find returns.
    na_rets = na_normalized_price.copy()

    # Calculate the daily returns of the prices. (Inplace calculation)
    # returnize0 works on ndarray and not dataframes.
    tsu.returnize0(na_rets)

# Try executing the function to test
simulate('2011-01-01', '2011-12-31', ['GOOG','AAPL','GLD','XOM'], [0.2,0.3,0.4,0.1])
<commit_msg>testing<commit_after>'''
(c) 2011, 2012 Georgia Tech Research Corporation
This source code is released under the New BSD license.  Please see
http://wiki.quantsoftware.org/index.php?title=QSTK_License
for license details.

Created on October 5, 2014 (fork of Examples/Basic/tutorial1.py)

@author: Bradley Wogsland
@contact: bradley@wogsland.org
@summary: Solution to Homework 1
'''

# QSTK Imports
import QSTK.qstkutil.qsdateutil as du
import QSTK.qstkutil.tsutil as tsu
import QSTK.qstkutil.DataAccess as da

# Third Party Imports
import datetime as dt
import matplotlib.pyplot as plt
import pandas as pd

# Example call of function:
# vol, daily_ret, sharpe, cum_ret = simulate(startdate, enddate, ['GOOG','AAPL','GLD','XOM'], [0.2,0.3,0.4,0.1])
# The function should return:
#   Standard deviation of daily returns of the total portfolio
#   Average daily return of the total portfolio
#   Sharpe ratio (Always assume you have 252 trading days in an year. And risk free rate = 0) of the total portfolio
#   Cumulative return of the total portfolio
def simulate(startdate, endate, symbols, allocations):
    ''' Main Function'''
    print "In the simulate function.\n"

    # We need closing prices so the timestamp should be hours=16.
    timeofday = dt.timedelta(hours=16)

    # Get a list of trading days between the start and the end.
    ldt_timestamps = du.getNYSEdays(startdate, endate, timeofday)

    # Creating an object of the dataaccess class with Yahoo as the source.
    c_dataobj = da.DataAccess('Yahoo')

    # Keys to be read from the data, it is good to read everything in one go.
    ls_keys = ['open', 'high', 'low', 'close', 'volume', 'actual_close']

    # Reading the data, now d_data is a dictionary with the keys above.
    # Timestamps and symbols are the ones that were specified before.
    ldf_data = c_dataobj.get_data(ldt_timestamps, ls_symbols, ls_keys)
    d_data = dict(zip(ls_keys, ldf_data))

    # Filling the data for NAN
    for s_key in ls_keys:
        d_data[s_key] = d_data[s_key].fillna(method='ffill')
        d_data[s_key] = d_data[s_key].fillna(method='bfill')
        d_data[s_key] = d_data[s_key].fillna(1.0)

    # Getting the numpy ndarray of close prices.
    na_price = d_data['close'].values

    # Normalizing the prices to start at 1 and see relative returns
    na_normalized_price = na_price / na_price[0, :]

    # Copy the normalized prices to a new ndarry to find returns.
    na_rets = na_normalized_price.copy()

    # Calculate the daily returns of the prices. (Inplace calculation)
    # returnize0 works on ndarray and not dataframes.
    tsu.returnize0(na_rets)

# Try executing the function to test
#simulate('2011-01-01', '2011-12-31', ['GOOG','AAPL','GLD','XOM'], [0.2,0.3,0.4,0.1])
simulate(20110101, 20111231, ['GOOG','AAPL','GLD','XOM'], [0.2,0.3,0.4,0.1])
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# encoding=utf-8
import urllib.request
import urllib.error
import urllib.parse
from PIL import Image

class ImageHandler(object):
    def __init__(self, url):
        # TODO Read 1000 bytes
        self.url = url
        try:
            self.r = urllib.request.Request(self.url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"})
            self.file = urllib.request.urlopen(self.r)
            self.image = Image.open(self.file)
        except:
            self.image = False

    def get_format(self, lower=False):
        if self.image:
            if lower:
                return self.image.format.lower()
            return self.image.format
        return False

    def get_size(self, formats="Width: %W Height: %H"):
        if self.image:
            (width, height) = self.image.size
            return formats.replace("%W", str(width)).replace("%H", str(height))
        return False

    def get_mode(self, lower=False):
        if self.image:
            if lower:
                return self.image.mode.lower()
            return self.image.mode
        return False

    def get_length(self):
        length = self.r.get_header("Content-Length")
        if length:
            return length
        return False
<commit_msg>fix filesize<commit_after>#!/usr/bin/env python3
# encoding=utf-8
import urllib.request
import urllib.error
import urllib.parse
import requests
from PIL import Image

class ImageHandler(object):
    def __init__(self, url):
        # TODO Read 1000 bytes
        self.url = url
        try:
            self.r = urllib.request.Request(self.url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"})
            self.file = urllib.request.urlopen(self.r)
            self.image = Image.open(self.file)
        except:
            self.image = False

    def get_format(self, lower=False):
        if self.image:
            if lower:
                return self.image.format.lower()
            return self.image.format
        return False

    def get_size(self, formats="Width: %W Height: %H"):
        if self.image:
            (width, height) = self.image.size
            return formats.replace("%W", str(width)).replace("%H", str(height))
        return False

    def get_mode(self, lower=False):
        if self.image:
            if lower:
                return self.image.mode.lower()
            return self.image.mode
        return False

    def get_length(self):
        req = requests.head(self.url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0"})
        length = r.headers.get("content-length")
        if length:
            return length
        return False
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
This script downloads the latest MTG card data from http://mtgjson.com/ and processes
it to turn the highly-structured data there into a flat list of card names to descriptions
formatted to send down the chat.
"""

import sys
import os
import urllib.request
import urllib.error
import contextlib
import time
import zipfile
import io
import json
import re
import datetime
import dateutil.parser
import sqlalchemy

from common import utils
import common.postgres
from common.cardname import clean_text

URL = 'http://mtgjson.com/json/AllSets.json.zip'
ZIP_FILENAME = 'AllSets.json.zip'
SOURCE_FILENAME = 'AllSets.json'
EXTRAS_FILENAME = 'extracards.json'
MAXLEN = 450

engine, metadata = common.postgres.get_engine_and_metadata()

def main():
	if not do_download_file(URL, ZIP_FILENAME) and not os.access(EXTRAS_FILENAME, os.F_OK):
		print("No new version of mtgjson data file")
		return

	print("Reading card data...")
	with zipfile.ZipFile(ZIP_FILENAME) as zfp:
		fp = io.TextIOWrapper(zfp.open(SOURCE_FILENAME))
		mtgjson = json.load(fp)

	try:
		with open(EXTRAS_FILENAME) as fp:
			extracards = json.load(fp)
	except IOError:
		pass
	else:
		# If the set is in both mtgjson and the extra data, use the one from mtgjson
		extracards.update(mtgjson)
		mtgjson = extracards
		del extracards

	print("Processing...")
	cards = metadata.tables["cards"]
	card_multiverse = metadata.tables["card_multiverse"]
	card_collector = metadata.tables["card_collector"]
	with engine.begin() as conn:
		conn.execute(card_multiverse.delete())
		conn.execute(card_collector.delete())
		conn.execute(cards.delete())
		cardid = 0
		for setid, expansion in mtgjson.items():
			release_date = dateutil.parser.parse(expansion.get('releaseDate', '1970-01-01')).date()
			for card in expansion['cards']:
				cardid += 1
				if card['layout'] in ('token', 'plane', 'scheme', 'phenomenon', 'vanguard'):  # don't care about these special cards for now
					continue
				if card['name'] == 'B.F.M. (Big Furry Monster)':  # do this card special
					continue

				filteredname, cardname, description, multiverseids, collector, hidden = process_card(card, expansion, include_reminder=setid in ('UGL', 'UNH'))
				if description is None:
					continue

				# Check if there's already a row for this card in the DB
				# (keep the one with the latest release date - it's more likely to have the accurate text in mtgjson)
				rows = conn.execute(sqlalchemy.select([cards.c.id, cards.c.lastprinted])
					.where(cards.c.filteredname == filteredname)).fetchall()
				if not rows:
					real_cardid = cardid
					conn.execute(cards.insert(),
						id=real_cardid,
						filteredname=filteredname,
						name=cardname,
						text=description,
						lastprinted=release_date,
						hidden=hidden,
					)
				elif rows[0][1] < release_date:
					real_cardid = rows[0][0]
					conn.execute(cards.update().where(cards.c.id == real_cardid),
						name=card["name"],
						text=description,
						lastprinted=release_date,
						hidden=hidden,
					)
				else:
					real_cardid = rows[0][0]

				for mid in multiverseids:
					rows = conn.execute(sqlalchemy.select([card_multiverse.c.cardid])
						.where(card_multiverse.c.id == mid)).fetchall()
					if not rows:
						conn.execute(card_multiverse.insert(),
							id=mid,
							cardid=real_cardid,
						)
					elif rows[0][0] != real_cardid:
						rows2 = conn.execute(sqlalchemy.select([cards.c.name]).where(cards.c.id == rows[0][0])).fetchall()
						print("Different names for multiverseid %d: \"%s\" and \"%s\"" % (mid, card['name'], rows2[0][0]))

				if collector:
					rows = conn.execute(sqlalchemy.select([card_collector.c.cardid])
						.where((card_collector.c.setid == setid) & (card_collector.c.collector == collector))).fetchall()
					if not rows:
						conn.execute(card_collector.insert(),
							setid=setid,
							collector=collector,
							cardid=real_cardid,
						)
					elif rows[0][0] != real_cardid:
						rows2 = conn.execute(sqlalchemy.select([cards.c.name]).where(cards.c.id == rows[0][0])).fetchall()
						print("Different names for set %s collector number %s: \"%s\" and \"%s\"" % (setid, collector, card['name'], rows2[0][0]))

		cardid += 1
		conn.execute(cards.insert(),
			id=cardid,
			filteredname="bfmbigfurrymonster",
			name="B.F.M. (Big Furry Monster)""B.F.M. (Big Furry Monster)",
			text="B.F.M. (Big Furry Monster) (BBBBBBBBBBBBBBB) | Summon \u2014 The Biggest, Baddest, Nastiest, Scariest Creature You'll Ever See [99/99] | You must play both B.F.M. cards to put B.F.M. into play. If either B.F.M. card leaves play, sacrifice the other. / B.F.M. can only be blocked by three or more creatures.",
			lastprinted=datetime.date(1998, 8, 11),
		)
		conn.execute(card_multiverse.insert(), [
			{"id": 9780, "cardid": cardid},
			{"id": 9844, "cardid": cardid},
		])
		conn.execute(card_collector.insert(), [
			{"setid": "UGL", "collector": "28", "cardid": cardid},
			{"setid": "UGL", "collector": "29", "cardid": cardid},
		])

def do_download_file(url, fn):
	"""
	Download a file, checking that there is a new version of the file on the
	server before doing so. Returns True if a download occurs.
	"""
	# Much of this code cribbed from urllib.request.urlretrieve, with If-Modified-Since logic added

	req = urllib.request.Request(url, headers={
		'User-Agent': "LRRbot/2.0 (https://lrrbot.mrphlip.com/)",
	})
	try:
		stat = os.stat(fn)
	except FileNotFoundError:
		pass
	else:
		mtime = time.strftime('%a, %d %b %Y %H:%M:%S %z', time.gmtime(stat.st_mtime))
		req.add_header('If-Modified-Since', mtime)

	try:
		fp = urllib.request.urlopen(req)
	except urllib.error.HTTPError as e:
		if e.code == 304: # Not Modified
			return False
		else:
			raise

	print("Downloading %s..." % url)
	with contextlib.closing(fp):
		headers = fp.info()

		with open(fn, 'wb') as tfp:
			bs = 1024*8
			size = None
			read = 0
			if "content-length" in headers:
				size = int(headers["Content-Length"])

			while True:
				block = fp.read(bs)
				if not block:
					break
				read += len(block)
				tfp.write(block)

	if size is not None and read < size:
		os.unlink(fn)
		raise urllib.error.ContentTooShortError(
			"retrieval incomplete: got only %i out of %i bytes"
			% (read, size), (fn, headers))

	if "last-modified" in headers:
		mtime = dateutil.parser.parse(headers['last-modified'])
		mtime = mtime.timestamp()
		os.utime(fn, (mtime, mtime))

	return True

re_check = re.compile(r"^[a-z0-9_]+$")
re_mana = re.compile(r"\{(.)\}")
re_newlines = re.compile(r"[\r\n]+")
re_multiplespaces = re.compile(r"\s{2,}")
re_remindertext = re.compile(r"( *)\([^()]*\)( *)")
def process_card(card, expansion, include_reminder=False):
	if card.get('layout') in ('split', 'aftermath'):
		# Return split cards as a single card... for all the other pieces, return nothing
		if card['name'] != card['names'][0]:
			return None, None, None, None, None, None
		splits = []
		for splitname in card['names']:
			candidates = [i for i in expansion['cards'] if i['name'] == splitname]
			if not candidates:
				print("Can't find split card piece: %s" % splitname)
				sys.exit(1)
			splits.append(candidates[0])
		filteredparts = []
		nameparts = []
		descparts = []
		allmultiverseids = []
		numbers = []
		anyhidden = False
		for s in splits:
			filtered, name, desc, multiverseids, number, hidden = process_single_card(s, expansion, include_reminder)
			filteredparts.append(filtered)
			nameparts.append(name)
			descparts.append(desc)
			allmultiverseids.extend(multiverseids)
			numbers.append(number)
			anyhidden = anyhidden or hidden

		separator = {'split': ' and ', 'aftermath': ' to '}.get(card['layout'], ' // ')
		filtseparator = {'split': 'and', 'aftermath': 'to'}.get(card['layout'], '')
		filteredname = filtseparator.join(filteredparts)
		cardname = separator.join(nameparts)
		description = "%s | %s" % (separator.join(card['names']), " // ".join(descparts))
		return filteredname, cardname, description, allmultiverseids, numbers[0], anyhidden
	else:
		return process_single_card(card, expansion, include_reminder)

def process_single_card(card, expansion, include_reminder=False):
	if card.get('layout') == 'flip':
		if card['name'] == card['names'][0] and card.get('multiverseid'):
			multiverseids = [card['multiverseid']]
		else:
			multiverseids = []
	else:
		if card.get('multiverseid'):
			multiverseids = [card['multiverseid']]
		else:
			multiverseids = []

	# sanitise card name
	filtered = clean_text(card.get('internalname', card["name"]))
	if not re_check.match(filtered):
		print("Still some junk left in name %s (%s)" % (card.get('internalname', card["name"]), json.dumps(filtered)))
		print(json.dumps(card))
		sys.exit(1)

	def build_description():
		yield card['name']
		if 'manaCost' in card:
			yield ' ['
			yield re_mana.sub(r"\1", card['manaCost'])
			yield ']'
		if card.get('layout') == 'flip':
			if card['name'] == card['names'][0]:
				yield ' (flip: '
				yield card['names'][1]
				yield ')'
			else:
				yield ' (unflip: '
				yield card['names'][0]
				yield ')'
		elif card.get('layout') == 'double-faced':
			if card['name'] == card['names'][0]:
				yield ' (back: '
				yield card['names'][1]
				yield ')'
			else:
				yield ' (front: '
				yield card['names'][0]
				yield ')'
		elif card.get('layout') == 'meld':
			# Only the melded card in MTGJSON has all three cards in the 'names' field
			# The front-face cards only have [me, back]
			melded_card = [i for i in expansion['cards'] if i['name'] == card['names'][-1]]
			if not melded_card:
				print("Can't find melded card: %s" % card['names'][-1])
				sys.exit(1)
			melded_card = melded_card[0]
			# MTGJSON doesn't have this field in a consistent order...
			# some cards have [top, bottom, back] some have [bottom, top, back]
			part_cards = [i for i in expansion['cards'] if i['name'] in melded_card['names'][:-1]]
			if len(part_cards) != 2:
				print("Can't find part-cards for melded card: %s" % melded_card['name'])
				sys.exit(1)
			if melded_card['number'][-1] != 'b':
				print("Melded card's number doesn't end in 'b': %s" % melded_card['name'])
				sys.exit(1)
			if part_cards[0]['number'] == melded_card['number'][:-1] + 'a':
				bottom_card, top_card = part_cards
			elif part_cards[1]['number'] == melded_card['number'][:-1] + 'a':
				top_card, bottom_card = part_cards
			else:
				print("Couldn't figure out which card was top and bottom for: %s" % melded_card['name'])
				sys.exit(1)
			if card['name'] == top_card['name']:
				# The names of what this melds with and into are in the card text
				pass
			elif card['name'] == bottom_card['name']:
				yield ' (melds with: '
				yield top_card['name']
				yield '; into: '
				yield melded_card['name']
				yield ')'
			elif card['name'] == melded_card['name']:
				yield ' (melds from: '
				yield top_card['name']
				yield '; '
				yield bottom_card['name']
				yield ')'
		yield ' | '
		yield card.get('type', '?Type missing?')
		if 'power' in card or 'toughness' in card:
			yield ' ['
			yield card.get('power', '?')
			yield '/'
			yield card.get('toughness', '?')
			yield ']'
		if 'loyalty' in card:
			yield ' ['
			yield str(card['loyalty'])
			yield ']'
		if 'hand' in card or 'life' in card:
			yield ' [hand:'
			yield str(card.get('hand', '?'))
			yield '/life:'
			yield str(card.get('life', '?'))
			yield ']'
		if 'text' in card:
			yield ' | '
			text = card['text']
			# Let UGL and UNH cards keep their reminder text, since there's joeks in there
			if not include_reminder:
				text = re_remindertext.sub(lambda match: ' ' if match.group(1) and match.group(2) else '', text)
			yield re_newlines.sub(' / ', text.strip())

	desc = ''.join(build_description())
	desc = re_multiplespaces.sub(' ', desc).strip()
	if len(desc) > MAXLEN:
		desc = desc[:MAXLEN-1] + "\u2026"

	return filtered, card['name'], desc, multiverseids, card.get('number'), 'internalname' in card

if __name__ == '__main__':
	main()
<commit_msg>86 the CPK warnings<commit_after>#!/usr/bin/env python3
"""
This script downloads the latest MTG card data from http://mtgjson.com/ and processes
it to turn the highly-structured data there into a flat list of card names to descriptions
formatted to send down the chat.
"""

import sys
import os
import urllib.request
import urllib.error
import contextlib
import time
import zipfile
import io
import json
import re
import datetime
import dateutil.parser
import sqlalchemy

from common import utils
import common.postgres
from common.cardname import clean_text

URL = 'http://mtgjson.com/json/AllSets.json.zip'
ZIP_FILENAME = 'AllSets.json.zip'
SOURCE_FILENAME = 'AllSets.json'
EXTRAS_FILENAME = 'extracards.json'
MAXLEN = 450

engine, metadata = common.postgres.get_engine_and_metadata()

def main():
	if not do_download_file(URL, ZIP_FILENAME) and not os.access(EXTRAS_FILENAME, os.F_OK):
		print("No new version of mtgjson data file")
		return

	print("Reading card data...")
	with zipfile.ZipFile(ZIP_FILENAME) as zfp:
		fp = io.TextIOWrapper(zfp.open(SOURCE_FILENAME))
		mtgjson = json.load(fp)

	try:
		with open(EXTRAS_FILENAME) as fp:
			extracards = json.load(fp)
	except IOError:
		pass
	else:
		# If the set is in both mtgjson and the extra data, use the one from mtgjson
		extracards.update(mtgjson)
		mtgjson = extracards
		del extracards

	print("Processing...")
	cards = metadata.tables["cards"]
	card_multiverse = metadata.tables["card_multiverse"]
	card_collector = metadata.tables["card_collector"]
	with engine.begin() as conn:
		conn.execute(card_multiverse.delete())
		conn.execute(card_collector.delete())
		conn.execute(cards.delete())
		cardid = 0
		for setid, expansion in mtgjson.items():
			release_date = dateutil.parser.parse(expansion.get('releaseDate', '1970-01-01')).date()
			for card in expansion['cards']:
				cardid += 1
				if card['layout'] in ('token', 'plane', 'scheme', 'phenomenon', 'vanguard'):  # don't care about these special cards for now
					continue
				if card['name'] == 'B.F.M. (Big Furry Monster)':  # do this card special
					continue

				filteredname, cardname, description, multiverseids, collector, hidden = process_card(card, expansion, include_reminder=setid in ('UGL', 'UNH'))
				if description is None:
					continue

				# Check if there's already a row for this card in the DB
				# (keep the one with the latest release date - it's more likely to have the accurate text in mtgjson)
				rows = conn.execute(sqlalchemy.select([cards.c.id, cards.c.lastprinted])
					.where(cards.c.filteredname == filteredname)).fetchall()
				if not rows:
					real_cardid = cardid
					conn.execute(cards.insert(),
						id=real_cardid,
						filteredname=filteredname,
						name=cardname,
						text=description,
						lastprinted=release_date,
						hidden=hidden,
					)
				elif rows[0][1] < release_date:
					real_cardid = rows[0][0]
					conn.execute(cards.update().where(cards.c.id == real_cardid),
						name=card["name"],
						text=description,
						lastprinted=release_date,
						hidden=hidden,
					)
				else:
					real_cardid = rows[0][0]

				for mid in multiverseids:
					rows = conn.execute(sqlalchemy.select([card_multiverse.c.cardid])
						.where(card_multiverse.c.id == mid)).fetchall()
					if not rows:
						conn.execute(card_multiverse.insert(),
							id=mid,
							cardid=real_cardid,
						)
					elif rows[0][0] != real_cardid:
						rows2 = conn.execute(sqlalchemy.select([cards.c.name]).where(cards.c.id == rows[0][0])).fetchall()
						print("Different names for multiverseid %d: \"%s\" and \"%s\"" % (mid, card['name'], rows2[0][0]))

				if collector:
					rows = conn.execute(sqlalchemy.select([card_collector.c.cardid])
						.where((card_collector.c.setid == setid) & (card_collector.c.collector == collector))).fetchall()
					if not rows:
						conn.execute(card_collector.insert(),
							setid=setid,
							collector=collector,
							cardid=real_cardid,
						)
					elif rows[0][0] != real_cardid and setid != 'CPK':
						rows2 = conn.execute(sqlalchemy.select([cards.c.name]).where(cards.c.id == rows[0][0])).fetchall()
						print("Different names for set %s collector number %s: \"%s\" and \"%s\"" % (setid, collector, card['name'], rows2[0][0]))

		cardid += 1
		conn.execute(cards.insert(),
			id=cardid,
			filteredname="bfmbigfurrymonster",
			name="B.F.M. (Big Furry Monster)""B.F.M. (Big Furry Monster)",
			text="B.F.M. (Big Furry Monster) (BBBBBBBBBBBBBBB) | Summon \u2014 The Biggest, Baddest, Nastiest, Scariest Creature You'll Ever See [99/99] | You must play both B.F.M. cards to put B.F.M. into play. If either B.F.M. card leaves play, sacrifice the other. / B.F.M. can only be blocked by three or more creatures.",
			lastprinted=datetime.date(1998, 8, 11),
		)
		conn.execute(card_multiverse.insert(), [
			{"id": 9780, "cardid": cardid},
			{"id": 9844, "cardid": cardid},
		])
		conn.execute(card_collector.insert(), [
			{"setid": "UGL", "collector": "28", "cardid": cardid},
			{"setid": "UGL", "collector": "29", "cardid": cardid},
		])

def do_download_file(url, fn):
	"""
	Download a file, checking that there is a new version of the file on the
	server before doing so. Returns True if a download occurs.
	"""
	# Much of this code cribbed from urllib.request.urlretrieve, with If-Modified-Since logic added

	req = urllib.request.Request(url, headers={
		'User-Agent': "LRRbot/2.0 (https://lrrbot.mrphlip.com/)",
	})
	try:
		stat = os.stat(fn)
	except FileNotFoundError:
		pass
	else:
		mtime = time.strftime('%a, %d %b %Y %H:%M:%S %z', time.gmtime(stat.st_mtime))
		req.add_header('If-Modified-Since', mtime)

	try:
		fp = urllib.request.urlopen(req)
	except urllib.error.HTTPError as e:
		if e.code == 304: # Not Modified
			return False
		else:
			raise

	print("Downloading %s..." % url)
	with contextlib.closing(fp):
		headers = fp.info()

		with open(fn, 'wb') as tfp:
			bs = 1024*8
			size = None
			read = 0
			if "content-length" in headers:
				size = int(headers["Content-Length"])

			while True:
				block = fp.read(bs)
				if not block:
					break
				read += len(block)
				tfp.write(block)

	if size is not None and read < size:
		os.unlink(fn)
		raise urllib.error.ContentTooShortError(
			"retrieval incomplete: got only %i out of %i bytes"
			% (read, size), (fn, headers))

	if "last-modified" in headers:
		mtime = dateutil.parser.parse(headers['last-modified'])
		mtime = mtime.timestamp()
		os.utime(fn, (mtime, mtime))

	return True

re_check = re.compile(r"^[a-z0-9_]+$")
re_mana = re.compile(r"\{(.)\}")
re_newlines = re.compile(r"[\r\n]+")
re_multiplespaces = re.compile(r"\s{2,}")
re_remindertext = re.compile(r"( *)\([^()]*\)( *)")
def process_card(card, expansion, include_reminder=False):
	if card.get('layout') in ('split', 'aftermath'):
		# Return split cards as a single card... for all the other pieces, return nothing
		if card['name'] != card['names'][0]:
			return None, None, None, None, None, None
		splits = []
		for splitname in card['names']:
			candidates = [i for i in expansion['cards'] if i['name'] == splitname]
			if not candidates:
				print("Can't find split card piece: %s" % splitname)
				sys.exit(1)
			splits.append(candidates[0])
		filteredparts = []
		nameparts = []
		descparts = []
		allmultiverseids = []
		numbers = []
		anyhidden = False
		for s in splits:
			filtered, name, desc, multiverseids, number, hidden = process_single_card(s, expansion, include_reminder)
			filteredparts.append(filtered)
			nameparts.append(name)
			descparts.append(desc)
			allmultiverseids.extend(multiverseids)
			numbers.append(number)
			anyhidden = anyhidden or hidden

		separator = {'split': ' and ', 'aftermath': ' to '}.get(card['layout'], ' // ')
		filtseparator = {'split': 'and', 'aftermath': 'to'}.get(card['layout'], '')
		filteredname = filtseparator.join(filteredparts)
		cardname = separator.join(nameparts)
		description = "%s | %s" % (separator.join(card['names']), " // ".join(descparts))
		return filteredname, cardname, description, allmultiverseids, numbers[0], anyhidden
	else:
		return process_single_card(card, expansion, include_reminder)

def process_single_card(card, expansion, include_reminder=False):
	if card.get('layout') == 'flip':
		if card['name'] == card['names'][0] and card.get('multiverseid'):
			multiverseids = [card['multiverseid']]
		else:
			multiverseids = []
	else:
		if card.get('multiverseid'):
			multiverseids = [card['multiverseid']]
		else:
			multiverseids = []

	# sanitise card name
	filtered = clean_text(card.get('internalname', card["name"]))
	if not re_check.match(filtered):
		print("Still some junk left in name %s (%s)" % (card.get('internalname', card["name"]), json.dumps(filtered)))
		print(json.dumps(card))
		sys.exit(1)

	def build_description():
		yield card['name']
		if 'manaCost' in card:
			yield ' ['
			yield re_mana.sub(r"\1", card['manaCost'])
			yield ']'
		if card.get('layout') == 'flip':
			if card['name'] == card['names'][0]:
				yield ' (flip: '
				yield card['names'][1]
				yield ')'
			else:
				yield ' (unflip: '
				yield card['names'][0]
				yield ')'
		elif card.get('layout') == 'double-faced':
			if card['name'] == card['names'][0]:
				yield ' (back: '
				yield card['names'][1]
				yield ')'
			else:
				yield ' (front: '
				yield card['names'][0]
				yield ')'
		elif card.get('layout') == 'meld':
			# Only the melded card in MTGJSON has all three cards in the 'names' field
			# The front-face cards only have [me, back]
			melded_card = [i for i in expansion['cards'] if i['name'] == card['names'][-1]]
			if not melded_card:
				print("Can't find melded card: %s" % card['names'][-1])
				sys.exit(1)
			melded_card = melded_card[0]
			# MTGJSON doesn't have this field in a consistent order...
			# some cards have [top, bottom, back] some have [bottom, top, back]
			part_cards = [i for i in expansion['cards'] if i['name'] in melded_card['names'][:-1]]
			if len(part_cards) != 2:
				print("Can't find part-cards for melded card: %s" % melded_card['name'])
				sys.exit(1)
			if melded_card['number'][-1] != 'b':
				print("Melded card's number doesn't end in 'b': %s" % melded_card['name'])
				sys.exit(1)
			if part_cards[0]['number'] == melded_card['number'][:-1] + 'a':
				bottom_card, top_card = part_cards
			elif part_cards[1]['number'] == melded_card['number'][:-1] + 'a':
				top_card, bottom_card = part_cards
			else:
				print("Couldn't figure out which card was top and bottom for: %s" % melded_card['name'])
				sys.exit(1)
			if card['name'] == top_card['name']:
				# The names of what this melds with and into are in the card text
				pass
			elif card['name'] == bottom_card['name']:
				yield ' (melds with: '
				yield top_card['name']
				yield '; into: '
				yield melded_card['name']
				yield ')'
			elif card['name'] == melded_card['name']:
				yield ' (melds from: '
				yield top_card['name']
				yield '; '
				yield bottom_card['name']
				yield ')'
		yield ' | '
		yield card.get('type', '?Type missing?')
		if 'power' in card or 'toughness' in card:
			yield ' ['
			yield card.get('power', '?')
			yield '/'
			yield card.get('toughness', '?')
			yield ']'
		if 'loyalty' in card:
			yield ' ['
			yield str(card['loyalty'])
			yield ']'
		if 'hand' in card or 'life' in card:
			yield ' [hand:'
			yield str(card.get('hand', '?'))
			yield '/life:'
			yield str(card.get('life', '?'))
			yield ']'
		if 'text' in card:
			yield ' | '
			text = card['text']
			# Let UGL and UNH cards keep their reminder text, since there's joeks in there
			if not include_reminder:
				text = re_remindertext.sub(lambda match: ' ' if match.group(1) and match.group(2) else '', text)
			yield re_newlines.sub(' / ', text.strip())

	desc = ''.join(build_description())
	desc = re_multiplespaces.sub(' ', desc).strip()
	if len(desc) > MAXLEN:
		desc = desc[:MAXLEN-1] + "\u2026"

	return filtered, card['name'], desc, multiverseids, card.get('number'), 'internalname' in card

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>import numpy as np
import gc
import argparse

if __name__ == "__main__":
	"""
	The routine works by reading in a file containig in each line the
	time stamp of the sub-matrix file (WARNING: they have to be in the
	correct order, 1st line -> sub-matrix 00; 2nd line -> sub-matrix 01
	etc.).
	The time stamp is inserted in the file name.
	"""

	parser = argparse.ArgumentParser(
        description = "SN lightcurve fitter and classifier: Bulding distance matrix from files.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

	parser.add_argument(
		'--path', '-p', dest=path,
		help='Path where to find files.')

	parser.add_argument(
		'--time-stamps', '-t', dest=timeStamps,
		default=''
		help='ASCII file containing time stamps of sub-matrix files. One per line in the correct order.')

	args = parser.parse_args()

if __name__ == "__main__":
	if args.timeStamps == '':
		raise SystemExit

	inFile = file(args.timeStamps, 'r')
	timeStamps = inFile.readlines()
	inFile.close()

	# path = 'results/SIMGEN_PUBLIC_FIT/RBF_test-length/distance_matrix/'
	fileName = 'dist_matrix_Sum_mothra_{:<14.3f}.txt'

	print 'mat00 ...'
	mat00 = np.loadtxt(args.path+fileName.format(timeStamps[0]))

	print 'mat01 ...'
	mat01 = np.loadtxt(args.path+fileName.format(timeStamps[1]))

	print 'mat02 ...'
	mat02 = np.loadtxt(args.path+fileName.format(timeStamps[2]))

	print 'mat03 ...'
	mat03 = np.loadtxt(args.path+fileName.format(timeStamps[3]))

	#---> hstacking ....
	print 'hstacking mat0 ...'
	mat0 = np.hstack((mat00, mat01, mat02, mat03))

	del mat00, mat01, mat02, mat03
	gc.collect()

	print 'mat10 ...'
	mat10 = np.transpose(mat01)
	# ----- line 1 -----

	print 'mat11 ...'
	mat11 = np.loadtxt(args.path+fileName.format(timeStamps[4]))

	print 'mat12 ...'
	mat12 = np.loadtxt(args.path+fileName.format(timeStamps[5]))

	print 'mat13 ...'
	mat13 = np.loadtxt(args.path+fileName.format(timeStamps[6]))

	#---> hstacking ....
	print 'stacking mat1 ...'
	mat1 = np.hstack((mat10, mat11, mat12, mat13))

	del mat10, mat11, mat12, mat13
	gc.collect()

	# ----- line 2 -----
	print 'mat20 ...'
	mat20 = np.transpose(mat02)

	print 'mat21 ...'
	mat21 = np.transpose(mat12)

	print 'mat22 ...'
	mat22 = np.loadtxt(args.path+fileName.format(timeStamps[7]))

	print 'mat23 ...'
	mat23 = np.loadtxt(args.path+fileName.format(timeStamps[8]))

	#---> stacking ....

	print 'stacking mat2 ...'
	mat2 = np.hstack((mat20, mat21, mat22, mat23))

	del mat20, mat21, mat22, mat23
	gc.collect()

	# ----- line 3 -----

	print 'mat30 ...'
	mat30 = np.transpose(mat03)

	print 'mat31 ...'
	mat31 = np.transpose(mat13)

	print 'mat32 ...'
	mat32 = np.transpose(mat23)

	print 'mat33 ...'
	mat33 = np.loadtxt(args.path+fileName.format(timeStamps[9]))

	#---> stacking ....
	print 'stacking mat3 ...'
	mat3 = np.hstack((mat30, mat31, mat32, mat33))

	del mat30, mat31, mat32, mat33
	gc.collect()

	#---> stacking ....
	print 'stacking mat ...'
	mat = np.vstack((mat0, mat1, mat2, mat3))

	del mat0, mat1, mat2, mat3
	gc.collect()

	print 'saveing mat ...'
	np.savetxt(args.path+'dist_matrix_Sum.txt', mat)

	del mat
	gc.collect()
	print 'END'
	#END
<commit_msg>Fixed missing comma<commit_after>import numpy as np
import gc
import argparse

if __name__ == "__main__":
	"""
	The routine works by reading in a file containig in each line the
	time stamp of the sub-matrix file (WARNING: they have to be in the
	correct order, 1st line -> sub-matrix 00; 2nd line -> sub-matrix 01
	etc.).
	The time stamp is inserted in the file name.
	"""

	parser = argparse.ArgumentParser(
        description = "SN lightcurve fitter and classifier: Bulding distance matrix from files.",
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

	parser.add_argument(
		'--path', '-p', dest=path,
		help='Path where to find files.')

	parser.add_argument(
		'--time-stamps', '-t', dest=timeStamps,
		default='',
		help='ASCII file containing time stamps of sub-matrix files. One per line in the correct order.')

	args = parser.parse_args()

if __name__ == "__main__":
	if args.timeStamps == '':
		raise SystemExit

	inFile = file(args.timeStamps, 'r')
	timeStamps = inFile.readlines()
	inFile.close()

	# path = 'results/SIMGEN_PUBLIC_FIT/RBF_test-length/distance_matrix/'
	fileName = 'dist_matrix_Sum_mothra_{:<14.3f}.txt'

	print 'mat00 ...'
	mat00 = np.loadtxt(args.path+fileName.format(timeStamps[0]))

	print 'mat01 ...'
	mat01 = np.loadtxt(args.path+fileName.format(timeStamps[1]))

	print 'mat02 ...'
	mat02 = np.loadtxt(args.path+fileName.format(timeStamps[2]))

	print 'mat03 ...'
	mat03 = np.loadtxt(args.path+fileName.format(timeStamps[3]))

	#---> hstacking ....
	print 'hstacking mat0 ...'
	mat0 = np.hstack((mat00, mat01, mat02, mat03))

	del mat00, mat01, mat02, mat03
	gc.collect()

	print 'mat10 ...'
	mat10 = np.transpose(mat01)
	# ----- line 1 -----

	print 'mat11 ...'
	mat11 = np.loadtxt(args.path+fileName.format(timeStamps[4]))

	print 'mat12 ...'
	mat12 = np.loadtxt(args.path+fileName.format(timeStamps[5]))

	print 'mat13 ...'
	mat13 = np.loadtxt(args.path+fileName.format(timeStamps[6]))

	#---> hstacking ....
	print 'stacking mat1 ...'
	mat1 = np.hstack((mat10, mat11, mat12, mat13))

	del mat10, mat11, mat12, mat13
	gc.collect()

	# ----- line 2 -----
	print 'mat20 ...'
	mat20 = np.transpose(mat02)

	print 'mat21 ...'
	mat21 = np.transpose(mat12)

	print 'mat22 ...'
	mat22 = np.loadtxt(args.path+fileName.format(timeStamps[7]))

	print 'mat23 ...'
	mat23 = np.loadtxt(args.path+fileName.format(timeStamps[8]))

	#---> stacking ....

	print 'stacking mat2 ...'
	mat2 = np.hstack((mat20, mat21, mat22, mat23))

	del mat20, mat21, mat22, mat23
	gc.collect()

	# ----- line 3 -----

	print 'mat30 ...'
	mat30 = np.transpose(mat03)

	print 'mat31 ...'
	mat31 = np.transpose(mat13)

	print 'mat32 ...'
	mat32 = np.transpose(mat23)

	print 'mat33 ...'
	mat33 = np.loadtxt(args.path+fileName.format(timeStamps[9]))

	#---> stacking ....
	print 'stacking mat3 ...'
	mat3 = np.hstack((mat30, mat31, mat32, mat33))

	del mat30, mat31, mat32, mat33
	gc.collect()

	#---> stacking ....
	print 'stacking mat ...'
	mat = np.vstack((mat0, mat1, mat2, mat3))

	del mat0, mat1, mat2, mat3
	gc.collect()

	print 'saveing mat ...'
	np.savetxt(args.path+'dist_matrix_Sum.txt', mat)

	del mat
	gc.collect()
	print 'END'
	#END
<|endoftext|>


<commit_before># Copyright 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
from platform import system
from os import makedirs
from os.path import basename, isdir, join

from SCons.Script import (ARGUMENTS, COMMAND_LINE_TARGETS, AlwaysBuild,
                          Builder, Default, DefaultEnvironment)

from platformio.util import get_serial_ports

def BeforeUpload(target, source, env):  # pylint: disable=W0613,W0621
    env.AutodetectUploadPort()

    upload_options = {}
    if "BOARD" in env:
        upload_options = env.BoardConfig().get("upload", {})

    if not bool(upload_options.get("disable_flushing", False)):
        env.FlushSerialBuffer("$UPLOAD_PORT")

    before_ports = get_serial_ports()

    if bool(upload_options.get("use_1200bps_touch", False)):
        env.TouchSerialPort("$UPLOAD_PORT", 1200)

    if bool(upload_options.get("wait_for_upload_port", False)):
        env.Replace(UPLOAD_PORT=env.WaitForNewSerialPort(before_ports))

env = DefaultEnvironment()
env.SConscript("compat.py", exports="env")
platform = env.PioPlatform()
board = env.BoardConfig()

env.Replace(
    AR="arm-none-eabi-gcc-ar",
    AS="arm-none-eabi-as",
    CC="arm-none-eabi-gcc",
    CXX="arm-none-eabi-g++",
    GDB="arm-none-eabi-gdb",
    OBJCOPY="arm-none-eabi-objcopy",
    RANLIB="arm-none-eabi-gcc-ranlib",
    SIZETOOL="arm-none-eabi-size",

    ARFLAGS=["rc"],

    SIZEPROGREGEXP=r"^(?:\.text|\.data|\.rodata|\.text.align|\.ARM.exidx)\s+(\d+).*",
    SIZEDATAREGEXP=r"^(?:\.data|\.bss|\.noinit)\s+(\d+).*",
    SIZECHECKCMD="$SIZETOOL -A -d $SOURCES",
    SIZEPRINTCMD='$SIZETOOL -B -d $SOURCES',

    PROGSUFFIX=".elf"
)

# Allow user to override via pre:script
if env.get("PROGNAME", "program") == "program":
    env.Replace(PROGNAME="firmware")

env.Append(
    BUILDERS=dict(
        ElfToBin=Builder(
            action=env.VerboseAction(" ".join([
                "$OBJCOPY",
                "-O",
                "binary",
                "$SOURCES",
                "$TARGET"
            ]), "Building $TARGET"),
            suffix=".bin"
        ),
        ElfToHex=Builder(
            action=env.VerboseAction(" ".join([
                "$OBJCOPY",
                "-O",
                "ihex",
                "-R",
                ".eeprom",
                "$SOURCES",
                "$TARGET"
            ]), "Building $TARGET"),
            suffix=".hex"
        )
    )
)

if not env.get("PIOFRAMEWORK"):
    env.SConscript("frameworks/_bare.py")

#
# Target: Build executable and linkable firmware
#

if "zephyr" in env.get("PIOFRAMEWORK", []):
    env.SConscript(
        join(platform.get_package_dir(
            "framework-zephyr"), "scripts", "platformio", "platformio-build-pre.py"),
        exports={"env": env}
    )

target_elf = None
if "nobuild" in COMMAND_LINE_TARGETS:
    target_elf = join("$BUILD_DIR", "${PROGNAME}.elf")
    target_firm = join("$BUILD_DIR", "${PROGNAME}.bin")
else:
    target_elf = env.BuildProgram()
    target_firm = env.ElfToBin(join("$BUILD_DIR", "${PROGNAME}"), target_elf)
    env.Depends(target_firm, "checkprogsize")

AlwaysBuild(env.Alias("nobuild", target_firm))
target_buildprog = env.Alias("buildprog", target_firm, target_firm)

#
# Target: Print binary size
#

target_size = env.Alias(
    "size", target_elf,
    env.VerboseAction("$SIZEPRINTCMD", "Calculating size $SOURCE"))
AlwaysBuild(target_size)

#
# Target: Upload by default .bin file
#

upload_protocol = env.subst("$UPLOAD_PROTOCOL")
debug_tools = board.get("debug.tools", {})
upload_source = target_firm
upload_actions = []

if upload_protocol == "mbed":
    upload_actions = [
        env.VerboseAction(env.AutodetectUploadPort, "Looking for upload disk..."),
        env.VerboseAction(env.UploadToDisk, "Uploading $SOURCE")
    ]

elif upload_protocol.startswith("blackmagic"):
    env.Replace(
        UPLOADER="$GDB",
        UPLOADERFLAGS=[
            "-nx",
            "--batch",
            "-ex", "target extended-remote $UPLOAD_PORT",
            "-ex", "monitor %s_scan" %
            ("jtag" if upload_protocol == "blackmagic-jtag" else "swdp"),
            "-ex", "attach 1",
            "-ex", "load",
            "-ex", "compare-sections",
            "-ex", "kill"
        ],
        UPLOADCMD="$UPLOADER $UPLOADERFLAGS $SOURCE"
    )
    upload_source = target_elf
    upload_actions = [
        env.VerboseAction(env.AutodetectUploadPort, "Looking for BlackMagic port..."),
        env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")
    ]

elif upload_protocol.startswith("jlink"):

    def _jlink_cmd_script(env, source):
        build_dir = env.subst("$BUILD_DIR")
        if not isdir(build_dir):
            makedirs(build_dir)
        script_path = join(build_dir, "upload.jlink")
        commands = [
            "h",
            "loadbin %s, %s" % (source, board.get(
                "upload.offset_address", "0x08000000")),
            "r",
            "q"
        ]
        with open(script_path, "w") as fp:
            fp.write("\n".join(commands))
        return script_path

    env.Replace(
        __jlink_cmd_script=_jlink_cmd_script,
        UPLOADER="JLink.exe" if system() == "Windows" else "JLinkExe",
        UPLOADERFLAGS=[
            "-device", board.get("debug", {}).get("jlink_device"),
            "-speed", env.GetProjectOption("debug_speed", "4000"),
            "-if", ("jtag" if upload_protocol == "jlink-jtag" else "swd"),
            "-autoconnect", "1",
            "-NoGui", "1"
        ],
        UPLOADCMD='$UPLOADER $UPLOADERFLAGS -CommanderScript "${__jlink_cmd_script(__env__, SOURCE)}"'
    )
    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

elif upload_protocol == "dfu":
    hwids = board.get("build.hwids", [["0x0483", "0xDF11"]])
    vid = hwids[0][0]
    pid = hwids[0][1]

    # default tool for all boards with embedded DFU bootloader over USB
    _upload_tool = '"%s"' % join(platform.get_package_dir(
        "tool-dfuutil") or "", "bin", "dfu-util"),
    _upload_flags = [
        "-d", ",".join(["%s:%s" % (hwid[0], hwid[1]) for hwid in hwids]),
        "-a", "0", "-s",
        "%s:leave" % board.get("upload.offset_address", "0x08000000"), "-D"
    ]

    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

    if "arduino" in env.get("PIOFRAMEWORK"):
        if env.subst("$BOARD").startswith("portenta"):
            upload_actions.insert(
                0,
                env.VerboseAction(
                    env.AutodetectUploadPort, "Looking for upload port..."
                ),
            )
        elif board.get("build.mcu").startswith("stm32f103"):
            # F103 series doesn't have embedded DFU over USB
            # stm32duino bootloader (v1, v2) is used instead
            def __configure_upload_port(env):
                return basename(env.subst("$UPLOAD_PORT"))

            _upload_tool = "maple_upload"
            _upload_flags = [
                "${__configure_upload_port(__env__)}",
                board.get("upload.boot_version", 2),
                "%s:%s" % (vid[2:], pid[2:])
            ]

            env.Replace(__configure_upload_port=__configure_upload_port)

            upload_actions.insert(
                0, env.VerboseAction(env.AutodetectUploadPort,
                                     "Looking for upload port..."))

    if "dfu-util" in _upload_tool:
        # Add special DFU header to the binary image
        env.AddPostAction(
            join("$BUILD_DIR", "${PROGNAME}.bin"),
            env.VerboseAction(
                " ".join([
                    '"%s"' % join(platform.get_package_dir("tool-dfuutil") or "",
                         "bin", "dfu-suffix"),
                    "-v %s" % vid,
                    "-p %s" % pid,
                    "-d 0xffff", "-a", "$TARGET"
                ]), "Adding dfu suffix to ${PROGNAME}.bin"))

    env.Replace(
        UPLOADER=_upload_tool,
        UPLOADERFLAGS=_upload_flags,
        UPLOADCMD='$UPLOADER $UPLOADERFLAGS "${SOURCE.get_abspath()}"')

    upload_source = target_firm

elif upload_protocol == "serial":
    def __configure_upload_port(env):
        return env.subst("$UPLOAD_PORT")

    env.Replace(
        __configure_upload_port=__configure_upload_port,
        UPLOADER=join(
            '"%s"' % platform.get_package_dir("tool-stm32duino") or "",
            "stm32flash", "stm32flash"),
        UPLOADERFLAGS=[
            "-g", board.get("upload.offset_address", "0x08000000"),
            "-b", "115200", "-w"
        ],
        UPLOADCMD='$UPLOADER $UPLOADERFLAGS "$SOURCE" "${__configure_upload_port(__env__)}"'
    )

    upload_actions = [
        env.VerboseAction(env.AutodetectUploadPort, "Looking for upload port..."),
        env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")
    ]

elif upload_protocol == "hid":
    def __configure_upload_port(env):
        return basename(env.subst("$UPLOAD_PORT"))

    env.Replace(
        __configure_upload_port=__configure_upload_port,
        UPLOADER="hid-flash",
        UPLOADCMD='$UPLOADER "$SOURCES" "${__configure_upload_port(__env__)}"'
    )
    upload_actions = [
        env.VerboseAction(env.AutodetectUploadPort, "Looking for upload port..."),
        env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")
    ]

elif upload_protocol in debug_tools:
    openocd_args = [
        "-d%d" % (2 if int(ARGUMENTS.get("PIOVERBOSE", 0)) else 1)
    ]
    openocd_args.extend(
        debug_tools.get(upload_protocol).get("server").get("arguments", []))
    if env.GetProjectOption("debug_speed"):
        openocd_args.extend(
            ["-c", "adapter speed %s" % env.GetProjectOption("debug_speed")]
        )
    openocd_args.extend([
        "-c", "program {$SOURCE} %s verify reset; shutdown;" %
        board.get("upload.offset_address", "")
    ])
    openocd_args = [
        f.replace("$PACKAGE_DIR",
                  platform.get_package_dir("tool-openocd") or "")
        for f in openocd_args
    ]
    env.Replace(
        UPLOADER="openocd",
        UPLOADERFLAGS=openocd_args,
        UPLOADCMD="$UPLOADER $UPLOADERFLAGS")

    if not board.get("upload").get("offset_address"):
        upload_source = target_elf
    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

# custom upload tool
elif upload_protocol == "custom":
    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

else:
    sys.stderr.write("Warning! Unknown upload protocol %s\n" % upload_protocol)

AlwaysBuild(env.Alias("upload", upload_source, upload_actions))

#
# Information about obsolete method of specifying linker scripts
#

if any("-Wl,-T" in f for f in env.get("LINKFLAGS", [])):
    print("Warning! '-Wl,-T' option for specifying linker scripts is deprecated. "
          "Please use 'board_build.ldscript' option in your 'platformio.ini' file.")

#
# Default targets
#

Default([target_buildprog, target_size])
<commit_msg>Typo fix in dfu-util path<commit_after># Copyright 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import sys
from platform import system
from os import makedirs
from os.path import basename, isdir, join

from SCons.Script import (ARGUMENTS, COMMAND_LINE_TARGETS, AlwaysBuild,
                          Builder, Default, DefaultEnvironment)

from platformio.util import get_serial_ports

def BeforeUpload(target, source, env):  # pylint: disable=W0613,W0621
    env.AutodetectUploadPort()

    upload_options = {}
    if "BOARD" in env:
        upload_options = env.BoardConfig().get("upload", {})

    if not bool(upload_options.get("disable_flushing", False)):
        env.FlushSerialBuffer("$UPLOAD_PORT")

    before_ports = get_serial_ports()

    if bool(upload_options.get("use_1200bps_touch", False)):
        env.TouchSerialPort("$UPLOAD_PORT", 1200)

    if bool(upload_options.get("wait_for_upload_port", False)):
        env.Replace(UPLOAD_PORT=env.WaitForNewSerialPort(before_ports))

env = DefaultEnvironment()
env.SConscript("compat.py", exports="env")
platform = env.PioPlatform()
board = env.BoardConfig()

env.Replace(
    AR="arm-none-eabi-gcc-ar",
    AS="arm-none-eabi-as",
    CC="arm-none-eabi-gcc",
    CXX="arm-none-eabi-g++",
    GDB="arm-none-eabi-gdb",
    OBJCOPY="arm-none-eabi-objcopy",
    RANLIB="arm-none-eabi-gcc-ranlib",
    SIZETOOL="arm-none-eabi-size",

    ARFLAGS=["rc"],

    SIZEPROGREGEXP=r"^(?:\.text|\.data|\.rodata|\.text.align|\.ARM.exidx)\s+(\d+).*",
    SIZEDATAREGEXP=r"^(?:\.data|\.bss|\.noinit)\s+(\d+).*",
    SIZECHECKCMD="$SIZETOOL -A -d $SOURCES",
    SIZEPRINTCMD='$SIZETOOL -B -d $SOURCES',

    PROGSUFFIX=".elf"
)

# Allow user to override via pre:script
if env.get("PROGNAME", "program") == "program":
    env.Replace(PROGNAME="firmware")

env.Append(
    BUILDERS=dict(
        ElfToBin=Builder(
            action=env.VerboseAction(" ".join([
                "$OBJCOPY",
                "-O",
                "binary",
                "$SOURCES",
                "$TARGET"
            ]), "Building $TARGET"),
            suffix=".bin"
        ),
        ElfToHex=Builder(
            action=env.VerboseAction(" ".join([
                "$OBJCOPY",
                "-O",
                "ihex",
                "-R",
                ".eeprom",
                "$SOURCES",
                "$TARGET"
            ]), "Building $TARGET"),
            suffix=".hex"
        )
    )
)

if not env.get("PIOFRAMEWORK"):
    env.SConscript("frameworks/_bare.py")

#
# Target: Build executable and linkable firmware
#

if "zephyr" in env.get("PIOFRAMEWORK", []):
    env.SConscript(
        join(platform.get_package_dir(
            "framework-zephyr"), "scripts", "platformio", "platformio-build-pre.py"),
        exports={"env": env}
    )

target_elf = None
if "nobuild" in COMMAND_LINE_TARGETS:
    target_elf = join("$BUILD_DIR", "${PROGNAME}.elf")
    target_firm = join("$BUILD_DIR", "${PROGNAME}.bin")
else:
    target_elf = env.BuildProgram()
    target_firm = env.ElfToBin(join("$BUILD_DIR", "${PROGNAME}"), target_elf)
    env.Depends(target_firm, "checkprogsize")

AlwaysBuild(env.Alias("nobuild", target_firm))
target_buildprog = env.Alias("buildprog", target_firm, target_firm)

#
# Target: Print binary size
#

target_size = env.Alias(
    "size", target_elf,
    env.VerboseAction("$SIZEPRINTCMD", "Calculating size $SOURCE"))
AlwaysBuild(target_size)

#
# Target: Upload by default .bin file
#

upload_protocol = env.subst("$UPLOAD_PROTOCOL")
debug_tools = board.get("debug.tools", {})
upload_source = target_firm
upload_actions = []

if upload_protocol == "mbed":
    upload_actions = [
        env.VerboseAction(env.AutodetectUploadPort, "Looking for upload disk..."),
        env.VerboseAction(env.UploadToDisk, "Uploading $SOURCE")
    ]

elif upload_protocol.startswith("blackmagic"):
    env.Replace(
        UPLOADER="$GDB",
        UPLOADERFLAGS=[
            "-nx",
            "--batch",
            "-ex", "target extended-remote $UPLOAD_PORT",
            "-ex", "monitor %s_scan" %
            ("jtag" if upload_protocol == "blackmagic-jtag" else "swdp"),
            "-ex", "attach 1",
            "-ex", "load",
            "-ex", "compare-sections",
            "-ex", "kill"
        ],
        UPLOADCMD="$UPLOADER $UPLOADERFLAGS $SOURCE"
    )
    upload_source = target_elf
    upload_actions = [
        env.VerboseAction(env.AutodetectUploadPort, "Looking for BlackMagic port..."),
        env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")
    ]

elif upload_protocol.startswith("jlink"):

    def _jlink_cmd_script(env, source):
        build_dir = env.subst("$BUILD_DIR")
        if not isdir(build_dir):
            makedirs(build_dir)
        script_path = join(build_dir, "upload.jlink")
        commands = [
            "h",
            "loadbin %s, %s" % (source, board.get(
                "upload.offset_address", "0x08000000")),
            "r",
            "q"
        ]
        with open(script_path, "w") as fp:
            fp.write("\n".join(commands))
        return script_path

    env.Replace(
        __jlink_cmd_script=_jlink_cmd_script,
        UPLOADER="JLink.exe" if system() == "Windows" else "JLinkExe",
        UPLOADERFLAGS=[
            "-device", board.get("debug", {}).get("jlink_device"),
            "-speed", env.GetProjectOption("debug_speed", "4000"),
            "-if", ("jtag" if upload_protocol == "jlink-jtag" else "swd"),
            "-autoconnect", "1",
            "-NoGui", "1"
        ],
        UPLOADCMD='$UPLOADER $UPLOADERFLAGS -CommanderScript "${__jlink_cmd_script(__env__, SOURCE)}"'
    )
    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

elif upload_protocol == "dfu":
    hwids = board.get("build.hwids", [["0x0483", "0xDF11"]])
    vid = hwids[0][0]
    pid = hwids[0][1]

    # default tool for all boards with embedded DFU bootloader over USB
    _upload_tool = '"%s"' % join(platform.get_package_dir(
        "tool-dfuutil") or "", "bin", "dfu-util")
    _upload_flags = [
        "-d", ",".join(["%s:%s" % (hwid[0], hwid[1]) for hwid in hwids]),
        "-a", "0", "-s",
        "%s:leave" % board.get("upload.offset_address", "0x08000000"), "-D"
    ]

    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

    if "arduino" in env.get("PIOFRAMEWORK"):
        if env.subst("$BOARD").startswith("portenta"):
            upload_actions.insert(
                0,
                env.VerboseAction(
                    env.AutodetectUploadPort, "Looking for upload port..."
                )
            )
        elif board.get("build.mcu").startswith("stm32f103"):
            # F103 series doesn't have embedded DFU over USB
            # stm32duino bootloader (v1, v2) is used instead
            def __configure_upload_port(env):
                return basename(env.subst("$UPLOAD_PORT"))

            _upload_tool = "maple_upload"
            _upload_flags = [
                "${__configure_upload_port(__env__)}",
                board.get("upload.boot_version", 2),
                "%s:%s" % (vid[2:], pid[2:])
            ]

            env.Replace(__configure_upload_port=__configure_upload_port)

            upload_actions.insert(
                0, env.VerboseAction(env.AutodetectUploadPort,
                                     "Looking for upload port..."))

    if "dfu-util" in _upload_tool:
        # Add special DFU header to the binary image
        env.AddPostAction(
            join("$BUILD_DIR", "${PROGNAME}.bin"),
            env.VerboseAction(
                " ".join([
                    '"%s"' % join(platform.get_package_dir("tool-dfuutil") or "",
                         "bin", "dfu-suffix"),
                    "-v %s" % vid,
                    "-p %s" % pid,
                    "-d 0xffff", "-a", "$TARGET"
                ]), "Adding dfu suffix to ${PROGNAME}.bin"))

    env.Replace(
        UPLOADER=_upload_tool,
        UPLOADERFLAGS=_upload_flags,
        UPLOADCMD='$UPLOADER $UPLOADERFLAGS "${SOURCE.get_abspath()}"')

    upload_source = target_firm

elif upload_protocol == "serial":
    def __configure_upload_port(env):
        return env.subst("$UPLOAD_PORT")

    env.Replace(
        __configure_upload_port=__configure_upload_port,
        UPLOADER=join(
            '"%s"' % platform.get_package_dir("tool-stm32duino") or "",
            "stm32flash", "stm32flash"),
        UPLOADERFLAGS=[
            "-g", board.get("upload.offset_address", "0x08000000"),
            "-b", "115200", "-w"
        ],
        UPLOADCMD='$UPLOADER $UPLOADERFLAGS "$SOURCE" "${__configure_upload_port(__env__)}"'
    )

    upload_actions = [
        env.VerboseAction(env.AutodetectUploadPort, "Looking for upload port..."),
        env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")
    ]

elif upload_protocol == "hid":
    def __configure_upload_port(env):
        return basename(env.subst("$UPLOAD_PORT"))

    env.Replace(
        __configure_upload_port=__configure_upload_port,
        UPLOADER="hid-flash",
        UPLOADCMD='$UPLOADER "$SOURCES" "${__configure_upload_port(__env__)}"'
    )
    upload_actions = [
        env.VerboseAction(env.AutodetectUploadPort, "Looking for upload port..."),
        env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")
    ]

elif upload_protocol in debug_tools:
    openocd_args = [
        "-d%d" % (2 if int(ARGUMENTS.get("PIOVERBOSE", 0)) else 1)
    ]
    openocd_args.extend(
        debug_tools.get(upload_protocol).get("server").get("arguments", []))
    if env.GetProjectOption("debug_speed"):
        openocd_args.extend(
            ["-c", "adapter speed %s" % env.GetProjectOption("debug_speed")]
        )
    openocd_args.extend([
        "-c", "program {$SOURCE} %s verify reset; shutdown;" %
        board.get("upload.offset_address", "")
    ])
    openocd_args = [
        f.replace("$PACKAGE_DIR",
                  platform.get_package_dir("tool-openocd") or "")
        for f in openocd_args
    ]
    env.Replace(
        UPLOADER="openocd",
        UPLOADERFLAGS=openocd_args,
        UPLOADCMD="$UPLOADER $UPLOADERFLAGS")

    if not board.get("upload").get("offset_address"):
        upload_source = target_elf
    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

# custom upload tool
elif upload_protocol == "custom":
    upload_actions = [env.VerboseAction("$UPLOADCMD", "Uploading $SOURCE")]

else:
    sys.stderr.write("Warning! Unknown upload protocol %s\n" % upload_protocol)

AlwaysBuild(env.Alias("upload", upload_source, upload_actions))

#
# Information about obsolete method of specifying linker scripts
#

if any("-Wl,-T" in f for f in env.get("LINKFLAGS", [])):
    print("Warning! '-Wl,-T' option for specifying linker scripts is deprecated. "
          "Please use 'board_build.ldscript' option in your 'platformio.ini' file.")

#
# Default targets
#

Default([target_buildprog, target_size])
<|endoftext|>


<commit_before>"""Functions to plot M/EEG data e.g. topographies
"""

# Author: Alexandre Gramfort <gramfort@nmr.mgh.harvard.edu>
#
# License: Simplified BSD

import copy
import numpy as np
from scipy import linalg

# XXX : don't import pylab here or you will break the doc

from .fiff.pick import channel_type, pick_types
from .fiff.proj import make_projector

def plot_topo(evoked, layout):
    """Plot 2D topographies
    """
    ch_names = evoked.info['ch_names']
    times = evoked.times
    data = evoked.data

    import pylab as pl
    pl.rcParams['axes.edgecolor'] = 'w'
    pl.figure(facecolor='k')
    for name in layout.names:
        if name in ch_names:
            idx = ch_names.index(name)
            ax = pl.axes(layout.pos[idx], axisbg='k')
            ax.plot(times, data[idx, :], 'w')
            pl.xticks([], ())
            pl.yticks([], ())

    pl.rcParams['axes.edgecolor'] = 'k'

def plot_evoked(evoked, picks=None, unit=True, show=True):
    """Plot evoked data

    Parameters
    ----------
    evoked : instance of Evoked
        The evoked data
    picks : None | array-like of int
        The indices of channels to plot. If None show all.
    unit : bool
        Scale plot with channel (SI) unit.
    show : bool
        Call pylab.show() as the end or not.
    """
    import pylab as pl
    pl.clf()
    if picks is None:
        picks = range(evoked.info['nchan'])
    types = [channel_type(evoked.info, idx) for idx in picks]
    n_channel_types = 0
    channel_types = []
    for t in ['eeg', 'grad', 'mag']:
        if t in types:
            n_channel_types += 1
            channel_types.append(t)

    counter = 1
    times = 1e3 * evoked.times  # time in miliseconds
    for t, scaling, name, ch_unit in zip(['eeg', 'grad', 'mag'],
                           [1e6, 1e13, 1e15],
                           ['EEG', 'Gradiometers', 'Magnetometers'],
                           ['uV', 'fT/cm', 'fT']):
        if unit is False:
            scaling = 1.0
            ch_unit = 'NA'  # no unit
        idx = [picks[i] for i in range(len(picks)) if types[i] == t]
        if len(idx) > 0:
            pl.subplot(n_channel_types, 1, counter)
            pl.plot(times, scaling * evoked.data[idx, :].T)
            pl.title(name)
            pl.xlabel('time (ms)')
            counter += 1
            pl.ylabel('data (%s)' % ch_unit)

    pl.subplots_adjust(0.175, 0.08, 0.94, 0.94, 0.2, 0.63)
    if show:
        pl.show()

def plot_cov(cov, info, exclude=None, colorbar=True, proj=False, show_svd=True,
             show=True):
    """Plot Covariance data

    Parameters
    ----------
    cov : instance of Covariance
        The covariance matrix
    info: dict
        Measurement info
    exclude : list of string
        List of channels to exclude. If empty do not exclude any channel.
    colorbar : bool
        Show colorbar or not
    proj : bool
        Apply projections or not
    show : bool
        Call pylab.show() as the end or not.
    show_svd : bool
        Plot also singular values of the noise covariance for each sensor type.
    """
    ch_names = [n for n in cov.ch_names if not n in exclude]
    ch_idx = [cov.ch_names.index(n) for n in ch_names]
    info_ch_names = info['ch_names']
    sel_eeg = pick_types(info, meg=False, eeg=True, exclude=exclude)
    sel_mag = pick_types(info, meg='mag', eeg=False, exclude=exclude)
    sel_grad = pick_types(info, meg='grad', eeg=False, exclude=exclude)
    idx_eeg = [ch_names.index(info_ch_names[c]) for c in sel_eeg]
    idx_mag = [ch_names.index(info_ch_names[c]) for c in sel_mag]
    idx_grad = [ch_names.index(info_ch_names[c]) for c in sel_grad]

    idx_names = [(idx_eeg, 'EEG covariance', 'uV', 1e6),
                 (idx_grad, 'Gradiometers', 'fT/cm', 1e13),
                 (idx_mag, 'Magnetometers', 'fT', 1e15)]
    idx_names = [(idx, name, unit, scaling)
                 for idx, name, unit, scaling in idx_names if len(idx) > 0]

    C = cov.data[ch_idx][:, ch_idx]

    if proj:
        projs = copy.deepcopy(info['projs'])

        #   Activate the projection items
        for p in projs:
            p['active'] = True

        P, ncomp, _ = make_projector(projs, ch_names)
        if ncomp > 0:
            print '    Created an SSP operator (subspace dimension = %d)' % \
                                                                        ncomp
            C = np.dot(P, np.dot(C, P.T))
        else:
            print '    The projection vectors do not apply to these channels.'

    import pylab as pl

    if show_svd:
        pl.figure()
        for k, (idx, name, unit, scaling) in enumerate(idx_names):
            _, s, _ = linalg.svd(C[idx][:, idx])
            pl.subplot(1, len(idx_names), k + 1)
            pl.ylabel('Noise std (%s)' % unit)
            pl.xlabel('Eigenvalue index')
            pl.semilogy(s * (scaling ** 2))
            pl.title(name)
        try:
            pl.tight_layout()  # XXX : recent pylab feature
        except:
            pass

    pl.figure(figsize=(2.5 * len(idx_names), 2.7))
    for k, (idx, name, _, _) in enumerate(idx_names):
        pl.subplot(1, len(idx_names), k + 1)
        pl.imshow(C[idx][:, idx], interpolation="nearest")
        pl.title(name)
    pl.subplots_adjust(0.04, 0.0, 0.98, 0.94, 0.2, 0.26)

    if show:
        pl.show()

def plot_source_estimate(src, stc, n_smooth=200, cmap='jet'):
    """Plot source estimates
    """
    from enthought.tvtk.api import tvtk
    from enthought.traits.api import HasTraits, Range, Instance, \
                                     on_trait_change
    from enthought.traits.ui.api import View, Item, Group

    from enthought.mayavi.core.api import PipelineBase
    from enthought.mayavi.core.ui.api import MayaviScene, SceneEditor, \
                    MlabSceneModel

    class SurfaceViewer(HasTraits):
        n_times = Range(0, 100, 0, )

        scene = Instance(MlabSceneModel, ())
        surf = Instance(PipelineBase)
        text = Instance(PipelineBase)

        def __init__(self, src, data, times, n_smooth=20, cmap='jet'):
            super(SurfaceViewer, self).__init__()
            self.src = src
            self.data = data
            self.times = times
            self.n_smooth = n_smooth
            self.cmap = cmap

            lh_points = src[0]['rr']
            rh_points = src[1]['rr']
            # lh_faces = src[0]['tris']
            # rh_faces = src[1]['tris']
            lh_faces = src[0]['use_tris']
            rh_faces = src[1]['use_tris']
            points = np.r_[lh_points, rh_points]
            points *= 200
            faces = np.r_[lh_faces, lh_points.shape[0] + rh_faces]

            lh_idx = np.where(src[0]['inuse'])[0]
            rh_idx = np.where(src[1]['inuse'])[0]
            use_idx = np.r_[lh_idx, lh_points.shape[0] + rh_idx]

            self.points = points[use_idx]
            self.faces = np.searchsorted(use_idx, faces)

        # When the scene is activated, or when the parameters are changed, we
        # update the plot.
        @on_trait_change('n_times,scene.activated')
        def update_plot(self):
            idx = int(self.n_times * len(self.times) / 100)
            t = self.times[idx]
            d = self.data[:, idx].astype(np.float)  # 8bits for mayavi
            points = self.points
            faces = self.faces
            info_time = "%d ms" % (1e3 * t)
            if self.surf is None:
                surface_mesh = self.scene.mlab.pipeline.triangular_mesh_source(
                                    points[:, 0], points[:, 1], points[:, 2],
                                    faces, scalars=d)
                smooth_ = tvtk.SmoothPolyDataFilter(
                                    number_of_iterations=self.n_smooth,
                                    relaxation_factor=0.18,
                                    feature_angle=70,
                                    feature_edge_smoothing=False,
                                    boundary_smoothing=False,
                                    convergence=0.)
                surface_mesh_smooth = self.scene.mlab.pipeline.user_defined(
                                                surface_mesh, filter=smooth_)
                self.surf = self.scene.mlab.pipeline.surface(
                                    surface_mesh_smooth, colormap=self.cmap)

                self.scene.mlab.colorbar()
                self.text = self.scene.mlab.text(0.7, 0.9, info_time,
                                                 width=0.2)
                self.scene.background = (.05, 0, .1)
            else:
                self.surf.mlab_source.set(scalars=d)
                self.text.set(text=info_time)

        # The layout of the dialog created
        view = View(Item('scene', editor=SceneEditor(scene_class=MayaviScene),
                         height=800, width=800, show_label=False),
                    Group('_', 'n_times',),
                    resizable=True,)

    viewer = SurfaceViewer(src, stc.data, stc.times, n_smooth=200)
    viewer.configure_traits()
    return viewer
<commit_msg>fix std not var<commit_after>"""Functions to plot M/EEG data e.g. topographies
"""

# Author: Alexandre Gramfort <gramfort@nmr.mgh.harvard.edu>
#
# License: Simplified BSD

import copy
import numpy as np
from scipy import linalg

# XXX : don't import pylab here or you will break the doc

from .fiff.pick import channel_type, pick_types
from .fiff.proj import make_projector

def plot_topo(evoked, layout):
    """Plot 2D topographies
    """
    ch_names = evoked.info['ch_names']
    times = evoked.times
    data = evoked.data

    import pylab as pl
    pl.rcParams['axes.edgecolor'] = 'w'
    pl.figure(facecolor='k')
    for name in layout.names:
        if name in ch_names:
            idx = ch_names.index(name)
            ax = pl.axes(layout.pos[idx], axisbg='k')
            ax.plot(times, data[idx, :], 'w')
            pl.xticks([], ())
            pl.yticks([], ())

    pl.rcParams['axes.edgecolor'] = 'k'

def plot_evoked(evoked, picks=None, unit=True, show=True):
    """Plot evoked data

    Parameters
    ----------
    evoked : instance of Evoked
        The evoked data
    picks : None | array-like of int
        The indices of channels to plot. If None show all.
    unit : bool
        Scale plot with channel (SI) unit.
    show : bool
        Call pylab.show() as the end or not.
    """
    import pylab as pl
    pl.clf()
    if picks is None:
        picks = range(evoked.info['nchan'])
    types = [channel_type(evoked.info, idx) for idx in picks]
    n_channel_types = 0
    channel_types = []
    for t in ['eeg', 'grad', 'mag']:
        if t in types:
            n_channel_types += 1
            channel_types.append(t)

    counter = 1
    times = 1e3 * evoked.times  # time in miliseconds
    for t, scaling, name, ch_unit in zip(['eeg', 'grad', 'mag'],
                           [1e6, 1e13, 1e15],
                           ['EEG', 'Gradiometers', 'Magnetometers'],
                           ['uV', 'fT/cm', 'fT']):
        if unit is False:
            scaling = 1.0
            ch_unit = 'NA'  # no unit
        idx = [picks[i] for i in range(len(picks)) if types[i] == t]
        if len(idx) > 0:
            pl.subplot(n_channel_types, 1, counter)
            pl.plot(times, scaling * evoked.data[idx, :].T)
            pl.title(name)
            pl.xlabel('time (ms)')
            counter += 1
            pl.ylabel('data (%s)' % ch_unit)

    pl.subplots_adjust(0.175, 0.08, 0.94, 0.94, 0.2, 0.63)
    if show:
        pl.show()

def plot_cov(cov, info, exclude=None, colorbar=True, proj=False, show_svd=True,
             show=True):
    """Plot Covariance data

    Parameters
    ----------
    cov : instance of Covariance
        The covariance matrix
    info: dict
        Measurement info
    exclude : list of string
        List of channels to exclude. If empty do not exclude any channel.
    colorbar : bool
        Show colorbar or not
    proj : bool
        Apply projections or not
    show : bool
        Call pylab.show() as the end or not.
    show_svd : bool
        Plot also singular values of the noise covariance for each sensor type.
        We show square roots ie. standard deviations.
    """
    ch_names = [n for n in cov.ch_names if not n in exclude]
    ch_idx = [cov.ch_names.index(n) for n in ch_names]
    info_ch_names = info['ch_names']
    sel_eeg = pick_types(info, meg=False, eeg=True, exclude=exclude)
    sel_mag = pick_types(info, meg='mag', eeg=False, exclude=exclude)
    sel_grad = pick_types(info, meg='grad', eeg=False, exclude=exclude)
    idx_eeg = [ch_names.index(info_ch_names[c]) for c in sel_eeg]
    idx_mag = [ch_names.index(info_ch_names[c]) for c in sel_mag]
    idx_grad = [ch_names.index(info_ch_names[c]) for c in sel_grad]

    idx_names = [(idx_eeg, 'EEG covariance', 'uV', 1e6),
                 (idx_grad, 'Gradiometers', 'fT/cm', 1e13),
                 (idx_mag, 'Magnetometers', 'fT', 1e15)]
    idx_names = [(idx, name, unit, scaling)
                 for idx, name, unit, scaling in idx_names if len(idx) > 0]

    C = cov.data[ch_idx][:, ch_idx]

    if proj:
        projs = copy.deepcopy(info['projs'])

        #   Activate the projection items
        for p in projs:
            p['active'] = True

        P, ncomp, _ = make_projector(projs, ch_names)
        if ncomp > 0:
            print '    Created an SSP operator (subspace dimension = %d)' % \
                                                                        ncomp
            C = np.dot(P, np.dot(C, P.T))
        else:
            print '    The projection vectors do not apply to these channels.'

    import pylab as pl

    if show_svd:
        pl.figure()
        for k, (idx, name, unit, scaling) in enumerate(idx_names):
            _, s, _ = linalg.svd(C[idx][:, idx])
            pl.subplot(1, len(idx_names), k + 1)
            pl.ylabel('Noise std (%s)' % unit)
            pl.xlabel('Eigenvalue index')
            pl.semilogy(np.sqrt(s) * scaling)
            pl.title(name)
        try:
            pl.tight_layout()  # XXX : recent pylab feature
        except:
            pass

    pl.figure(figsize=(2.5 * len(idx_names), 2.7))
    for k, (idx, name, _, _) in enumerate(idx_names):
        pl.subplot(1, len(idx_names), k + 1)
        pl.imshow(C[idx][:, idx], interpolation="nearest")
        pl.title(name)
    pl.subplots_adjust(0.04, 0.0, 0.98, 0.94, 0.2, 0.26)

    if show:
        pl.show()

def plot_source_estimate(src, stc, n_smooth=200, cmap='jet'):
    """Plot source estimates
    """
    from enthought.tvtk.api import tvtk
    from enthought.traits.api import HasTraits, Range, Instance, \
                                     on_trait_change
    from enthought.traits.ui.api import View, Item, Group

    from enthought.mayavi.core.api import PipelineBase
    from enthought.mayavi.core.ui.api import MayaviScene, SceneEditor, \
                    MlabSceneModel

    class SurfaceViewer(HasTraits):
        n_times = Range(0, 100, 0, )

        scene = Instance(MlabSceneModel, ())
        surf = Instance(PipelineBase)
        text = Instance(PipelineBase)

        def __init__(self, src, data, times, n_smooth=20, cmap='jet'):
            super(SurfaceViewer, self).__init__()
            self.src = src
            self.data = data
            self.times = times
            self.n_smooth = n_smooth
            self.cmap = cmap

            lh_points = src[0]['rr']
            rh_points = src[1]['rr']
            # lh_faces = src[0]['tris']
            # rh_faces = src[1]['tris']
            lh_faces = src[0]['use_tris']
            rh_faces = src[1]['use_tris']
            points = np.r_[lh_points, rh_points]
            points *= 200
            faces = np.r_[lh_faces, lh_points.shape[0] + rh_faces]

            lh_idx = np.where(src[0]['inuse'])[0]
            rh_idx = np.where(src[1]['inuse'])[0]
            use_idx = np.r_[lh_idx, lh_points.shape[0] + rh_idx]

            self.points = points[use_idx]
            self.faces = np.searchsorted(use_idx, faces)

        # When the scene is activated, or when the parameters are changed, we
        # update the plot.
        @on_trait_change('n_times,scene.activated')
        def update_plot(self):
            idx = int(self.n_times * len(self.times) / 100)
            t = self.times[idx]
            d = self.data[:, idx].astype(np.float)  # 8bits for mayavi
            points = self.points
            faces = self.faces
            info_time = "%d ms" % (1e3 * t)
            if self.surf is None:
                surface_mesh = self.scene.mlab.pipeline.triangular_mesh_source(
                                    points[:, 0], points[:, 1], points[:, 2],
                                    faces, scalars=d)
                smooth_ = tvtk.SmoothPolyDataFilter(
                                    number_of_iterations=self.n_smooth,
                                    relaxation_factor=0.18,
                                    feature_angle=70,
                                    feature_edge_smoothing=False,
                                    boundary_smoothing=False,
                                    convergence=0.)
                surface_mesh_smooth = self.scene.mlab.pipeline.user_defined(
                                                surface_mesh, filter=smooth_)
                self.surf = self.scene.mlab.pipeline.surface(
                                    surface_mesh_smooth, colormap=self.cmap)

                self.scene.mlab.colorbar()
                self.text = self.scene.mlab.text(0.7, 0.9, info_time,
                                                 width=0.2)
                self.scene.background = (.05, 0, .1)
            else:
                self.surf.mlab_source.set(scalars=d)
                self.text.set(text=info_time)

        # The layout of the dialog created
        view = View(Item('scene', editor=SceneEditor(scene_class=MayaviScene),
                         height=800, width=800, show_label=False),
                    Group('_', 'n_times',),
                    resizable=True,)

    viewer = SurfaceViewer(src, stc.data, stc.times, n_smooth=200)
    viewer.configure_traits()
    return viewer
<|endoftext|>


<commit_before>import types
from static_mocker import *

_STUBBING_ = -2
_STUBBING_STATICS_ = -3

_STATIC_MOCKER_ = StaticMocker()

_RETURNS_ = 1
_THROWS_ = 2

class Mock:

  def __init__(self):
    self.invocations = []
    self.stubbed_invocations = []
    self.mocking_mode = None
    self.mocked_obj = None

  def __getattr__(self, method_name):
    if self.mocking_mode == _STUBBING_ or self.mocking_mode == _STUBBING_STATICS_:
      return InvocationStubber(self, method_name)

    if self.mocking_mode >= 0:
      return InvocationVerifier(self, method_name)

    return InvocationMemorizer(self, method_name)

  def finishStubbing(self, invocation):
    if (self.stubbed_invocations.count(invocation)):
      self.stubbed_invocations.remove(invocation)

    self.stubbed_invocations.append(invocation)

    if (self.mocking_mode == _STUBBING_STATICS_):
      _STATIC_MOCKER_.stub(invocation)

    self.mocking_mode = None

class Invocation:
  def __init__(self, mock, method_name):
    self.method_name = method_name
    self.mock = mock
    self.answers = []
    self.verified = False

  def getMockedObj(self):
    return self.mock.mocked_obj

  def __cmp__(self, other):
    return 0 if self.matches(other) else 1

  def matches(self, invocation):
    if self.method_name == invocation.method_name and self.params == invocation.params:
      return True
    if len(self.params) != len(invocation.params):
      return False
    return self.__compareUsingMatchers(invocation)

  def __compareUsingMatchers(self, invocation):
    for x, p1 in enumerate(self.params):
      p2 = invocation.params[x]
      if isinstance(p1, Matcher):
        if not p1.satisfies(p2): return False
      elif p1 != p2: return False
    return True

  def stubWith(self, answer, chained_mode):
    if chained_mode:
        self.answers[-1].append(answer.current())
    else:
        self.answers.append(answer)

    self.mock.finishStubbing(self)

class InvocationMemorizer(Invocation):
  def __call__(self, *params, **named_params):
    self.params = params
    self.mock.invocations.append(self)

    for invocation in self.mock.stubbed_invocations:
      if self.matches(invocation):
        return invocation.answers[0].answer()

    return None

class InvocationVerifier(Invocation):
  def __call__(self, *params, **named_params):
    self.params = params
    matches = 0
    for invocation in self.mock.invocations:
      if self.matches(invocation):
        matches += 1
        invocation.verified = True

    if (matches != self.mock.mocking_mode):
      raise VerificationError("Wanted times: " + str(self.mock.mocking_mode) + ", actual times: " + str(matches))

class InvocationStubber(Invocation):
  def __call__(self, *params, **named_params):
    self.params = params
    return AnswerSelector(self)

class AnswerSelector():
  def __init__(self, invocation):
    self.invocation = invocation
    self.chained_mode = False

  def thenReturn(self, return_value):
    return self.__then(Answer(return_value, _RETURNS_))

  def thenRaise(self, exception):
    return self.__then(Answer(exception, _THROWS_))

  def __then(self, answer):
    self.invocation.stubWith(answer, self.chained_mode)
    self.chained_mode = True
    return self

class Answer():
  def __init__(self, value, type):
    self.answers = [[value, type]]
    self.index = 0

  def current(self):
    return self.answers[self.index]

  def append(self, answer):
    self.answers.append(answer)

  def answer(self):
    answer = self.current()[0]
    type = self.current()[1]
    if self.index < len(self.answers) - 1: self.index += 1
    if type == _THROWS_: raise answer
    return answer

class VerificationError(AssertionError):
  pass

class ArgumentError(Exception):
  pass

def verify(obj, times=1):
  if times < 0:
    raise ArgumentError("'times' argument has invalid value. It should be at least 0. You wanted to set it to: " + str(times))

  if (isinstance(obj, types.ClassType)):
    mock = _STATIC_MOCKER_.getMockFor(obj)
  else:
    mock = obj

  mock.mocking_mode = times
  return mock

def times(count):
  return count

def when(obj):
  #TODO verify obj is a class or a mock
  if (isinstance(obj, types.ClassType)):
    mock = Mock()
    mock.mocking_mode = _STUBBING_STATICS_
    mock.mocked_obj = obj
    return mock

  obj.mocking_mode = _STUBBING_
  return obj

def unstub():
  """Unstubs all stubbed static methods / class methods"""

  _STATIC_MOCKER_.unstub()

def verifyNoMoreInteractions(*mocks):
  for mock in mocks:
    for i in mock.invocations:
      if not i.verified:
        raise VerificationError("Unwanted interaction: " + i.method_name)

class Matcher:
  def satisfies(self, arg):
    pass

class any(Matcher):
  def __init__(self, type=None):
    self.type = type

  def satisfies(self, arg):
    return isinstance(arg, self.type) if self.type else True

class contains(Matcher):
  def __init__(self, sub):
    self.sub = sub

  def satisfies(self, arg):
    return self.sub and len(self.sub) > 0 and arg.find(self.sub) > -1<commit_msg>changed satisfies to matches (yupiiii!)<commit_after>import types
from static_mocker import *

_STUBBING_ = -2
_STUBBING_STATICS_ = -3

_STATIC_MOCKER_ = StaticMocker()

_RETURNS_ = 1
_THROWS_ = 2

class Mock:

  def __init__(self):
    self.invocations = []
    self.stubbed_invocations = []
    self.mocking_mode = None
    self.mocked_obj = None

  def __getattr__(self, method_name):
    if self.mocking_mode == _STUBBING_ or self.mocking_mode == _STUBBING_STATICS_:
      return InvocationStubber(self, method_name)

    if self.mocking_mode >= 0:
      return InvocationVerifier(self, method_name)

    return InvocationMemorizer(self, method_name)

  def finishStubbing(self, invocation):
    if (self.stubbed_invocations.count(invocation)):
      self.stubbed_invocations.remove(invocation)

    self.stubbed_invocations.append(invocation)

    if (self.mocking_mode == _STUBBING_STATICS_):
      _STATIC_MOCKER_.stub(invocation)

    self.mocking_mode = None

class Invocation:
  def __init__(self, mock, method_name):
    self.method_name = method_name
    self.mock = mock
    self.answers = []
    self.verified = False

  def getMockedObj(self):
    return self.mock.mocked_obj

  def __cmp__(self, other):
    return 0 if self.matches(other) else 1

  def matches(self, invocation):
    if self.method_name == invocation.method_name and self.params == invocation.params:
      return True
    if len(self.params) != len(invocation.params):
      return False
    return self.__compareUsingMatchers(invocation)

  def __compareUsingMatchers(self, invocation):
    for x, p1 in enumerate(self.params):
      p2 = invocation.params[x]
      if isinstance(p1, Matcher):
        if not p1.matches(p2): return False
      elif p1 != p2: return False
    return True

  def stubWith(self, answer, chained_mode):
    if chained_mode:
        self.answers[-1].append(answer.current())
    else:
        self.answers.append(answer)

    self.mock.finishStubbing(self)

class InvocationMemorizer(Invocation):
  def __call__(self, *params, **named_params):
    self.params = params
    self.mock.invocations.append(self)

    for invocation in self.mock.stubbed_invocations:
      if self.matches(invocation):
        return invocation.answers[0].answer()

    return None

class InvocationVerifier(Invocation):
  def __call__(self, *params, **named_params):
    self.params = params
    matches = 0
    for invocation in self.mock.invocations:
      if self.matches(invocation):
        matches += 1
        invocation.verified = True

    if (matches != self.mock.mocking_mode):
      raise VerificationError("Wanted times: " + str(self.mock.mocking_mode) + ", actual times: " + str(matches))

class InvocationStubber(Invocation):
  def __call__(self, *params, **named_params):
    self.params = params
    return AnswerSelector(self)

class AnswerSelector():
  def __init__(self, invocation):
    self.invocation = invocation
    self.chained_mode = False

  def thenReturn(self, return_value):
    return self.__then(Answer(return_value, _RETURNS_))

  def thenRaise(self, exception):
    return self.__then(Answer(exception, _THROWS_))

  def __then(self, answer):
    self.invocation.stubWith(answer, self.chained_mode)
    self.chained_mode = True
    return self

class Answer():
  def __init__(self, value, type):
    self.answers = [[value, type]]
    self.index = 0

  def current(self):
    return self.answers[self.index]

  def append(self, answer):
    self.answers.append(answer)

  def answer(self):
    answer = self.current()[0]
    type = self.current()[1]
    if self.index < len(self.answers) - 1: self.index += 1
    if type == _THROWS_: raise answer
    return answer

class VerificationError(AssertionError):
  pass

class ArgumentError(Exception):
  pass

def verify(obj, times=1):
  if times < 0:
    raise ArgumentError("'times' argument has invalid value. It should be at least 0. You wanted to set it to: " + str(times))

  if (isinstance(obj, types.ClassType)):
    mock = _STATIC_MOCKER_.getMockFor(obj)
  else:
    mock = obj

  mock.mocking_mode = times
  return mock

def times(count):
  return count

def when(obj):
  #TODO verify obj is a class or a mock
  if (isinstance(obj, types.ClassType)):
    mock = Mock()
    mock.mocking_mode = _STUBBING_STATICS_
    mock.mocked_obj = obj
    return mock

  obj.mocking_mode = _STUBBING_
  return obj

def unstub():
  """Unstubs all stubbed static methods and class methods"""

  _STATIC_MOCKER_.unstub()

def verifyNoMoreInteractions(*mocks):
  for mock in mocks:
    for i in mock.invocations:
      if not i.verified:
        raise VerificationError("Unwanted interaction: " + i.method_name)

class Matcher:
  def matches(self, arg):
    pass

class any(Matcher):
  def __init__(self, type=None):
    self.type = type

  def matches(self, arg):
    return isinstance(arg, self.type) if self.type else True

class contains(Matcher):
  def __init__(self, sub):
    self.sub = sub

  def matches(self, arg):
    return self.sub and len(self.sub) > 0 and arg.find(self.sub) > -1<|endoftext|>