<commit_before># encoding: utf-8

from __future__ import unicode_literals

"""Convienent utilities."""

# ## Imports

import sys

from inspect import isfunction, isclass
from collections import deque, namedtuple, Sized, Iterable
from xml.sax.saxutils import quoteattr

# ## Python Cross-Compatibility

try:  # pragma: no cover
	from types import StringTypes as stringy
	bytes = str
	str = unicode
	py = 2
except:
	stringy = str
	bytes = bytes
	str = str
	py = 3

# ## Type Definitions

Iteration = namedtuple('Iteration', ['first', 'last', 'index', 'total', 'value'])

# ## Simple Utility Functions

def interruptable(iterable):
	"""Allow easy catching of a generator interrupting operation when using "yield from"."""

	for i in iterable:
		if i is None:
			return

		yield i

def iterate(obj):
	"""Loop over an iterable and track progress, including first and last state.

	On each iteration yield an Iteration named tuple with the first and last flags, current element index, total
	iterable length (if possible to acquire), and value, in that order.

		for iteration in iterate(something):
			iteration.value  # Do something.

	You can unpack these safely:

		for first, last, index, total, value in iterate(something):
			pass

	If you want to unpack the values you are iterating across, you can by wrapping the nested unpacking in parenthesis:

		for first, last, index, total, (foo, bar, baz) in iterate(something):
			pass

	Even if the length of the iterable can't be reliably determined this function will still capture the "last" state
	of the final loop iteration.  (Basically: this works with generators.)

	This process is about 10x slower than simple enumeration on CPython 3.4, so only use it where you actually need to
	track state.  Use `enumerate()` elsewhere.
	"""

	global next, Iteration
	next = next
	Iteration = Iteration

	total = len(obj) if isinstance(obj, Sized) else None
	iterator = iter(obj)
	first = True
	last = False
	i = 0

	try:
		value = next(iterator)
	except StopIteration:
		return

	while True:
		try:
			next_value = next(iterator)
		except StopIteration:
			last = True

		yield Iteration(first, last, i, total, value)
		if last: return

		value = next_value
		i += 1
		first = False

def xmlargs(_source=None, **values):
	from cinje.helpers import bless

	# Optimize by binding these names to the local scope, saving a lookup on each call.
	global str, Iterable, stringy
	str = str
	Iterable = Iterable
	stringy = stringy
	ejoin = " ".join

	parts = []
	pappend = parts.append

	# If a data source is provided it overrides the keyword arguments which are treated as defaults.
	if _source:
		values.update(_source)

	for k in sorted(values):
		# We technically allow non-string values for keys.  They're just converted to strings first.
		key = str(k).rstrip('_').replace('__', ':').replace('_', '-')
		value = values[k]

		# We skip empty, None, False, and other falsy values other than zero.
		if k[0] == '_' or (not value and (value is False or value != 0)):  # False == 0, so, uh, work around that.
			continue

		if value is True:  # For explicitly True values, we don't have a value for the attribute.
			pappend(key)
			continue

		# Non-string iterables (such as lists, sets, tuples, etc.) are treated as space-separated strings.
		if isinstance(value, Iterable) and not isinstance(value, stringy):
			value = ejoin(str(i) for i in value)

		pappend(key + "=" + quoteattr(str(value)))

	return bless(" " + ejoin(parts)) if parts else ''

def chunk(text, mapping={None: 'text', '${': '_escape', '#{': '_bless', '&{': '_args', '%{': 'format', '@{': '_json'}):
	"""Chunkify and "tag" a block of text into plain text and code sections.

	The first delimeter is blank to represent text sections, and keep the indexes aligned with the tags.

	Values are yielded in the form (tag, text).
	"""

	skipping = 0  # How many closing parenthesis will we need to skip?
	start = None  # Starting position of current match.
	last = 0

	i = 0

	while i < len(text):
		if start is not None:
			if text[i] == '{':
				skipping += 1

			elif text[i] == '}':
				if skipping:
					skipping -= 1
				else:
					yield mapping[text[start-2:start]], text[start:i]
					start = None
					last = i = i + 1
					continue

		elif text[i:i+2] in mapping:
			if last is not None and last != i:
				yield mapping[None], text[last:i]
				last = None

			start = i = i + 2
			continue

		i += 1

	if last < len(text):
		yield mapping[None], text[last:]

def ensure_buffer(context):
	if 'text' in context.flag:
		return

	# TODO: Determine if the Python 2-required approach (reassignment) is faster than the .clear() approach.

	yield Line(0, "")
	yield Line(0, "_buffer = []")
	yield Line(0, "__w, __ws = _buffer.extend, _buffer.append")
	yield Line(0, "")

	context.flag.add('text')

# ## Common Classes

class Line(object):
	"""A rich description for a line of input, allowing for annotation."""

	__slots__ = ('number', 'line', 'scope', 'kind', 'continued')

	def __init__(self, number, line, scope=None):
		self.number = number
		self.line = line
		self.scope = scope
		self.kind = None
		self.continued = self.stripped.endswith('\\')

		self.process()

		super(Line, self).__init__()

	def process(self):
		if self.stripped.startswith('#'):
			self.kind = 'comment'
		elif self.stripped.startswith(':'):
			self.kind = 'code'
			self.line = self.stripped[1:].lstrip()
		else:
			self.kind = 'text'

	@property
	def stripped(self):
		return self.line.strip()

	@property
	def partitioned(self):
		prefix, _, remainder = self.stripped.partition(' ')
		return prefix.rstrip(), remainder.lstrip()

	def __repr__(self):
		return '{0.__class__.__name__}({0.number}, {0.kind}, "{0.stripped}")'.format(self)

	def __bytes__(self):
		return str(self).encode('utf8')

	def __str__(self):
		if self.scope is None:
			return self.line

		return '\t' * self.scope + self.line.lstrip()

	if py == 2:  # pragma: no cover
		__unicode__ = __str__
		__str__ = __bytes__
		del __bytes__

	def clone(self, **kw):
		values = dict(
				number = self.number,
				line = self.line,
				scope = self.scope,
			)

		values.update(kw)

		instance = self.__class__(**values)

		return instance

class Lines(object):
	"""Iterate input lines of source, with the ability to push lines back."""

	__slots__ = ['Line', 'source', 'buffer']

	def __init__(self, input=None, Line=Line):
		self.Line = Line

		if input is None:
			self.source = None
			self.buffer = deque()

		elif hasattr(input, 'readlines'):
			self.source = list(self.Line(i + 1, j) for i, j in enumerate(input.readlines()))
			self.buffer = deque(self.source)

		else:
			self.source = list(self.Line(i + 1, j) for i, j in enumerate(input.split('\n')))
			self.buffer = deque(self.source)

		super(Lines, self).__init__()

	@property
	def count(self):
		return len(self.buffer)

	def __len__(self):
		return self.count

	def __repr__(self):
		return 'Lines({0.count})'.format(self)

	def __iter__(self):
		return self

	def __next__(self):
		return self.next()

	def __str__(self):
		return "\n".join(str(i) for i in self)

	def next(self):
		if not self.buffer:
			raise StopIteration()

		return self.buffer.popleft()

	def peek(self):
		return self.buffer[0] if self.buffer else None

	def push(self, *lines):
		self.buffer.extendleft((i if isinstance(i, self.Line) else self.Line(self.buffer[0].number if self.buffer else 0, i)) for i in reversed(lines))

	def reset(self):
		self.buffer = deque(self.source)

	def append(self, *lines):
		self.buffer.extend((i if isinstance(i, self.Line) else self.Line(self.buffer[-1].number if self.buffer else 0, i)) for i in lines)

class Context(object):
	"""The processing context for translating cinje source into Python source.

	This is the primary entry point for translation.
	"""

	__slots__ = ('input', 'scope', 'flag', '_handler', 'templates')

	handlers = []

	def __init__(self, input):
		self.input = Lines(input.decode('utf8') if hasattr(input, 'decode') else input)
		self.scope = 0
		self.flag = set()
		self._handler = []
		self.templates = []

	def __repr__(self):
		return "Context({!r}, {}, {})".format(self.input, self.scope, self.flag)

	@classmethod
	def register(cls, handler):
		"""Register a line transformer class with the processing context."""

		assert isclass(handler), "Must supply handler class for registration, not instance."

		cls.handlers.append(handler)

		return handler  # Allow certain types of chaining, i.e. use as a decorator.

	def prepare(self):
		"""Prepare the ordered list of transformers and reset context state to initial."""
		self.scope = 0
		self._handler = [i() for i in sorted(self.handlers, key=lambda handler: handler.priority)]

	@property
	def stream(self):
		"""The workhorse of cinje: transform input lines and emit output lines.

		After constructing an instance with a set of input lines iterate this property to generate the template.
		"""

		if 'init' not in self.flag:
			self.prepare()

		for line in self.input:
			handler = self.classify(line)

			if line.kind == 'code' and line.stripped == 'end':  # Exit the current child scope.
				return

			assert handler, "Unable to identify handler for line; this should be impossible!"

			self.input.push(line)  # Put it back so it can be consumed by the handler.

			for line in handler(self):  # This re-indents the code to match, if missing explicit scope.
				if line.scope is None:
					line = line.clone(scope=self.scope)

				yield line

	def classify(self, line):
		"""Identify the correct handler for a given line of input."""

		for handler in self._handler:
			if handler.match(self, line):
				return handler

		return None

class Pipe(object):
	"""An object representing a pipe-able callable, optionally with preserved arguments.

	Using this you can custruct custom subclasses (define a method named "callable") or use it as a decorator:

		@Pipe
		def s(text):
			return str(text)

	"""

	__slots__ = ('callable', 'args', 'kwargs')

	def __init__(self, callable, *args, **kw):
		super(Pipe, self).__init__()

		self.callable = callable
		self.args = args if args else ()
		self.kwargs = kw if kw else {}

	def __ror__(self, other):
		"""The main machinery of the Pipe, calling the chosen callable with the recorded arguments."""

		return self.callable(*(self.args + (other, )), **self.kwargs)

	def __call__(self, *args, **kw):
		"""Allow for the preserved args and kwargs to be updated, returning a mutated copy.

		This allows for usage with arguments, as in the following example:

			"Hello!" | encode('utf8')

		This also allows for easy construction of custom mutated copies for use later, a la:

			utf8 = encode('utf8')
			"Hello!" | utf8
		"""

		return self.__class__(self.callable, *args, **kw)
<commit_msg>Added handy utility to strip HTML from text.<commit_after># encoding: utf-8

from __future__ import unicode_literals

"""Convienent utilities."""

# ## Imports

import sys

from inspect import isfunction, isclass
from collections import deque, namedtuple, Sized, Iterable
from xml.sax.saxutils import quoteattr
from html.parser import HTMLParser

# ## Python Cross-Compatibility

try:  # pragma: no cover
	from types import StringTypes as stringy
	bytes = str
	str = unicode
	py = 2
except:
	stringy = str
	bytes = bytes
	str = str
	py = 3

# ## Type Definitions

Iteration = namedtuple('Iteration', ['first', 'last', 'index', 'total', 'value'])

# ## Simple Utility Functions

def interruptable(iterable):
	"""Allow easy catching of a generator interrupting operation when using "yield from"."""

	for i in iterable:
		if i is None:
			return

		yield i

def iterate(obj):
	"""Loop over an iterable and track progress, including first and last state.

	On each iteration yield an Iteration named tuple with the first and last flags, current element index, total
	iterable length (if possible to acquire), and value, in that order.

		for iteration in iterate(something):
			iteration.value  # Do something.

	You can unpack these safely:

		for first, last, index, total, value in iterate(something):
			pass

	If you want to unpack the values you are iterating across, you can by wrapping the nested unpacking in parenthesis:

		for first, last, index, total, (foo, bar, baz) in iterate(something):
			pass

	Even if the length of the iterable can't be reliably determined this function will still capture the "last" state
	of the final loop iteration.  (Basically: this works with generators.)

	This process is about 10x slower than simple enumeration on CPython 3.4, so only use it where you actually need to
	track state.  Use `enumerate()` elsewhere.
	"""

	global next, Iteration
	next = next
	Iteration = Iteration

	total = len(obj) if isinstance(obj, Sized) else None
	iterator = iter(obj)
	first = True
	last = False
	i = 0

	try:
		value = next(iterator)
	except StopIteration:
		return

	while True:
		try:
			next_value = next(iterator)
		except StopIteration:
			last = True

		yield Iteration(first, last, i, total, value)
		if last: return

		value = next_value
		i += 1
		first = False

def xmlargs(_source=None, **values):
	from cinje.helpers import bless

	# Optimize by binding these names to the local scope, saving a lookup on each call.
	global str, Iterable, stringy
	str = str
	Iterable = Iterable
	stringy = stringy
	ejoin = " ".join

	parts = []
	pappend = parts.append

	# If a data source is provided it overrides the keyword arguments which are treated as defaults.
	if _source:
		values.update(_source)

	for k in sorted(values):
		# We technically allow non-string values for keys.  They're just converted to strings first.
		key = str(k).rstrip('_').replace('__', ':').replace('_', '-')
		value = values[k]

		# We skip empty, None, False, and other falsy values other than zero.
		if k[0] == '_' or (not value and (value is False or value != 0)):  # False == 0, so, uh, work around that.
			continue

		if value is True:  # For explicitly True values, we don't have a value for the attribute.
			pappend(key)
			continue

		# Non-string iterables (such as lists, sets, tuples, etc.) are treated as space-separated strings.
		if isinstance(value, Iterable) and not isinstance(value, stringy):
			value = ejoin(str(i) for i in value)

		pappend(key + "=" + quoteattr(str(value)))

	return bless(" " + ejoin(parts)) if parts else ''

def chunk(text, mapping={None: 'text', '${': '_escape', '#{': '_bless', '&{': '_args', '%{': 'format', '@{': '_json'}):
	"""Chunkify and "tag" a block of text into plain text and code sections.

	The first delimeter is blank to represent text sections, and keep the indexes aligned with the tags.

	Values are yielded in the form (tag, text).
	"""

	skipping = 0  # How many closing parenthesis will we need to skip?
	start = None  # Starting position of current match.
	last = 0

	i = 0

	while i < len(text):
		if start is not None:
			if text[i] == '{':
				skipping += 1

			elif text[i] == '}':
				if skipping:
					skipping -= 1
				else:
					yield mapping[text[start-2:start]], text[start:i]
					start = None
					last = i = i + 1
					continue

		elif text[i:i+2] in mapping:
			if last is not None and last != i:
				yield mapping[None], text[last:i]
				last = None

			start = i = i + 2
			continue

		i += 1

	if last < len(text):
		yield mapping[None], text[last:]

def ensure_buffer(context):
	if 'text' in context.flag:
		return

	# TODO: Determine if the Python 2-required approach (reassignment) is faster than the .clear() approach.

	yield Line(0, "")
	yield Line(0, "_buffer = []")
	yield Line(0, "__w, __ws = _buffer.extend, _buffer.append")
	yield Line(0, "")

	context.flag.add('text')

# ## Common Classes

class Line(object):
	"""A rich description for a line of input, allowing for annotation."""

	__slots__ = ('number', 'line', 'scope', 'kind', 'continued')

	def __init__(self, number, line, scope=None):
		self.number = number
		self.line = line
		self.scope = scope
		self.kind = None
		self.continued = self.stripped.endswith('\\')

		self.process()

		super(Line, self).__init__()

	def process(self):
		if self.stripped.startswith('#'):
			self.kind = 'comment'
		elif self.stripped.startswith(':'):
			self.kind = 'code'
			self.line = self.stripped[1:].lstrip()
		else:
			self.kind = 'text'

	@property
	def stripped(self):
		return self.line.strip()

	@property
	def partitioned(self):
		prefix, _, remainder = self.stripped.partition(' ')
		return prefix.rstrip(), remainder.lstrip()

	def __repr__(self):
		return '{0.__class__.__name__}({0.number}, {0.kind}, "{0.stripped}")'.format(self)

	def __bytes__(self):
		return str(self).encode('utf8')

	def __str__(self):
		if self.scope is None:
			return self.line

		return '\t' * self.scope + self.line.lstrip()

	if py == 2:  # pragma: no cover
		__unicode__ = __str__
		__str__ = __bytes__
		del __bytes__

	def clone(self, **kw):
		values = dict(
				number = self.number,
				line = self.line,
				scope = self.scope,
			)

		values.update(kw)

		instance = self.__class__(**values)

		return instance

class Lines(object):
	"""Iterate input lines of source, with the ability to push lines back."""

	__slots__ = ['Line', 'source', 'buffer']

	def __init__(self, input=None, Line=Line):
		self.Line = Line

		if input is None:
			self.source = None
			self.buffer = deque()

		elif hasattr(input, 'readlines'):
			self.source = list(self.Line(i + 1, j) for i, j in enumerate(input.readlines()))
			self.buffer = deque(self.source)

		else:
			self.source = list(self.Line(i + 1, j) for i, j in enumerate(input.split('\n')))
			self.buffer = deque(self.source)

		super(Lines, self).__init__()

	@property
	def count(self):
		return len(self.buffer)

	def __len__(self):
		return self.count

	def __repr__(self):
		return 'Lines({0.count})'.format(self)

	def __iter__(self):
		return self

	def __next__(self):
		return self.next()

	def __str__(self):
		return "\n".join(str(i) for i in self)

	def next(self):
		if not self.buffer:
			raise StopIteration()

		return self.buffer.popleft()

	def peek(self):
		return self.buffer[0] if self.buffer else None

	def push(self, *lines):
		self.buffer.extendleft((i if isinstance(i, self.Line) else self.Line(self.buffer[0].number if self.buffer else 0, i)) for i in reversed(lines))

	def reset(self):
		self.buffer = deque(self.source)

	def append(self, *lines):
		self.buffer.extend((i if isinstance(i, self.Line) else self.Line(self.buffer[-1].number if self.buffer else 0, i)) for i in lines)

class Context(object):
	"""The processing context for translating cinje source into Python source.

	This is the primary entry point for translation.
	"""

	__slots__ = ('input', 'scope', 'flag', '_handler', 'templates')

	handlers = []

	def __init__(self, input):
		self.input = Lines(input.decode('utf8') if hasattr(input, 'decode') else input)
		self.scope = 0
		self.flag = set()
		self._handler = []
		self.templates = []

	def __repr__(self):
		return "Context({!r}, {}, {})".format(self.input, self.scope, self.flag)

	@classmethod
	def register(cls, handler):
		"""Register a line transformer class with the processing context."""

		assert isclass(handler), "Must supply handler class for registration, not instance."

		cls.handlers.append(handler)

		return handler  # Allow certain types of chaining, i.e. use as a decorator.

	def prepare(self):
		"""Prepare the ordered list of transformers and reset context state to initial."""
		self.scope = 0
		self._handler = [i() for i in sorted(self.handlers, key=lambda handler: handler.priority)]

	@property
	def stream(self):
		"""The workhorse of cinje: transform input lines and emit output lines.

		After constructing an instance with a set of input lines iterate this property to generate the template.
		"""

		if 'init' not in self.flag:
			self.prepare()

		for line in self.input:
			handler = self.classify(line)

			if line.kind == 'code' and line.stripped == 'end':  # Exit the current child scope.
				return

			assert handler, "Unable to identify handler for line; this should be impossible!"

			self.input.push(line)  # Put it back so it can be consumed by the handler.

			for line in handler(self):  # This re-indents the code to match, if missing explicit scope.
				if line.scope is None:
					line = line.clone(scope=self.scope)

				yield line

	def classify(self, line):
		"""Identify the correct handler for a given line of input."""

		for handler in self._handler:
			if handler.match(self, line):
				return handler

		return None

class Pipe(object):
	"""An object representing a pipe-able callable, optionally with preserved arguments.

	Using this you can custruct custom subclasses (define a method named "callable") or use it as a decorator:

		@Pipe
		def s(text):
			return str(text)

	"""

	__slots__ = ('callable', 'args', 'kwargs')

	def __init__(self, callable, *args, **kw):
		super(Pipe, self).__init__()

		self.callable = callable
		self.args = args if args else ()
		self.kwargs = kw if kw else {}

	def __ror__(self, other):
		"""The main machinery of the Pipe, calling the chosen callable with the recorded arguments."""

		return self.callable(*(self.args + (other, )), **self.kwargs)

	def __call__(self, *args, **kw):
		"""Allow for the preserved args and kwargs to be updated, returning a mutated copy.

		This allows for usage with arguments, as in the following example:

			"Hello!" | encode('utf8')

		This also allows for easy construction of custom mutated copies for use later, a la:

			utf8 = encode('utf8')
			"Hello!" | utf8
		"""

		return self.__class__(self.callable, *args, **kw)

# ## Tag Stripper

class MLStripper(HTMLParser):
	def __init__(self):
		self.reset()
		self.strict = False
		self.convert_charrefs = True
		self.fed = []

	def handle_data(self, d):
		self.fed.append(d)

	def get_data(self):
		return ''.join(self.fed)

def strip_tags(html):
	s = MLStripper()
	s.feed(html)
	return s.get_data()

<|endoftext|>


<commit_before>import usb.core
import time
import json

def read_command():
    f = open('/tmp/panicbutton/config.json')
    config = json.load(f)
    f.close()
    return config['command']

def pressed(device):
    """ Read the USB device
    Return 1 if pressed and released, 0 otherwise.
    """
    # Magic numbers are from http://search.cpan.org/~bkendi/Device-USB-PanicButton-0.04/lib/Device/USB/PanicButton.pm
    try:
        device.set_configuration()
        ct = device.ctrl_transfer(bmRequestType=0xA1, bRequest=1, wValue=0x300, data_or_wLength=8, timeout=250)
        result = ct[0]
    except Exception, e:
        result = 0
        print e
    return result

#    Product ID:    0x0202
#    Vendor ID:     0x1130    (Tenx Technology, Inc.)
#    Version:       1.00
#    Speed:         Up to 1.5 Mb/sec
device = usb.core.find(idVendor=0x1130, idProduct=0x0202)

print "Panic Button time! Press Ctrl-C to quit."
print "Looking for the Panic Button... please wait"
# loop until device is found
while 1:
    try:
        if not device:
            raise ValueError("Panic Button not found: Are you sure it's plugged in?")
    except Exception, e:
        print e
        time.sleep(1)
        device = usb.core.find(idVendor=0x1130, idProduct=0x0202)
        continue
    print "Panic Button found. Press away!"
    break

# do an initial read to clear out any stale button presses
pressed(device)

if __name__ == "__main__":
    try:
        while 1:
            if pressed(device):
                print "Command to execute: " + read_command()
            time.sleep(.15)
    except KeyboardInterrupt, e:
        exit(0)
<commit_msg>lets rather get the config from your home folder<commit_after>import usb.core
import time
import json
import os

def read_command():
    home = os.path.expanduser('~')
    f = open(home + '/.panicbutton/config.json')
    config = json.load(f)
    f.close()
    return config['command']

def pressed(device):
    """ Read the USB device
    Return 1 if pressed and released, 0 otherwise.
    """
    # Magic numbers are from http://search.cpan.org/~bkendi/Device-USB-PanicButton-0.04/lib/Device/USB/PanicButton.pm
    try:
        device.set_configuration()
        ct = device.ctrl_transfer(bmRequestType=0xA1, bRequest=1, wValue=0x300, data_or_wLength=8, timeout=250)
        result = ct[0]
    except Exception, e:
        result = 0
        print e
    return result

#    Product ID:    0x0202
#    Vendor ID:     0x1130    (Tenx Technology, Inc.)
#    Version:       1.00
#    Speed:         Up to 1.5 Mb/sec
device = usb.core.find(idVendor=0x1130, idProduct=0x0202)

print "Panic Button time! Press Ctrl-C to quit."
print "Looking for the Panic Button... please wait"
# loop until device is found
while 1:
    try:
        if not device:
            raise ValueError("Panic Button not found: Are you sure it's plugged in?")
    except Exception, e:
        print e
        time.sleep(1)
        device = usb.core.find(idVendor=0x1130, idProduct=0x0202)
        continue
    print "Panic Button found. Press away!"
    break

# do an initial read to clear out any stale button presses
pressed(device)

if __name__ == "__main__":
    try:
        while 1:
            if pressed(device):
                print "Command to execute: " + read_command()
            time.sleep(.15)
    except KeyboardInterrupt, e:
        exit(0)
<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
import numpy as np
import scipy.sparse as sparse

############################################################
# Label Matrix Plotting
############################################################

def view_label_matrix(L, colorbar=True):
    """Display an [n, m] matrix of labels"""
    L = L.todense() if sparse.issparse(L) else L
    plt.imshow(L, aspect="auto")
    plt.title("Label Matrix")
    if colorbar:
        labels = sorted(np.unique(np.asarray(L).reshape(-1, 1).squeeze()))
        boundaries = np.array(labels + [max(labels) + 1]) - 0.5
        plt.colorbar(boundaries=boundaries, ticks=labels)
    plt.show()

def view_overlaps(L, self_overlaps=False, normalize=True, colorbar=True):
    """Display an [m, m] matrix of overlaps"""
    L = L.todense() if sparse.issparse(L) else L
    G = _get_overlaps_matrix(L, normalize=normalize)
    if not self_overlaps:
        np.fill_diagonal(G, 0)  # Zero out self-overlaps
    plt.imshow(G, aspect="auto")
    plt.title("Overlaps")
    if colorbar:
        plt.colorbar()
    plt.show()

def view_conflicts(L, normalize=True, colorbar=True):
    """Display an [m, m] matrix of conflicts"""
    L = L.todense() if sparse.issparse(L) else L
    C = _get_conflicts_matrix(L, normalize=normalize)
    plt.imshow(C, aspect="auto")
    plt.title("Conflicts")
    if colorbar:
        plt.colorbar()
    plt.show()

def _get_overlaps_matrix(L, normalize=True):
    n, m = L.shape
    X = np.where(L != 0, 1, 0).T
    G = X @ X.T

    if normalize:
        G = G / n
    return G

def _get_conflicts_matrix(L, normalize=True):
    n, m = L.shape
    C = np.zeros((m, m))

    # Iterate over the pairs of LFs
    for i in range(m):
        for j in range(m):
            # Get the overlapping non-zero indices
            overlaps = list(
                set(np.where(L[:, i] != 0)[0]).intersection(np.where(L[:, j] != 0)[0])
            )
            C[i, j] = np.where(L[overlaps, i] != L[overlaps, j], 1, 0).sum()

    if normalize:
        C = C / n
    return C

############################################################
# Classifier Diagnostics
############################################################

def plot_probabilities_histogram(Y_p, title=None):
    """Plot a histogram from a numpy array of probabilities

    Args:
        Y_p: An [n] or [n, 1] np.ndarray of probabilities (floats in [0,1])
    """
    if Y_p.ndim > 1:
        msg = (
            f"Arg Y_p should be a 1-dimensional np.ndarray, not of shape "
            f"{Y_p.shape}."
        )
        raise ValueError(msg)
    plt.hist(Y_p, bins=20)
    plt.xlim((0, 1.025))
    plt.xlabel("Probability")
    plt.ylabel("# Predictions")
    if isinstance(title, str):
        plt.title(title)
    plt.show()

def plot_predictions_histogram(Y_ph, Y, title=None):
    """Plot a histogram comparing int predictions vs true labels by class

    Args:
        Y_ph: An [n] or [n, 1] np.ndarray of predicted int labels
        Y: An [n] or [n, 1] np.ndarray of gold labels
    """
    labels = list(set(Y).union(set(Y_ph)))
    edges = [x - 0.5 for x in range(min(labels), max(labels) + 2)]

    plt.hist([Y_ph, Y], bins=edges, label=["Predicted", "Gold"])
    ax = plt.gca()
    ax.set_xticks(labels)
    plt.xlabel("Label")
    plt.ylabel("# Predictions")
    plt.legend(loc="upper right")
    if isinstance(title, str):
        plt.title(title)
    plt.show()

def plot_calibration_histogram(Y_p, Y, title=None, legend=True):
    """Plot a histogram of correct predictions from an array of probabilities

    Args:
        Y_p: An [n] or [n, 1] np.ndarray of probabilities (floats in [0,1])
        Y: An [n] or [n, 1] np.ndarray of gold labels
    """
    if Y_p.ndim > 1:
        Y_ph = np.argmax(np.array(Y_p), axis=1).astype(float) + 1.
        Y_p = Y_p[:, 0]
    else:
        Y_ph = (np.sign(Y_p - 0.5) + 1.) / 2.

    correct_idx = list(np.where(Y_ph == Y)[0])
    print("Accuracy: ", len(correct_idx) / (1.0 * len(Y_p)))

    Y_p_correct = Y_p[correct_idx]
    plt.hist(Y_p, bins=20, alpha=0.75)
    plt.hist(Y_p_correct, bins=20, alpha=0.75)
    plt.xlim((0, 1.025))
    plt.xlabel("Probability")
    plt.ylabel("# Predictions")
    if isinstance(title, str):
        plt.title(title)
    if legend:
        plt.legend(["All", "Correct"])
<commit_msg>Fix formatting issues (thanks Paroma!)<commit_after>import matplotlib.pyplot as plt
import numpy as np
import scipy.sparse as sparse

############################################################
# Label Matrix Plotting
############################################################

def view_label_matrix(L, colorbar=True):
    """Display an [n, m] matrix of labels"""
    L = L.todense() if sparse.issparse(L) else L
    plt.imshow(L, aspect="auto")
    plt.title("Label Matrix")
    if colorbar:
        labels = sorted(np.unique(np.asarray(L).reshape(-1, 1).squeeze()))
        boundaries = np.array(labels + [max(labels) + 1]) - 0.5
        plt.colorbar(boundaries=boundaries, ticks=labels)
    plt.show()

def view_overlaps(L, self_overlaps=False, normalize=True, colorbar=True):
    """Display an [m, m] matrix of overlaps"""
    L = L.todense() if sparse.issparse(L) else L
    G = _get_overlaps_matrix(L, normalize=normalize)
    if not self_overlaps:
        np.fill_diagonal(G, 0)  # Zero out self-overlaps
    plt.imshow(G, aspect="auto")
    plt.title("Overlaps")
    if colorbar:
        plt.colorbar()
    plt.show()

def view_conflicts(L, normalize=True, colorbar=True):
    """Display an [m, m] matrix of conflicts"""
    L = L.todense() if sparse.issparse(L) else L
    C = _get_conflicts_matrix(L, normalize=normalize)
    plt.imshow(C, aspect="auto")
    plt.title("Conflicts")
    if colorbar:
        plt.colorbar()
    plt.show()

def _get_overlaps_matrix(L, normalize=True):
    n, m = L.shape
    X = np.where(L != 0, 1, 0).T
    G = X @ X.T

    if normalize:
        G = G / n
    return G

def _get_conflicts_matrix(L, normalize=True):
    n, m = L.shape
    C = np.zeros((m, m))

    # Iterate over the pairs of LFs
    for i in range(m):
        for j in range(m):
            # Get the overlapping non-zero indices
            overlaps = list(
                set(np.where(L[:, i] != 0)[0]).intersection(np.where(L[:, j] != 0)[0])
            )
            C[i, j] = np.where(L[overlaps, i] != L[overlaps, j], 1, 0).sum()

    if normalize:
        C = C / n
    return C

############################################################
# Classifier Diagnostics
############################################################

def plot_probabilities_histogram(Y_p, title=None):
    """Plot a histogram from a numpy array of probabilities

    Args:
        Y_p: An [n] or [n, 1] np.ndarray of probabilities (floats in [0,1])
    """
    if Y_p.ndim > 1:
        msg = (
            f"Arg Y_p should be a 1-dimensional np.ndarray, not of shape "
            f"{Y_p.shape}."
        )
        raise ValueError(msg)
    plt.hist(Y_p, bins=20)
    plt.xlim((0, 1.025))
    plt.xlabel("Probability")
    plt.ylabel("# Predictions")
    if isinstance(title, str):
        plt.title(title)
    plt.show()

def plot_predictions_histogram(Y_ph, Y, title=None):
    """Plot a histogram comparing int predictions vs true labels by class

    Args:
        Y_ph: An [n] or [n, 1] np.ndarray of predicted int labels
        Y: An [n] or [n, 1] np.ndarray of gold labels
    """
    labels = list(set(Y).union(set(Y_ph)))
    edges = [x - 0.5 for x in range(min(labels), max(labels) + 2)]

    plt.hist([Y_ph, Y], bins=edges, label=["Predicted", "Gold"])
    ax = plt.gca()
    ax.set_xticks(labels)
    plt.xlabel("Label")
    plt.ylabel("# Predictions")
    plt.legend(loc="upper right")
    if isinstance(title, str):
        plt.title(title)
    plt.show()

def plot_calibration_histogram(Y_p, Y, title=None, legend=True):
    """Plot a histogram of correct predictions from an array of probabilities

    Args:
        Y_p: An [n] or [n, 1] np.ndarray of probabilities (floats in [0,1])
        Y: An [n] or [n, 1] np.ndarray of gold labels
    """
    if Y_p.ndim > 1:
        Y_ph = np.argmax(np.array(Y_p), axis=1).astype(float) + 1.0
        Y_p = Y_p[:, 0]
    else:
        Y_ph = (np.sign(Y_p - 0.5) + 1.0) / 2.0

    correct_idx = list(np.where(Y_ph == Y)[0])
    print("Accuracy: ", len(correct_idx) / (1.0 * len(Y_p)))

    Y_p_correct = Y_p[correct_idx]
    plt.hist(Y_p, bins=20, alpha=0.75)
    plt.hist(Y_p_correct, bins=20, alpha=0.75)
    plt.xlim((0, 1.025))
    plt.xlabel("Probability")
    plt.ylabel("# Predictions")
    if isinstance(title, str):
        plt.title(title)
    if legend:
        plt.legend(["All", "Correct"])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import subprocess
import sys
import re

class Color:
    PATH_BG = 237
    PATH_FG = 250
    CWD_BG = 237
    CWD_FG = 254
    SEPARATOR_FG = 244

    REPO_CLEAN_BG = 148 # a light green color
    REPO_CLEAN_FG = 0 # black
    REPO_DIRTY_BG = 161 # pink/red
    REPO_DIRTY_FG = 15 # white

    CMD_PASSED_BG = 236
    CMD_PASSED_FG = 15
    CMD_FAILED_BG = 161
    CMD_FAILED_FG = 15

class Powerline:
    symbols = {
        'compatible': {
            'separator': u'\u25B6',
            'separator_thin': u'\u276F'
        },
        'patched': {
            'separator': u'\u2B80',
            'separator_thin': u'\u2B81'
        }
    }
    LSQESCRSQ = '\\[\\e%s\\]'
    reset = LSQESCRSQ % '[0m'

    def __init__(self, mode='compatible'):
        self.separator = Powerline.symbols[mode]['separator']
        self.separator_thin = Powerline.symbols[mode]['separator_thin']
        self.segments = []

    def color(self, prefix, code):
        return self.LSQESCRSQ % ('[%s;5;%sm' % (prefix, code))

    def fgcolor(self, code):
        return self.color('38', code)

    def bgcolor(self, code):
        return self.color('48', code)

    def append(self, segment):
        self.segments.append(segment)

    def draw(self):
        return (''.join((s[0].draw(s[1]) for s in zip(self.segments, self.segments[1:]+[None])))
            + self.reset).encode('utf-8')

class Segment:
    def __init__(self, powerline, content, fg, bg, separator=None, separator_fg=None):
        self.powerline = powerline
        self.content = content
        self.fg = fg
        self.bg = bg
        self.separator = separator or powerline.separator
        self.separator_fg = separator_fg or bg

    def draw(self, next_segment=None):
        if next_segment:
            separator_bg = self.powerline.bgcolor(next_segment.bg)
        else:
            separator_bg = self.powerline.reset

        return ''.join((
            self.powerline.fgcolor(self.fg),
            self.powerline.bgcolor(self.bg),
            self.content,
            separator_bg,
            self.powerline.fgcolor(self.separator_fg),
            self.separator))

def add_cwd_segment(powerline, cwd, maxdepth):
    #powerline.append(' \\w ', 15, 237)
    home = os.getenv('HOME')
    cwd = os.getenv('PWD')

    if cwd.find(home) == 0:
        cwd = cwd.replace(home, '~', 1)

    if cwd[0] == '/':
        cwd = cwd[1:]

    names = cwd.split('/')
    if len(names) > maxdepth:
        names = names[:2] + [u'\u2026'] + names[2-maxdepth:]

    for n in names[:-1]:
        powerline.append(Segment(powerline, ' %s ' % n, Color.PATH_FG, Color.PATH_BG, powerline.separator_thin, Color.SEPARATOR_FG))
    powerline.append(Segment(powerline, ' %s ' % names[-1], Color.CWD_FG, Color.CWD_BG))

def get_hg_status():
    has_modified_files = False
    has_untracked_files = False
    has_missing_files = False
    output = subprocess.Popen(['hg', 'status'], stdout=subprocess.PIPE).communicate()[0]
    for line in output.split('\n'):
        if line == '':
            continue
        elif line[0] == '?':
            has_untracked_files = True
        elif line[0] == '!':
            has_missing_files = True
        else:
            has_modified_files = True
    return has_modified_files, has_untracked_files, has_missing_files

def add_hg_segment(powerline, cwd):
    branch = os.popen('hg branch 2> /dev/null').read().rstrip()
    if len(branch) == 0:
        return False
    bg = Color.REPO_CLEAN_BG
    fg = Color.REPO_CLEAN_FG
    has_modified_files, has_untracked_files, has_missing_files = get_hg_status()
    if has_modified_files or has_untracked_files or has_missing_files:
        bg = Color.REPO_DIRTY_BG
        fg = Color.REPO_DIRTY_FG
        extra = ''
        if has_untracked_files:
            extra += '+'
        if has_missing_files:
            extra += '!'
        branch += (' ' + extra if extra != '' else '')
    powerline.append(Segment(powerline, ' %s ' % branch, fg, bg))
    return True

def get_git_status():
    has_pending_commits = True
    has_untracked_files = False
    origin_position = ""
    output = subprocess.Popen(['git', 'status', '--ignore-submodules'], stdout=subprocess.PIPE).communicate()[0]
    for line in output.split('\n'):
        origin_status = re.findall("Your branch is (ahead|behind).*?(\d+) comm", line)
        if len(origin_status) > 0:
            origin_position = " %d" % int(origin_status[0][1])
            if origin_status[0][0] == 'behind':
                origin_position += u'\u21E3'
            if origin_status[0][0] == 'ahead':
                origin_position += u'\u21E1'

        if line.find('nothing to commit') >= 0:
            has_pending_commits = False
        if line.find('Untracked files') >= 0:
            has_untracked_files = True
    return has_pending_commits, has_untracked_files, origin_position

def add_git_segment(powerline, cwd):
    #cmd = "git branch 2> /dev/null | grep -e '\\*'"
    p1 = subprocess.Popen(['git', 'branch'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    p2 = subprocess.Popen(['grep', '-e', '\\*'], stdin=p1.stdout, stdout=subprocess.PIPE)
    output = p2.communicate()[0].strip()
    if len(output) == 0:
        return False
    branch = output.rstrip()[2:]
    has_pending_commits, has_untracked_files, origin_position = get_git_status()
    branch += origin_position
    if has_untracked_files:
        branch += ' +'
    bg = Color.REPO_CLEAN_BG
    fg = Color.REPO_CLEAN_FG
    if has_pending_commits:
        bg = Color.REPO_DIRTY_BG
        fg = Color.REPO_DIRTY_FG
    powerline.append(Segment(powerline, ' %s ' % branch, fg, bg))
    return True

def add_svn_segment(powerline, cwd):
    if not os.path.exists(os.path.join(cwd,'.svn')):
        return
    '''svn info:
        First column: Says if item was added, deleted, or otherwise changed
        ' ' no modifications
        'A' Added
        'C' Conflicted
        'D' Deleted
        'I' Ignored
        'M' Modified
        'R' Replaced
        'X' an unversioned directory created by an externals definition
        '?' item is not under version control
        '!' item is missing (removed by non-svn command) or incomplete
         '~' versioned item obstructed by some item of a different kind
    '''
    #TODO: Color segment based on above status codes
    try:
        #cmd = '"svn status | grep -c "^[ACDIMRX\\!\\~]"'
        p1 = subprocess.Popen(['svn', 'status'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        p2 = subprocess.Popen(['grep', '-c', '^[ACDIMRX\\!\\~]'], stdin=p1.stdout, stdout=subprocess.PIPE)
        output = p2.communicate()[0].strip()
        if len(output) > 0 and int(output) > 0:
            changes = output.strip()
            powerline.append(Segment(powerline, ' %s ' % changes, 22, Color.REPO_CLEAN_BG))
    except OSError:
        return False
    except subprocess.CalledProcessError:
        return False
    return True

def add_repo_segment(powerline, cwd):
    for add_repo_segment in [add_git_segment, add_svn_segment, add_hg_segment]:
        try:
            if add_repo_segment(p, cwd): return
        except subprocess.CalledProcessError:
            pass
        except OSError:
            pass

def add_virtual_env_segment(powerline, cwd):
    env = os.getenv("VIRTUAL_ENV")
    if env == None:
        return False
    env_name = os.path.basename(env)
    bg = 35
    fg = 22
    powerline.append(Segment(powerline,' %s ' % env_name, fg, bg))
    return True

def add_root_indicator(powerline, error):
    bg = Color.CMD_PASSED_BG
    fg = Color.CMD_PASSED_FG
    if int(error) != 0:
        fg = Color.CMD_FAILED_FG
        bg = Color.CMD_FAILED_BG
    powerline.append(Segment(powerline, ' \\$ ', fg, bg))

if __name__ == '__main__':
    p = Powerline(mode='patched')
    cwd = os.getcwd()
    add_virtual_env_segment(p, cwd)
    #p.append(Segment(powerline, ' \\u ', 250, 240))
    #p.append(Segment(powerline, ' \\h ', 250, 238))
    add_cwd_segment(p, cwd, 5)
    add_repo_segment(p, cwd)
    add_root_indicator(p, sys.argv[1] if len(sys.argv) > 1 else 0)
    sys.stdout.write(p.draw())

# vim: set expandtab:
<commit_msg>special color for svn<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import subprocess
import sys
import re

class Color:
    PATH_BG = 237
    PATH_FG = 250
    CWD_BG = 237
    CWD_FG = 254
    SEPARATOR_FG = 244

    REPO_CLEAN_BG = 148 # a light green color
    REPO_CLEAN_FG = 0 # black
    REPO_DIRTY_BG = 161 # pink/red
    REPO_DIRTY_FG = 15 # white

    CMD_PASSED_BG = 236
    CMD_PASSED_FG = 15
    CMD_FAILED_BG = 161
    CMD_FAILED_FG = 15

    SVN_CHANGES_FG = 22 # dark green

class Powerline:
    symbols = {
        'compatible': {
            'separator': u'\u25B6',
            'separator_thin': u'\u276F'
        },
        'patched': {
            'separator': u'\u2B80',
            'separator_thin': u'\u2B81'
        }
    }
    LSQESCRSQ = '\\[\\e%s\\]'
    reset = LSQESCRSQ % '[0m'

    def __init__(self, mode='compatible'):
        self.separator = Powerline.symbols[mode]['separator']
        self.separator_thin = Powerline.symbols[mode]['separator_thin']
        self.segments = []

    def color(self, prefix, code):
        return self.LSQESCRSQ % ('[%s;5;%sm' % (prefix, code))

    def fgcolor(self, code):
        return self.color('38', code)

    def bgcolor(self, code):
        return self.color('48', code)

    def append(self, segment):
        self.segments.append(segment)

    def draw(self):
        return (''.join((s[0].draw(s[1]) for s in zip(self.segments, self.segments[1:]+[None])))
            + self.reset).encode('utf-8')

class Segment:
    def __init__(self, powerline, content, fg, bg, separator=None, separator_fg=None):
        self.powerline = powerline
        self.content = content
        self.fg = fg
        self.bg = bg
        self.separator = separator or powerline.separator
        self.separator_fg = separator_fg or bg

    def draw(self, next_segment=None):
        if next_segment:
            separator_bg = self.powerline.bgcolor(next_segment.bg)
        else:
            separator_bg = self.powerline.reset

        return ''.join((
            self.powerline.fgcolor(self.fg),
            self.powerline.bgcolor(self.bg),
            self.content,
            separator_bg,
            self.powerline.fgcolor(self.separator_fg),
            self.separator))

def add_cwd_segment(powerline, cwd, maxdepth):
    #powerline.append(' \\w ', 15, 237)
    home = os.getenv('HOME')
    cwd = os.getenv('PWD')

    if cwd.find(home) == 0:
        cwd = cwd.replace(home, '~', 1)

    if cwd[0] == '/':
        cwd = cwd[1:]

    names = cwd.split('/')
    if len(names) > maxdepth:
        names = names[:2] + [u'\u2026'] + names[2-maxdepth:]

    for n in names[:-1]:
        powerline.append(Segment(powerline, ' %s ' % n, Color.PATH_FG, Color.PATH_BG, powerline.separator_thin, Color.SEPARATOR_FG))
    powerline.append(Segment(powerline, ' %s ' % names[-1], Color.CWD_FG, Color.CWD_BG))

def get_hg_status():
    has_modified_files = False
    has_untracked_files = False
    has_missing_files = False
    output = subprocess.Popen(['hg', 'status'], stdout=subprocess.PIPE).communicate()[0]
    for line in output.split('\n'):
        if line == '':
            continue
        elif line[0] == '?':
            has_untracked_files = True
        elif line[0] == '!':
            has_missing_files = True
        else:
            has_modified_files = True
    return has_modified_files, has_untracked_files, has_missing_files

def add_hg_segment(powerline, cwd):
    branch = os.popen('hg branch 2> /dev/null').read().rstrip()
    if len(branch) == 0:
        return False
    bg = Color.REPO_CLEAN_BG
    fg = Color.REPO_CLEAN_FG
    has_modified_files, has_untracked_files, has_missing_files = get_hg_status()
    if has_modified_files or has_untracked_files or has_missing_files:
        bg = Color.REPO_DIRTY_BG
        fg = Color.REPO_DIRTY_FG
        extra = ''
        if has_untracked_files:
            extra += '+'
        if has_missing_files:
            extra += '!'
        branch += (' ' + extra if extra != '' else '')
    powerline.append(Segment(powerline, ' %s ' % branch, fg, bg))
    return True

def get_git_status():
    has_pending_commits = True
    has_untracked_files = False
    origin_position = ""
    output = subprocess.Popen(['git', 'status', '--ignore-submodules'], stdout=subprocess.PIPE).communicate()[0]
    for line in output.split('\n'):
        origin_status = re.findall("Your branch is (ahead|behind).*?(\d+) comm", line)
        if len(origin_status) > 0:
            origin_position = " %d" % int(origin_status[0][1])
            if origin_status[0][0] == 'behind':
                origin_position += u'\u21E3'
            if origin_status[0][0] == 'ahead':
                origin_position += u'\u21E1'

        if line.find('nothing to commit') >= 0:
            has_pending_commits = False
        if line.find('Untracked files') >= 0:
            has_untracked_files = True
    return has_pending_commits, has_untracked_files, origin_position

def add_git_segment(powerline, cwd):
    #cmd = "git branch 2> /dev/null | grep -e '\\*'"
    p1 = subprocess.Popen(['git', 'branch'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    p2 = subprocess.Popen(['grep', '-e', '\\*'], stdin=p1.stdout, stdout=subprocess.PIPE)
    output = p2.communicate()[0].strip()
    if len(output) == 0:
        return False
    branch = output.rstrip()[2:]
    has_pending_commits, has_untracked_files, origin_position = get_git_status()
    branch += origin_position
    if has_untracked_files:
        branch += ' +'
    bg = Color.REPO_CLEAN_BG
    fg = Color.REPO_CLEAN_FG
    if has_pending_commits:
        bg = Color.REPO_DIRTY_BG
        fg = Color.REPO_DIRTY_FG
    powerline.append(Segment(powerline, ' %s ' % branch, fg, bg))
    return True

def add_svn_segment(powerline, cwd):
    if not os.path.exists(os.path.join(cwd,'.svn')):
        return
    '''svn info:
        First column: Says if item was added, deleted, or otherwise changed
        ' ' no modifications
        'A' Added
        'C' Conflicted
        'D' Deleted
        'I' Ignored
        'M' Modified
        'R' Replaced
        'X' an unversioned directory created by an externals definition
        '?' item is not under version control
        '!' item is missing (removed by non-svn command) or incomplete
         '~' versioned item obstructed by some item of a different kind
    '''
    #TODO: Color segment based on above status codes
    try:
        #cmd = '"svn status | grep -c "^[ACDIMRX\\!\\~]"'
        p1 = subprocess.Popen(['svn', 'status'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        p2 = subprocess.Popen(['grep', '-c', '^[ACDIMRX\\!\\~]'], stdin=p1.stdout, stdout=subprocess.PIPE)
        output = p2.communicate()[0].strip()
        if len(output) > 0 and int(output) > 0:
            changes = output.strip()
            powerline.append(Segment(powerline, ' %s ' % changes, Color.SVN_CHANGES_FG, Color.REPO_CLEAN_BG))
    except OSError:
        return False
    except subprocess.CalledProcessError:
        return False
    return True

def add_repo_segment(powerline, cwd):
    for add_repo_segment in [add_git_segment, add_svn_segment, add_hg_segment]:
        try:
            if add_repo_segment(p, cwd): return
        except subprocess.CalledProcessError:
            pass
        except OSError:
            pass

def add_virtual_env_segment(powerline, cwd):
    env = os.getenv("VIRTUAL_ENV")
    if env == None:
        return False
    env_name = os.path.basename(env)
    bg = 35
    fg = 22
    powerline.append(Segment(powerline,' %s ' % env_name, fg, bg))
    return True

def add_root_indicator(powerline, error):
    bg = Color.CMD_PASSED_BG
    fg = Color.CMD_PASSED_FG
    if int(error) != 0:
        fg = Color.CMD_FAILED_FG
        bg = Color.CMD_FAILED_BG
    powerline.append(Segment(powerline, ' \\$ ', fg, bg))

if __name__ == '__main__':
    p = Powerline(mode='patched')
    cwd = os.getcwd()
    add_virtual_env_segment(p, cwd)
    #p.append(Segment(powerline, ' \\u ', 250, 240))
    #p.append(Segment(powerline, ' \\h ', 250, 238))
    add_cwd_segment(p, cwd, 5)
    add_repo_segment(p, cwd)
    add_root_indicator(p, sys.argv[1] if len(sys.argv) > 1 else 0)
    sys.stdout.write(p.draw())

# vim: set expandtab:
<|endoftext|>


<commit_before>"""
Encrypted DynamoDB config loader using credstash.

Code adapted with much appreciation to credstash. As much as possible, uses credstash code,
but currently prevented from full reuse because of difficulty configurating credstash's KMS key.

See: https://github.com/fugue/credstash/blob/master/credstash.py

"""
from base64 import b64decode, b64encode
from collections import namedtuple

from boto3 import Session
from credstash import KeyService, open_aes_ctr_legacy, seal_aes_ctr_legacy

from microcosm_dynamodb.loaders.base import DynamoDBLoader

# match credstash key names
EncryptedValue = namedtuple("EncryptedValue", ["key", "contents", "hmac"])

class EncryptedDynamoDBLoader(DynamoDBLoader):
    """
    A credstash-compatible microcosm config loader using KMS-encrypted DynamoDB data.

    Usage:

        from microcosm.metadata import Metadata
        from microcosm_dynamodb.loaders.encrypted import EncryptedDynamoDBLoader

        metadata = Metadata("myservice")
        loader = EncryptedDynamoDBLoader(kms_key="alias/mykey")
        loader.put(metadata.name, "key", "value")
        print loader.get(metadata.name, "key")
        print loader(metadata)
        loader.delete(metadata.name, "key")

    """
    def __init__(self, kms_key, **kwargs):
        super(EncryptedDynamoDBLoader, self).__init__(**kwargs)
        self.kms_key = kms_key

    @property
    def value_type(self):
        return EncryptedValue

    def decode(self, value_type):
        return self.decrypt(value_type)

    def encode(self, value):
        return self.encrypt(value)

    def decrypt(self, value, context=None):
        if not context:
            context = {}
        session = Session(profile_name=self.profile_name)
        kms = session.client('kms', region_name=self.region)
        key_service = KeyService(kms, None, context)
        return open_aes_ctr_legacy(
            key_service,
            dict(
                key=value.key,
                contents=value.contents,
                hmac=value.hmac,
            )
        )

    def encrypt(self, plaintext, context=None):
        if not context:
            context = {}
        session = Session(profile_name=self.profile_name)
        kms = session.client('kms', region_name=self.region)
        key_service = KeyService(kms, None, context)
        sealed = seal_aes_ctr_legacy(
            key_service,
            plaintext,
        )
        return EncryptedValue(
            sealed["key"],
            sealed["contents"],
            sealed["hmac"],
        )
<commit_msg>flake8 unnecessary imports<commit_after>"""
Encrypted DynamoDB config loader using credstash.

Code adapted with much appreciation to credstash. As much as possible, uses credstash code,
but currently prevented from full reuse because of difficulty configurating credstash's KMS key.

See: https://github.com/fugue/credstash/blob/master/credstash.py

"""
from collections import namedtuple

from boto3 import Session
from credstash import KeyService, open_aes_ctr_legacy, seal_aes_ctr_legacy

from microcosm_dynamodb.loaders.base import DynamoDBLoader

# match credstash key names
EncryptedValue = namedtuple("EncryptedValue", ["key", "contents", "hmac"])

class EncryptedDynamoDBLoader(DynamoDBLoader):
    """
    A credstash-compatible microcosm config loader using KMS-encrypted DynamoDB data.

    Usage:

        from microcosm.metadata import Metadata
        from microcosm_dynamodb.loaders.encrypted import EncryptedDynamoDBLoader

        metadata = Metadata("myservice")
        loader = EncryptedDynamoDBLoader(kms_key="alias/mykey")
        loader.put(metadata.name, "key", "value")
        print loader.get(metadata.name, "key")
        print loader(metadata)
        loader.delete(metadata.name, "key")

    """
    def __init__(self, kms_key, **kwargs):
        super(EncryptedDynamoDBLoader, self).__init__(**kwargs)
        self.kms_key = kms_key

    @property
    def value_type(self):
        return EncryptedValue

    def decode(self, value_type):
        return self.decrypt(value_type)

    def encode(self, value):
        return self.encrypt(value)

    def decrypt(self, value, context=None):
        if not context:
            context = {}
        session = Session(profile_name=self.profile_name)
        kms = session.client('kms', region_name=self.region)
        key_service = KeyService(kms, None, context)
        return open_aes_ctr_legacy(
            key_service,
            dict(
                key=value.key,
                contents=value.contents,
                hmac=value.hmac,
            )
        )

    def encrypt(self, plaintext, context=None):
        if not context:
            context = {}
        session = Session(profile_name=self.profile_name)
        kms = session.client('kms', region_name=self.region)
        key_service = KeyService(kms, None, context)
        sealed = seal_aes_ctr_legacy(
            key_service,
            plaintext,
        )
        return EncryptedValue(
            sealed["key"],
            sealed["contents"],
            sealed["hmac"],
        )
<|endoftext|>


<commit_before>
#
# rpclib - Copyright (C) Rpclib contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

"""This module contains the HttpRpc protocol implementation. This is not exactly
Rest, because it ignores Http verbs.
"""

import logging
logger = logging.getLogger(__name__)

try:
    from urlparse import parse_qs
except ImportError: # Python 3
    from urllib.parse import parse_qs

from rpclib.error import ValidationError
from rpclib.model.complex import Array
from rpclib.model.fault import Fault
from rpclib.protocol import ProtocolBase

def _get_http_headers(req_env):
    retval = {}

    for k, v in req_env.items():
        if k.startswith("HTTP_"):
            retval[k[5:].lower()]= [v]

    return retval

class HttpRpc(ProtocolBase):
    """The so-called REST-minus-the-verbs HttpRpc protocol implementation.
    It only works with the http server (wsgi) transport.

    It only parses requests where the whole data is in the 'QUERY_STRING', i.e.
    the part after '?' character in a URI string.
    """

    def set_validator(self, validator):
        if validator == 'soft' or validator is self.SOFT_VALIDATION:
            self.validator = self.SOFT_VALIDATION
        elif validator is None:
            self.validator = None
        else:
            raise ValueError(validator)

    def create_in_document(self, ctx, in_string_encoding=None):
        assert ctx.transport.type == 'wsgi', ("This protocol only works with "
                                              "the wsgi api.")

        ctx.in_document = ctx.transport.req_env

    def decompose_incoming_envelope(self, ctx):
        ctx.method_request_string = '{%s}%s' % (self.app.interface.get_tns(),
                              ctx.in_document['PATH_INFO'].split('/')[-1])
        logger.debug("\033[92mMethod name: %r\033[0m" % ctx.method_request_string)

        ctx.in_header_doc = _get_http_headers(ctx.in_document)
        ctx.in_body_doc = parse_qs(ctx.in_document['QUERY_STRING'])

        logger.debug('\theader : %r' % (ctx.in_header_doc))
        logger.debug('\tbody   : %r' % (ctx.in_body_doc))

    def dict_to_object(self, doc, inst_class):
        simple_type_info = inst_class.get_simple_type_info(inst_class)
        inst = inst_class.get_deserialization_instance()

        # this is for validating cls.Attributes.{min,max}_occurs
        frequencies = {}

        for k, v in doc.items():
            member = simple_type_info.get(k, None)
            if member is None:
                logger.debug("discarding field %r" % k)
                continue

            mo = member.type.Attributes.max_occurs
            value = getattr(inst, k, None)
            if value is None:
                value = []
            elif mo == 1:
                raise Fault('Client.ValidationError',
                        '"%s" member must occur at most %d times' % (k, max_o))

            for v2 in v:
                if (self.validator is self.SOFT_VALIDATION and not
                            member.type.validate_string(member.type, v2)):
                    raise ValidationError(v2)
                native_v2 = member.type.from_string(v2)
                if (self.validator is self.SOFT_VALIDATION and not
                            member.type.validate_native(member.type, native_v2)):
                    raise ValidationError(v2)

                value.append(native_v2)

                # set frequencies of parents.
                if not (member.path[:-1] in frequencies):
                    for i in range(1,len(member.path)):
                        logger.debug("\tset freq %r = 1" % (member.path[:i],))
                        frequencies[member.path[:i]] = 1

                freq = frequencies.get(member.path, 0)
                freq += 1
                frequencies[member.path] = freq
                logger.debug("\tset freq %r = %d" % (member.path, freq))

            if mo == 1:
                value = value[0]

            cinst = inst
            ctype_info = inst_class.get_flat_type_info(inst_class)
            pkey = member.path[0]
            for i in range(len(member.path) - 1):
                pkey = member.path[i]
                if not (ctype_info[pkey].Attributes.max_occurs in (0,1)):
                    raise Exception("HttpRpc deserializer does not support "
                                    "non-primitives with max_occurs > 1")

                ninst = getattr(cinst, pkey, None)
                if ninst is None:
                    ninst = ctype_info[pkey].get_deserialization_instance()
                    setattr(cinst, pkey, ninst)
                cinst = ninst

                ctype_info = ctype_info[pkey]._type_info

            if isinstance(cinst, list):
                cinst.extend(value)
                logger.debug("\tset array   %r(%r) = %r" %
                                                    (member.path, pkey, value))
            else:
                setattr(cinst, member.path[-1], value)
                logger.debug("\tset default %r(%r) = %r" %
                                                    (member.path, pkey, value))

        try:
            print inst.qo.vehicles
        except Exception,e:
            print e

        if self.validator is self.SOFT_VALIDATION:
            sti = simple_type_info.values()
            sti.sort(key=lambda x: (len(x.path), x.path))
            pfrag = None
            for s in sti:
                if len(s.path) > 1 and pfrag != s.path[:-1]:
                    pfrag = s.path[:-1]
                    ctype_info = inst_class.get_flat_type_info(inst_class)
                    for i in range(len(pfrag)):
                        f = pfrag[i]
                        ntype_info = ctype_info[f]

                        min_o = ctype_info[f].Attributes.min_occurs
                        max_o = ctype_info[f].Attributes.max_occurs
                        val = frequencies.get(pfrag[:i+1], 0)
                        if val < min_o:
                            raise Fault('Client.ValidationError',
                                '"%s" member must occur at least %d times'
                                              % ('_'.join(pfrag[:i+1]), min_o))

                        if max_o != 'unbounded' and val > max_o:
                            raise Fault('Client.ValidationError',
                                '"%s" member must occur at most %d times'
                                             % ('_'.join(pfrag[:i+1]), max_o))

                        ctype_info = ntype_info.get_flat_type_info(ntype_info)

                val = frequencies.get(s.path, 0)
                min_o = s.type.Attributes.min_occurs
                max_o = s.type.Attributes.max_occurs
                if val < min_o:
                    raise Fault('Client.ValidationError',
                                '"%s" member must occur at least %d times'
                                                    % ('_'.join(s.path), min_o))
                if max_o != 'unbounded' and val > max_o:
                    raise Fault('Client.ValidationError',
                                '"%s" member must occur at most %d times'
                                                    % ('_'.join(s.path), max_o))

        return inst

    def deserialize(self, ctx, message):
        assert message in (self.REQUEST,)

        self.event_manager.fire_event('before_deserialize', ctx)

        if len(ctx.in_header_doc) > 0:
            ctx.in_header = self.dict_to_object(ctx.in_header_doc,
                                                    ctx.descriptor.in_header)
        else:
            ctx.in_header = [None] * len(
                        ctx.descriptor.in_header.get_flat_type_info(
                                                    ctx.descriptor.in_message))

        if ctx.in_body_doc is not None:
            ctx.in_object = self.dict_to_object(ctx.in_body_doc,
                                                    ctx.descriptor.in_message)
        else:
            ctx.in_object = [None] * len(
                        ctx.descriptor.in_message.get_flat_type_info(
                                                    ctx.descriptor.in_message))

        self.event_manager.fire_event('after_deserialize', ctx)

    def serialize(self, ctx, message):
        assert message in (self.RESPONSE,)

        if ctx.out_error is None:
            result_message_class = ctx.descriptor.out_message

            # assign raw result to its wrapper, result_message
            out_type_info = result_message_class.get_flat_type_info(result_message_class)
            if len(out_type_info) == 1:
                out_class = out_type_info.values()[0]
                if ctx.out_object is None:
                    ctx.out_document = ['']
                else:
                    if hasattr(out_class, 'to_string_iterable'):
                        ctx.out_document = out_class.to_string_iterable(ctx.out_object[0])
                    else:
                        raise ValueError("HttpRpc protocol can only serialize primitives. %r" % out_class)
            else:
                raise ValueError("HttpRpc protocol can only serialize simple return values.")
        else:
            ctx.transport.mime_type = 'text/plain'
            ctx.out_document = ctx.out_error.to_string_iterable(ctx.out_error)

        self.event_manager.fire_event('serialize', ctx)

    def create_out_string(self, ctx, out_string_encoding='utf8'):
        ctx.out_string = ctx.out_document
<commit_msg>use werkzeug to parse HTTP POST data.<commit_after>
#
# rpclib - Copyright (C) Rpclib contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

"""This module contains the HttpRpc protocol implementation. This is not exactly
Rest, because it ignores Http verbs.
"""

import logging
logger = logging.getLogger(__name__)

try:
    from urlparse import parse_qs
except ImportError: # Python 3
    from urllib.parse import parse_qs

from rpclib.error import ValidationError
from rpclib.model.binary import ByteArray
from rpclib.model.fault import Fault
from rpclib.protocol import ProtocolBase

from werkzeug.formparser import parse_form_data

STREAM_READ_BLOCK_SIZE = 16384

def yield_stream(istr):
    data = istr.read(STREAM_READ_BLOCK_SIZE)
    while len(data) > 0:
        yield data
        data = istr.read(STREAM_READ_BLOCK_SIZE)

def _get_http_headers(req_env):
    retval = {}

    for k, v in req_env.items():
        if k.startswith("HTTP_"):
            retval[k[5:].lower()]= [v]

    return retval

class HttpRpc(ProtocolBase):
    """The so-called REST-minus-the-verbs HttpRpc protocol implementation.
    It only works with the http server (wsgi) transport.

    It only parses requests where the whole data is in the 'QUERY_STRING', i.e.
    the part after '?' character in a URI string.
    """

    def set_validator(self, validator):
        if validator == 'soft' or validator is self.SOFT_VALIDATION:
            self.validator = self.SOFT_VALIDATION
        elif validator is None:
            self.validator = None
        else:
            raise ValueError(validator)

    def create_in_document(self, ctx, in_string_encoding=None):
        assert ctx.transport.type == 'wsgi', ("This protocol only works with "
                                              "the wsgi api.")

        ctx.in_document = ctx.transport.req_env

    def decompose_incoming_envelope(self, ctx):
        ctx.method_request_string = '{%s}%s' % (self.app.interface.get_tns(),
                              ctx.in_document['PATH_INFO'].split('/')[-1])

        logger.debug("\033[92mMethod name: %r\033[0m" % ctx.method_request_string)

        ctx.in_header_doc = _get_http_headers(ctx.in_document)
        ctx.in_body_doc = parse_qs(ctx.in_document['QUERY_STRING'])

        if ctx.transport.req_env['REQUEST_METHOD'].lower() in ('post', 'put', 'patch'):
            stream, form, files = parse_form_data(ctx.transport.req_env)

            for k, v in form.lists():
                val = ctx.in_body_doc.get(k, [])
                val.extend(v)
                ctx.in_body_doc[k] = val

            for k, v in files.items():
                val = ctx.in_body_doc.get(k, [])
                val.append(yield_stream(v.stream))
                ctx.in_body_doc[k] = val

                # FIXME: some proper variable matching is needed here.
                k2 = k + "_name"
                val = ctx.in_body_doc.get(k2, [])
                val.append(v.filename)
                ctx.in_body_doc[k2] = val

                k2 = k + "_type"
                val = ctx.in_body_doc.get(k2, [])
                val.append(v.headers.get('Content-Type','application/octet-stream'))
                ctx.in_body_doc[k2] = val

        logger.debug('\theader : %r' % (ctx.in_header_doc))
        logger.debug('\tbody   : %r' % (ctx.in_body_doc))

    def dict_to_object(self, doc, inst_class):
        simple_type_info = inst_class.get_simple_type_info(inst_class)
        inst = inst_class.get_deserialization_instance()

        # this is for validating cls.Attributes.{min,max}_occurs
        frequencies = {}

        for k, v in doc.items():
            member = simple_type_info.get(k, None)
            if member is None:
                logger.debug("discarding field %r" % k)
                continue

            mo = member.type.Attributes.max_occurs
            value = getattr(inst, k, None)
            if value is None:
                value = []
            elif mo == 1:
                raise Fault('Client.ValidationError',
                        '"%s" member must occur at most %d times' % (k, max_o))

            for v2 in v:
                if (self.validator is self.SOFT_VALIDATION and not
                            member.type.validate_string(member.type, v2)):
                    raise ValidationError(v2)

                if member.type is ByteArray or \
                        getattr(member.type, '_is_clone_of', None) is ByteArray:
                    if isinstance(v2, str) or isinstance(v2, unicode):
                        native_v2 = member.type.from_string(v2)
                    else:
                        native_v2 = v2
                else:
                    native_v2 = member.type.from_string(v2)

                if (self.validator is self.SOFT_VALIDATION and not
                            member.type.validate_native(member.type, native_v2)):
                    raise ValidationError(v2)

                value.append(native_v2)

                # set frequencies of parents.
                if not (member.path[:-1] in frequencies):
                    for i in range(1,len(member.path)):
                        logger.debug("\tset freq %r = 1" % (member.path[:i],))
                        frequencies[member.path[:i]] = 1

                freq = frequencies.get(member.path, 0)
                freq += 1
                frequencies[member.path] = freq
                logger.debug("\tset freq %r = %d" % (member.path, freq))

            if mo == 1:
                value = value[0]

            cinst = inst
            ctype_info = inst_class.get_flat_type_info(inst_class)
            pkey = member.path[0]
            for i in range(len(member.path) - 1):
                pkey = member.path[i]
                if not (ctype_info[pkey].Attributes.max_occurs in (0,1)):
                    raise Exception("HttpRpc deserializer does not support "
                                    "non-primitives with max_occurs > 1")

                ninst = getattr(cinst, pkey, None)
                if ninst is None:
                    ninst = ctype_info[pkey].get_deserialization_instance()
                    setattr(cinst, pkey, ninst)
                cinst = ninst

                ctype_info = ctype_info[pkey]._type_info

            if isinstance(cinst, list):
                cinst.extend(value)
                logger.debug("\tset array   %r(%r) = %r" %
                                                    (member.path, pkey, value))
            else:
                setattr(cinst, member.path[-1], value)
                logger.debug("\tset default %r(%r) = %r" %
                                                    (member.path, pkey, value))

        if self.validator is self.SOFT_VALIDATION:
            sti = simple_type_info.values()
            sti.sort(key=lambda x: (len(x.path), x.path))
            pfrag = None
            for s in sti:
                if len(s.path) > 1 and pfrag != s.path[:-1]:
                    pfrag = s.path[:-1]
                    ctype_info = inst_class.get_flat_type_info(inst_class)
                    for i in range(len(pfrag)):
                        f = pfrag[i]
                        ntype_info = ctype_info[f]

                        min_o = ctype_info[f].Attributes.min_occurs
                        max_o = ctype_info[f].Attributes.max_occurs
                        val = frequencies.get(pfrag[:i+1], 0)
                        if val < min_o:
                            raise Fault('Client.ValidationError',
                                '"%s" member must occur at least %d times'
                                              % ('_'.join(pfrag[:i+1]), min_o))

                        if max_o != 'unbounded' and val > max_o:
                            raise Fault('Client.ValidationError',
                                '"%s" member must occur at most %d times'
                                             % ('_'.join(pfrag[:i+1]), max_o))

                        ctype_info = ntype_info.get_flat_type_info(ntype_info)

                val = frequencies.get(s.path, 0)
                min_o = s.type.Attributes.min_occurs
                max_o = s.type.Attributes.max_occurs
                if val < min_o:
                    raise Fault('Client.ValidationError',
                                '"%s" member must occur at least %d times'
                                                    % ('_'.join(s.path), min_o))
                if max_o != 'unbounded' and val > max_o:
                    raise Fault('Client.ValidationError',
                                '"%s" member must occur at most %d times'
                                                    % ('_'.join(s.path), max_o))

        return inst

    def deserialize(self, ctx, message):
        assert message in (self.REQUEST,)

        self.event_manager.fire_event('before_deserialize', ctx)

        if len(ctx.in_header_doc) > 0:
            ctx.in_header = self.dict_to_object(ctx.in_header_doc,
                                                    ctx.descriptor.in_header)
        else:
            ctx.in_header = [None] * len(
                        ctx.descriptor.in_header.get_flat_type_info(
                                                    ctx.descriptor.in_message))

        if ctx.in_body_doc is not None:
            ctx.in_object = self.dict_to_object(ctx.in_body_doc,
                                                    ctx.descriptor.in_message)
        else:
            ctx.in_object = [None] * len(
                        ctx.descriptor.in_message.get_flat_type_info(
                                                    ctx.descriptor.in_message))

        self.event_manager.fire_event('after_deserialize', ctx)

    def serialize(self, ctx, message):
        assert message in (self.RESPONSE,)

        if ctx.out_error is None:
            result_message_class = ctx.descriptor.out_message

            # assign raw result to its wrapper, result_message
            out_type_info = result_message_class.get_flat_type_info(result_message_class)
            if len(out_type_info) == 1:
                out_class = out_type_info.values()[0]
                if ctx.out_object is None:
                    ctx.out_document = ['']
                else:
                    if hasattr(out_class, 'to_string_iterable'):
                        ctx.out_document = out_class.to_string_iterable(ctx.out_object[0])
                    else:
                        raise ValueError("HttpRpc protocol can only serialize primitives. %r" % out_class)
            else:
                raise ValueError("HttpRpc protocol can only serialize simple return values.")
        else:
            ctx.transport.mime_type = 'text/plain'
            ctx.out_document = ctx.out_error.to_string_iterable(ctx.out_error)

        self.event_manager.fire_event('serialize', ctx)

    def create_out_string(self, ctx, out_string_encoding='utf8'):
        ctx.out_string = ctx.out_document
<|endoftext|>


<commit_before>import numpy as n
import pylab as p

def sync(length, amp=16382):
    s = n.zeros((length,), dtype=n.uint16)
    s[1::2] = 1
    return s * amp

def zigzagify(A):
    """reshape a 2d array to have alternate rows going backward"""
    A[1::2] = A[1::2][:,::-1]

def pairify(a):
    """ return an array that's twice as long, with each value duplicated"""
    return n.asarray((a,a)).T.ravel()

def make_waveforms(X, Y, zigzag=False):

    x = pairify(n.array(X, dtype=n.uint16))
    y = n.array(Y, dtype=n.uint16)

    X,Y = n.meshgrid(x,y)

    assert X.size == Y.size <= 2**17

    if zigzag:
        zigzagify(X)

    return X.ravel(), Y.ravel(), sync(X.size)

# AFG controls

class AFG_channel(object):
    """
    Class encapsulating a AFG channel
    """

    def __init__(self, bus, channel=None):
        self.bus = bus
        self.channel = channel

    def cmd(self, command):
        if self.channel is not None:
            prefix = 'source%d:' % self.channel
        else:
            prefix = ''
        return prefix + command

    def get_mode(self):
        return self.bus.ask(self.cmd('func?'))

    def set_mode(self, val):
        self.bus.write(self.cmd('func ' + val))

    mode = property(get_mode, set_mode)

    def get_freq(self):
        return float(self.bus.ask(self.cmd('freq?')))

    def set_freq(self, val):
        self.bus.write(self.cmd('freq ' + str(val)))

    freq = property(get_freq, set_freq)

from io import BytesIO

def decode_array(data):
    b = BytesIO(data)
    assert b.read(1) == b'#'
    meta_len = int(b.read(1))
    data_len = int(b.read(meta_len))
    result = b.read(data_len)
    array = n.fromstring(result, dtype='>u2')
    return array

def encode_array(array):
    #assert array.dtype == n.dtype('>u2')
    data = array.astype('>u2').tostring()
    data_len = str(len(data))
    meta_len = str(len(data_len))
    assert len(meta_len) == 1
    return b'#' + meta_len + data_len + data

class Arb(object):
    """
    Class encapsulating the AFG's arbitrary edit memory
    """

    YMAX = 16382

    def __init__(self, bus):
        self.bus = bus

    def get_selected(self):
        return 'EMEM' in self.bus.ask('func?').upper()

    def set_selected(self, val):
        if val:
            self.bus.write('func emem')

    selected = property(get_selected, set_selected)

    def initialize(self, npts, val=None):
        """
        Wipe the edit meory register.

        Sets the register to be `npts` points long,
        where npts is between 2 and 131072 points.

        Optionally, provide `val` to initialize
        all points to a given value of y. Y can be a value
        between 0 and 16382. The default is 8191
        """
        self.bus.write('data:define emem,%d' % npts)
        if val is not None:
            self.add_line(1,npts,int(val))

    def add_line(self, x0, x1, y0, y1=None):
        # 1-based indexing because nothing is sacred
        if y1 is None:
            y1 = y0
        self.bus.write('data:line emem,%d,%d,%d,%d' % (x0, y0, x1, y1))

    def get_npts(self):
        return int(self.bus.ask('data:points? emem'))

    def set_npts(self, val):
        self.bus.write('data:points emem,%d' % val)

    npts = property(get_npts, set_npts)

    def get_data(self):
        self.bus.write('data:data? emem')
        data = self.bus.read_raw()
        return decode_array(data)

    def set_data(self, array):
        block = encode_array(array)
        self.bus.write_raw(b'data:data emem,' + block)

    def copy_to(self, register):
        #register is either USER1, USER2, etc.
        self.bus.write('data:copy %s,emem' % register)

    def copy_from(self, register):
        #register is either USER1, USER2, etc.
        self.bus.write('data:copy emem,%s' % register)

# load it

import visa
rm = visa.ResourceManager()

det_afg = rm.get_instrument('GPIB::11')
det_afg.ask('*IDN?')
rf_afg = rm.get_instrument('GPIB::10')
rf_afg.ask('*IDN?')

det_arb = Arb(det_afg)
rf_arb = Arb(rf_afg)

x_ch = AFG_channel(det_afg, 1)
y_ch = AFG_channel(det_afg, 2)
s_ch = AFG_channel(rf_afg)

def make_and_load_waveforms(X, Y, t=.01):
    X,Y,s = make_waveforms(X, Y)

    npts = len(s)     # number of waveform points
    naqs = npts // 2  # number of acquisitions (pixels)

    # load into the two AFGs
    det_arb.npts = npts
    det_arb.set_data(X)
    det_arb.copy_to('user3') # ch1
    det_arb.set_data(Y)
    #det_arb.copy_to('user2') # ch2
    rf_arb.npts = npts
    rf_arb.set_data(s)
    #rf_arb.copy_to('user2') # only channel

    # configure AFGs
    x_ch.set_mode('user3')
    y_ch.set_mode('emem')
    s_ch.set_mode('emem')

    total_time = naqs * t
    for ch in (x_ch, y_ch, s_ch):
        ch.freq = 1. / total_time

    return naqs

from PyDAQmx import *

def make_pulse(duration=.1):
    pulse = Task()
    pulse.CreateCOPulseChanTime(
        "Dev1/ctr1", "",         # physical channel, name to assign
        DAQmx_Val_Seconds,       # units:seconds
        DAQmx_Val_Low,           # idle state: low
        0.00, .0001, duration,  # initial delay, low time, high time
    )
    return pulse

def make_buffered_counter(samps):
    ctr = Task()
    ctr.CreateCICountEdgesChan("Dev1/ctr0", "",
                               DAQmx_Val_Rising,
                               0,  # initial count
                               DAQmx_Val_CountUp)

    # configure sample clock (http://www.ni.com/white-paper/5404/en/)
    ctr.CfgSampClkTiming("PFI34",   # source of the sample clock
                         10000,  # rate of the sample clock.
                                 # not sure about units, lifted from white paper
                         DAQmx_Val_Rising,       # activeEdge
                         DAQmx_Val_FiniteSamps,  # sampleMode
                         samps)                    # sampsPerChanToAcquire

    return ctr

def many_samples(samps, timeout=120.):
    ctr = make_buffered_counter(samps)
    co = make_pulse(.01)  # this will trigger the burst on AFGs
    ctr.StartTask()  # start counting
    co.StartTask()   # send trigger
    # while that's going, allocate some memory
    spcr = c_long()
    data = n.zeros((samps,), dtype=numpy.uint32)
    ctr.WaitUntilTaskDone(timeout)
    # now read out the data
    ctr.ReadCounterU32(samps, # number of samples to read
                       10.0,  # timeout
                       data,  # readArray
                       samps, # arraySizeInSamps
                       spcr,  # sampsPerChanRead
                       None)  # reserved
    ctr.StopTask()
    co.StopTask()
    #print spcr.value
    assert spcr.value == samps
    return data

def decode_image(result, shape, zigzag=False):
    im = n.diff(n.insert(result,0,0))
    im = im.reshape(*shape)
    if zigzag:
        im[1::2] = im[1::2][:,::-1]  #zigzag
    return im

#from time import sleep

def do_everything(X, Y, t=.01):
    naqs = make_and_load_waveforms(X, Y, t)
    #sleep(1)
    outshape = Y.shape + X.shape
    timeout = 5. + naqs * t
    print(timeout)
    result = many_samples(naqs, timeout=timeout)
    return decode_image(result, shape=outshape)

# DO IT

#from wanglib.pylab_extensions.density import density_plot
#p.mpl.rc('image', origin='lower', interpolation='nearest')
#
#X = n.arange(4300, 4700, 2)
#Y = n.arange(4100, 4400, 2)
#
#result = do_everything(X,Y, t=.005)
#density_plot(result, X, Y)

# generators for the GUI

def generate_frames(X, Y, t=.01):
    naqs = make_and_load_waveforms(X, Y, t)
    outshape = Y.shape + X.shape
    timeout = 5. + naqs * t
    while True:
        result = many_samples(naqs, timeout=timeout)
        yield decode_image(result, shape=outshape)/t

def update_result(gen, resultarray):
    """
    plays the same role as yielding_fromiter,
    but expects gen to yield the whole result each time

    """
    for result in gen:
        resultarray[:] = result
        yield

def make_generator_factory(xgalvo, ygalvo):
    # don't do anything with the galvos, they're fake

    def make_data_generator(X, Y, t, vector):
        return update_result(generate_frames(X, Y, t), vector)

    return make_data_generator

def main():
    import wx
    from acquisition import FakeGalvoPixel, AcquisitionWindow

    xgalvo = FakeGalvoPixel("Dev2/ao0", reverse=True)
    ygalvo = FakeGalvoPixel("Dev2/ao1")

    make_data_generator = make_generator_factory(xgalvo,ygalvo)

    # gui app
    app = wx.App(False)
    frame = AcquisitionWindow(None, xgalvo, ygalvo,
                              make_data_generator, nvals=16383)
    frame.Show(True)
    app.MainLoop()

if __name__ == "__main__":
    main()
<commit_msg>Trigger sequence on ctr2<commit_after>import numpy as n
import pylab as p

def sync(length, amp=16382):
    s = n.zeros((length,), dtype=n.uint16)
    s[1::2] = 1
    return s * amp

def zigzagify(A):
    """reshape a 2d array to have alternate rows going backward"""
    A[1::2] = A[1::2][:,::-1]

def pairify(a):
    """ return an array that's twice as long, with each value duplicated"""
    return n.asarray((a,a)).T.ravel()

def make_waveforms(X, Y, zigzag=False):

    x = pairify(n.array(X, dtype=n.uint16))
    y = n.array(Y, dtype=n.uint16)

    X,Y = n.meshgrid(x,y)

    assert X.size == Y.size <= 2**17

    if zigzag:
        zigzagify(X)

    return X.ravel(), Y.ravel(), sync(X.size)

# AFG controls

class AFG_channel(object):
    """
    Class encapsulating a AFG channel
    """

    def __init__(self, bus, channel=None):
        self.bus = bus
        self.channel = channel

    def cmd(self, command):
        if self.channel is not None:
            prefix = 'source%d:' % self.channel
        else:
            prefix = ''
        return prefix + command

    def get_mode(self):
        return self.bus.ask(self.cmd('func?'))

    def set_mode(self, val):
        self.bus.write(self.cmd('func ' + val))

    mode = property(get_mode, set_mode)

    def get_freq(self):
        return float(self.bus.ask(self.cmd('freq?')))

    def set_freq(self, val):
        self.bus.write(self.cmd('freq ' + str(val)))

    freq = property(get_freq, set_freq)

from io import BytesIO

def decode_array(data):
    b = BytesIO(data)
    assert b.read(1) == b'#'
    meta_len = int(b.read(1))
    data_len = int(b.read(meta_len))
    result = b.read(data_len)
    array = n.fromstring(result, dtype='>u2')
    return array

def encode_array(array):
    #assert array.dtype == n.dtype('>u2')
    data = array.astype('>u2').tostring()
    data_len = str(len(data))
    meta_len = str(len(data_len))
    assert len(meta_len) == 1
    return b'#' + meta_len + data_len + data

class Arb(object):
    """
    Class encapsulating the AFG's arbitrary edit memory
    """

    YMAX = 16382

    def __init__(self, bus):
        self.bus = bus

    def get_selected(self):
        return 'EMEM' in self.bus.ask('func?').upper()

    def set_selected(self, val):
        if val:
            self.bus.write('func emem')

    selected = property(get_selected, set_selected)

    def initialize(self, npts, val=None):
        """
        Wipe the edit meory register.

        Sets the register to be `npts` points long,
        where npts is between 2 and 131072 points.

        Optionally, provide `val` to initialize
        all points to a given value of y. Y can be a value
        between 0 and 16382. The default is 8191
        """
        self.bus.write('data:define emem,%d' % npts)
        if val is not None:
            self.add_line(1,npts,int(val))

    def add_line(self, x0, x1, y0, y1=None):
        # 1-based indexing because nothing is sacred
        if y1 is None:
            y1 = y0
        self.bus.write('data:line emem,%d,%d,%d,%d' % (x0, y0, x1, y1))

    def get_npts(self):
        return int(self.bus.ask('data:points? emem'))

    def set_npts(self, val):
        self.bus.write('data:points emem,%d' % val)

    npts = property(get_npts, set_npts)

    def get_data(self):
        self.bus.write('data:data? emem')
        data = self.bus.read_raw()
        return decode_array(data)

    def set_data(self, array):
        block = encode_array(array)
        self.bus.write_raw(b'data:data emem,' + block)

    def copy_to(self, register):
        #register is either USER1, USER2, etc.
        self.bus.write('data:copy %s,emem' % register)

    def copy_from(self, register):
        #register is either USER1, USER2, etc.
        self.bus.write('data:copy emem,%s' % register)

# load it

import visa
rm = visa.ResourceManager()

det_afg = rm.get_instrument('GPIB::11')
det_afg.ask('*IDN?')
rf_afg = rm.get_instrument('GPIB::10')
rf_afg.ask('*IDN?')

det_arb = Arb(det_afg)
rf_arb = Arb(rf_afg)

x_ch = AFG_channel(det_afg, 1)
y_ch = AFG_channel(det_afg, 2)
s_ch = AFG_channel(rf_afg)

def make_and_load_waveforms(X, Y, t=.01):
    X,Y,s = make_waveforms(X, Y)

    npts = len(s)     # number of waveform points
    naqs = npts // 2  # number of acquisitions (pixels)

    # load into the two AFGs
    det_arb.npts = npts
    det_arb.set_data(X)
    det_arb.copy_to('user3') # ch1
    det_arb.set_data(Y)
    #det_arb.copy_to('user2') # ch2
    rf_arb.npts = npts
    rf_arb.set_data(s)
    #rf_arb.copy_to('user2') # only channel

    # configure AFGs
    x_ch.set_mode('user3')
    y_ch.set_mode('emem')
    s_ch.set_mode('emem')

    total_time = naqs * t
    for ch in (x_ch, y_ch, s_ch):
        ch.freq = 1. / total_time

    return naqs

from PyDAQmx import *

def make_pulse(duration=.1):
    pulse = Task()
    pulse.CreateCOPulseChanTime(
        "Dev1/ctr2", "",         # physical channel, name to assign
        DAQmx_Val_Seconds,       # units:seconds
        DAQmx_Val_Low,           # idle state: low
        0.00, .0001, duration,  # initial delay, low time, high time
    )
    return pulse

def make_buffered_counter(samps):
    ctr = Task()
    ctr.CreateCICountEdgesChan("Dev1/ctr0", "",
                               DAQmx_Val_Rising,
                               0,  # initial count
                               DAQmx_Val_CountUp)

    # configure sample clock (http://www.ni.com/white-paper/5404/en/)
    ctr.CfgSampClkTiming("PFI34",   # source of the sample clock
                         10000,  # rate of the sample clock.
                                 # not sure about units, lifted from white paper
                         DAQmx_Val_Rising,       # activeEdge
                         DAQmx_Val_FiniteSamps,  # sampleMode
                         samps)                    # sampsPerChanToAcquire

    return ctr

def many_samples(samps, timeout=120.):
    ctr = make_buffered_counter(samps)
    co = make_pulse(.01)  # this will trigger the burst on AFGs
    ctr.StartTask()  # start counting
    co.StartTask()   # send trigger
    # while that's going, allocate some memory
    spcr = c_long()
    data = n.zeros((samps,), dtype=numpy.uint32)
    ctr.WaitUntilTaskDone(timeout)
    # now read out the data
    ctr.ReadCounterU32(samps, # number of samples to read
                       10.0,  # timeout
                       data,  # readArray
                       samps, # arraySizeInSamps
                       spcr,  # sampsPerChanRead
                       None)  # reserved
    ctr.StopTask()
    co.StopTask()
    #print spcr.value
    assert spcr.value == samps
    return data

def decode_image(result, shape, zigzag=False):
    im = n.diff(n.insert(result,0,0))
    im = im.reshape(*shape)
    if zigzag:
        im[1::2] = im[1::2][:,::-1]  #zigzag
    return im

#from time import sleep

def do_everything(X, Y, t=.01):
    naqs = make_and_load_waveforms(X, Y, t)
    #sleep(1)
    outshape = Y.shape + X.shape
    timeout = 5. + naqs * t
    print(timeout)
    result = many_samples(naqs, timeout=timeout)
    return decode_image(result, shape=outshape)

# DO IT

#from wanglib.pylab_extensions.density import density_plot
#p.mpl.rc('image', origin='lower', interpolation='nearest')
#
#X = n.arange(4300, 4700, 2)
#Y = n.arange(4100, 4400, 2)
#
#result = do_everything(X,Y, t=.005)
#density_plot(result, X, Y)

# generators for the GUI

def generate_frames(X, Y, t=.01):
    naqs = make_and_load_waveforms(X, Y, t)
    outshape = Y.shape + X.shape
    timeout = 5. + naqs * t
    while True:
        result = many_samples(naqs, timeout=timeout)
        yield decode_image(result, shape=outshape)/t

def update_result(gen, resultarray):
    """
    plays the same role as yielding_fromiter,
    but expects gen to yield the whole result each time

    """
    for result in gen:
        resultarray[:] = result
        yield

def make_generator_factory(xgalvo, ygalvo):
    # don't do anything with the galvos, they're fake

    def make_data_generator(X, Y, t, vector):
        return update_result(generate_frames(X, Y, t), vector)

    return make_data_generator

def main():
    import wx
    from acquisition import FakeGalvoPixel, AcquisitionWindow

    xgalvo = FakeGalvoPixel("Dev2/ao0", reverse=True)
    ygalvo = FakeGalvoPixel("Dev2/ao1")

    make_data_generator = make_generator_factory(xgalvo,ygalvo)

    # gui app
    app = wx.App(False)
    frame = AcquisitionWindow(None, xgalvo, ygalvo,
                              make_data_generator, nvals=16383)
    frame.Show(True)
    app.MainLoop()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>
def countup(n):
    if n > 10:
        print "10"
    else:
        print n
        countup(n + 1)

def countup_2(start, stop):
    if start > stop:
        print "Done"
    else:
        print start
        countup_2(start + 1, stop)

def countdown(start, stop):
    if stop < start:
        print "Done"
    else:
        print stop
        countdown(start, stop - 1)

def adder(running_total):
    number = raw_input("Next number: ")
    if number == "":
        print "Running total: " + str(running_total)
    else:
        total = running_total + float(number)
        print "Running total: " + str(total)
        adder(running_total + float(number))

def biggest(number):
    new = raw_input("Next: ")
    if new == "":
        print number
    elif float(new) > number:
        biggest(float(new))
    elif number > float(new):
        biggest(number)

def smallest(number):
    new = raw_input("Next: ")
    if new == "":
        print number
    elif float(new) < number:
        smallest(float(new))
    elif number < float(new):
        smallest(number)

def

def main():
    running_total = 0
    number = 0
    number2 =
    a = biggest(number)
    b = smallest(number2)

main()

<commit_msg>recursion<commit_after>
def countup(n):
    if n > 10:
        print "10"
    else:
        print n
        countup(n + 1)

def countup_2(start, stop):
    if start > stop:
        print "Done"
    else:
        print start
        countup_2(start + 1, stop)

def countdown(start, stop):
    if stop < start:
        print "Done"
    else:
        print stop
        countdown(start, stop - 1)

def adder(running_total):
    number = raw_input("Next number: ")
    if number == "":
        print "Running total: " + str(running_total)
    else:
        total = running_total + float(number)
        print "Running total: " + str(total)
        adder(running_total + float(number))

def biggest(number):
    new = raw_input("Next: ")
    if new == "":
        print number
    elif float(new) > number:
        biggest(float(new))
    elif number > float(new):
        biggest(number)

def smallest(number):
    new = raw_input("Next: ")
    if new == "":
        print number
    elif float(new) < number:
        smallest(float(new))
    elif number < float(new):
        smallest(number)

def pow(x, y):
    if x:
    else:
        print x * (x ** (n-1))

def main():
    running_total = 0
    number = 0
    number2 = float("inf")
    a = biggest(number)
    b = smallest(number2)

main()

<|endoftext|>