<commit_before># SimpleCV Feature library
#
# Tools return basic features in feature sets

#load system libraries
from SimpleCV.base import *

class FeatureSet(list):
    """
    FeatureSet is a class extended from Python's list which has special
    functions so that it is useful for handling feature metadata on an image.

    In general, functions dealing with attributes will return numpy arrays, and
    functions dealing with sorting or filtering will return new FeatureSets.

    Example:

    image = Image("/path/to/image.png")
    lines = image.findLines()  #lines are the feature set
    lines.draw()
    lines.x()
    lines.crop()
    """

    def draw(self, color = (255, 0, 0)):
        """
        Call draw() on each feature in the FeatureSet.
        """
        for f in self:
            f.draw(color)

    def x(self):
        """
        Returns a numpy array of the x (horizontal) coordinate of each feature.
        """
        return np.array([f.x for f in self])

    def y(self):
        """
        Returns a numpy array of the y (vertical) coordinate of each feature.
        """
        return np.array([f.y for f in self])

    def coordinates(self):
        """
        Returns a 2d numpy array of the x,y coordinates of each feature.  This
        is particularly useful if you want to use Scipy's Spatial Distance module
        """
        return np.array([[f.x, f.y] for f in self])

    def area(self):
        """
        Returns a numpy array of the area of each feature in pixels.
        """
        return np.array([f.area() for f in self])

    def sortArea(self):
        """
        Returns a new FeatureSet, with the largest area features first.
        """
        return FeatureSet(sorted(self, key = lambda f: f.area()))

    def distanceFrom(self, point = (-1, -1)):
        """
        Returns a numpy array of the distance each Feature is from a given coordinate.
        Default is the center of the image.
        """
        if (point[0] == -1 or point[1] == -1 and len(self)):
            point = self[0].image.size()

        return spsd.cdist(self.coordinates(), [point])[:,0]

    def sortDistance(self, point = (-1, -1)):
        """
        Returns a sorted FeatureSet with the features closest to a given coordinate first.
        Default is from the center of the image.
        """
        return FeatureSet(sorted(self, key = lambda f: f.distanceFrom(point)))

    def distancePairs(self):
        """
        Returns the square-form of pairwise distances for the featureset.
        The resulting N x N array can be used to quickly look up distances
        between features.
        """
        return spsd.squareform(spsd.pdist(self.coordinates()))

    def angle(self):
        """
        Return a numpy array of the angles (theta) of each feature.
        Note that theta is given in radians, with 0 being horizontal.
        """
        return np.array([f.angle() for f in self])

    def sortAngle(self, theta = 0):
        """
        Return a sorted FeatureSet with the features closest to a given angle first.
        Note that theta is given in radians, with 0 being horizontal.
        """
        return FeatureSet(sorted(self, key = lambda f: abs(f.angle() - theta)))

    def length(self):
        """
        Return a numpy array of the length (longest dimension) of each feature.
        """

        return np.array([f.length() for f in self])

    def sortLength(self):
        """
        Return a sorted FeatureSet with the longest features first.
        """
        return FeatureSet(sorted(self, key = lambda f: f.length()))

    def meanColor(self):
        """
        Return a numpy array of the average color of the area covered by each Feature.
        """
        return np.array([f.meanColor() for f in self])

    def colorDistance(self, color = (0, 0, 0)):
        """
        Return a numpy array of the distance each features average color is from
        a given color tuple (default black, so colorDistance() returns intensity)
        """
        return spsd.cdist(self.meanColor(), [color])[:,0]

    def sortColorDistance(self, color = (0, 0, 0)):
        """
        Return a sorted FeatureSet with features closest to a given color first.
        Default is black, so sortColorDistance() will return darkest to brightest
        """
        return FeatureSet(sorted(self, key = lambda f: f.colorDistance(color)))

    def filter(self, filterarray):
        """
        Return a FeatureSet which is filtered on a numpy boolean array.  This
        will let you use the attribute functions to easily screen Features out
        of return FeatureSets.

        Some examples::

        my_lines.filter(my_lines.length() < 200) # returns all lines < 200px
        my_blobs.filter(my_blobs.area() > 0.9 * my_blobs.length**2) # returns blobs that are nearly square
        my_lines.filter(abs(my_lines.angle()) < numpy.pi / 4) #any lines within 45 degrees of horizontal
        my_corners.filter(my_corners.x() - my_corners.y() > 0) #only return corners in the upper diagonal of the image

        """
        return FeatureSet(list(np.array(self)[filterarray]))

    def width(self):
        """
        Returns a nparray which is the width of all the objects in the FeatureSet
        """
        return np.array([f.width() for f in self])

    def height(self):
        """
        Returns a nparray which is the height of all the objects in the FeatureSet
        """
        return np.array([f.height() for f in self])

    def crop(self):
        """
        Returns a nparray with the cropped features as Imges
        """
        return np.array([f.crop() for f in self])

class Feature(object):
    """
    The Feature object is an abstract class which real features descend from.
    Each feature object has:

    * a draw() method,
    * an image property, referencing the originating Image object
    * x and y coordinates
    * default functions for determining angle, area, meanColor, etc for FeatureSets
    * in the Feature class, these functions assume the feature is 1px
    """
    x = 0.0
    y = 0.0
    image = "" #parent image
    points = []

    def __init__(self, i, at_x, at_y):
        self.x = at_x
        self.y = at_y
        self.image = i

    def coordinates(self):
        """
        Return a an array of x,y
        """
        return np.array([self.x, self.y])

    def draw(self, color = (255.0, 0.0, 0.0)):
        """
        With no dimension information, color the x,y point for the featuer
        """
        self.image[self.x, self.y] = color

    def distanceFrom(self, point = (-1, -1)):
        """
        Given a point (default to center of the image), return the euclidean distance of x,y from this point
        """
        if (point[0] == -1 or point[1] == -1):
            point = np.array(self.image.size()) / 2
        return spsd.euclidean(point, [self.x, self.y])

    def meanColor(self):
        """
          Return the color tuple from x,y
        """
        return self.image[self.x, self.y]

    def colorDistance(self, color = (0, 0, 0)):
        """
          Return the euclidean color distance of the color tuple at x,y from a given color (default black)
        """
        return spsd.euclidean(np.array(color), np.array(self.meanColor()))

    def angle(self):
        """
        Return the angle (theta) of the feature -- default 0 (horizontal)
        """
        return 0

    def length(self):
        """
        Longest dimension of the feature -- for a pixel, 1
        """
        return 1

    def area(self):
        """
        Area covered by the feature -- for a pixel, 1
        """
        return 1

    def width(self):
        """
        Width of the feature -- defaults to 1
        """
        maxX = float("-infinity")
        minX = float("infinity")
        if(len(self.points) <= 0):
            return 1

        for p in self.points:
            if(p[0] > maxX):
                maxX = p[0]
            if(p[0] < minX):
                minX = p[0]

        return maxX - minX

    def height(self):
        """
        Height of the feature -- defaults to 1
        """
        maxY = float("-infinity")
        minY = float("infinity")
        if(len(self.points) <= 0):
            return 1

        for p in self.points:
            if(p[1] > maxY):
                maxY = p[1]
            if(p[1] < minY):
                minY = p[1]

        return maxY - minY

    def crop(self):
        """
        This function returns the largest bounding box for an image.

        Returns Image
        """

        return self.image.crop(self.x, self.y, self.width(), self.height(), centered = True)
<commit_msg>cast FeatureSet.filter() input into numpy<commit_after># SimpleCV Feature library
#
# Tools return basic features in feature sets

#load system libraries
from SimpleCV.base import *

class FeatureSet(list):
    """
    FeatureSet is a class extended from Python's list which has special
    functions so that it is useful for handling feature metadata on an image.

    In general, functions dealing with attributes will return numpy arrays, and
    functions dealing with sorting or filtering will return new FeatureSets.

    Example:

    image = Image("/path/to/image.png")
    lines = image.findLines()  #lines are the feature set
    lines.draw()
    lines.x()
    lines.crop()
    """

    def draw(self, color = (255, 0, 0)):
        """
        Call draw() on each feature in the FeatureSet.
        """
        for f in self:
            f.draw(color)

    def x(self):
        """
        Returns a numpy array of the x (horizontal) coordinate of each feature.
        """
        return np.array([f.x for f in self])

    def y(self):
        """
        Returns a numpy array of the y (vertical) coordinate of each feature.
        """
        return np.array([f.y for f in self])

    def coordinates(self):
        """
        Returns a 2d numpy array of the x,y coordinates of each feature.  This
        is particularly useful if you want to use Scipy's Spatial Distance module
        """
        return np.array([[f.x, f.y] for f in self])

    def area(self):
        """
        Returns a numpy array of the area of each feature in pixels.
        """
        return np.array([f.area() for f in self])

    def sortArea(self):
        """
        Returns a new FeatureSet, with the largest area features first.
        """
        return FeatureSet(sorted(self, key = lambda f: f.area()))

    def distanceFrom(self, point = (-1, -1)):
        """
        Returns a numpy array of the distance each Feature is from a given coordinate.
        Default is the center of the image.
        """
        if (point[0] == -1 or point[1] == -1 and len(self)):
            point = self[0].image.size()

        return spsd.cdist(self.coordinates(), [point])[:,0]

    def sortDistance(self, point = (-1, -1)):
        """
        Returns a sorted FeatureSet with the features closest to a given coordinate first.
        Default is from the center of the image.
        """
        return FeatureSet(sorted(self, key = lambda f: f.distanceFrom(point)))

    def distancePairs(self):
        """
        Returns the square-form of pairwise distances for the featureset.
        The resulting N x N array can be used to quickly look up distances
        between features.
        """
        return spsd.squareform(spsd.pdist(self.coordinates()))

    def angle(self):
        """
        Return a numpy array of the angles (theta) of each feature.
        Note that theta is given in radians, with 0 being horizontal.
        """
        return np.array([f.angle() for f in self])

    def sortAngle(self, theta = 0):
        """
        Return a sorted FeatureSet with the features closest to a given angle first.
        Note that theta is given in radians, with 0 being horizontal.
        """
        return FeatureSet(sorted(self, key = lambda f: abs(f.angle() - theta)))

    def length(self):
        """
        Return a numpy array of the length (longest dimension) of each feature.
        """

        return np.array([f.length() for f in self])

    def sortLength(self):
        """
        Return a sorted FeatureSet with the longest features first.
        """
        return FeatureSet(sorted(self, key = lambda f: f.length()))

    def meanColor(self):
        """
        Return a numpy array of the average color of the area covered by each Feature.
        """
        return np.array([f.meanColor() for f in self])

    def colorDistance(self, color = (0, 0, 0)):
        """
        Return a numpy array of the distance each features average color is from
        a given color tuple (default black, so colorDistance() returns intensity)
        """
        return spsd.cdist(self.meanColor(), [color])[:,0]

    def sortColorDistance(self, color = (0, 0, 0)):
        """
        Return a sorted FeatureSet with features closest to a given color first.
        Default is black, so sortColorDistance() will return darkest to brightest
        """
        return FeatureSet(sorted(self, key = lambda f: f.colorDistance(color)))

    def filter(self, filterarray):
        """
        Return a FeatureSet which is filtered on a numpy boolean array.  This
        will let you use the attribute functions to easily screen Features out
        of return FeatureSets.

        Some examples::

        my_lines.filter(my_lines.length() < 200) # returns all lines < 200px
        my_blobs.filter(my_blobs.area() > 0.9 * my_blobs.length**2) # returns blobs that are nearly square
        my_lines.filter(abs(my_lines.angle()) < numpy.pi / 4) #any lines within 45 degrees of horizontal
        my_corners.filter(my_corners.x() - my_corners.y() > 0) #only return corners in the upper diagonal of the image

        """
        return FeatureSet(list(np.array(self)[np.array(filterarray)]))

    def width(self):
        """
        Returns a nparray which is the width of all the objects in the FeatureSet
        """
        return np.array([f.width() for f in self])

    def height(self):
        """
        Returns a nparray which is the height of all the objects in the FeatureSet
        """
        return np.array([f.height() for f in self])

    def crop(self):
        """
        Returns a nparray with the cropped features as Imges
        """
        return np.array([f.crop() for f in self])

class Feature(object):
    """
    The Feature object is an abstract class which real features descend from.
    Each feature object has:

    * a draw() method,
    * an image property, referencing the originating Image object
    * x and y coordinates
    * default functions for determining angle, area, meanColor, etc for FeatureSets
    * in the Feature class, these functions assume the feature is 1px
    """
    x = 0.0
    y = 0.0
    image = "" #parent image
    points = []

    def __init__(self, i, at_x, at_y):
        self.x = at_x
        self.y = at_y
        self.image = i

    def coordinates(self):
        """
        Return a an array of x,y
        """
        return np.array([self.x, self.y])

    def draw(self, color = (255.0, 0.0, 0.0)):
        """
        With no dimension information, color the x,y point for the featuer
        """
        self.image[self.x, self.y] = color

    def distanceFrom(self, point = (-1, -1)):
        """
        Given a point (default to center of the image), return the euclidean distance of x,y from this point
        """
        if (point[0] == -1 or point[1] == -1):
            point = np.array(self.image.size()) / 2
        return spsd.euclidean(point, [self.x, self.y])

    def meanColor(self):
        """
          Return the color tuple from x,y
        """
        return self.image[self.x, self.y]

    def colorDistance(self, color = (0, 0, 0)):
        """
          Return the euclidean color distance of the color tuple at x,y from a given color (default black)
        """
        return spsd.euclidean(np.array(color), np.array(self.meanColor()))

    def angle(self):
        """
        Return the angle (theta) of the feature -- default 0 (horizontal)
        """
        return 0

    def length(self):
        """
        Longest dimension of the feature -- for a pixel, 1
        """
        return 1

    def area(self):
        """
        Area covered by the feature -- for a pixel, 1
        """
        return 1

    def width(self):
        """
        Width of the feature -- defaults to 1
        """
        maxX = float("-infinity")
        minX = float("infinity")
        if(len(self.points) <= 0):
            return 1

        for p in self.points:
            if(p[0] > maxX):
                maxX = p[0]
            if(p[0] < minX):
                minX = p[0]

        return maxX - minX

    def height(self):
        """
        Height of the feature -- defaults to 1
        """
        maxY = float("-infinity")
        minY = float("infinity")
        if(len(self.points) <= 0):
            return 1

        for p in self.points:
            if(p[1] > maxY):
                maxY = p[1]
            if(p[1] < minY):
                minY = p[1]

        return maxY - minY

    def crop(self):
        """
        This function returns the largest bounding box for an image.

        Returns Image
        """

        return self.image.crop(self.x, self.y, self.width(), self.height(), centered = True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Benchmark cos."""

import timeit

name = "cos"
repeats = 3
iterations = 1000000

def print_version():
    """Print the TAP version."""

    print("TAP version 13")

def print_plan(total):
    """Print the TAP plan.

    # Arguments

    * `total`: total number of tests

    """

    print("1.." + str(total))

def print_summary(total, passing):
    """Print the benchmark summary.

    # Arguments

    * `total`: total number of tests
    * `passing`: number of passing tests

    """

    print("#")
    print_plan(total)
    print("# total " + str(total))
    print("# pass " + str(passing))
    print("#")
    print("# ok")

def print_results(elapsed):
    """Print benchmark results.

    # Arguments

    * `elapsed`: elapsed time (in seconds)

    # Examples

    ``` python
    python> print_results(0.131009101868)
    ```
    """

    rate = iterations / elapsed

    print("  ---")
    print("  iterations: " + str(iterations))
    print("  elapsed: " + str(elapsed))
    print("  rate: " + str(rate))
    print("  ...")

def benchmark():
    """Run the benchmark and print benchmark results."""

    setup = "from math import cos; from random import random;"
    stmt = "y = cos(random())"

    t = timeit.Timer(stmt, setup=setup)

    print_version()

    for i in xrange(3):
        print("# python::" + name)
        elapsed = t.timeit(number=iterations)
        print_results(elapsed)
        print("ok " + str(i+1) + " benchmark finished")

    print_summary(repeats, repeats)

def main():
    """Run the benchmark."""
    benchmark()

if __name__ == "__main__":
    main()
<commit_msg>Simplify printing of TAP summary<commit_after>#!/usr/bin/env python
"""Benchmark cos."""

import timeit

name = "cos"
repeats = 3
iterations = 1000000

def print_version():
    """Print the TAP version."""

    print("TAP version 13")

def print_summary(total, passing):
    """Print the benchmark summary.

    # Arguments

    * `total`: total number of tests
    * `passing`: number of passing tests

    """

    print("#")
    print("1.." + str(total)) # TAP plan
    print("# total " + str(total))
    print("# pass " + str(passing))
    print("#")
    print("# ok")

def print_results(elapsed):
    """Print benchmark results.

    # Arguments

    * `elapsed`: elapsed time (in seconds)

    # Examples

    ``` python
    python> print_results(0.131009101868)
    ```
    """

    rate = iterations / elapsed

    print("  ---")
    print("  iterations: " + str(iterations))
    print("  elapsed: " + str(elapsed))
    print("  rate: " + str(rate))
    print("  ...")

def benchmark():
    """Run the benchmark and print benchmark results."""

    setup = "from math import cos; from random import random;"
    stmt = "y = cos(random())"

    t = timeit.Timer(stmt, setup=setup)

    print_version()

    for i in xrange(3):
        print("# python::" + name)
        elapsed = t.timeit(number=iterations)
        print_results(elapsed)
        print("ok " + str(i+1) + " benchmark finished")

    print_summary(repeats, repeats)

def main():
    """Run the benchmark."""
    benchmark()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>from ctypes import LittleEndianStructure, Union, sizeof, c_ubyte, c_uint16, c_uint32
import calendar
import itertools
import time

from msfat import ATTR_READ_ONLY, ATTR_HIDDEN, ATTR_SYSTEM, ATTR_VOLUME_ID, \
	ATTR_DIRECTORY, ATTR_ARCHIVE, ATTR_LONG_NAME, ATTR_LONG_NAME_MASK, \
	ATTR_RESERVED_MASK

THISDIR_NAME = ".          "
UPDIR_NAME =   "..         "

def allowed_in_short_name(char):
	return char == '\x05' or char >= '\x20' and char not in '"*+,./:;<=>?[\\]|'

def is_valid_short_name(s):
	return s[0] != ' ' and all(allowed_in_short_name(c) for c in s)

def allowed_in_long_name(char):
	return char >= '\x20' and char not in '"*/:<>?\\|'

def is_valid_long_name(s):
	return all(allowed_in_long_name(c) for c in s)

def is_long_name_correctly_padded(s):
	null_pos = s.find(u"\0")
	if null_pos == -1:
		return True
	return all(c == u"\uFFFF" for c in s[null_pos + 1:])

def short_name_checksum(short_name_bytes):
	"""Calculates a single-byte checksum of a short name for use in
	its associated long filename entries. The short name should be an iterable
	of exactly 11 unsigned 8-bit ints, in the space-padded format as is stored
	on disk."""
	if len(short_name_bytes) != 11:
		raise ValueError("short_name_bytes must be exactly 11 bytes long")
	s = 0
	for b in short_name_bytes:
		# 8-bit rotate right and add (the add is 8 bit too!!)
		s = ((s >> 1) + (s << 7) + b) & 0xFF
	return s

def unpack_fat_date(date16):
	day =    date16 & 0x001F
	month = (date16 & 0x01E0) >> 5
	year =  (date16 & 0xFE00) >> 9
	return (year + 1980, month, day)

def unpack_fat_time(time16):
	seconds = (time16 & 0x001F) << 1
	minutes = (time16 & 0x07E0) >> 5
	hours =   (time16 & 0xF800) >> 11
	return (hours, minutes, seconds)

def fat_time_to_unix(date16, time16, add_seconds=0, timezone=None):
	"""Given a FAT date Uint16, a FAT time Uint16, an optional number of seconds
	to add, and a timezone, returns the time expressed as the number of seconds
	since the Unix epoch.

	timezone should be the number of seconds difference from UTC. Positive is
	West of UTC, negative is East. This is the same convention as time.timezone.
	If it is omitted or None, the timestamp is interpreted in the current
	timezone."""

	struct_time = unpack_fat_date(date16) + unpack_fat_time(date16)
	if timezone is None:
		epoch = time.mktime(struct_time)
	else:
		epoch = calendar.timegm(struct_time) - timezone

	return epoch + add_seconds

_LAST_MARKER = 0x00
_FREE_MARKER = 0xE5
_INITIAL_0XE5_PLACEHOLDER = 0x05

class _FATShortDirEntry(LittleEndianStructure):
	_pack_ = 1
	_fields_ = [
		# 8.3 name. restrictions:
		# 0x20 can't be in [0]

		# allowed nowhere:
		# 0x00-0x04, 0x06-0x1F, 0x22, 0x2A-0x2C, 0x2E, 0x2F, 0x3A-0x3F, 0x5B-0x5D, 0x7C

		# 0x2E (.) IS allowed in the thisdir an updir entries, which should be
		# the first two entries in a non-root dir

		# must be unique within a directory
		("DIR_Name",         c_ubyte * 11),

		("DIR_Attr",         c_ubyte), # 0xC0 bits are reserved, set to 0
		("DIR_NTRes",        c_ubyte), # Reserved by Win NT, set to 0
		("DIR_CrtTimeTenth", c_ubyte), # / 100 and add to create time
		("DIR_CrtTime",      c_uint16),
		("DIR_CrtDate",      c_uint16),
		("DIR_LstAccDate",   c_uint16),

		# << 16 and | with first cluster. should be 0 on FAT12/16
		# must be 0 when ATTR_VOLUME_ID set
		("DIR_FstClusHI",    c_uint16),
		("DIR_WrtTime",      c_uint16),
		("DIR_WrtDate",      c_uint16),

		# must be 0 when ATTR_VOLUME_ID set
		# if this is a .. entry and the parent is root, must be 0
		# (along with DIR_FstClusHI)
		("DIR_FstClusLO",    c_uint16),

		# must be 0 when ATTR_DIRECTORY set
		("DIR_FileSize",     c_uint32)
	]

_LAST_LONG_ENTRY = 0x40
_LONG_ENTRY_ORD_MASK = 0x3F

class _FATLongDirEntry(LittleEndianStructure):
	_pack_ = 1
	_fields_ = [
		("LDIR_Ord",         c_ubyte),      # ORed with LAST_LONG_ENTRY
		("LDIR_Name1",       c_ubyte * 10), # UCS2 chars 1-5 of this segment
		("LDIR_Attr",        c_ubyte),      # Must be ATTR_LONG_NAME
		("LDIR_Type",        c_ubyte),      # Must be zero. Non-zero for future expansion (which is safe to say never happened)
		("LDIR_Chksum",      c_ubyte),      # Check byte
		("LDIR_Name2",       c_ubyte * 12), # UCS2 chars 6-11 of this segment
		("LDIR_FstClusLO",   c_uint16),     # Must be 0 for non LFN-aware disk utils
		("LDIR_Name3",       c_ubyte * 4)   # UCS2 chars 12-13 of this segment
	]

class FATDirEntry(Union):
	_anonymous_ = ("short", "long")
	_fields_ = [
		("short", _FATShortDirEntry),
		("long",  _FATLongDirEntry)
	]

	def short_name_checksum(self):
		return short_name_checksum(self.DIR_Name)

	def is_free_entry(self):
		return self.DIR_Name[0] == _FREE_MARKER

	def is_last_in_dir(self):
		return self.DIR_Name[0] == _LAST_MARKER

	def is_long_name_segment(self):
		return (self.DIR_Attr & ATTR_LONG_NAME_MASK) == ATTR_LONG_NAME

	def is_final_long_name_segment(self):
		return (self.LDIR_Ord & _LAST_LONG_ENTRY) != 0

	def long_name_ordinal(self):
		return self.LDIR_Ord & _LONG_ENTRY_ORD_MASK

	def is_read_only(self):
		return self.DIR_Attr & ATTR_READ_ONLY != 0

	def is_hidden(self):
		return self.DIR_Attr & ATTR_HIDDEN != 0

	def is_system(self):
		return self.DIR_Attr & ATTR_SYSTEM != 0

	def is_volume_id(self):
		return self.DIR_Attr & ATTR_VOLUME_ID != 0

	def is_directory(self):
		return self.DIR_Attr & ATTR_DIRECTORY != 0

	def is_archive(self):
		return self.DIR_Attr & ATTR_ARCHIVE != 0

	def start_cluster(self):
		return (
			(self.DIR_FstClusHI << 16) |
			self.DIR_FstClusLO
		)

	def iter_long_name_bytes(self):
		for field in (self.LDIR_Name1, self.LDIR_Name2, self.LDIR_Name3):
			for b in field:
				yield b

	def long_name_segment(self):
		return bytearray(self.iter_long_name_bytes()).decode("utf_16_le")

	def type_string(self):
		if self.is_free_entry():
			return "non-terminal free entry"
		elif self.is_last_in_dir():
			return "terminal free entry"
		elif self.is_long_name_segment():
			return "long filename segment"
		elif self.is_volume_id():
			return "volume id"
		elif self.is_directory():
			return "directory"
		else:
			return "file"

class FATAggregateDirEntry(object):
	def __init__(self, short_entry, long_name=None):
		self.short_entry = short_entry
		self.long_name = long_name

	def name(self):
		if self.long_name is None:
			return self.short_name()
		return self.long_name

	def short_name(self):
		name = bytearray(self.short_entry.DIR_Name)
		if name[0] == _INITIAL_0XE5_PLACEHOLDER:
			name[0] = 0xE5

		prefix = str(name[:8]).rstrip()
		suffix = str(name[8:]).rstrip()

		if suffix:
			return prefix + "." + suffix
		return prefix

	def attr_string(self):
		return "".join(
			(b and ch or "-") for ch, b in [
				("A", self.short_entry.is_archive()),
				("D", self.short_entry.is_directory()),
				("V", self.short_entry.is_volume_id()),
				("S", self.short_entry.is_system()),
				("H", self.short_entry.is_hidden()),
				("R", self.short_entry.is_read_only())
			]
		)

	def create_time(self, timezone=None):
		return fat_time_to_unix(
			self.short_entry.DIR_CrtDate,
			self.short_entry.DIR_CrtTime,
			self.short_entry.DIR_CrtTimeTenth * 0.01,
			timezone
		)

	def access_time(self, timezone=None):
		return fat_time_to_unix(
			self.short_entry.DIR_LstAccDate,
			0, 0, timezone
		)

	def start_cluster(self):
		return self.short_entry.start_cluster()

	def write_time(self, timezone=None):
		return fat_time_to_unix(
			self.short_entry.DIR_WrtDate,
			self.short_entry.DIR_WrtTime,
			0, timezone
		)

	def file_size(self):
		return self.short_entry.DIR_FileSize

	def __str__(self):
		return "{0!r} [{1}] {2}".format(
			self.name(),
			self.attr_string(),
			self.file_size()
		)

def _can_append_long_entry(e, f):
	"""True if f is a FATDirEntry that can follow e. For f to follow e, f and e
	must have the same checksum, and f's ordinal must be one less than e"""
	return (
		f.LDIR_Chksum == e.LDIR_Chksum and
		f.long_name_ordinal() + 1 == e.long_name_ordinal()
	)

def _long_entry_set_belongs_to_short_entry(long_entries, short_entry):
	"""True if a list of long entries is associated with the short_entry.
	The last member of long_entries must be numbered 1, and its checksum value
	must match the calculated checksum of short_entry's name. Only the last
	checksum in long_entries is checked. It's assumed that long_entries is
	accumulated with the use of _can_append_long_entry"""
	return (
		len(long_entries) > 0 and
		long_entries[-1].long_name_ordinal() == 1 and
		long_entries[-1].LDIR_Chksum == short_entry.short_name_checksum()
	)

def assemble_long_entries(long_entries):
	name_bytes = bytearray()
	for e in reversed(long_entries):
		name_bytes.extend(e.iter_long_name_bytes())

	# append a 16-bit null in case the last entry segment is filled exactly
	name_bytes.extend((0, 0))

	name = name_bytes.decode("utf_16_le", "replace")
	return name[:name.find(u"\0")]

def read_dir(stream):
	long_entries = [ ]
	while True:
		bytes = stream.read(sizeof(FATDirEntry))
		if len(bytes) == 0:
			# this is unexpected if we're not intentionally reading beyond end
			break

		entry = FATDirEntry.from_buffer_copy(bytes)
		if entry.is_last_in_dir():
			break
		elif entry.is_free_entry():
			continue
		elif entry.is_long_name_segment():
			if len(long_entries) == 0:
				long_entries.append(entry)
			elif entry.is_final_long_name_segment():
				# begins a new entry
				long_entries = [ entry ]
			elif _can_append_long_entry(long_entries[-1], entry):
				long_entries.append(entry)
			else:
				# it's not the last in a new sequence, it's not part of
				# this one. what is it? dunno
				long_entries = [ ]
		else:
			long_name = None
			# we should have hit long entry 1 (they're 1-based to prevent 0s
			# in the first byte)
			if _long_entry_set_belongs_to_short_entry(long_entries, entry):
				long_name = assemble_long_entries(long_entries)

			yield FATAggregateDirEntry(entry, long_name)
			long_entries = [ ]
<commit_msg>don't count spaces, let the computer do it<commit_after>from ctypes import LittleEndianStructure, Union, sizeof, c_ubyte, c_uint16, c_uint32
import calendar
import itertools
import time

from msfat import ATTR_READ_ONLY, ATTR_HIDDEN, ATTR_SYSTEM, ATTR_VOLUME_ID, \
	ATTR_DIRECTORY, ATTR_ARCHIVE, ATTR_LONG_NAME, ATTR_LONG_NAME_MASK, \
	ATTR_RESERVED_MASK

THISDIR_NAME = "{0:11s}".format(".")
UPDIR_NAME =   "{0:11s}".format("..")

def allowed_in_short_name(char):
	return char == '\x05' or char >= '\x20' and char not in '"*+,./:;<=>?[\\]|'

def is_valid_short_name(s):
	return s[0] != ' ' and all(allowed_in_short_name(c) for c in s)

def allowed_in_long_name(char):
	return char >= '\x20' and char not in '"*/:<>?\\|'

def is_valid_long_name(s):
	return all(allowed_in_long_name(c) for c in s)

def is_long_name_correctly_padded(s):
	null_pos = s.find(u"\0")
	if null_pos == -1:
		return True
	return all(c == u"\uFFFF" for c in s[null_pos + 1:])

def short_name_checksum(short_name_bytes):
	"""Calculates a single-byte checksum of a short name for use in
	its associated long filename entries. The short name should be an iterable
	of exactly 11 unsigned 8-bit ints, in the space-padded format as is stored
	on disk."""
	if len(short_name_bytes) != 11:
		raise ValueError("short_name_bytes must be exactly 11 bytes long")
	s = 0
	for b in short_name_bytes:
		# 8-bit rotate right and add (the add is 8 bit too!!)
		s = ((s >> 1) + (s << 7) + b) & 0xFF
	return s

def unpack_fat_date(date16):
	day =    date16 & 0x001F
	month = (date16 & 0x01E0) >> 5
	year =  (date16 & 0xFE00) >> 9
	return (year + 1980, month, day)

def unpack_fat_time(time16):
	seconds = (time16 & 0x001F) << 1
	minutes = (time16 & 0x07E0) >> 5
	hours =   (time16 & 0xF800) >> 11
	return (hours, minutes, seconds)

def fat_time_to_unix(date16, time16, add_seconds=0, timezone=None):
	"""Given a FAT date Uint16, a FAT time Uint16, an optional number of seconds
	to add, and a timezone, returns the time expressed as the number of seconds
	since the Unix epoch.

	timezone should be the number of seconds difference from UTC. Positive is
	West of UTC, negative is East. This is the same convention as time.timezone.
	If it is omitted or None, the timestamp is interpreted in the current
	timezone."""

	struct_time = unpack_fat_date(date16) + unpack_fat_time(date16)
	if timezone is None:
		epoch = time.mktime(struct_time)
	else:
		epoch = calendar.timegm(struct_time) - timezone

	return epoch + add_seconds

_LAST_MARKER = 0x00
_FREE_MARKER = 0xE5
_INITIAL_0XE5_PLACEHOLDER = 0x05

class _FATShortDirEntry(LittleEndianStructure):
	_pack_ = 1
	_fields_ = [
		# 8.3 name. restrictions:
		# 0x20 can't be in [0]

		# allowed nowhere:
		# 0x00-0x04, 0x06-0x1F, 0x22, 0x2A-0x2C, 0x2E, 0x2F, 0x3A-0x3F, 0x5B-0x5D, 0x7C

		# 0x2E (.) IS allowed in the thisdir an updir entries, which should be
		# the first two entries in a non-root dir

		# must be unique within a directory
		("DIR_Name",         c_ubyte * 11),

		("DIR_Attr",         c_ubyte), # 0xC0 bits are reserved, set to 0
		("DIR_NTRes",        c_ubyte), # Reserved by Win NT, set to 0
		("DIR_CrtTimeTenth", c_ubyte), # / 100 and add to create time
		("DIR_CrtTime",      c_uint16),
		("DIR_CrtDate",      c_uint16),
		("DIR_LstAccDate",   c_uint16),

		# << 16 and | with first cluster. should be 0 on FAT12/16
		# must be 0 when ATTR_VOLUME_ID set
		("DIR_FstClusHI",    c_uint16),
		("DIR_WrtTime",      c_uint16),
		("DIR_WrtDate",      c_uint16),

		# must be 0 when ATTR_VOLUME_ID set
		# if this is a .. entry and the parent is root, must be 0
		# (along with DIR_FstClusHI)
		("DIR_FstClusLO",    c_uint16),

		# must be 0 when ATTR_DIRECTORY set
		("DIR_FileSize",     c_uint32)
	]

_LAST_LONG_ENTRY = 0x40
_LONG_ENTRY_ORD_MASK = 0x3F

class _FATLongDirEntry(LittleEndianStructure):
	_pack_ = 1
	_fields_ = [
		("LDIR_Ord",         c_ubyte),      # ORed with LAST_LONG_ENTRY
		("LDIR_Name1",       c_ubyte * 10), # UCS2 chars 1-5 of this segment
		("LDIR_Attr",        c_ubyte),      # Must be ATTR_LONG_NAME
		("LDIR_Type",        c_ubyte),      # Must be zero. Non-zero for future expansion (which is safe to say never happened)
		("LDIR_Chksum",      c_ubyte),      # Check byte
		("LDIR_Name2",       c_ubyte * 12), # UCS2 chars 6-11 of this segment
		("LDIR_FstClusLO",   c_uint16),     # Must be 0 for non LFN-aware disk utils
		("LDIR_Name3",       c_ubyte * 4)   # UCS2 chars 12-13 of this segment
	]

class FATDirEntry(Union):
	_anonymous_ = ("short", "long")
	_fields_ = [
		("short", _FATShortDirEntry),
		("long",  _FATLongDirEntry)
	]

	def short_name_checksum(self):
		return short_name_checksum(self.DIR_Name)

	def is_free_entry(self):
		return self.DIR_Name[0] == _FREE_MARKER

	def is_last_in_dir(self):
		return self.DIR_Name[0] == _LAST_MARKER

	def is_long_name_segment(self):
		return (self.DIR_Attr & ATTR_LONG_NAME_MASK) == ATTR_LONG_NAME

	def is_final_long_name_segment(self):
		return (self.LDIR_Ord & _LAST_LONG_ENTRY) != 0

	def long_name_ordinal(self):
		return self.LDIR_Ord & _LONG_ENTRY_ORD_MASK

	def is_read_only(self):
		return self.DIR_Attr & ATTR_READ_ONLY != 0

	def is_hidden(self):
		return self.DIR_Attr & ATTR_HIDDEN != 0

	def is_system(self):
		return self.DIR_Attr & ATTR_SYSTEM != 0

	def is_volume_id(self):
		return self.DIR_Attr & ATTR_VOLUME_ID != 0

	def is_directory(self):
		return self.DIR_Attr & ATTR_DIRECTORY != 0

	def is_archive(self):
		return self.DIR_Attr & ATTR_ARCHIVE != 0

	def start_cluster(self):
		return (
			(self.DIR_FstClusHI << 16) |
			self.DIR_FstClusLO
		)

	def iter_long_name_bytes(self):
		for field in (self.LDIR_Name1, self.LDIR_Name2, self.LDIR_Name3):
			for b in field:
				yield b

	def long_name_segment(self):
		return bytearray(self.iter_long_name_bytes()).decode("utf_16_le")

	def type_string(self):
		if self.is_free_entry():
			return "non-terminal free entry"
		elif self.is_last_in_dir():
			return "terminal free entry"
		elif self.is_long_name_segment():
			return "long filename segment"
		elif self.is_volume_id():
			return "volume id"
		elif self.is_directory():
			return "directory"
		else:
			return "file"

class FATAggregateDirEntry(object):
	def __init__(self, short_entry, long_name=None):
		self.short_entry = short_entry
		self.long_name = long_name

	def name(self):
		if self.long_name is None:
			return self.short_name()
		return self.long_name

	def short_name(self):
		name = bytearray(self.short_entry.DIR_Name)
		if name[0] == _INITIAL_0XE5_PLACEHOLDER:
			name[0] = 0xE5

		prefix = str(name[:8]).rstrip()
		suffix = str(name[8:]).rstrip()

		if suffix:
			return prefix + "." + suffix
		return prefix

	def attr_string(self):
		return "".join(
			(b and ch or "-") for ch, b in [
				("A", self.short_entry.is_archive()),
				("D", self.short_entry.is_directory()),
				("V", self.short_entry.is_volume_id()),
				("S", self.short_entry.is_system()),
				("H", self.short_entry.is_hidden()),
				("R", self.short_entry.is_read_only())
			]
		)

	def create_time(self, timezone=None):
		return fat_time_to_unix(
			self.short_entry.DIR_CrtDate,
			self.short_entry.DIR_CrtTime,
			self.short_entry.DIR_CrtTimeTenth * 0.01,
			timezone
		)

	def access_time(self, timezone=None):
		return fat_time_to_unix(
			self.short_entry.DIR_LstAccDate,
			0, 0, timezone
		)

	def start_cluster(self):
		return self.short_entry.start_cluster()

	def write_time(self, timezone=None):
		return fat_time_to_unix(
			self.short_entry.DIR_WrtDate,
			self.short_entry.DIR_WrtTime,
			0, timezone
		)

	def file_size(self):
		return self.short_entry.DIR_FileSize

	def __str__(self):
		return "{0!r} [{1}] {2}".format(
			self.name(),
			self.attr_string(),
			self.file_size()
		)

def _can_append_long_entry(e, f):
	"""True if f is a FATDirEntry that can follow e. For f to follow e, f and e
	must have the same checksum, and f's ordinal must be one less than e"""
	return (
		f.LDIR_Chksum == e.LDIR_Chksum and
		f.long_name_ordinal() + 1 == e.long_name_ordinal()
	)

def _long_entry_set_belongs_to_short_entry(long_entries, short_entry):
	"""True if a list of long entries is associated with the short_entry.
	The last member of long_entries must be numbered 1, and its checksum value
	must match the calculated checksum of short_entry's name. Only the last
	checksum in long_entries is checked. It's assumed that long_entries is
	accumulated with the use of _can_append_long_entry"""
	return (
		len(long_entries) > 0 and
		long_entries[-1].long_name_ordinal() == 1 and
		long_entries[-1].LDIR_Chksum == short_entry.short_name_checksum()
	)

def assemble_long_entries(long_entries):
	name_bytes = bytearray()
	for e in reversed(long_entries):
		name_bytes.extend(e.iter_long_name_bytes())

	# append a 16-bit null in case the last entry segment is filled exactly
	name_bytes.extend((0, 0))

	name = name_bytes.decode("utf_16_le", "replace")
	return name[:name.find(u"\0")]

def read_dir(stream):
	long_entries = [ ]
	while True:
		bytes = stream.read(sizeof(FATDirEntry))
		if len(bytes) == 0:
			# this is unexpected if we're not intentionally reading beyond end
			break

		entry = FATDirEntry.from_buffer_copy(bytes)
		if entry.is_last_in_dir():
			break
		elif entry.is_free_entry():
			continue
		elif entry.is_long_name_segment():
			if len(long_entries) == 0:
				long_entries.append(entry)
			elif entry.is_final_long_name_segment():
				# begins a new entry
				long_entries = [ entry ]
			elif _can_append_long_entry(long_entries[-1], entry):
				long_entries.append(entry)
			else:
				# it's not the last in a new sequence, it's not part of
				# this one. what is it? dunno
				long_entries = [ ]
		else:
			long_name = None
			# we should have hit long entry 1 (they're 1-based to prevent 0s
			# in the first byte)
			if _long_entry_set_belongs_to_short_entry(long_entries, entry):
				long_name = assemble_long_entries(long_entries)

			yield FATAggregateDirEntry(entry, long_name)
			long_entries = [ ]
<|endoftext|>


<commit_before>__author__ = 'ggdhines'
from ouroboros_api import MarkingProject
from agglomerative2 import Agglomerative
from classification import MajorityVote,AllOrNothing
import numpy

class FloatingForests(MarkingProject):
    def __init__(self,date="2015-05-06",pickle_directory="/tmp/"):
        MarkingProject.__init__(self, "plankton", date,experts=["yshish"],pickle_directory=pickle_directory)

    def __annotations_to_markings__(self,annotations):
        """
        This is the main function projects will have to override - given a set of annotations, we need to return the list
        of all markings in that annotation
        """
        markings = []
        for ann in annotations:
            if "finished_at" in ann:
                break
            x_pts=[]
            y_pts=[]
            for point in ["p0","p1","p2","p3"]:
                x_pts.append(float(ann[point][0]))
                y_pts.append(float(ann[point][1]))

            x = numpy.mean(x_pts)
            y = numpy.mean(y_pts)

            try:
                species = ann["species"]
            except KeyError:
                continue
            markings.append(((x,y),species))

        # assert False
        return markings

project = FloatingForests()
project.__top_users__()
agglomerative = Agglomerative(project)
classifier = AllOrNothing(project,agglomerative)

for subject_id in project.gold_standard_subjects:
    agglomerative.__cluster_subject__(subject_id,gold_standard =True)

print classifier.__fit__(project.gold_standard_subjects,gold_standard=True)
# project.__set_gold_standard__(limit=100)
#
# agglomerative = Agglomerative(project)
# majority = MajorityVote(project,agglomerative)
#
# for subject_id in project.gold_standard_subjects:
#     print subject_id
#     project.__store_markings__(subject_id,max_users=20)
#     agglomerative.__cluster_subject__(subject_id)
#
#
# results = majority.__classify__(project.gold_standard_subjects)
# project.__evaluate__(results,agglomerative)
#
<commit_msg>blog update<commit_after>__author__ = 'ggdhines'
from ouroboros_api import MarkingProject
from agglomerative import Agglomerative
from classification import MajorityVote,IBCC
import numpy
import matplotlib.pyplot as plt

class PlanktonPortal(MarkingProject):
    def __init__(self,date="2015-05-08"):
        MarkingProject.__init__(self, "plankton", date,experts=["yshish"])

    def __annotations_to_markings__(self,annotations):
        """
        This is the main function projects will have to override - given a set of annotations, we need to return the list
        of all markings in that annotation
        """
        markings = []
        for ann in annotations:
            if "finished_at" in ann:
                break
            x_pts=[]
            y_pts=[]
            for point in ["p0","p1","p2","p3"]:
                x_pts.append(float(ann[point][0]))
                y_pts.append(float(ann[point][1]))

            x = numpy.mean(x_pts)
            y = numpy.mean(y_pts)

            try:
                species = ann["species"]
            except KeyError:
                continue

            if species == "":
                continue
            markings.append(((x,y),species))

        # assert False
        return markings

project = PlanktonPortal()
# project.__top_users__()
project.__set_gold_standard__(max_subjects=500)

clustering = Agglomerative(project)
classifier = IBCC(project,clustering)

for subject_id in project.gold_standard_subjects:
    print subject_id
    project.__store_markings__(subject_id,max_users=20)
    clustering.__fit__(subject_id)
    clustering.__fit__(subject_id,gold_standard=True)

candidates,results = classifier.__classify__(project.gold_standard_subjects,gold_standard=False)
errors,percentage = project.__evaluate__(candidates,results,clustering)

sorted_percentage = sorted(list(set(percentage)))
c = zip(errors,percentage)

avg_error = [numpy.mean([e for e,p in c if p >= s]) for s in sorted_percentage]
coverage = [sum([1 for e,p in c if p >= s])/float(len(c)) for s in sorted_percentage]
plt.plot(coverage,avg_error)
plt.show()
# project.__set_gold_standard__(limit=100)
#
# agglomerative = Agglomerative(project)
# majority = MajorityVote(project,agglomerative)
#
# for subject_id in project.gold_standard_subjects:
#     print subject_id
#     project.__store_markings__(subject_id,max_users=20)
#     agglomerative.__cluster_subject__(subject_id)
#
#
# results = majority.__classify__(project.gold_standard_subjects)
# project.__evaluate__(results,agglomerative)
#
<|endoftext|>


<commit_before>import copy, os, csv, string, fpformat
import numpy as N
import enthought.traits as traits
import image
from neuroimaging.reference import grid
from neuroimaging.statistics.regression import RegressionOutput
from neuroimaging.statistics import utils

class ImageRegressionOutput(RegressionOutput):
    """
    A class to output things in GLM passes through Image data. It
    uses the image\'s iterator values to output to an image.

    """

    nout = traits.Int(1)
    arraygrid = traits.Any()
    clobber = traits.false

    def __init__(self, grid, outgrid=None, **keywords):
        traits.HasTraits.__init__(self, **keywords)
        self.grid = grid
        if outgrid is None:
            self.outgrid = grid
        else:
            self.outgrid = outgrid

        if self.nout > 1:
            self.grid = grid.DuplicatedGrids([self.grid]*self.nout)
        if self.arraygrid is not None:
            self.img = iter(image.Image(N.zeros(self.arraygrid.shape, N.Float), grid=self.arraygrid))

    def sync_grid(self, img=None):
        """
        Synchronize an image's grid iterator to self.grid's iterator.
        """
        if img is None:
            img = self.img
        img.grid.itertype = self.grid.itertype
        img.grid.labels = self.grid.labels
        img.grid.labelset = self.grid.labelset
        iter(img)

    def __iter__(self):
        return self

    def next(self, data=None):
        value = self.grid.itervalue
        self.img.next(data=data, value=value)

    def extract(self, results):
        return 0.

class TContrastOutput(ImageRegressionOutput):

    contrast = traits.Any() # should really start specifying classes with traits, too
    effect = traits.true
    sd = traits.true
    t = traits.true
    outdir = traits.Str()
    ext = traits.Str('.img')
    subpath = traits.Str('contrasts')

    def __init__(self, grid, contrast, path='.', **keywords):
        ImageRegressionOutput.__init__(self, grid, **keywords)
        self.contrast = contrast
        self.outdir = os.path.join(path, self.subpath, self.contrast.name)
        self.path = path
        self.setup_contrast()
        self.setup_output(time=self.frametimes)

    def setup_contrast(self, **extra):
        self.contrast.getmatrix(**extra)

    def setup_output(self, **extra):

        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)

        outname = os.path.join(self.outdir, 't%s' % self.ext)
        self.timg = image.Image(outname, mode='w', grid=self.outgrid,
                                clobber=self.clobber)

        self.sync_grid(img=self.timg)

        if self.effect:
            outname = os.path.join(self.outdir, 'effect%s' % self.ext)
            self.effectimg = image.Image(outname, mode='w', grid=self.outgrid,
                                         clobber=self.clobber)

            self.sync_grid(img=self.effectimg)
        if self.sd:
            outname = os.path.join(self.outdir, 'sd%s' % self.ext)
            self.sdimg = iter(image.Image(outname, mode='w', grid=self.outgrid,
                                          clobber=self.clobber))
            self.sync_grid(img=self.sdimg)

        outname = os.path.join(self.outdir, 'matrix.csv')
        outfile = file(outname, 'w')
        outfile.write(string.join([fpformat.fix(x,4) for x in self.contrast.matrix], ',') + '\n')
        outfile.close()

        outname = os.path.join(self.outdir, 'matrix.bin')
        outfile = file(outname, 'w')
        self.contrast.matrix = self.contrast.matrix.astype('<f8')
        self.contrast.matrix.tofile(outfile)
        outfile.close()

    def extract(self, results):
        return results.Tcontrast(self.contrast.matrix, sd=self.sd, t=self.t)

    def next(self, data=None):
        value = self.grid.itervalue

        self.timg.next(data=data.t, value=value)
        if self.effect:
            self.effectimg.next(data=data.effect, value=value)
        if self.sd:
            self.sdimg.next(data=data.effect, value=value)

class FContrastOutput(ImageRegressionOutput):

    contrast = traits.Any()
    outdir = traits.Str()
    ext = traits.Str('.img')
    subpath = traits.Str('contrasts')

    def __init__(self, grid, contrast, path='.', **keywords):
        ImageRegressionOutput.__init__(self, grid, **keywords)
        self.contrast = contrast
        self.path = path
        self.outdir = os.path.join(self.path, self.subpath, self.contrast.name)
        self.setup_contrast()
        self.setup_output()

    def setup_contrast(self, **extra):
        self.contrast.getmatrix(**extra)

    def setup_output(self):

        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)

        outname = os.path.join(self.outdir, 'F%s' % self.ext)
        self.img = iter(image.Image(outname, mode='w', grid=self.outgrid,
                                    clobber=self.clobber))
        self.sync_grid()

        outname = os.path.join(self.outdir, 'matrix.csv')
        outfile = file(outname, 'w')
        writer = csv.writer(outfile)
        for row in self.contrast.matrix:
            writer.writerow([fpformat.fix(x, 4) for x in row])
        outfile.close()

        outname = os.path.join(self.outdir, 'matrix.bin')
        outfile = file(outname, 'w')
        self.contrast.matrix = self.contrast.matrix.astype('<f8')
        self.contrast.matrix.tofile(outfile)
        outfile.close()

    def extract(self, results):
        F = results.Fcontrast(self.contrast.matrix).F
        return results.Fcontrast(self.contrast.matrix).F

class ResidOutput(ImageRegressionOutput):

    outdir = traits.Str()
    ext = traits.Str('.img')
    basename = traits.Str('resid')

    def __init__(self, grid, path='.', nout=1, **keywords):
        ImageRegressionOutput.__init__(self, grid, nout=nout, **keywords)
        self.outdir = os.path.join(path)
        self.path = path

        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)
        outname = os.path.join(self.outdir, '%s%s' % (self.basename, self.ext))
        self.img = image.Image(outname, mode='w', grid=self.grid,
                               clobber=self.clobber)
        self.nout = self.grid.shape[0]
        self.sync_grid()

    def extract(self, results):
        return results.resid

    def next(self, data=None):
        value = self.grid.next()
        self.img.next(data=data, value=value)

<commit_msg>moved Image definition from image.image module into image package<commit_after>import copy, os, csv, string, fpformat

import numpy as N
import enthought.traits as traits

from neuroimaging.image import Image
from neuroimaging.reference import grid
from neuroimaging.statistics.regression import RegressionOutput
from neuroimaging.statistics import utils

class ImageRegressionOutput(RegressionOutput):
    """
    A class to output things in GLM passes through Image data. It
    uses the image\'s iterator values to output to an image.

    """

    nout = traits.Int(1)
    arraygrid = traits.Any()
    clobber = traits.false

    def __init__(self, grid, outgrid=None, **keywords):
        traits.HasTraits.__init__(self, **keywords)
        self.grid = grid
        if outgrid is None:
            self.outgrid = grid
        else:
            self.outgrid = outgrid

        if self.nout > 1:
            self.grid = grid.DuplicatedGrids([self.grid]*self.nout)
        if self.arraygrid is not None:
            self.img = iter(Image(N.zeros(self.arraygrid.shape, N.Float), grid=self.arraygrid))

    def sync_grid(self, img=None):
        """
        Synchronize an image's grid iterator to self.grid's iterator.
        """
        if img is None:
            img = self.img
        img.grid.itertype = self.grid.itertype
        img.grid.labels = self.grid.labels
        img.grid.labelset = self.grid.labelset
        iter(img)

    def __iter__(self):
        return self

    def next(self, data=None):
        value = self.grid.itervalue
        self.img.next(data=data, value=value)

    def extract(self, results):
        return 0.

class TContrastOutput(ImageRegressionOutput):

    contrast = traits.Any() # should really start specifying classes with traits, too
    effect = traits.true
    sd = traits.true
    t = traits.true
    outdir = traits.Str()
    ext = traits.Str('.img')
    subpath = traits.Str('contrasts')

    def __init__(self, grid, contrast, path='.', **keywords):
        ImageRegressionOutput.__init__(self, grid, **keywords)
        self.contrast = contrast
        self.outdir = os.path.join(path, self.subpath, self.contrast.name)
        self.path = path
        self.setup_contrast()
        self.setup_output(time=self.frametimes)

    def setup_contrast(self, **extra):
        self.contrast.getmatrix(**extra)

    def setup_output(self, **extra):

        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)

        outname = os.path.join(self.outdir, 't%s' % self.ext)
        self.timg = Image(outname, mode='w', grid=self.outgrid,
                                clobber=self.clobber)

        self.sync_grid(img=self.timg)

        if self.effect:
            outname = os.path.join(self.outdir, 'effect%s' % self.ext)
            self.effectimg = Image(outname, mode='w', grid=self.outgrid,
                                         clobber=self.clobber)

            self.sync_grid(img=self.effectimg)
        if self.sd:
            outname = os.path.join(self.outdir, 'sd%s' % self.ext)
            self.sdimg = iter(Image(outname, mode='w', grid=self.outgrid,
                                          clobber=self.clobber))
            self.sync_grid(img=self.sdimg)

        outname = os.path.join(self.outdir, 'matrix.csv')
        outfile = file(outname, 'w')
        outfile.write(string.join([fpformat.fix(x,4) for x in self.contrast.matrix], ',') + '\n')
        outfile.close()

        outname = os.path.join(self.outdir, 'matrix.bin')
        outfile = file(outname, 'w')
        self.contrast.matrix = self.contrast.matrix.astype('<f8')
        self.contrast.matrix.tofile(outfile)
        outfile.close()

    def extract(self, results):
        return results.Tcontrast(self.contrast.matrix, sd=self.sd, t=self.t)

    def next(self, data=None):
        value = self.grid.itervalue

        self.timg.next(data=data.t, value=value)
        if self.effect:
            self.effectimg.next(data=data.effect, value=value)
        if self.sd:
            self.sdimg.next(data=data.effect, value=value)

class FContrastOutput(ImageRegressionOutput):

    contrast = traits.Any()
    outdir = traits.Str()
    ext = traits.Str('.img')
    subpath = traits.Str('contrasts')

    def __init__(self, grid, contrast, path='.', **keywords):
        ImageRegressionOutput.__init__(self, grid, **keywords)
        self.contrast = contrast
        self.path = path
        self.outdir = os.path.join(self.path, self.subpath, self.contrast.name)
        self.setup_contrast()
        self.setup_output()

    def setup_contrast(self, **extra):
        self.contrast.getmatrix(**extra)

    def setup_output(self):

        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)

        outname = os.path.join(self.outdir, 'F%s' % self.ext)
        self.img = iter(Image(outname, mode='w', grid=self.outgrid,
                                    clobber=self.clobber))
        self.sync_grid()

        outname = os.path.join(self.outdir, 'matrix.csv')
        outfile = file(outname, 'w')
        writer = csv.writer(outfile)
        for row in self.contrast.matrix:
            writer.writerow([fpformat.fix(x, 4) for x in row])
        outfile.close()

        outname = os.path.join(self.outdir, 'matrix.bin')
        outfile = file(outname, 'w')
        self.contrast.matrix = self.contrast.matrix.astype('<f8')
        self.contrast.matrix.tofile(outfile)
        outfile.close()

    def extract(self, results):
        F = results.Fcontrast(self.contrast.matrix).F
        return results.Fcontrast(self.contrast.matrix).F

class ResidOutput(ImageRegressionOutput):

    outdir = traits.Str()
    ext = traits.Str('.img')
    basename = traits.Str('resid')

    def __init__(self, grid, path='.', nout=1, **keywords):
        ImageRegressionOutput.__init__(self, grid, nout=nout, **keywords)
        self.outdir = os.path.join(path)
        self.path = path

        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)
        outname = os.path.join(self.outdir, '%s%s' % (self.basename, self.ext))
        self.img = Image(outname, mode='w', grid=self.grid,
                               clobber=self.clobber)
        self.nout = self.grid.shape[0]
        self.sync_grid()

    def extract(self, results):
        return results.resid

    def next(self, data=None):
        value = self.grid.next()
        self.img.next(data=data, value=value)

<|endoftext|>


<commit_before>from django.shortcuts import render

# Create your views here.

from django.contrib.auth.models import User, Group
from rest_framework import viewsets
from msg.serializers import UserSerializer, GroupSerializer
from msg.models import Msg
from rest_framework.generics import ListCreateAPIView, RetrieveUpdateDestroyAPIView

class UserViewSet(viewsets.ModelViewSet):
    """
    API endpoint that allows users to be viewed or edited.
    """
    queryset = User.objects.all()
    serializer_class = UserSerializer

class GroupViewSet(viewsets.ModelViewSet):
    """
    API endpoint that allows groups to be viewed or edited.
    """
    queryset = Group.objects.all()
    serializer_class = GroupSerializer

from msg.models import Msg
from msg.serializers import MsgSerializer
from django.http import Http404
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status, permissions, renderers
from rest_framework.decorators import api_view

from rest_framework import generics
from datetime import datetime, timedelta

class MsgList(generics.ListCreateAPIView):
    queryset = Msg.objects.all().order_by('-timestamp')
    serializer_class = MsgSerializer
    paginate_by = 10
    paginate_by_param = 'page_size'
    max_paginate_by = 100

class MsgListLast(generics.ListCreateAPIView):
    backtime = datetime.now()-timedelta(minutes=10)
    now = datetime.now()

    queryset = Msg.objects.filter(timestamp__range=[backtime,now])
    serializer_class = MsgSerializer

    """
    paginate_by = 10
    paginate_by_param = 'page_size'
    max_paginate_by = 100
    """

# here you could easily disconnect Update/Destroy functionality
class MsgDetail(generics.RetrieveUpdateDestroyAPIView):
    queryset = Msg.objects.all()
    serializer_class = MsgSerializer

<commit_msg>time overlap<commit_after>from django.shortcuts import render

# Create your views here.

from django.contrib.auth.models import User, Group
from rest_framework import viewsets
from msg.serializers import UserSerializer, GroupSerializer
from msg.models import Msg
from rest_framework.generics import ListCreateAPIView, RetrieveUpdateDestroyAPIView

class UserViewSet(viewsets.ModelViewSet):
    """
    API endpoint that allows users to be viewed or edited.
    """
    queryset = User.objects.all()
    serializer_class = UserSerializer

class GroupViewSet(viewsets.ModelViewSet):
    """
    API endpoint that allows groups to be viewed or edited.
    """
    queryset = Group.objects.all()
    serializer_class = GroupSerializer

from msg.models import Msg
from msg.serializers import MsgSerializer
from django.http import Http404
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework import status, permissions, renderers
from rest_framework.decorators import api_view

from rest_framework import generics
from datetime import datetime, timedelta

class MsgList(generics.ListCreateAPIView):
    queryset = Msg.objects.all().order_by('-timestamp')
    serializer_class = MsgSerializer
    paginate_by = 10
    paginate_by_param = 'page_size'
    max_paginate_by = 100

class MsgListLast(generics.ListCreateAPIView):
    backtime = datetime.now()-timedelta(minutes=10)
    now = datetime.now()+timedelta(minutes=2)

    queryset = Msg.objects.filter(timestamp__range=[backtime,now])
    serializer_class = MsgSerializer

    """
    paginate_by = 10
    paginate_by_param = 'page_size'
    max_paginate_by = 100
    """

# here you could easily disconnect Update/Destroy functionality
class MsgDetail(generics.RetrieveUpdateDestroyAPIView):
    queryset = Msg.objects.all()
    serializer_class = MsgSerializer

<|endoftext|>


<commit_before>"""
Python wrapper to download data through the Bloomberg Open API
Written by Alexandre Almosni
(C) 2014-2015 Alexandre Almosni
Released under Apache 2.0 license. More info at http://www.apache.org/licenses/LICENSE-2.0
"""

import blpapi
import datetime
import pandas
import threading
from abc import ABCMeta, abstractmethod

class BLP():
    """Naive implementation of the Request/Response Paradigm closely matching the Excel API.
    Sharing one session for subsequent requests is faster, however it is not thread-safe, as some events can come faster than others.
    bdp returns a string, bdh returns a pandas DataFrame.
    """

    def __init__(self):
        self.session = blpapi.Session()
        self.session.start()
        self.session.openService('//BLP/refdata')
        self.refDataSvc = self.session.getService('//BLP/refdata')

    def bdp(self, strSecurity='US900123AL40 Govt', strData='PX_LAST', strOverrideField='', strOverrideValue=''):
        request = self.refDataSvc.createRequest('ReferenceDataRequest')
        request.append('securities', strSecurity)
        request.append('fields',strData)
        if strOverrideField != '':
            o = request.getElement('overrides').appendElement()
            o.setElement('fieldId',strOverrideField)
            o.setElement('value',strOverrideValue)
        requestID = self.session.sendRequest(request)
        while True:
            event = self.session.nextEvent()
            if event.eventType() == blpapi.event.Event.RESPONSE:
                break
        try:
            output = blpapi.event.MessageIterator(event).next().getElement('securityData').getValueAsElement(0).getElement('fieldData').getElementAsString(strData)
            if output == '#N/A':
                output = pandas.np.nan
        except:
            print 'error with '+strSecurity+' '+strData
            output = pandas.np.nan
        return output

    def bdh(self, strSecurity='SPX Index', strData='PX_LAST', startdate=datetime.date(2014,1,1), enddate=datetime.date(2014,1,9), periodicity='DAILY'):
        request = self.refDataSvc.createRequest('HistoricalDataRequest')
        request.append('securities', strSecurity)
        if type(strData) == str:
            strData=[strData]
        for strD in strData:
            request.append('fields',strD)
        request.set('startDate',startdate.strftime('%Y%m%d'))
        request.set('endDate',enddate.strftime('%Y%m%d'))
        request.set('periodicitySelection', periodicity);
        requestID = self.session.sendRequest(request)
        while True:
            event = self.session.nextEvent()
            if event.eventType() == blpapi.event.Event.RESPONSE:
                break
        fieldDataArray = blpapi.event.MessageIterator(event).next().getElement('securityData').getElement('fieldData')
        fieldDataList = [fieldDataArray.getValueAsElement(i) for i in range(0,fieldDataArray.numValues())]
        outDates = [x.getElementAsDatetime('date') for x in fieldDataList]
        output = pandas.DataFrame(index=outDates,columns=strData)
        for strD in strData:
            output[strD] = [x.getElementAsFloat(strD) for x in fieldDataList]
        output.replace('#N/A History',pandas.np.nan,inplace=True)
        output.index = output.index.to_datetime()
        return output

    def bdhOHLC(self, strSecurity='SPX Index', startdate=datetime.date(2014,1,1), enddate=datetime.date(2014,1,9), periodicity='DAILY'):
        return self.bdh(strSecurity, ['PX_OPEN','PX_HIGH','PX_LOW','PX_LAST'], startdate, enddate, periodicity)

    def closeSession(self):
        self.session.stop()

class BLPTS():
    """Thread-safe implementation of the Request/Response Paradigm.
    The functions don't return anything but notify observers of results.
    Including startDate as a keyword argument will define a HistoricalDataRequest, otherwise it will be a ReferenceDataRequest.
    HistoricalDataRequest sends observers a pandas DataFrame, whereas ReferenceDataRequest sends a pandas Series.
    Override seems to only work when there's one security, one field, and one override.
    Examples:
    BLPTS(['ESA Index','VGA Index'],['BID','ASK'])
    BLPTSNew('US900123AL40 Govt','YLD_YTM_BID',strOverrideField='PX_BID',strOverrideValue='200')
    BLPTSNew(['SPX Index','SX5E Index','EUR Curncy'],['PX_LAST','VOLUME'],startDate=datetime.datetime(2014,1,1),endDate=datetime.datetime(2015,5,14),periodicity='DAILY')
    """

    def __init__(self,securities=[], fields=[], **kwargs):
        self.session = blpapi.Session()
        self.session.start()
        self.session.openService('//BLP/refdata')
        self.refDataSvc = self.session.getService('//BLP/refdata')
        self.observers = []
        self.kwargs=kwargs
        if len(securities)>0 and len(fields)>0:#also works if securities and fields are a string
            self.fillRequest(securities,fields,**kwargs)

    def fillRequest(self, securities, fields, **kwargs):
        self.kwargs=kwargs
        if 'startDate' in kwargs:
            self.request = self.refDataSvc.createRequest('HistoricalDataRequest')
            self.startDate = kwargs['startDate']
            self.endDate = kwargs['endDate']
            if 'periodicity' in kwargs:
                self.periodicity = kwargs['periodicity']
            else:
                self.periodicity='DAILY'
            self.request.set('startDate', self.startDate.strftime('%Y%m%d'))
            self.request.set('endDate', self.endDate.strftime('%Y%m%d'))
            self.request.set('periodicitySelection', self.periodicity)
        else:
            self.request = self.refDataSvc.createRequest('ReferenceDataRequest')
            if 'strOverrideField' in kwargs:
                o = self.request.getElement('overrides').appendElement()
                o.setElement('fieldId',kwargs['strOverrideField'])
                o.setElement('value',kwargs['strOverrideValue'])
        if type(securities)==str:
            securities=[securities]
        if type(fields)==str:
            fields=[fields]
        self.securities=securities
        self.fields=fields
        for s in securities:
            self.request.append('securities', s)
        for f in fields:
            self.request.append('fields',f)

    def get(self,newSecurities=[], newFields=[], **kwargs):
        if len(newSecurities)>0 or len(newFields)>0:
            self.fillRequest(newSecurities,newFields,**kwargs)
        self.requestID = self.session.sendRequest(self.request)
        while True:
            event = self.session.nextEvent()
            if event.eventType() == blpapi.event.Event.RESPONSE or event.eventType() == blpapi.event.Event.PARTIAL_RESPONSE:
                responseSize=blpapi.event.MessageIterator(event).next().getElement('securityData').numValues()
                for i in range(0,responseSize):
                    try:
                        if 'startDate' in self.kwargs:#HistoricalDataRequest
                            output = blpapi.event.MessageIterator(event).next().getElement('securityData')
                            security = output.getElement('security').getValueAsString()
                            fieldDataArray = output.getElement('fieldData')
                            fieldDataList = [fieldDataArray.getValueAsElement(i) for i in range(0,fieldDataArray.numValues())]
                            dates=map(lambda x:x.getElement('date').getValueAsString(),fieldDataList)
                            outDF=pandas.DataFrame(index=dates,columns=self.fields)
                            outDF.index=outDF.index.to_datetime()
                            for field in self.fields:
                                data=[]
                                for row in fieldDataList:
                                    if row.hasElement(field):
                                        data.append(row.getElement(field).getValueAsFloat())
                                    else:
                                        data.append(pandas.np.nan)
                                outDF[field]=data
                                self.updateObservers(security=security, field=field, data=outDF)
                        else:#ReferenceDataRequest
                            output = blpapi.event.MessageIterator(event).next().getElement('securityData').getValueAsElement(i)
                            outDF=pandas.Series(index=self.fields)
                            for j in range(0,output.getElement('fieldData').numElements()):
                                data=output.getElement('fieldData').getElement(j)
                                security=output.getElement('security').getValueAsString()
                                field=str(data.name())
                                outData=data.getValueAsString()
                                outDF[field]=outData
                                self.updateObservers(security=security, field=field, data=outData)
                    except:
                        #print 'Error with security '+security+' and field '+field
                        self.updateObservers(security=security, field=field, data=pandas.np.nan)
                    self.updateObservers(security=security, field='ALL', data=outDF)
                    #print outDF
            if event.eventType() == blpapi.event.Event.RESPONSE:
                break

    def register(self, observer):
        if not observer in self.observers:
            self.observers.append(observer)

    def unregister(self, observer):
        if observer in self.observers:
            self.observers.remove(observer)

    def unregisterAll(self):
        if self.observers:
            del self.observers[:]

    def updateObservers(self, *args, **kwargs):
        for observer in self.observers:
            observer.update(*args, **kwargs)

    def closeSession(self):
        self.session.stop()

class BLPStream(threading.Thread):
    """The Subscription Paradigm
    The subscribed data will be sitting in self.output and update automatically. Observers will be notified.
    floatInterval is the minimum amount of time before updates - sometimes needs to be set at 0 for things to work properly
    intCorrID is a user defined ID for the request
    It is sometimes safer to ask for each data (for instance BID and ASK) in a separate stream.
    Note that for corporate bonds, a change in the ASK price will still trigger a BID event.
    """

    def __init__(self, strSecurityList=['ESM5 Index','VGM5 Index'], strDataList=['BID','ASK'], floatInterval=0, intCorrIDList=[0,1]):
        threading.Thread.__init__(self)
        self.session = blpapi.Session()
        self.session.start()
        self.session.openService("//BLP/mktdata")
        if type(strSecurityList)== str:
            strSecurityList=[strSecurityList]
        if type(intCorrIDList)== int:
            intCorrIDList=[intCorrIDList]
        if type(strDataList)== str:
            strDataList=[strDataList]
        self.strSecurityList=strSecurityList
        self.strDataList=strDataList
        if len(strSecurityList)!=len(intCorrIDList):
            print 'Number of securities needs to match number of Correlation IDs, overwriting IDs'
            self.intCorrIDList=range(0,len(strSecurityList))
        else:
            self.intCorrIDList=intCorrIDList
        self.subscriptionList=blpapi.subscriptionlist.SubscriptionList()
        for (security, intCorrID) in zip(self.strSecurityList,self.intCorrIDList):
            self.subscriptionList.add(security, self.strDataList, "interval="+str(floatInterval), blpapi.CorrelationId(intCorrID))
        self.output=pandas.DataFrame(index=self.strSecurityList, columns=self.strDataList)
        self.dictCorrID=dict(zip(self.intCorrIDList,self.strSecurityList))
        self.lastUpdateTimeBlmbrg=''#Warning - if you mix live and delayed data you could have non increasing data
        self.lastUpdateTime=datetime.datetime(1900,1,1)
        self.observers = []

    def register(self, observer):
        if not observer in self.observers:
            self.observers.append(observer)

    def unregister(self, observer):
        if observer in self.observers:
            self.observers.remove(observer)

    def unregisterAll(self):
        if self.observers:
            del self.observers[:]

    def updateObservers(self, *args, **kwargs):
        for observer in self.observers:
            observer.update(*args, **kwargs)

    def run(self):
        self.session.subscribe(self.subscriptionList)
        while True:
            event = self.session.nextEvent()
            if event.eventType() == blpapi.event.Event.SUBSCRIPTION_DATA:
                self.handleDataEvent(event)
            else:
                self.handleOtherEvent(event)

    def handleDataEvent(self,event):
        output=blpapi.event.MessageIterator(event).next()
        self.lastUpdateTime=datetime.datetime.now()
        #print output.toString()
        if output.hasElement("EVENT_TIME"):
            self.lastUpdateTimeBlmbrg=output.getElement("EVENT_TIME").toString()
        for i in range(0,len(self.strDataList)):
            field=self.strDataList[i]
            if output.hasElement(field):
                corrID=output.correlationIds()[0].value()
                security=self.dictCorrID[corrID]
                data=output.getElement(field).getValueAsFloat()
                self.output.loc[security,field]=data
                self.updateObservers(time=self.lastUpdateTime, security=security, field=field, corrID=corrID, data=data, bbgTime=self.lastUpdateTimeBlmbrg)

    def handleOtherEvent(self,event):
        #print "Other event: event "+str(event.eventType())
        pass

    def closeSubscription(self):
        self.session.unsubscribe(self.subscriptionList)

class Observer(object):
    __metaclass__ = ABCMeta

    @abstractmethod
    def update(self, *args, **kwargs):
        pass

#############################################################################
def excelEmulationExample():
    ##Examples of the Request/Response Paradigm
    bloomberg=BLP()
    print bloomberg.bdp()
    print ''
    print bloomberg.bdp('US900123AL40 Govt','YLD_YTM_BID','PX_BID','200')
    print ''
    print bloomberg.bdh()
    print ''
    print bloomberg.bdhOHLC()
    bloomberg.closeSession()
#############################################################################
class ObserverExample(Observer):
    def update(self, *args, **kwargs):
        output = kwargs['time'].strftime("%Y-%m-%d %H:%M:%S")+' received '+ kwargs['security'] + ' ' + kwargs['field'] + '=' + str(kwargs['data'])
        output = output + '. CorrID '+str(kwargs['corrID']) + ' bbgTime ' + kwargs['bbgTime']
        print output

def streamPatternExample():
    stream=BLPStream('ESM5 Index',['BID','ASK'],0,1)
    #stream=BLPStream('XS1151974877 CORP',['BID','ASK'],0,1) #Note that for a bond only BID gets updated even if ASK moves.
    obs=ObserverExample()
    stream.register(obs)
    stream.start()
#############################################################################

def main():
    pass

if __name__ == '__main__':
    main()
<commit_msg>Update blpapiwrapper.py<commit_after>"""
Python wrapper to download data through the Bloomberg Open API
Written by Alexandre Almosni
(C) 2014-2015 Alexandre Almosni
Released under Apache 2.0 license. More info at http://www.apache.org/licenses/LICENSE-2.0
"""

import blpapi
import datetime
import pandas
import threading
from abc import ABCMeta, abstractmethod

#This makes successive requests faster
SECURITY_DATA = blpapi.Name("securityData")
SECURITY = blpapi.Name("security")
FIELD_DATA = blpapi.Name("fieldData")
FIELD_EXCEPTIONS = blpapi.Name("fieldExceptions")
FIELD_ID = blpapi.Name("fieldId")
ERROR_INFO = blpapi.Name("errorInfo")

class BLP():
    """Naive implementation of the Request/Response Paradigm closely matching the Excel API.
    Sharing one session for subsequent requests is faster, however it is not thread-safe, as some events can come faster than others.
    bdp returns a string, bdh returns a pandas DataFrame.
    """

    def __init__(self):
        self.session = blpapi.Session()
        self.session.start()
        self.session.openService('//BLP/refdata')
        self.refDataSvc = self.session.getService('//BLP/refdata')

    def bdp(self, strSecurity='US900123AL40 Govt', strData='PX_LAST', strOverrideField='', strOverrideValue=''):
        request = self.refDataSvc.createRequest('ReferenceDataRequest')
        request.append('securities', strSecurity)
        request.append('fields',strData)
        if strOverrideField != '':
            o = request.getElement('overrides').appendElement()
            o.setElement('fieldId',strOverrideField)
            o.setElement('value',strOverrideValue)
        requestID = self.session.sendRequest(request)
        while True:
            event = self.session.nextEvent()
            if event.eventType() == blpapi.event.Event.RESPONSE:
                break
        try:
            output = blpapi.event.MessageIterator(event).next().getElement(SECURITY_DATA).getValueAsElement(0).getElement(FIELD_DATA).getElementAsString(strData)
            if output == '#N/A':
                output = pandas.np.nan
        except:
            print 'error with '+strSecurity+' '+strData
            output = pandas.np.nan
        return output

    def bdh(self, strSecurity='SPX Index', strData='PX_LAST', startdate=datetime.date(2014,1,1), enddate=datetime.date(2014,1,9), periodicity='DAILY'):
        request = self.refDataSvc.createRequest('HistoricalDataRequest')
        request.append('securities', strSecurity)
        if type(strData) == str:
            strData=[strData]
        for strD in strData:
            request.append('fields',strD)
        request.set('startDate',startdate.strftime('%Y%m%d'))
        request.set('endDate',enddate.strftime('%Y%m%d'))
        request.set('periodicitySelection', periodicity);
        requestID = self.session.sendRequest(request)
        while True:
            event = self.session.nextEvent()
            if event.eventType() == blpapi.event.Event.RESPONSE:
                break
        fieldDataArray = blpapi.event.MessageIterator(event).next().getElement(SECURITY_DATA).getElement(FIELD_DATA)
        fieldDataList = [fieldDataArray.getValueAsElement(i) for i in range(0,fieldDataArray.numValues())]
        outDates = [x.getElementAsDatetime('date') for x in fieldDataList]
        output = pandas.DataFrame(index=outDates,columns=strData)
        for strD in strData:
            output[strD] = [x.getElementAsFloat(strD) for x in fieldDataList]
        output.replace('#N/A History',pandas.np.nan,inplace=True)
        output.index = output.index.to_datetime()
        return output

    def bdhOHLC(self, strSecurity='SPX Index', startdate=datetime.date(2014,1,1), enddate=datetime.date(2014,1,9), periodicity='DAILY'):
        return self.bdh(strSecurity, ['PX_OPEN','PX_HIGH','PX_LOW','PX_LAST'], startdate, enddate, periodicity)

    def closeSession(self):
        self.session.stop()

class BLPTS():
    """Thread-safe implementation of the Request/Response Paradigm.
    The functions don't return anything but notify observers of results.
    Including startDate as a keyword argument will define a HistoricalDataRequest, otherwise it will be a ReferenceDataRequest.
    HistoricalDataRequest sends observers a pandas DataFrame, whereas ReferenceDataRequest sends a pandas Series.
    Override seems to only work when there's one security, one field, and one override.
    Examples:
    BLPTS(['ESA Index','VGA Index'],['BID','ASK'])
    BLPTSNew('US900123AL40 Govt','YLD_YTM_BID',strOverrideField='PX_BID',strOverrideValue='200')
    BLPTSNew(['SPX Index','SX5E Index','EUR Curncy'],['PX_LAST','VOLUME'],startDate=datetime.datetime(2014,1,1),endDate=datetime.datetime(2015,5,14),periodicity='DAILY')
    """

    def __init__(self,securities=[], fields=[], **kwargs):
        self.session = blpapi.Session()
        self.session.start()
        self.session.openService('//BLP/refdata')
        self.refDataSvc = self.session.getService('//BLP/refdata')
        self.observers = []
        self.kwargs=kwargs
        if len(securities)>0 and len(fields)>0:#also works if securities and fields are a string
            self.fillRequest(securities,fields,**kwargs)

    def fillRequest(self, securities, fields, **kwargs):
        self.kwargs=kwargs
        if type(securities)==str:
            securities=[securities]
        if type(fields)==str:
            fields=[fields]
        if 'startDate' in kwargs:
            self.request = self.refDataSvc.createRequest('HistoricalDataRequest')
            self.startDate = kwargs['startDate']
            self.endDate = kwargs['endDate']
            if 'periodicity' in kwargs:
                self.periodicity = kwargs['periodicity']
            else:
                self.periodicity='DAILY'
            self.request.set('startDate', self.startDate.strftime('%Y%m%d'))
            self.request.set('endDate', self.endDate.strftime('%Y%m%d'))
            self.request.set('periodicitySelection', self.periodicity)
        else:
            self.request = self.refDataSvc.createRequest('ReferenceDataRequest')
            self.output = pandas.DataFrame(index=securities,columns=fields)
            if 'strOverrideField' in kwargs:
                o = self.request.getElement('overrides').appendElement()
                o.setElement('fieldId',kwargs['strOverrideField'])
                o.setElement('value',kwargs['strOverrideValue'])
        self.securities=securities
        self.fields=fields
        for s in securities:
            self.request.append('securities', s)
        for f in fields:
            self.request.append('fields',f)

    def get(self,newSecurities=[], newFields=[], **kwargs):
        if len(newSecurities)>0 or len(newFields)>0:
            self.fillRequest(newSecurities,newFields,**kwargs)
        self.requestID = self.session.sendRequest(self.request)
        while True:
            event = self.session.nextEvent()
            if event.eventType() == blpapi.event.Event.RESPONSE or event.eventType() == blpapi.event.Event.PARTIAL_RESPONSE:
                responseSize=blpapi.event.MessageIterator(event).next().getElement(SECURITY_DATA).numValues()
                for i in range(0,responseSize):
                    try:
                        if 'startDate' in self.kwargs:#HistoricalDataRequest
                            output = blpapi.event.MessageIterator(event).next().getElement(SECURITY_DATA)
                            security = output.getElement(SECURITY).getValueAsString()
                            fieldDataArray = output.getElement(FIELD_DATA)
                            fieldDataList = [fieldDataArray.getValueAsElement(i) for i in range(0,fieldDataArray.numValues())]
                            dates=map(lambda x:x.getElement('date').getValueAsString(),fieldDataList)
                            outDF=pandas.DataFrame(index=dates,columns=self.fields)
                            outDF.index=outDF.index.to_datetime()
                            for field in self.fields:
                                data=[]
                                for row in fieldDataList:
                                    if row.hasElement(field):
                                        data.append(row.getElement(field).getValueAsFloat())
                                    else:
                                        data.append(pandas.np.nan)
                                outDF[field]=data
                                self.updateObservers(security=security, field=field, data=outDF)
                        else:#ReferenceDataRequest
                            output = blpapi.event.MessageIterator(event).next().getElement(SECURITY_DATA).getValueAsElement(i)
                            outDF=pandas.Series(index=self.fields)
                            for j in range(0,output.getElement(FIELD_DATA).numElements()):
                                data=output.getElement(FIELD_DATA).getElement(j)
                                security=output.getElement(SECURITY).getValueAsString()
                                field=str(data.name())
                                outData=data.getValueAsString()
                                outDF[field]=outData
                                self.updateObservers(security=security, field=field, data=outData)
                            self.output.loc[security]=outDF
                    except:
                        #print 'Error with security '+security+' and field '+field
                        self.updateObservers(security=security, field=field, data=pandas.np.nan)
                    self.updateObservers(security=security, field='ALL', data=outDF)
                    #print outDF
            if event.eventType() == blpapi.event.Event.RESPONSE:
                break

    def register(self, observer):
        if not observer in self.observers:
            self.observers.append(observer)

    def unregister(self, observer):
        if observer in self.observers:
            self.observers.remove(observer)

    def unregisterAll(self):
        if self.observers:
            del self.observers[:]

    def updateObservers(self, *args, **kwargs):
        for observer in self.observers:
            observer.update(*args, **kwargs)

    def closeSession(self):
        self.session.stop()

class BLPStream(threading.Thread):
    """The Subscription Paradigm
    The subscribed data will be sitting in self.output and update automatically. Observers will be notified.
    floatInterval is the minimum amount of time before updates - sometimes needs to be set at 0 for things to work properly
    intCorrID is a user defined ID for the request
    It is sometimes safer to ask for each data (for instance BID and ASK) in a separate stream.
    Note that for corporate bonds, a change in the ASK price will still trigger a BID event.
    """

    def __init__(self, strSecurityList=['ESM5 Index','VGM5 Index'], strDataList=['BID','ASK'], floatInterval=0, intCorrIDList=[0,1]):
        threading.Thread.__init__(self)
        self.session = blpapi.Session()
        self.session.start()
        self.session.openService("//BLP/mktdata")
        if type(strSecurityList)== str:
            strSecurityList=[strSecurityList]
        if type(intCorrIDList)== int:
            intCorrIDList=[intCorrIDList]
        if type(strDataList)== str:
            strDataList=[strDataList]
        self.strSecurityList=strSecurityList
        self.strDataList=strDataList
        if len(strSecurityList)!=len(intCorrIDList):
            print 'Number of securities needs to match number of Correlation IDs, overwriting IDs'
            self.intCorrIDList=range(0,len(strSecurityList))
        else:
            self.intCorrIDList=intCorrIDList
        self.subscriptionList=blpapi.subscriptionlist.SubscriptionList()
        for (security, intCorrID) in zip(self.strSecurityList,self.intCorrIDList):
            self.subscriptionList.add(security, self.strDataList, "interval="+str(floatInterval), blpapi.CorrelationId(intCorrID))
        self.output=pandas.DataFrame(index=self.strSecurityList, columns=self.strDataList)
        self.dictCorrID=dict(zip(self.intCorrIDList,self.strSecurityList))
        self.lastUpdateTimeBlmbrg=''#Warning - if you mix live and delayed data you could have non increasing data
        self.lastUpdateTime=datetime.datetime(1900,1,1)
        self.observers = []

    def register(self, observer):
        if not observer in self.observers:
            self.observers.append(observer)

    def unregister(self, observer):
        if observer in self.observers:
            self.observers.remove(observer)

    def unregisterAll(self):
        if self.observers:
            del self.observers[:]

    def updateObservers(self, *args, **kwargs):
        for observer in self.observers:
            observer.update(*args, **kwargs)

    def run(self):
        self.session.subscribe(self.subscriptionList)
        while True:
            event = self.session.nextEvent()
            if event.eventType() == blpapi.event.Event.SUBSCRIPTION_DATA:
                self.handleDataEvent(event)
            else:
                self.handleOtherEvent(event)

    def handleDataEvent(self,event):
        output=blpapi.event.MessageIterator(event).next()
        self.lastUpdateTime=datetime.datetime.now()
        #print output.toString()
        if output.hasElement("EVENT_TIME"):
            self.lastUpdateTimeBlmbrg=output.getElement("EVENT_TIME").toString()
        for i in range(0,len(self.strDataList)):
            field=self.strDataList[i]
            if output.hasElement(field):
                corrID=output.correlationIds()[0].value()
                security=self.dictCorrID[corrID]
                data=output.getElement(field).getValueAsFloat()
                self.output.loc[security,field]=data
                self.updateObservers(time=self.lastUpdateTime, security=security, field=field, corrID=corrID, data=data, bbgTime=self.lastUpdateTimeBlmbrg)

    def handleOtherEvent(self,event):
        if event.eventType() == blpapi.event.Event.AUTHORIZATION_STATUS:
            output=blpapi.event.MessageIterator(event).next()
            output.toString()
        #print "Other event: event "+str(event.eventType())
        pass

    def closeSubscription(self):
        self.session.unsubscribe(self.subscriptionList)

class Observer(object):
    __metaclass__ = ABCMeta

    @abstractmethod
    def update(self, *args, **kwargs):
        pass

#############################################################################
def excelEmulationExample():
    ##Examples of the Request/Response Paradigm
    bloomberg=BLP()
    print bloomberg.bdp()
    print ''
    print bloomberg.bdp('US900123AL40 Govt','YLD_YTM_BID','PX_BID','200')
    print ''
    print bloomberg.bdh()
    print ''
    print bloomberg.bdhOHLC()
    bloomberg.closeSession()
#############################################################################

class ObserverStreamExample(Observer):
    def update(self, *args, **kwargs):
        output = kwargs['time'].strftime("%Y-%m-%d %H:%M:%S")+' received '+ kwargs['security'] + ' ' + kwargs['field'] + '=' + str(kwargs['data'])
        output = output + '. CorrID '+str(kwargs['corrID']) + ' bbgTime ' + kwargs['bbgTime']
        print output

def streamPatternExample():
    stream=BLPStream('ESM5 Index',['BID','ASK'],0,1)
    #stream=BLPStream('XS1151974877 CORP',['BID','ASK'],0,1) #Note that for a bond only BID gets updated even if ASK moves.
    obs=ObserverStreamExample()
    stream.register(obs)
    stream.start()

class ObserverRequestExample(Observer):
    def update(self, *args, **kwargs):
        if kwargs['field']=='ALL':
            print kwargs['security']
            print kwargs['data']

def BLPTSExample():
    result=BLPTS(['XS0316524130 Corp','US900123CG37 Corp'],['PX_BID','INT_ACC','DAYS_TO_NEXT_COUPON'])
    result.get()
    print result.output
    result.closeSession()

#############################################################################

def main():
    pass

if __name__ == '__main__':
    main()
<|endoftext|>