<commit_before>import codecs
import re
from mafan import simplify

p = re.compile('^Q[0-9]+:.*$')

def join_elements(candidate):
    seen = set()
    for name in candidate:
        name = simplify(''.join(name.split()))
        seen.add(name)
    return seen

def write_file(output_filename, phrase_list):
    out = codecs.open(output_filename, 'w', 'utf-8')
    for name in phrase_list:
        out.write(name + '\n')
    out.close()

def get_entry_version(token):
    dots_count = token.count(':')
    has_qid = (p.match(token) != None)

    if (dots_count == 4) and has_qid:
        return "new"
    elif (dots_count == 2) and not has_qid:
        return "old"
    else:
        return "undef"

def get_document_version(entities_filename):
    entries_count = {'new': 0, 'old': 0, 'undef': 0}

    MIN_ITERS = 100
    for i, line in enumerate(codecs.open(entities_filename, 'r', 'utf-8')):
        tokens = line.strip().split('\t')
        entries_count[get_entry_version(tokens[-1])] += 1

        if i >= MIN_ITERS:
            if entries_count['new'] > entries_count['old'] + entries_count['undef']:
                return "new"
            elif entries_count['old'] > entries_count['new'] + entries_count['undef']:
                return "old"

    return "undef" # probably wont happen

def get_name_new(entity):
    return ':'.join(entity.split(':')[1:-3])

def get_name_old(entity):
    return ':'.join(entity.split(':')[:-2])

def get_support(entity):
    return int(entity.split(':')[-2])

def get_perc(entity):
    return float(entity.split(':')[-1][:-1])

def iter_entries_new(entities_filename):
    for line in codecs.open(entities_filename, 'r', 'utf-8'):
        tokens = line.strip().split('\t')
        yield (tokens[0], [(get_name_new(x), get_support(x), get_perc(x)) for x in tokens[3:]])

def iter_entries_old(entities_filename):
    for line in codecs.open(entities_filename, 'r', 'utf-8'):
        tokens = line.strip().split('\t')
        yield (tokens[0], [(get_name_old(x), get_support(x), get_perc(x)) for x in tokens[2:]])

def get_file_loader(entities_filename):
    version = get_document_version(entities_filename)

    print(f"Version detected: {version}")

    if version == 'new':
        return iter_entries_new
    elif version == 'old':
        return iter_entries_old
    else:
        raise Exception<commit_msg>renamed variable to be more readable<commit_after>import codecs
import re
from mafan import simplify

NEW_VERSION_PREFIX_REGEXP = re.compile('^Q[0-9]+:.*$')

def join_elements(candidate):
    seen = set()
    for name in candidate:
        name = simplify(''.join(name.split()))
        seen.add(name)
    return seen

def write_file(output_filename, phrase_list):
    out = codecs.open(output_filename, 'w', 'utf-8')
    for name in phrase_list:
        out.write(name + '\n')
    out.close()

def get_entry_version(token):
    dots_count = token.count(':')
    has_qid = (NEW_VERSION_PREFIX_REGEXP.match(token) != None)

    if (dots_count == 4) and has_qid:
        return "new"
    elif (dots_count == 2) and not has_qid:
        return "old"
    else:
        return "undef"

def get_document_version(entities_filename):
    entries_count = {'new': 0, 'old': 0, 'undef': 0}

    MIN_ITERS = 100
    for i, line in enumerate(codecs.open(entities_filename, 'r', 'utf-8')):
        tokens = line.strip().split('\t')
        entries_count[get_entry_version(tokens[-1])] += 1

        if i >= MIN_ITERS:
            if entries_count['new'] > entries_count['old'] + entries_count['undef']:
                return "new"
            elif entries_count['old'] > entries_count['new'] + entries_count['undef']:
                return "old"

    return "undef" # probably wont happen

def get_name_new(entity):
    return ':'.join(entity.split(':')[1:-3])

def get_name_old(entity):
    return ':'.join(entity.split(':')[:-2])

def get_support(entity):
    return int(entity.split(':')[-2])

def get_perc(entity):
    return float(entity.split(':')[-1][:-1])

def iter_entries_new(entities_filename):
    for line in codecs.open(entities_filename, 'r', 'utf-8'):
        tokens = line.strip().split('\t')
        yield (tokens[0], [(get_name_new(x), get_support(x), get_perc(x)) for x in tokens[3:]])

def iter_entries_old(entities_filename):
    for line in codecs.open(entities_filename, 'r', 'utf-8'):
        tokens = line.strip().split('\t')
        yield (tokens[0], [(get_name_old(x), get_support(x), get_perc(x)) for x in tokens[2:]])

def get_file_loader(entities_filename):
    version = get_document_version(entities_filename)

    print(f"Version detected: {version}")

    if version == 'new':
        return iter_entries_new
    elif version == 'old':
        return iter_entries_old
    else:
        raise Exception<|endoftext|>


<commit_before>"""
Bounding regions and bounding boxes.

$Id$
"""

__version__='$Revision$'

from numpy import inf

### JABALERT: The aarect information should probably be rewritten in
### matrix notation, not list notation, so that it can be scaled,
### translated, etc. easily.
###
from parameterizedobject import ParameterizedObject, Parameter
from parameterclasses import Number

class BoundingRegion(ParameterizedObject):
    """
    Abstract bounding region class, for any portion of a 2D plane.

    Only subclasses can be instantiated directly.
    """
    __abstract = True

    def __init__(self,**args):
        super(BoundingRegion,self).__init__(**args)
    def contains(self,x,y):
        raise NotImplementedError
    def scale(self,xs,ys):
        raise NotImplementedError
    def translate(self,xoff,yoff):
        l,b,r,t = self.aarect().lbrt()
        self._aarect = AARectangle((l+xoff,b+yoff),(r+xoff,t+yoff))
    def rotate(self,theta):
        raise NotImplementedError
    def aarect(self):
        raise NotImplementedError
    def centroid(self):
        """
        Return the coordinates of the center of this BoundingBox
        """
        return self.aarect().centroid()

class BoundingBox(BoundingRegion):
    """
    A rectangular bounding box defined either by two points forming
    an axis-aligned rectangle (or simply a radius for a square).
    """
    def __str__(self):
        """
        Return BoundingBox(points=((left,bottom),(right,top)))

        Reimplemented here so that 'print' for a BoundingBox
        will display the bounds.
        """
        l,b,r,t = self._aarect.lbrt()
        if r == -l and t == -b and r == t:
            return 'BoundingBox(radius=%s)'%(r)
        else:
            return 'BoundingBox(points=((%s,%s),(%s,%s)))'%(l,b,r,t)

    def __repr__(self):
        return self.__str__()

    def script_repr(self,imports=[],prefix="    "):
        # Generate import statement
        cls = self.__class__.__name__
        mod = self.__module__
        imports.append("from %s import %s" % (mod,cls))
        return self.__str__()

    def __init__(self,**args):
        """
        Create a BoundingBox.

        Either 'radius' or 'points' can be specified for the
        AARectangle.

        If radius is passed in, the BoundingBox will use min_radius
        (which defaults to 0.0) if it's larger than radius - so by
        passing min_radius=1.25/density, a BoundingBox of at least 3x3
        matrix units can be guaranteed.

        If neither radius nor points is passed in, create a default
        AARectangle defined by (-0.5,-0.5),(0.5,0.5).
        """
        # if present, 'radius', 'min_radius', and 'points' are deleted from
        # args before they're passed to the superclass (because they
        # aren't parameters to be set)
        if 'radius' in args:
            radius = args['radius']
            del args['radius']

            if 'min_radius' in args:
                min_radius = args['min_radius']
                del args['min_radius']
            else:
                min_radius = 0.0

            r=max(radius,min_radius)
            self._aarect=AARectangle((-r,-r),(r,r))

        elif 'points' in args:
            self._aarect = AARectangle(*args['points'])
            del args['points']
        else:
            self._aarect = AARectangle((-0.5,-0.5),(0.5,0.5))

        super(BoundingBox,self).__init__(**args)

    def contains(self,x,y):
        """
        Returns true if the given point is contained within the
        bounding box, where all boundaries of the box are
        considered to be inclusive.
        """
        left,bottom,right,top = self.aarect().lbrt()
        return (left <= x <= right) and (bottom <= y <= top)

    def contains_exclusive(self,x,y):
        """
        Return True if the given point is contained within the
        bounding box, where the bottom and right boundaries are
        considered exclusive.
        """
        left,bottom,right,top = self.aarect().lbrt()
        return (left <= x < right) and (bottom < y <= top)

    def containsbb_exclusive(self,x):
        """
        Returns true if the given BoundingBox x is contained within the
        bounding box, where at least one of the boundaries of the box has
        to be exclusive.
        """
        left,bottom,right,top = self.aarect().lbrt()
        leftx,bottomx,rightx,topx = x.aarect().lbrt()
        return (left <= leftx) and (bottom <= bottomx) and (right >= rightx) and (top >= topx) and (not ((left == leftx) and (bottom == bottomx) and (right == rightx) and (top == topx)))

    def containsbb_inclusive(self,x):
        """
        Returns true if the given BoundingBox x is contained within the
        bounding box, including cases of exact match.
        """
        left,bottom,right,top = self.aarect().lbrt()
        leftx,bottomx,rightx,topx = x.aarect().lbrt()
        return (left <= leftx) and (bottom <= bottomx) and (right >= rightx) and (top >= topx)

    def upperexclusive_contains(self,x,y):
        """
        Returns true if the given point is contained within the
        bounding box, where the right and upper boundaries
        are exclusive, and the left and lower boundaries are
        inclusive.  Useful for tiling a plane into non-overlapping
        regions.
        """
        left,bottom,right,top = self.aarect().lbrt()
        return (left <= x < right) and (bottom <= y < top)

    def aarect(self):
        return self._aarect

    def lbrt(self):
        """
        return left,bottom,right,top values for the BoundingBox.
        """
        return self._aarect.lbrt()

class Cartesian2DPoint(Parameter):
    """
    Parameter whose value represents a point in a 2D Cartesian plane.
    """
    ### JABALERT: Should accept and respect a BoundingBox bounds.
    def __set__(self,obj,val):
        try: ## Test that it is a 2-tuple
            (x,y) = val
            super(Cartesian2DPoint,self).__set__(obj,val)
        except:
            raise ValueError("Parameter must be a 2D point (an x,y tuple).")

class BoundingEllipse(BoundingBox):
    """
    Similar to BoundingBox, but the region is the ellipse
    inscribed within the rectangle.
    """
    def __init__(self,**args):
        super(BoundingEllipse,self).__init__(**args)

    def contains(self,x,y):
        left,bottom,right,top = self.aarect().lbrt()
        xr = (right-left)/2.0
        yr = (top-bottom)/2.0
        xc = left + xr
        yc = bottom + yr

        xd = x-xc
        yd = y-yc

        return (xd**2/xr**2 + yd**2/yr**2) <= 1

class BoundingCircle(BoundingRegion):
    """
    A circular BoundingRegion.

    Takes parameters center (a single 2D point (x,y)) and radius (a
    scalar radius).
    """
    radius = Number(0.5,bounds=(0.0,None))
    center = Cartesian2DPoint((0.0,0.0))

    def __init__(self,**args):
        super(BoundingCircle,self).__init__(**args)

    def contains(self,x,y):
        xc,yc = self.center
        xd = x-xc
        yd = y-yc
        return xd*xd + yd*yd <= self.radius*self.radius

    def aarect(self):
        xc,yc = self.center
        r = self.radius
        return AARectangle((xc-r,yc-r),(xc+r,yc+r))

class Unbounded(BoundingRegion):
    def __init__(self,**args):
        super(Unbounded,self).__init__(**args)
    def contains(self,x,y):
        return True
    def scale(self,xs,ys):
        pass
    def translate(self,xoff,yoff):
        pass
    def rotate(self,theta):
        pass
    def aarect(self):
        return AARectangle((-inf,-inf),(inf,inf))
    def centroid(self):
        return 0.0,0.0

### This class is valid only for BoundingBoxes, because it
### only deals with the aarect(), ignoring arbitrarily shaped
### BoundingRegions.  To be a real Intersection(BoundingRegion) class,
### it would need to have a contains() function that computes a
### logical AND of the contains() for each of the regions supplied.
### Scale, rotate, and translate would also need to be applied to the
### individual regions each time.
### Could be simpler to write this as a function, because it just
### ends up with a simple BoundingBox after construction.
class BoundingBoxIntersection(BoundingBox):
    """A BoundingBox initialized as the intersection of the supplied list of BoundingBoxes."""

    def __init__(self,*boxes,**params):
        """
        Given a list of BoundingBoxes, computes a new BoundingBox that is
        the intersection of all of the supplied boxes.
        """
        super(BoundingBoxIntersection,self).__init__(**params)

        bounds = [r.aarect().lbrt() for r in boxes]
        left = max([l for (l,b,r,t) in bounds])
        bottom = max([b for (l,b,r,t) in bounds])
        right = min([r for (l,b,r,t) in bounds])
        top = min([t for (l,b,r,t) in bounds])

        # JABALERT: Why is this one __aarect, and BoundingBox
        # _aarect?  Probably should change this one to _aarect and
        # eliminate aarect(self).
        self.__aarect = AARectangle((left,bottom),(right,top))

    def aarect(self):
        return self.__aarect

# JABALERT: Should probably remove top, bottom, etc. accessor functions,
# and use the slot itself instead.
###################################################
class AARectangle(object):
    """
    Axis-aligned rectangle class.

    Defines the smallest axis-aligned rectangle that encloses a set of
    points.

    Usage:  aar = AARectangle( (x1,y1),(x2,y2), ... , (xN,yN) )
    """
    __slots__ = ['_left','_bottom','_right','_top']
    def __init__(self,*points):
        self._top = max([y for x,y in points])
        self._bottom = min([y for x,y in points])
        self._left = min([x for x,y in points])
        self._right = max([x for x,y in points])

    ## CB: are __gestate__ and __setstate__ needed?
    ## According to http://www.python.org/dev/peps/pep-0307/, pickling
    ## should be taken care of automatically without these methods:
    ##
    ##   For a new-style class that has __slots__ and no instance
    ##   __dict__, the default state [returned by __getstate__] is a
    ##   tuple whose first item is None and whose second item is a
    ##   dictionary mapping slot names to slot values described in the
    ##   previous bullet.
    ##
    ## The default __setstate__ would then handle this state automatically.
    ##
    ## But I think if we remove __getstate__ we get a TypeError saying
    ## that classes with __slots__ must have __getstate__ for pickling.
    ## So I'm not sure. Anyway, I guess this could change in the future,
    ## so this note could remain for information (and also applies to
    ## Parameter -- in fact it should probably be there rather than here).

    # Basic support for pickling
    def __getstate__(self):
        state={}
        try:
            for k in self.__slots__:
                state[k] = getattr(self,k)
        except AttributeError:
            pass
        return state

    def __setstate__(self,state):
        for k,v in state.items():
            setattr(self,k,v)
        self.unpickle()

    def unpickle(self):
        pass

    def top(self):
        """
        Return the y-coordinate of the top of the rectangle.
        """
        return self._top
    def bottom(self):
        """
        Return the y-coordinate of the bottom of the rectangle.
        """
        return self._bottom
    def left(self):
        """
        Return the x-coordinate of the left side of the rectangle.
        """
        return self._left
    def right(self):
        """
        Return the x-coordinate of the right side of the rectangle.
        """
        return self._right
    def lbrt(self):
        """
        Return (left,bottom,right,top) as a tuple
        """
        return (self._left,
                self._bottom,
                self._right,
                self._top)

    def centroid(self):
        """
        Return the centroid of the rectangle.
        """
        left,bottom,right,top = self.lbrt()
        return (right+left)/2.0,(top+bottom)/2.0

    def intersect(self,other):

        l1,b1,r1,t1 = self.lbrt()
        l2,b2,r2,t2 = other.lbrt()

        l = max(l1,l2)
        b = max(b1,b2)
        r = min(r1,r2)
        t = min(t1,t2)

        return AARectangle(points=((l,b),(r,t)))

    def width(self):
        return self._right - self._left
    def height(self):
        return self._top - self._bottom

    def empty(self):
        l,b,r,t = self.lbrt()
        return (r <= l) or (t <= b)

### JABALERT: Should classes like this inherit from something like
### ClassInstanceParameter, which takes a class name and verifies that
### the value is in that class?
###
### Do we also need a BoundingBoxParameter?
class BoundingRegionParameter(Parameter):
    """
    Parameter whose value can be any BoundingRegion instance, enclosing a region in a 2D plane.
    """
    def __init__(self,default=BoundingBox(radius=0.5),**params):
        super(BoundingRegionParameter,self).__init__(default=default,instantiate=True,**params)

    def __set__(self,obj,val):
        if not isinstance(val,BoundingRegion):
            raise ValueError("Parameter must be a BoundingRegion.")
        else:
            super(BoundingRegionParameter,self).__set__(obj,val)

<commit_msg>Simplified AARectangle's pickling; should be no visible change.<commit_after>"""
Bounding regions and bounding boxes.

$Id$
"""

__version__='$Revision$'

from numpy import inf

### JABALERT: The aarect information should probably be rewritten in
### matrix notation, not list notation, so that it can be scaled,
### translated, etc. easily.
###
from parameterizedobject import ParameterizedObject, Parameter
from parameterclasses import Number

class BoundingRegion(ParameterizedObject):
    """
    Abstract bounding region class, for any portion of a 2D plane.

    Only subclasses can be instantiated directly.
    """
    __abstract = True

    def __init__(self,**args):
        super(BoundingRegion,self).__init__(**args)
    def contains(self,x,y):
        raise NotImplementedError
    def scale(self,xs,ys):
        raise NotImplementedError
    def translate(self,xoff,yoff):
        l,b,r,t = self.aarect().lbrt()
        self._aarect = AARectangle((l+xoff,b+yoff),(r+xoff,t+yoff))
    def rotate(self,theta):
        raise NotImplementedError
    def aarect(self):
        raise NotImplementedError
    def centroid(self):
        """
        Return the coordinates of the center of this BoundingBox
        """
        return self.aarect().centroid()

class BoundingBox(BoundingRegion):
    """
    A rectangular bounding box defined either by two points forming
    an axis-aligned rectangle (or simply a radius for a square).
    """
    def __str__(self):
        """
        Return BoundingBox(points=((left,bottom),(right,top)))

        Reimplemented here so that 'print' for a BoundingBox
        will display the bounds.
        """
        l,b,r,t = self._aarect.lbrt()
        if r == -l and t == -b and r == t:
            return 'BoundingBox(radius=%s)'%(r)
        else:
            return 'BoundingBox(points=((%s,%s),(%s,%s)))'%(l,b,r,t)

    def __repr__(self):
        return self.__str__()

    def script_repr(self,imports=[],prefix="    "):
        # Generate import statement
        cls = self.__class__.__name__
        mod = self.__module__
        imports.append("from %s import %s" % (mod,cls))
        return self.__str__()

    def __init__(self,**args):
        """
        Create a BoundingBox.

        Either 'radius' or 'points' can be specified for the
        AARectangle.

        If radius is passed in, the BoundingBox will use min_radius
        (which defaults to 0.0) if it's larger than radius - so by
        passing min_radius=1.25/density, a BoundingBox of at least 3x3
        matrix units can be guaranteed.

        If neither radius nor points is passed in, create a default
        AARectangle defined by (-0.5,-0.5),(0.5,0.5).
        """
        # if present, 'radius', 'min_radius', and 'points' are deleted from
        # args before they're passed to the superclass (because they
        # aren't parameters to be set)
        if 'radius' in args:
            radius = args['radius']
            del args['radius']

            if 'min_radius' in args:
                min_radius = args['min_radius']
                del args['min_radius']
            else:
                min_radius = 0.0

            r=max(radius,min_radius)
            self._aarect=AARectangle((-r,-r),(r,r))

        elif 'points' in args:
            self._aarect = AARectangle(*args['points'])
            del args['points']
        else:
            self._aarect = AARectangle((-0.5,-0.5),(0.5,0.5))

        super(BoundingBox,self).__init__(**args)

    def contains(self,x,y):
        """
        Returns true if the given point is contained within the
        bounding box, where all boundaries of the box are
        considered to be inclusive.
        """
        left,bottom,right,top = self.aarect().lbrt()
        return (left <= x <= right) and (bottom <= y <= top)

    def contains_exclusive(self,x,y):
        """
        Return True if the given point is contained within the
        bounding box, where the bottom and right boundaries are
        considered exclusive.
        """
        left,bottom,right,top = self.aarect().lbrt()
        return (left <= x < right) and (bottom < y <= top)

    def containsbb_exclusive(self,x):
        """
        Returns true if the given BoundingBox x is contained within the
        bounding box, where at least one of the boundaries of the box has
        to be exclusive.
        """
        left,bottom,right,top = self.aarect().lbrt()
        leftx,bottomx,rightx,topx = x.aarect().lbrt()
        return (left <= leftx) and (bottom <= bottomx) and (right >= rightx) and (top >= topx) and (not ((left == leftx) and (bottom == bottomx) and (right == rightx) and (top == topx)))

    def containsbb_inclusive(self,x):
        """
        Returns true if the given BoundingBox x is contained within the
        bounding box, including cases of exact match.
        """
        left,bottom,right,top = self.aarect().lbrt()
        leftx,bottomx,rightx,topx = x.aarect().lbrt()
        return (left <= leftx) and (bottom <= bottomx) and (right >= rightx) and (top >= topx)

    def upperexclusive_contains(self,x,y):
        """
        Returns true if the given point is contained within the
        bounding box, where the right and upper boundaries
        are exclusive, and the left and lower boundaries are
        inclusive.  Useful for tiling a plane into non-overlapping
        regions.
        """
        left,bottom,right,top = self.aarect().lbrt()
        return (left <= x < right) and (bottom <= y < top)

    def aarect(self):
        return self._aarect

    def lbrt(self):
        """
        return left,bottom,right,top values for the BoundingBox.
        """
        return self._aarect.lbrt()

class Cartesian2DPoint(Parameter):
    """
    Parameter whose value represents a point in a 2D Cartesian plane.
    """
    ### JABALERT: Should accept and respect a BoundingBox bounds.
    def __set__(self,obj,val):
        try: ## Test that it is a 2-tuple
            (x,y) = val
            super(Cartesian2DPoint,self).__set__(obj,val)
        except:
            raise ValueError("Parameter must be a 2D point (an x,y tuple).")

class BoundingEllipse(BoundingBox):
    """
    Similar to BoundingBox, but the region is the ellipse
    inscribed within the rectangle.
    """
    def __init__(self,**args):
        super(BoundingEllipse,self).__init__(**args)

    def contains(self,x,y):
        left,bottom,right,top = self.aarect().lbrt()
        xr = (right-left)/2.0
        yr = (top-bottom)/2.0
        xc = left + xr
        yc = bottom + yr

        xd = x-xc
        yd = y-yc

        return (xd**2/xr**2 + yd**2/yr**2) <= 1

class BoundingCircle(BoundingRegion):
    """
    A circular BoundingRegion.

    Takes parameters center (a single 2D point (x,y)) and radius (a
    scalar radius).
    """
    radius = Number(0.5,bounds=(0.0,None))
    center = Cartesian2DPoint((0.0,0.0))

    def __init__(self,**args):
        super(BoundingCircle,self).__init__(**args)

    def contains(self,x,y):
        xc,yc = self.center
        xd = x-xc
        yd = y-yc
        return xd*xd + yd*yd <= self.radius*self.radius

    def aarect(self):
        xc,yc = self.center
        r = self.radius
        return AARectangle((xc-r,yc-r),(xc+r,yc+r))

class Unbounded(BoundingRegion):
    def __init__(self,**args):
        super(Unbounded,self).__init__(**args)
    def contains(self,x,y):
        return True
    def scale(self,xs,ys):
        pass
    def translate(self,xoff,yoff):
        pass
    def rotate(self,theta):
        pass
    def aarect(self):
        return AARectangle((-inf,-inf),(inf,inf))
    def centroid(self):
        return 0.0,0.0

### This class is valid only for BoundingBoxes, because it
### only deals with the aarect(), ignoring arbitrarily shaped
### BoundingRegions.  To be a real Intersection(BoundingRegion) class,
### it would need to have a contains() function that computes a
### logical AND of the contains() for each of the regions supplied.
### Scale, rotate, and translate would also need to be applied to the
### individual regions each time.
### Could be simpler to write this as a function, because it just
### ends up with a simple BoundingBox after construction.
class BoundingBoxIntersection(BoundingBox):
    """A BoundingBox initialized as the intersection of the supplied list of BoundingBoxes."""

    def __init__(self,*boxes,**params):
        """
        Given a list of BoundingBoxes, computes a new BoundingBox that is
        the intersection of all of the supplied boxes.
        """
        super(BoundingBoxIntersection,self).__init__(**params)

        bounds = [r.aarect().lbrt() for r in boxes]
        left = max([l for (l,b,r,t) in bounds])
        bottom = max([b for (l,b,r,t) in bounds])
        right = min([r for (l,b,r,t) in bounds])
        top = min([t for (l,b,r,t) in bounds])

        # JABALERT: Why is this one __aarect, and BoundingBox
        # _aarect?  Probably should change this one to _aarect and
        # eliminate aarect(self).
        self.__aarect = AARectangle((left,bottom),(right,top))

    def aarect(self):
        return self.__aarect

# JABALERT: Should probably remove top, bottom, etc. accessor functions,
# and use the slot itself instead.
###################################################
class AARectangle(object):
    """
    Axis-aligned rectangle class.

    Defines the smallest axis-aligned rectangle that encloses a set of
    points.

    Usage:  aar = AARectangle( (x1,y1),(x2,y2), ... , (xN,yN) )
    """
    __slots__ = ['_left','_bottom','_right','_top']
    def __init__(self,*points):
        self._top = max([y for x,y in points])
        self._bottom = min([y for x,y in points])
        self._left = min([x for x,y in points])
        self._right = max([x for x,y in points])

    # Basic support for pickling
    def __getstate__(self):
        state={}
        for k in self.__slots__:
            state[k] = getattr(self,k)
        return state

    def __setstate__(self,state):
        for k,v in state.items():
            setattr(self,k,v)

    def top(self):
        """
        Return the y-coordinate of the top of the rectangle.
        """
        return self._top
    def bottom(self):
        """
        Return the y-coordinate of the bottom of the rectangle.
        """
        return self._bottom
    def left(self):
        """
        Return the x-coordinate of the left side of the rectangle.
        """
        return self._left
    def right(self):
        """
        Return the x-coordinate of the right side of the rectangle.
        """
        return self._right
    def lbrt(self):
        """
        Return (left,bottom,right,top) as a tuple
        """
        return (self._left,
                self._bottom,
                self._right,
                self._top)

    def centroid(self):
        """
        Return the centroid of the rectangle.
        """
        left,bottom,right,top = self.lbrt()
        return (right+left)/2.0,(top+bottom)/2.0

    def intersect(self,other):

        l1,b1,r1,t1 = self.lbrt()
        l2,b2,r2,t2 = other.lbrt()

        l = max(l1,l2)
        b = max(b1,b2)
        r = min(r1,r2)
        t = min(t1,t2)

        return AARectangle(points=((l,b),(r,t)))

    def width(self):
        return self._right - self._left
    def height(self):
        return self._top - self._bottom

    def empty(self):
        l,b,r,t = self.lbrt()
        return (r <= l) or (t <= b)

### JABALERT: Should classes like this inherit from something like
### ClassInstanceParameter, which takes a class name and verifies that
### the value is in that class?
###
### Do we also need a BoundingBoxParameter?
class BoundingRegionParameter(Parameter):
    """
    Parameter whose value can be any BoundingRegion instance, enclosing a region in a 2D plane.
    """
    def __init__(self,default=BoundingBox(radius=0.5),**params):
        super(BoundingRegionParameter,self).__init__(default=default,instantiate=True,**params)

    def __set__(self,obj,val):
        if not isinstance(val,BoundingRegion):
            raise ValueError("Parameter must be a BoundingRegion.")
        else:
            super(BoundingRegionParameter,self).__set__(obj,val)

<|endoftext|>


<commit_before>"""
Construct PlotGroups, Plots, and the occasional SheetView, for saving to a
file or for a GUI to display.

This class should be the connection between the Simulator or GUI, and
any Plot generation routines.

PlotEngine will create the requested plot group type from a plot_key
(dictionary key name for the SheetView dictionaries on each sheet) or
a plot dictionary definition.

A Plot that contains the old LISSOM style plot information, (in a
different displayed form):

Define PlotGroup HuePreference
    Define Plot HuePref
        Channels:
            Strength   = Null
	    Hue        = HueP   (Predefined SheetView)
	    Confidence = Null
    Define Plot HuePrefAndSel
        Channels:
	    Strength   = HueSel (Predefined SheetView)
	    Hue        = HueP   (Predefined SheetView)
	    Confidence = Null
    Define Plot HueSelect
        Channels:
	    Strength   = HueSel (Predefined SheetView)
	    Hue	       = Null
	    Confidence = Null

in the new syntax would look like:

    hue_template = PlotGroupTemplate(
        [('HuePref', PlotTemplate({'Strength'   : None,
                                   'Hue'        : 'HueP',
                                   'Confidence' : None})),
         ('HuePrefAndSel', PlotTemplate({'Strength'   : 'HueSel',
                                         'Hue'        : 'HueP',
                                         'Confidence' : None})),
         ('HueSelect', PlotTemplate({'Strength'   : 'HueSel',
                                     'Hue'        : None,
                                     'Confidence' : None}))])

NOTE: get_plot_group() function interface could be cleaned up, since
the lambda functions are a bit confusing, and the user should probably
be protected as much as possible while remaining flexible.

$Id$
"""
__version__='$Revision$'

### JABHACKALERT!
###
### The code in this file has not yet been reviewed, and may need
### substantial changes.

from copy import deepcopy
from topo.base.topoobject import TopoObject
from topo.base.utils import flatten, dict_sort
from plot import Plot, SHC,HSV,RGB,COLORMAP
from plotgroup import *
from topo.base.sheet import Sheet
from topo.base.connectionfield import CFSheet
from topo.base.sheetview import SheetView
from topo.base.sheet import Sheet

def sheet_filter(sheet):
    """
    Example sheet filter that can be used to limit which sheets are
    displayed through make_plot_group().  Default filter used
    by the built-in get_plot_group(..) when there is no plot of the
    correct key name in the PlotGroup dictionary.

    SHOULD BE EXPANDED OR REPLACED FOR MORE (ANY?) DEFAULT FUNCTIONALITY
    """
    if sheet == sheet:
        return True

class PlotEngine(TopoObject):
    """
    Stores the main list of plots available to the simulation.  There are
    some default plot types such as 'Activity'.  Use this interface when
    generating plots.
    """

    def __init__(self, simulation, **params):
        """
        Create a new plot engine that is linked to a particular
        simulation.  The link is necessary since the PlotEngine will
        iterate over all the event processors in the simulation,
        requesting Plots from all EPs that are also Sheets.  This
        approach ensures that new Plot objects will show up
        automatically even in previously defined PlotGroups.

        Example calling style:
            s = topo.simulation.Simulation()
            new_plot_engine = plotengine.PlotEngine(s)
        """
        super(PlotEngine,self).__init__(**params)
        self.simulation = simulation
        self.plot_group_dict = {}

    def add_plot_group(self, name, group):
        """
        Add a constructed PlotGroup to the local dictionary for later
        reuse.  User defined plots should be stored here for later use.
        """
        self.plot_group_dict[name] = group

    def get_plot_group(self, name, group_type = 'BasicPlotGroup',
                       filter=sheet_filter, class_type='BasicPlotGroup'):
        """
        Return the PlotGroup registered in self.plot_group_dict with
        the provided key 'name'.  If the name does not exist, then
        generate a PlotGroup using the generic dynamic group creator.
        This default construction allows for certain types of plots to
        be defined automatically, such as 'Activity'.  If a SheetView
        dictionary entry has a key entry with 'name' then it will be part
        of the new plot.
        """
        if filter == None:
            filter = sheet_filter
        elif isinstance(filter,str):     # Allow sheet string name as input.
            target_string = filter
            filter = lambda s: s.name == target_string

        if self.plot_group_dict.has_key(name):
            self.debug(name, "key match in PlotEngine's PlotGroup list")
            requested_plot = self.plot_group_dict[name]
        else:
            # Current filter is sheet_filter(..).
            requested_plot = self.make_plot_group(name,group_type,filter,class_type)
        return requested_plot

     ### JCALERT! This function has been commented out because it does not seem to be of any use
     ### It is called on a line that is commented out in make_plot_group at the end of the file

#     def lambda_flat_dynamic_list(self, name, filter_lam):
#         """
#         lambda_single_view_per_name() expanded so that the a
#         sheet_view dictionary key entry can have a list of SheetViews,
#         and not just one.  Each SheetView in the list will be set with
#         a Plot object.
#         """
#         sheet_list = [each for each in self.simulation.objects(Sheet).values() if filter_lam(each)]
#         self.debug('sheet_list =' + str(sheet_list) + 'name = ' + str(name))

#         view_list = [(each, each.sheet_view(name)) for each in sheet_list
#                      if each is not None]

#         ### A possible list from the Sheet.sheet_view dictionary is
#         ### converted into multiple Plot generation requests.
#         plot_list = []
#         for (sheet,views) in view_list:
#             if not isinstance(views,list):
#                 views = [views]
#             for each in views:
#                 if isinstance(each,SheetView):
#                     plot_list.append(Plot((each,None,None),COLORMAP,None))
#                 else:
#                     plot_list.append(Plot((name,None,None),COLORMAP,sheet))
#         self.debug('plot_list =' + str(plot_list))
#         return plot_list

      ### JCALERT! This function has been commented out because it does not seem to be of any use
      ### It is called on a line that is commented out in make_plot_group at the end of the file

#     def lambda_for_weight_view(self, (w,sheet_target,sheet_x,sheet_y), filter_lam):
#         """
#         To have the UnitViews stored on the Projection source sheet,
#         the weights must be requested from each Projection on the
#         target sheets.
#         """
#         sheet_list = [each for each in self.simulation.objects(Sheet).values()if filter_lam(each) and each.name== sheet_target]

#         projection_list = []
#         for s in sheet_list:
#             if not isinstance(s,CFSheet):
#                 self.warning('Requested weights view from other than CFSheet.')
#             else:
#                 for p in set(flatten(s.in_projections.values())):
#                     key = ('Weights',sheet_target,p.name,sheet_x,sheet_y)
#                     v = p.src.sheet_view(key)
#                     if v: projection_list += [(s,p,each) for each in v if each.projection.name == p.name]
#         projection_list.sort(key=lambda x: x[2].name,reverse=True)

#         plot_list = []
#         for (sheet,projection,views) in projection_list:
#             if not isinstance(views,list):
#                 views = [views]
#             for each in views:
#                 if isinstance(each,SheetView):
#                     plot_list.append(Plot((each,None,None),COLORMAP,None))
#                 else:
#                     key = ('Weights',sheet,projection,sheet_x,sheet_y)
#                     plot_list.append(Plot((key,None,None),COLORMAP,p.src))
#         self.debug('plot_list =' + str(plot_list))
#         return plot_list

    def lambda_for_templates(self, template, filter_lam):
        """
        This assumes that a PlotGroupTemplate named 'template' has
        been passed in that describes a list of plots.  An example is
        given at the top of plotengine.py.

        It is possible for a requested SheetView to in fact have a
        list of Views under that dictionary key instead of a single
        SheetView.  The default behavior will be to only look at the
        first one and dispose of any others if they exist.
        """
        ### JCHACKALERT! It was necessary here to sort the sheet (with the function sort_dict
        ### so that (and by chance that our display is currently following the alphabetical order)
        ### the plot are placed in a logical way from left to right (i.e. LeftRetina, RightRetina, V1, V2
        ### but also Orientation Preference, Orientation Preference&Selectivity, Orientation Selectivity).
        ### This has to be changed so that each classes has a number that roughly determine where we
        ### wants it on the plot (i.e. GeneratorSheet: 5, CFSheet: 50)
        ### Note that the same problem has to be solved for PatternGenerator in order to determine
        ### in which order to display scale, offset...
        ### (When the problem will be solved, it can be sapred to import dict_sort from topo/base/utils.py)
        sheet_list = [each for each in dict_sort(self.simulation.objects(Sheet)) if filter_lam(each)]

        # Loop over all sheets that passed the filter.
        #     Loop over each individual PlotTemplate:
        #         Loop over each channel in the Plot.  Add needed Nones.
        #         Create new Plot and add to list.
        plot_list = []
        for each in sheet_list:
            for (k,pt) in template.plot_templates:
                c = pt.channels
                if 'Strength' in c or 'Hue' in c or 'Confidence' in c:     # SHC
                    plot_list.append(self.make_SHC_plot(k,pt,each))
                elif 'Location' in c and 'Sheet_name' in c:                # Unit Weights
                    if c['Sheet_name'] == each.name:
                        # Multiple Plots can be generated from one Sheet.
                        plot_list = plot_list + self.make_unitweights_plot(k,pt,each)
                elif 'Density' in c and 'Projection_name' in c:            # Projections
                    projection = each.get_in_projection_by_name(c['Projection_name'])
                    if projection:
                        plot_list = plot_list + self.make_projection_plot(k,pt,projection[0].src)
                else:
                    self.warning("Only SHC, Unit Weights, and Projection plots currently implemented.")
        return plot_list

    def make_projection_plot(self, k, pt, s):
        """
        k is the name of the plot template passed in.
        pt is the PlotTemplate being constructed.
        s is the sheet to request the sheet views from.
            It should be the source of the Projection connection.
        """
        c = pt.channels
        key = (k,c['Projection_name'],c['Density'])
        view_list = s.sheet_view(key)
        if not isinstance(view_list,list):
            view_list = [view_list]

        ### A probable list from the Sheet.sheet_view dictionary is
        ### converted into multiple Plot generation requests.
        plot_list = []
        for each in view_list:
            if isinstance(each,SheetView):
                plot_list.append(Plot((each,None,None),COLORMAP,None,pt.channels['Normalize']))
            else:
                plot_list.append(Plot((name,None,None),COLORMAP,s,pt.channels['Normalize']))
        return plot_list

    def make_unitweights_plot(self, k, pt, s):
        sheet_target = pt.channels['Sheet_name']
        (sheet_x,sheet_y) = pt.channels['Location']

        projection_list = []
        if not isinstance(s,CFSheet):
            self.warning('Requested weights view from other than CFSheet.')
        else:
            for p in set(flatten(s.in_projections.values())):
                key = ('Weights',sheet_target,p.name,sheet_x,sheet_y)
                v = p.src.sheet_view(key)
                if v:
                    projection_list += [(s,p,each) for each in v if each.projection.name == p.name]

        # JUDAH HERE IS WHERE WE ADD ADDITIONAL SORTING INFORMATION.
        projection_list.sort(key=lambda x: x[2].name,reverse=True)

        plot_list = []
        for (sheet,projection,views) in projection_list:
            if not isinstance(views,list):
                views = [views]
            for each in views:
                if isinstance(each,SheetView):
                    plot_list.append(Plot((each,None,None),COLORMAP,None,pt.channels['Normalize']))
                else:
                    key = ('Weights',sheet,projection,sheet_x,sheet_y)
                    plot_list.append(Plot((key,None,None),COLORMAP,p.src,pt.channels['Normalize']))
        self.debug('plot_list =' + str(plot_list))
        return plot_list

    def make_SHC_plot(self, k, pt, sheet):
        """
        Create and return a single Plot object matching the passed in
        PlotTemplate (parameter 'pt'), using the Sheet (parameter 'sheet').
        """
        strength = pt.channels.get('Strength',None)
        hue = pt.channels.get('Hue',None)
        confidence = pt.channels.get('Confidence',None)
        n = pt.channels.get('Normalize',False)
        p = Plot((strength,hue,confidence),SHC,sheet,n,name=k)
        return p

    def make_plot_group(self, name='None', group_type='BasicPlotGroup',
                             filter_lam=sheet_filter, class_type='BasicPlotGroup'):
        """
        name : The key to look under in the SheetView dictionaries.
        group_type: 2 Valid inputs:
               1. The string name of the PlotGroup subclass to create.
                  The actual name is passed in instead of a class
                  pointer so the function can be used from the
                  command-line, and also so a full list of class names
                  is not required.
               2. If group_type is a PlotGroupTemplate, then the
                  lambda function to handle templates is called.
        filter_lam: Optional lambda function to filter which sheets to
               ask for SheetViews.
        """
        if isinstance(group_type,PlotGroupTemplate):
            dynamic_list = lambda: self.lambda_for_templates(group_type,filter_lam)
            try:
                exec 'ptr = ' + class_type in globals()
            except Exception, e:
                self.warning('Exception raised:', e)
                self.warning('Invalid PlotGroup subclass: ', group_type)
                return PlotGroup(dynamic_list)
            new_group = ptr(name,filter_lam,dynamic_list)

            # Just copying the pointer.  Not currently sure if we want to
            # promote side-effects by not doing a deepcopy(), but assuming
            # we do for now.  If not, use deepcopy(group_type).
            new_group.template = group_type
            self.add_plot_group(name,new_group)
        else:
            print 'Template was not passed in.  This code deprecated and disabled.'
            import inspect
            print inspect.stack(), '\n'

            ### JCALERT! If you decide to uncomment these lines, you also have to uncomment
            ### the two function that are commented out above

            #if isinstance(name,tuple) and name[0] == 'Weights':
            #    dynamic_list = lambda : self.lambda_for_weight_view(name,filter_lam)
            #else:
            #    dynamic_list = lambda : self.lambda_flat_dynamic_list(name,filter_lam)
            #try:
            #    exec 'ptr = ' + group_type in globals()
            #except Exception, e:
            #    self.warning('Exception:', e)
            #    self.warning('Invalid PlotGroup subclass: ', group_type)
            #    return PlotGroup(dynamic_list)
            #new_group = ptr(name,filter_lam,dynamic_list)

        self.debug('Type of new_group is', type(new_group))
        return new_group
<commit_msg>Added an hack so that the unit weights are plotted in the right order (afferent, lat.Excitatory and Lat.Inhibitory) in the unitweight panel<commit_after>"""
Construct PlotGroups, Plots, and the occasional SheetView, for saving to a
file or for a GUI to display.

This class should be the connection between the Simulator or GUI, and
any Plot generation routines.

PlotEngine will create the requested plot group type from a plot_key
(dictionary key name for the SheetView dictionaries on each sheet) or
a plot dictionary definition.

A Plot that contains the old LISSOM style plot information, (in a
different displayed form):

Define PlotGroup HuePreference
    Define Plot HuePref
        Channels:
            Strength   = Null
	    Hue        = HueP   (Predefined SheetView)
	    Confidence = Null
    Define Plot HuePrefAndSel
        Channels:
	    Strength   = HueSel (Predefined SheetView)
	    Hue        = HueP   (Predefined SheetView)
	    Confidence = Null
    Define Plot HueSelect
        Channels:
	    Strength   = HueSel (Predefined SheetView)
	    Hue	       = Null
	    Confidence = Null

in the new syntax would look like:

    hue_template = PlotGroupTemplate(
        [('HuePref', PlotTemplate({'Strength'   : None,
                                   'Hue'        : 'HueP',
                                   'Confidence' : None})),
         ('HuePrefAndSel', PlotTemplate({'Strength'   : 'HueSel',
                                         'Hue'        : 'HueP',
                                         'Confidence' : None})),
         ('HueSelect', PlotTemplate({'Strength'   : 'HueSel',
                                     'Hue'        : None,
                                     'Confidence' : None}))])

NOTE: get_plot_group() function interface could be cleaned up, since
the lambda functions are a bit confusing, and the user should probably
be protected as much as possible while remaining flexible.

$Id$
"""
__version__='$Revision$'

### JABHACKALERT!
###
### The code in this file has not yet been reviewed, and may need
### substantial changes.

from copy import deepcopy
from topo.base.topoobject import TopoObject
from topo.base.utils import flatten, dict_sort
from plot import Plot, SHC,HSV,RGB,COLORMAP
from plotgroup import *
from topo.base.sheet import Sheet
from topo.base.connectionfield import CFSheet
from topo.base.sheetview import SheetView
from topo.base.sheet import Sheet

def sheet_filter(sheet):
    """
    Example sheet filter that can be used to limit which sheets are
    displayed through make_plot_group().  Default filter used
    by the built-in get_plot_group(..) when there is no plot of the
    correct key name in the PlotGroup dictionary.

    SHOULD BE EXPANDED OR REPLACED FOR MORE (ANY?) DEFAULT FUNCTIONALITY
    """
    if sheet == sheet:
        return True

class PlotEngine(TopoObject):
    """
    Stores the main list of plots available to the simulation.  There are
    some default plot types such as 'Activity'.  Use this interface when
    generating plots.
    """

    def __init__(self, simulation, **params):
        """
        Create a new plot engine that is linked to a particular
        simulation.  The link is necessary since the PlotEngine will
        iterate over all the event processors in the simulation,
        requesting Plots from all EPs that are also Sheets.  This
        approach ensures that new Plot objects will show up
        automatically even in previously defined PlotGroups.

        Example calling style:
            s = topo.simulation.Simulation()
            new_plot_engine = plotengine.PlotEngine(s)
        """
        super(PlotEngine,self).__init__(**params)
        self.simulation = simulation
        self.plot_group_dict = {}

    def add_plot_group(self, name, group):
        """
        Add a constructed PlotGroup to the local dictionary for later
        reuse.  User defined plots should be stored here for later use.
        """
        self.plot_group_dict[name] = group

    def get_plot_group(self, name, group_type = 'BasicPlotGroup',
                       filter=sheet_filter, class_type='BasicPlotGroup'):
        """
        Return the PlotGroup registered in self.plot_group_dict with
        the provided key 'name'.  If the name does not exist, then
        generate a PlotGroup using the generic dynamic group creator.
        This default construction allows for certain types of plots to
        be defined automatically, such as 'Activity'.  If a SheetView
        dictionary entry has a key entry with 'name' then it will be part
        of the new plot.
        """
        if filter == None:
            filter = sheet_filter
        elif isinstance(filter,str):     # Allow sheet string name as input.
            target_string = filter
            filter = lambda s: s.name == target_string

        if self.plot_group_dict.has_key(name):
            self.debug(name, "key match in PlotEngine's PlotGroup list")
            requested_plot = self.plot_group_dict[name]
        else:
            # Current filter is sheet_filter(..).
            requested_plot = self.make_plot_group(name,group_type,filter,class_type)
        return requested_plot

     ### JCALERT! This function has been commented out because it does not seem to be of any use
     ### It is called on a line that is commented out in make_plot_group at the end of the file

#     def lambda_flat_dynamic_list(self, name, filter_lam):
#         """
#         lambda_single_view_per_name() expanded so that the a
#         sheet_view dictionary key entry can have a list of SheetViews,
#         and not just one.  Each SheetView in the list will be set with
#         a Plot object.
#         """
#         sheet_list = [each for each in self.simulation.objects(Sheet).values() if filter_lam(each)]
#         self.debug('sheet_list =' + str(sheet_list) + 'name = ' + str(name))

#         view_list = [(each, each.sheet_view(name)) for each in sheet_list
#                      if each is not None]

#         ### A possible list from the Sheet.sheet_view dictionary is
#         ### converted into multiple Plot generation requests.
#         plot_list = []
#         for (sheet,views) in view_list:
#             if not isinstance(views,list):
#                 views = [views]
#             for each in views:
#                 if isinstance(each,SheetView):
#                     plot_list.append(Plot((each,None,None),COLORMAP,None))
#                 else:
#                     plot_list.append(Plot((name,None,None),COLORMAP,sheet))
#         self.debug('plot_list =' + str(plot_list))
#         return plot_list

      ### JCALERT! This function has been commented out because it does not seem to be of any use
      ### It is called on a line that is commented out in make_plot_group at the end of the file

#     def lambda_for_weight_view(self, (w,sheet_target,sheet_x,sheet_y), filter_lam):
#         """
#         To have the UnitViews stored on the Projection source sheet,
#         the weights must be requested from each Projection on the
#         target sheets.
#         """
#         sheet_list = [each for each in self.simulation.objects(Sheet).values()if filter_lam(each) and each.name== sheet_target]

#         projection_list = []
#         for s in sheet_list:
#             if not isinstance(s,CFSheet):
#                 self.warning('Requested weights view from other than CFSheet.')
#             else:
#                 for p in set(flatten(s.in_projections.values())):
#                     key = ('Weights',sheet_target,p.name,sheet_x,sheet_y)
#                     v = p.src.sheet_view(key)
#                     if v: projection_list += [(s,p,each) for each in v if each.projection.name == p.name]
#         projection_list.sort(key=lambda x: x[2].name,reverse=True)

#         plot_list = []
#         for (sheet,projection,views) in projection_list:
#             if not isinstance(views,list):
#                 views = [views]
#             for each in views:
#                 if isinstance(each,SheetView):
#                     plot_list.append(Plot((each,None,None),COLORMAP,None))
#                 else:
#                     key = ('Weights',sheet,projection,sheet_x,sheet_y)
#                     plot_list.append(Plot((key,None,None),COLORMAP,p.src))
#         self.debug('plot_list =' + str(plot_list))
#         return plot_list

    def lambda_for_templates(self, template, filter_lam):
        """
        This assumes that a PlotGroupTemplate named 'template' has
        been passed in that describes a list of plots.  An example is
        given at the top of plotengine.py.

        It is possible for a requested SheetView to in fact have a
        list of Views under that dictionary key instead of a single
        SheetView.  The default behavior will be to only look at the
        first one and dispose of any others if they exist.
        """
        ### JCHACKALERT! It was necessary here to sort the sheet (with the function sort_dict
        ### so that (and by chance that our display is currently following the alphabetical order)
        ### the plot are placed in a logical way from left to right (i.e. LeftRetina, RightRetina, V1, V2
        ### but also Orientation Preference, Orientation Preference&Selectivity, Orientation Selectivity).
        ### This has to be changed so that each classes has a number that roughly determine where we
        ### wants it on the plot (i.e. GeneratorSheet: 5, CFSheet: 50)
        ### Note that the same problem has to be solved for PatternGenerator in order to determine
        ### in which order to display scale, offset...
        ### (When the problem will be solved, it can be sapred to import dict_sort from topo/base/utils.py)
        sheet_list = [each for each in dict_sort(self.simulation.objects(Sheet)) if filter_lam(each)]

        # Loop over all sheets that passed the filter.
        #     Loop over each individual PlotTemplate:
        #         Loop over each channel in the Plot.  Add needed Nones.
        #         Create new Plot and add to list.
        plot_list = []
        for each in sheet_list:
            for (k,pt) in template.plot_templates:
                c = pt.channels
                if 'Strength' in c or 'Hue' in c or 'Confidence' in c:     # SHC
                    plot_list.append(self.make_SHC_plot(k,pt,each))
                elif 'Location' in c and 'Sheet_name' in c:                # Unit Weights
                    if c['Sheet_name'] == each.name:
                        # Multiple Plots can be generated from one Sheet.
                        plot_list = plot_list + self.make_unitweights_plot(k,pt,each)
                elif 'Density' in c and 'Projection_name' in c:            # Projections
                    projection = each.get_in_projection_by_name(c['Projection_name'])
                    if projection:
                        plot_list = plot_list + self.make_projection_plot(k,pt,projection[0].src)
                else:
                    self.warning("Only SHC, Unit Weights, and Projection plots currently implemented.")
        return plot_list

    def make_projection_plot(self, k, pt, s):
        """
        k is the name of the plot template passed in.
        pt is the PlotTemplate being constructed.
        s is the sheet to request the sheet views from.
            It should be the source of the Projection connection.
        """
        c = pt.channels
        key = (k,c['Projection_name'],c['Density'])
        view_list = s.sheet_view(key)
        if not isinstance(view_list,list):
            view_list = [view_list]

        ### A probable list from the Sheet.sheet_view dictionary is
        ### converted into multiple Plot generation requests.
        plot_list = []
        for each in view_list:
            if isinstance(each,SheetView):
                plot_list.append(Plot((each,None,None),COLORMAP,None,pt.channels['Normalize']))
            else:
                plot_list.append(Plot((name,None,None),COLORMAP,s,pt.channels['Normalize']))
        return plot_list

    def make_unitweights_plot(self, k, pt, s):
        sheet_target = pt.channels['Sheet_name']
        (sheet_x,sheet_y) = pt.channels['Location']

        projection_list = []
        if not isinstance(s,CFSheet):
            self.warning('Requested weights view from other than CFSheet.')
        else:
            for p in set(flatten(s.in_projections.values())):
                key = ('Weights',sheet_target,p.name,sheet_x,sheet_y)
                v = p.src.sheet_view(key)
                if v:
                    projection_list += [(s,p,each) for each in v if each.projection.name == p.name]

        # JUDAH HERE IS WHERE WE ADD ADDITIONAL SORTING INFORMATION.
        projection_list.sort(key=lambda x: x[2].name,reverse=True)

        plot_list = []
        for (sheet,projection,views) in projection_list:
            if not isinstance(views,list):
                views = [views]
            for each in views:
                if isinstance(each,SheetView):
                    plot_list.append(Plot((each,None,None),COLORMAP,None,pt.channels['Normalize']))
                else:
                    key = ('Weights',sheet,projection,sheet_x,sheet_y)
                    plot_list.append(Plot((key,None,None),COLORMAP,p.src,pt.channels['Normalize']))
        self.debug('plot_list =' + str(plot_list))
        ### JCHACKALERT! This is an Hack so that the plot are appearing in alphabetical order
        ### which, luckily, corresponds to the order we want them to be displayed in
        ### This is something that needs more fixing (also see PatternGenerators and the JCHACKALERT!
        ### alert above in the file for similar problem)
        plot_list.sort(reverse=True)
        return plot_list

    def make_SHC_plot(self, k, pt, sheet):
        """
        Create and return a single Plot object matching the passed in
        PlotTemplate (parameter 'pt'), using the Sheet (parameter 'sheet').
        """
        strength = pt.channels.get('Strength',None)
        hue = pt.channels.get('Hue',None)
        confidence = pt.channels.get('Confidence',None)
        n = pt.channels.get('Normalize',False)
        p = Plot((strength,hue,confidence),SHC,sheet,n,name=k)
        return p

    def make_plot_group(self, name='None', group_type='BasicPlotGroup',
                             filter_lam=sheet_filter, class_type='BasicPlotGroup'):
        """
        name : The key to look under in the SheetView dictionaries.
        group_type: 2 Valid inputs:
               1. The string name of the PlotGroup subclass to create.
                  The actual name is passed in instead of a class
                  pointer so the function can be used from the
                  command-line, and also so a full list of class names
                  is not required.
               2. If group_type is a PlotGroupTemplate, then the
                  lambda function to handle templates is called.
        filter_lam: Optional lambda function to filter which sheets to
               ask for SheetViews.
        """
        if isinstance(group_type,PlotGroupTemplate):
            dynamic_list = lambda: self.lambda_for_templates(group_type,filter_lam)
            try:
                exec 'ptr = ' + class_type in globals()
            except Exception, e:
                self.warning('Exception raised:', e)
                self.warning('Invalid PlotGroup subclass: ', group_type)
                return PlotGroup(dynamic_list)
            new_group = ptr(name,filter_lam,dynamic_list)

            # Just copying the pointer.  Not currently sure if we want to
            # promote side-effects by not doing a deepcopy(), but assuming
            # we do for now.  If not, use deepcopy(group_type).
            new_group.template = group_type
            self.add_plot_group(name,new_group)
        else:
            print 'Template was not passed in.  This code deprecated and disabled.'
            import inspect
            print inspect.stack(), '\n'

            ### JCALERT! If you decide to uncomment these lines, you also have to uncomment
            ### the two function that are commented out above

            #if isinstance(name,tuple) and name[0] == 'Weights':
            #    dynamic_list = lambda : self.lambda_for_weight_view(name,filter_lam)
            #else:
            #    dynamic_list = lambda : self.lambda_flat_dynamic_list(name,filter_lam)
            #try:
            #    exec 'ptr = ' + group_type in globals()
            #except Exception, e:
            #    self.warning('Exception:', e)
            #    self.warning('Invalid PlotGroup subclass: ', group_type)
            #    return PlotGroup(dynamic_list)
            #new_group = ptr(name,filter_lam,dynamic_list)

        self.debug('Type of new_group is', type(new_group))
        return new_group
<|endoftext|>


<commit_before>import logging
import itertools
from datetime import datetime, timedelta

import conf
from .abstract import AbstractController
from .icon import IconController
from web.models import User, Feed
from web.lib.utils import clear_string

logger = logging.getLogger(__name__)
DEFAULT_LIMIT = 5

class FeedController(AbstractController):
    _db_cls = Feed

    def __get_art_contr(self):
        from .article import ArticleController
        return ArticleController(self.user_id)

    def list_late(self, delta, max_error=conf.FEED_ERROR_MAX,
                  limit=DEFAULT_LIMIT):
        """Will list either late feed (which have been retrieved for the last
        time sooner than now minus the delta (default to 1h)) or the feed with
        articles recentrly created (later than now minus a quarter the delta
        (default to 15 logically)).

        The idea is to keep very active feed up to date and to avoid missing
        articles du to high activity (when, for example, the feed only displays
        its 30 last entries and produces more than one per minutes).
        """
        tenth = delta / 10
        feed_last_retrieved = datetime.utcnow() - delta
        art_last_retr = datetime.utcnow() - (2 * tenth)
        min_wait = datetime.utcnow() - tenth
        ac = self.__get_art_contr()
        new_art_feed = (ac.read(retrieved_date__gt=art_last_retr,
                                retrieved_date__lt=min_wait)
                          .with_entities(ac._db_cls.feed_id)
                          .distinct())

        query = (self.read(error_count__lt=max_error, enabled=True,
                           __or__=[{'last_retrieved__lt': feed_last_retrieved},
                                   {'last_retrieved__lt': min_wait,
                                    'id__in': new_art_feed}])
                     .join(User).filter(User.is_active == True)
                     .order_by(Feed.last_retrieved))
        if limit:
            query = query.limit(limit)
        yield from query

    def list_fetchable(self, max_error=conf.FEED_ERROR_MAX,
            limit=DEFAULT_LIMIT, refresh_rate=conf.FEED_REFRESH_RATE):
        now, delta = datetime.utcnow(), timedelta(minutes=refresh_rate)
        feeds = list(self.list_late(delta, max_error, limit))
        if feeds:
            self.update({'id__in': [feed.id for feed in feeds]},
                        {'last_retrieved': now})
        return feeds

    def get_duplicates(self, feed_id):
        """
        Compare a list of documents by pair.
        Pairs of duplicates are sorted by "retrieved date".
        """
        feed = self.get(id=feed_id)
        duplicates = []
        for pair in itertools.combinations(feed.articles, 2):
            date1, date2 = pair[0].date, pair[1].date
            if clear_string(pair[0].title) == clear_string(pair[1].title) \
                    and (date1 - date2) < timedelta(days=1):
                if pair[0].retrieved_date < pair[1].retrieved_date:
                    duplicates.append((pair[0], pair[1]))
                else:
                    duplicates.append((pair[1], pair[0]))
        return feed, duplicates

    def get_inactives(self, nb_days):
        today = datetime.utcnow()
        inactives = []
        for feed in self.read():
            try:
                last_post = feed.articles[0].date
            except IndexError:
                continue
            elapsed = today - last_post
            if elapsed > timedelta(days=nb_days):
                inactives.append((feed, elapsed))
        inactives.sort(key=lambda tup: tup[1], reverse=True)
        return inactives

    def count_by_category(self, **filters):
        return self._count_by(Feed.category_id, filters)

    def _ensure_icon(self, attrs):
        if not attrs.get('icon_url'):
            return
        icon_contr = IconController()
        if not icon_contr.read(url=attrs['icon_url']).count():
            icon_contr.create(**{'url': attrs['icon_url']})

    def create(self, **attrs):
        self._ensure_icon(attrs)
        return super().create(**attrs)

    def update(self, filters, attrs):
        self._ensure_icon(attrs)
        if 'category_id' in attrs:
            for feed in self.read(**filters):
                self.__get_art_contr().update({'feed_id': feed.id},
                        {'category_id': attrs['category_id']})
        return super().update(filters, attrs)
<commit_msg>allowing removing category from feeds<commit_after>import logging
import itertools
from datetime import datetime, timedelta

import conf
from .abstract import AbstractController
from .icon import IconController
from web.models import User, Feed
from web.lib.utils import clear_string

logger = logging.getLogger(__name__)
DEFAULT_LIMIT = 5

class FeedController(AbstractController):
    _db_cls = Feed

    def __get_art_contr(self):
        from .article import ArticleController
        return ArticleController(self.user_id)

    def list_late(self, delta, max_error=conf.FEED_ERROR_MAX,
                  limit=DEFAULT_LIMIT):
        """Will list either late feed (which have been retrieved for the last
        time sooner than now minus the delta (default to 1h)) or the feed with
        articles recentrly created (later than now minus a quarter the delta
        (default to 15 logically)).

        The idea is to keep very active feed up to date and to avoid missing
        articles du to high activity (when, for example, the feed only displays
        its 30 last entries and produces more than one per minutes).
        """
        tenth = delta / 10
        feed_last_retrieved = datetime.utcnow() - delta
        art_last_retr = datetime.utcnow() - (2 * tenth)
        min_wait = datetime.utcnow() - tenth
        ac = self.__get_art_contr()
        new_art_feed = (ac.read(retrieved_date__gt=art_last_retr,
                                retrieved_date__lt=min_wait)
                          .with_entities(ac._db_cls.feed_id)
                          .distinct())

        query = (self.read(error_count__lt=max_error, enabled=True,
                           __or__=[{'last_retrieved__lt': feed_last_retrieved},
                                   {'last_retrieved__lt': min_wait,
                                    'id__in': new_art_feed}])
                     .join(User).filter(User.is_active == True)
                     .order_by(Feed.last_retrieved))
        if limit:
            query = query.limit(limit)
        yield from query

    def list_fetchable(self, max_error=conf.FEED_ERROR_MAX,
            limit=DEFAULT_LIMIT, refresh_rate=conf.FEED_REFRESH_RATE):
        now, delta = datetime.utcnow(), timedelta(minutes=refresh_rate)
        feeds = list(self.list_late(delta, max_error, limit))
        if feeds:
            self.update({'id__in': [feed.id for feed in feeds]},
                        {'last_retrieved': now})
        return feeds

    def get_duplicates(self, feed_id):
        """
        Compare a list of documents by pair.
        Pairs of duplicates are sorted by "retrieved date".
        """
        feed = self.get(id=feed_id)
        duplicates = []
        for pair in itertools.combinations(feed.articles, 2):
            date1, date2 = pair[0].date, pair[1].date
            if clear_string(pair[0].title) == clear_string(pair[1].title) \
                    and (date1 - date2) < timedelta(days=1):
                if pair[0].retrieved_date < pair[1].retrieved_date:
                    duplicates.append((pair[0], pair[1]))
                else:
                    duplicates.append((pair[1], pair[0]))
        return feed, duplicates

    def get_inactives(self, nb_days):
        today = datetime.utcnow()
        inactives = []
        for feed in self.read():
            try:
                last_post = feed.articles[0].date
            except IndexError:
                continue
            elapsed = today - last_post
            if elapsed > timedelta(days=nb_days):
                inactives.append((feed, elapsed))
        inactives.sort(key=lambda tup: tup[1], reverse=True)
        return inactives

    def count_by_category(self, **filters):
        return self._count_by(Feed.category_id, filters)

    def _ensure_icon(self, attrs):
        if not attrs.get('icon_url'):
            return
        icon_contr = IconController()
        if not icon_contr.read(url=attrs['icon_url']).count():
            icon_contr.create(**{'url': attrs['icon_url']})

    def create(self, **attrs):
        self._ensure_icon(attrs)
        return super().create(**attrs)

    def update(self, filters, attrs):
        self._ensure_icon(attrs)
        if 'category_id' in attrs:
            if attrs['category_id'] == 0:
                attrs['category_id'] = None
            for feed in self.read(**filters):
                self.__get_art_contr().update({'feed_id': feed.id},
                        {'category_id': attrs['category_id']})
        return super().update(filters, attrs)
<|endoftext|>


<commit_before>"""Command Generator Application (GET)"""
from pysnmp.carrier.asynsock.dispatch import AsynsockDispatcher
from pysnmp.carrier.asynsock.dgram import udp
from pyasn1.codec.ber import encoder, decoder
from pysnmp.proto import api
from time import time

# Protocol version to use
pMod = api.protoModules[api.protoVersion1]

# Build PDU
reqPDU =  pMod.GetRequestPDU()
pMod.apiPDU.setDefaults(reqPDU)
pMod.apiPDU.setVarBinds(
    reqPDU,
    # A list of Var-Binds to SET
    (((1,3,6,1,2,1,1,1,0), pMod.Integer(123456)),
     ((1,3,6,1,2,1,1,2,0), pMod.IpAddress('127.0.0.1')))
    )

# Build message
reqMsg = pMod.Message()
pMod.apiMessage.setDefaults(reqMsg)
pMod.apiMessage.setCommunity(reqMsg, 'public')
pMod.apiMessage.setPDU(reqMsg, reqPDU)

def cbTimerFun(timeNow, startedAt=time()):
    if timeNow - startedAt > 3:
        raise "Request timed out"

def cbRecvFun(transportDispatcher, transportDomain, transportAddress,
              wholeMsg, reqPDU=reqPDU):
    while wholeMsg:
        rspMsg, wholeMsg = decoder.decode(wholeMsg, asn1Spec=pMod.Message())
        rspPDU = pMod.apiMessage.getPDU(rspMsg)
        # Match response to request
        if pMod.apiPDU.getRequestID(reqPDU)==pMod.apiPDU.getRequestID(rspPDU):
            # Check for SNMP errors reported
            errorStatus = pMod.apiPDU.getErrorStatus(rspPDU)
            if errorStatus:
                print 'Error: ', errorStatus
            else:
                for oid, val in pMod.apiPDU.getVarBinds(rspPDU):
                    print oid, val
            transportDispatcher.jobFinished(1)
    return wholeMsg

transportDispatcher = AsynsockDispatcher()
transportDispatcher.registerTransport(
    udp.domainName, udp.UdpSocketTransport().openClientMode()
    )
transportDispatcher.registerRecvCbFun(cbRecvFun)
transportDispatcher.registerTimerCbFun(cbTimerFun)
transportDispatcher.sendMessage(
    encoder.encode(reqMsg), udp.domainName, ('localhost', 161)
    )
transportDispatcher.jobStarted(1)
transportDispatcher.runDispatcher()
transportDispatcher.closeDispatcher()
<commit_msg>* use SET PDU for Set Command Generator example * pretty print Agent response<commit_after>"""Command Generator Application (GET)"""
from pysnmp.carrier.asynsock.dispatch import AsynsockDispatcher
from pysnmp.carrier.asynsock.dgram import udp
from pyasn1.codec.ber import encoder, decoder
from pysnmp.proto import api
from time import time

# Protocol version to use
pMod = api.protoModules[api.protoVersion1]

# Build PDU
reqPDU =  pMod.SetRequestPDU()
pMod.apiPDU.setDefaults(reqPDU)
pMod.apiPDU.setVarBinds(
    reqPDU,
    # A list of Var-Binds to SET
    (((1,3,6,1,2,1,1,1,0), pMod.OctetString('New system description')),
     ((1,3,6,1,2,1,1,3,0), pMod.TimeTicks(12)))
    )

# Build message
reqMsg = pMod.Message()
pMod.apiMessage.setDefaults(reqMsg)
pMod.apiMessage.setCommunity(reqMsg, 'public')
pMod.apiMessage.setPDU(reqMsg, reqPDU)

def cbTimerFun(timeNow, startedAt=time()):
    if timeNow - startedAt > 3:
        raise "Request timed out"

def cbRecvFun(transportDispatcher, transportDomain, transportAddress,
              wholeMsg, reqPDU=reqPDU):
    while wholeMsg:
        rspMsg, wholeMsg = decoder.decode(wholeMsg, asn1Spec=pMod.Message())
        rspPDU = pMod.apiMessage.getPDU(rspMsg)
        # Match response to request
        if pMod.apiPDU.getRequestID(reqPDU)==pMod.apiPDU.getRequestID(rspPDU):
            # Check for SNMP errors reported
            errorStatus = pMod.apiPDU.getErrorStatus(rspPDU)
            if errorStatus:
                print errorStatus.prettyOut(errorStatus)
            else:
                for oid, val in pMod.apiPDU.getVarBinds(rspPDU):
                    print oid, val.prettyOut(val)
            transportDispatcher.jobFinished(1)
    return wholeMsg

transportDispatcher = AsynsockDispatcher()
transportDispatcher.registerTransport(
    udp.domainName, udp.UdpSocketTransport().openClientMode()
    )
transportDispatcher.registerRecvCbFun(cbRecvFun)
transportDispatcher.registerTimerCbFun(cbTimerFun)
transportDispatcher.sendMessage(
    encoder.encode(reqMsg), udp.domainName, ('localhost', 161)
    )
transportDispatcher.jobStarted(1)
transportDispatcher.runDispatcher()
transportDispatcher.closeDispatcher()
<|endoftext|>