<commit_before>from __future__ import generators
"""
General utility functions and classes.

$Id$
"""
__version__='$Revision$'

import __main__
import math
import re

def wrap(lower, upper, x):
    """
    Circularly alias the numeric value x into the range [lower,upper).

    Valid for cyclic quantities like orientations or hues.
    """
    #I have no idea how I came up with this algorithm; it should be simplified.
    range=upper-lower
    return lower + math.fmod(x-lower + 2*range*(1-math.floor(x/(2*range))), range)

def NxN(tuple):
    """
    Converts a tuple (X1,X2,...,Xn) to a string 'X1xX2x...xXn'
    """
    return 'x'.join([`N` for N in tuple])

def enumerate(seq):
    """
    For each element Xi of the given sequence, produces the tuple (i,Xi).

    (A copy of the Python 2.3 enumeration generator.)
    """
    Generator
    i = 0
    for x in seq:
        yield i,x
        i+=1

enum = enumerate

class Struct:
    """
    A simple structure class, taking keyword args and assigning them to attributes.

    For instance:

    s = Struct(foo='a',bar=1)
    >>> s.foo
    'a'
    >>> s.bar
    1
    """
    def __init__(self,**fields):
        for name,value in fields.items():
            setattr(self,name,value)

def flat_indices(shape):
    """
    Returns a list of the indices needed to address or loop over all
    the elements of a 1D or 2D matrix with the given shape. E.g.:

    flat_indices((3,)) == [0,1,2]
    """
    if len(shape) == 1:
        return [(x,) for x in range(shape[0])]
    else:
        rows,cols = shape
        return [(r,c) for r in range(rows) for c in range(cols)]

def flatten(l):
    """
    Flattens a list.

    Written by Bearophile as published on the www.python.org newsgroups.
    Pulled into Topographica 3/5/2005.
    """
    if type(l) != list:
        return l
    else:
        result = []
        stack = []
        stack.append((l,0))
        while len(stack) != 0:
            sequence, j = stack.pop(-1)
            while j < len(sequence):
                if type(sequence[j]) != list:
                    k, j, lens = j, j+1, len(sequence)
                    while j < len(sequence) and \
                          (type(sequence[j]) != list):
                        j += 1
                    result.extend(sequence[k:j])
                else:
                    stack.append((sequence, j+1))
                    sequence, j = sequence[j], 0
        return result

def eval_atof(in_string):
    """
    Create a float from in_string by using string.atof(), but first
    check __main__.__dict__ for the presence of a variable named
    in_string, and if present, use its contents as the argument of
    string.atof() instead of in_string itself.

    See string.atof() for more details about the conversion from string
    to float.
    """
    # There are two cases to consider. The first is that in_string is
    # simply a string representing a number (e.g. "4"). In this case,
    # eval()ing in_string in __main__ has no effect. The second is
    # that in_string is the name of a variable that's already defined
    # in __main__. In this case, eval()ing in_string means we get the
    # contents of that variable as input to string.atof().
    return string.atof(eval(in_string,__main__.__dict__))

# CEBHACKALERT: see the problem in ParametersFrame with its translator_dictionary;
# I also think this function can probably be simplified.
def dict_translator(in_string, name = '', translator_dictionary = {}) :
    """
    Looks for an entry for the string in the dictionary. If it can't be
    found the string is evaluated in __main__.__dict__.
    """
    if translator_dictionary.has_key(name) :
        if translator_dictionary[name].has_key(in_string) :
            val = translator_dictionary[name][in_string]
            from topo.base.parameterizedobject import ParameterizedObject
            if isinstance(val, ParameterizedObject) :
                return val
            else :
                try :
                    val = val()
                    return val
                except : pass
    elif translator_dictionary.has_key(in_string) :
        return translator_dictionary[in_string]

    val = eval(in_string, __main__.__dict__)
    return val

"""
Return the cross-product of a variable number of lists (e.g. of a list of lists).

Use to obtain permutations, e.g.
l1=[a,b]
l2=[c,d]
cross_product([l1,l2]) =
[[a,c], [a,d], [b,c], [b,d]]

From:
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/159975
"""
# Need to re-write so someone other than Python knows what might happen when this runs
cross_product=lambda ss,row=[],level=0: len(ss)>1 \
   and reduce(lambda x,y:x+y,[cross_product(ss[1:],row+[i],level+1) for i in ss[0]]) \
   or [row+[i] for i in ss[0]]

def frange(start, end=None, inc=1.0, inclusive=False):
    """
    A range function that accepts float increments.

    Otherwise, works just as the inbuilt range() function.  If
    inclusive is False, as in the default, the range is exclusive (not
    including the end value), as in the inbuilt range(). If inclusive
    is true, the range may include the end value.

    'All theoretic restrictions apply, but in practice this is
    more useful than in theory.'

    From: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66472
    """
    if end == None:
        end = start + 0.0
        start = 0.0

    # Increments of zero would lead to an infinite loop, which can happen if
    # this is mistakenly called with a integer-based rational expression like 1/2.
    assert ((inc>0 and start<=end) or (inc<0 and start>=end))

    L = []
    while 1:
        next = start + len(L) * inc
        if inclusive:
          if inc > 0 and next > end: break
          elif inc < 0 and next < end: break
        else:
          if inc > 0 and next >= end: break
          elif inc < 0 and next <= end: break
        L.append(next)

    return L

# CB: this function isn't used, as of 2006/06/22
def values_sorted_by_key(d):
    """
    Return the values of dictionary d sorted by key.
    """
    # By Alex Martelli, 2001/04/08
    # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52306
    keys = d.keys()
    keys.sort()
    return map(d.get, keys)

def keys_sorted_by_value(d):
    """
    Return the keys of dictionary d sorted by value.
    """
    # By Daniel Schult, 2004/01/23
    # http://aspn.activestate.com/ASPN/Python/Cookbook/Recipe/52306
    items=d.items()
    backitems=[ [v[1],v[0]] for v in items]
    backitems.sort()
    return [ backitems[i][1] for i in range(0,len(backitems))]

# CEBHACKALERT: this could be improved!
import inspect
from topo.base.parameterizedobject import Parameter, ParameterizedObject
def get_states_of_classes_from_module(module,states_of_classes,processed_modules,exclude=None):
    """
    Recursively search module and get states of classes within it.

    states_of_classes is a dictionary {module.path.and.Classname: state}

    Something is considered a module for our purposes if inspect says it's a module,
    and it defines __all__. We only search through modules listed in __all__.

    Keeps a list of processed modules to avoid looking at the same one
    more than once (since e.g. __main__ contains __main__ contains
    __main__...)

    Packages can be specifically excluded if listed in exclude.
    """
    if not exclude:
        exclude = []

#    print "process",module

    dict_ = module.__dict__
    for (k,v) in dict_.items():
        if '__all__' in dict_ and inspect.ismodule(v) and k not in exclude:
            if k in dict_['__all__'] and v not in processed_modules:
                #print "pickling classes in",k
                get_states_of_classes_from_module(v,states_of_classes,processed_modules,exclude)
            processed_modules.append(v)

        else:
            if isinstance(v,type) and issubclass(v,ParameterizedObject):

                # Note: we take the class name as v.__name__, not k, because
                # k might be just a label for the true class. For example,
                # if Topographica falls back to the unoptimized components,
                # k could be "CFPRF_DotProduct_opt", but v.__name__
                # - and the actual class - is "CFPRF_DotProduct". It
                # is correct to set the attributes on the true class.
                full_class_path = v.__module__+'.'+v.__name__
                states_of_classes[full_class_path] = {}
                # class ALWAYS has __dict__, right? And no P.O. has slots.
                for (name,obj) in v.__dict__.items():
                    if isinstance(obj,Parameter):
                        states_of_classes[full_class_path][name] = obj

def shortclassname(x):
    """
    Returns the class name of x as a string with the leading package information removed.

    E.g. if x is of type "<class 'topo.base.sheet.Sheet'>", returns
    "Sheet"
    """
    return re.sub("'>","",re.sub(".*[.]","",repr(type(x))))

def array_to_ascii(A,outfile):
    """
    Save the given array A to disk as outfile in an ascii format suitable for
    use with Matlab or Octave.

    After using this function to create 'outfile', the following Matlab code
    would create a variable 'outfile' containing the array A:
    load -ascii outfile
    """
    # CB: someone might be concerned with precision; I have taken no notice
    # of it.
    # CEBALERT: there should be a complementary function (for reading in
    # arrays saved by matlab), but the plan is for a proper interface with
    # Matlab...

    # Matlab:
    # - columns separated by whitespace, rows by newline
    # - same number of columns in each row
    # - can't have comments
    # (Octave's more flexible but most people use Matlab.)
    f = open(outfile,'w')
    for row in A:
        for col in row:
            #f.write("%f ")%(col)
            f.write(str(col)+" ")
        f.write("\n")

    f.close()

# CEBHACKALERT: temporary solution for saving savespace state of arrays.
### Python's pickle.Pickler and Numeric's Pickler do not save the
### savespace attribute of arrays, so we implement our own Pickler and
### Unpickler to do this.  These are subclasses of pickle.Pickler and
### pickle.Unpickler, but we override the save_array and load_array
### methods.
###
### These methods are modified ones taken from Numeric's Pickler and
### Unpickler for the moment, but really they should just call
### pickle.Pickler/pickle.Unpickler's own load_array and save_array
### methods, but also save the savespace attribute.

from Numeric import array,multiply,ArrayType,LittleEndian,fromstring,reshape
import string

### CB: code from here until the end mark is almost completely lifted from Numeric.py;
### all modifications are prefaced by 'CB' ###

# These two functions are used in my modified pickle.py so that
# matrices can be pickled.  Notice that matrices are written in
# binary format for efficiency, but that they pay attention to
# byte-order issues for  portability.

def DumpArray(m, fp):
    if m.typecode() == 'O':
        raise TypeError, "Numeric Pickler can't pickle arrays of Objects"
    s = m.shape
    if LittleEndian: endian = "L"
    else: endian = "B"
    # CB: as well as writing typecode, write savespace
    fp.write("A%s%s%d%d " % (m.typecode(), endian, m.spacesaver(), m.itemsize()))
    for d in s:
        fp.write("%d "% d)
    fp.write('\n')
    fp.write(m.tostring())

def LoadArray(fp):
    ln = string.split(fp.readline())
    if ln[0][0] == 'A': ln[0] = ln[0][1:] # Nasty hack showing my ignorance of pickle
    typecode = ln[0][0]
    endian = ln[0][1]
    # CB: read back savespace
    savespace = int(ln[0][2])

    shape = map(lambda x: string.atoi(x), ln[1:])
    itemsize = string.atoi(ln[0][3:])

    sz = reduce(multiply, shape, 1)*itemsize
    data = fp.read(sz)

    m = fromstring(data, typecode)
    m = reshape(m, shape)
    # CB: set savespace
    m.savespace(savespace)

    if (LittleEndian and endian == 'B') or (not LittleEndian and endian == 'L'):
        return m.byteswapped()
    else:
        return m

import pickle, copy
class ExtraUnpickler(pickle.Unpickler):
    def load_array(self):
        self.stack.append(LoadArray(self))

    dispatch = copy.copy(pickle.Unpickler.dispatch)
    dispatch['A'] = load_array

class ExtraPickler(pickle.Pickler):
    def save_array(self, object):
        DumpArray(object, self)

    dispatch = copy.copy(pickle.Pickler.dispatch)
    dispatch[ArrayType] = save_array

### CB: end code from Numeric.py ###
<commit_msg>Clarified comments<commit_after>from __future__ import generators
"""
General utility functions and classes.

$Id$
"""
__version__='$Revision$'

import __main__
import math
import re

def wrap(lower, upper, x):
    """
    Circularly alias the numeric value x into the range [lower,upper).

    Valid for cyclic quantities like orientations or hues.
    """
    #I have no idea how I came up with this algorithm; it should be simplified.
    range=upper-lower
    return lower + math.fmod(x-lower + 2*range*(1-math.floor(x/(2*range))), range)

def NxN(tuple):
    """
    Converts a tuple (X1,X2,...,Xn) to a string 'X1xX2x...xXn'
    """
    return 'x'.join([`N` for N in tuple])

def enumerate(seq):
    """
    For each element Xi of the given sequence, produces the tuple (i,Xi).

    (A copy of the Python 2.3 enumeration generator.)
    """
    Generator
    i = 0
    for x in seq:
        yield i,x
        i+=1

enum = enumerate

class Struct:
    """
    A simple structure class, taking keyword args and assigning them to attributes.

    For instance:

    s = Struct(foo='a',bar=1)
    >>> s.foo
    'a'
    >>> s.bar
    1
    """
    def __init__(self,**fields):
        for name,value in fields.items():
            setattr(self,name,value)

def flat_indices(shape):
    """
    Returns a list of the indices needed to address or loop over all
    the elements of a 1D or 2D matrix with the given shape. E.g.:

    flat_indices((3,)) == [0,1,2]
    """
    if len(shape) == 1:
        return [(x,) for x in range(shape[0])]
    else:
        rows,cols = shape
        return [(r,c) for r in range(rows) for c in range(cols)]

def flatten(l):
    """
    Flattens a list.

    Written by Bearophile as published on the www.python.org newsgroups.
    Pulled into Topographica 3/5/2005.
    """
    if type(l) != list:
        return l
    else:
        result = []
        stack = []
        stack.append((l,0))
        while len(stack) != 0:
            sequence, j = stack.pop(-1)
            while j < len(sequence):
                if type(sequence[j]) != list:
                    k, j, lens = j, j+1, len(sequence)
                    while j < len(sequence) and \
                          (type(sequence[j]) != list):
                        j += 1
                    result.extend(sequence[k:j])
                else:
                    stack.append((sequence, j+1))
                    sequence, j = sequence[j], 0
        return result

def eval_atof(str):
    """
    Evaluates the given string in __main__, and converts it to a float.

    The string can contain any expression that will evaluate to a
    number.  The expression can use any variables or functions that
    are defined in the main namespace.

    See string.atof() for more details about the conversion from the
    evaluated string into a float.
    """
    return string.atof(eval(str,__main__.__dict__))

# CEBHACKALERT: see the problem in ParametersFrame with its translator_dictionary;
# I also think this function can probably be simplified.
def dict_translator(in_string, name = '', translator_dictionary = {}) :
    """
    Looks for an entry for the string in the dictionary. If it can't be
    found the string is evaluated in __main__.__dict__.
    """
    if translator_dictionary.has_key(name) :
        if translator_dictionary[name].has_key(in_string) :
            val = translator_dictionary[name][in_string]
            from topo.base.parameterizedobject import ParameterizedObject
            if isinstance(val, ParameterizedObject) :
                return val
            else :
                try :
                    val = val()
                    return val
                except : pass
    elif translator_dictionary.has_key(in_string) :
        return translator_dictionary[in_string]

    val = eval(in_string, __main__.__dict__)
    return val

"""
Return the cross-product of a variable number of lists (e.g. of a list of lists).

Use to obtain permutations, e.g.
l1=[a,b]
l2=[c,d]
cross_product([l1,l2]) =
[[a,c], [a,d], [b,c], [b,d]]

From:
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/159975
"""
# Need to re-write so someone other than Python knows what might happen when this runs
cross_product=lambda ss,row=[],level=0: len(ss)>1 \
   and reduce(lambda x,y:x+y,[cross_product(ss[1:],row+[i],level+1) for i in ss[0]]) \
   or [row+[i] for i in ss[0]]

def frange(start, end=None, inc=1.0, inclusive=False):
    """
    A range function that accepts float increments.

    Otherwise, works just as the inbuilt range() function.  If
    inclusive is False, as in the default, the range is exclusive (not
    including the end value), as in the inbuilt range(). If inclusive
    is true, the range may include the end value.

    'All theoretic restrictions apply, but in practice this is
    more useful than in theory.'

    From: http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66472
    """
    if end == None:
        end = start + 0.0
        start = 0.0

    # Increments of zero would lead to an infinite loop, which can happen if
    # this is mistakenly called with a integer-based rational expression like 1/2.
    assert ((inc>0 and start<=end) or (inc<0 and start>=end))

    L = []
    while 1:
        next = start + len(L) * inc
        if inclusive:
          if inc > 0 and next > end: break
          elif inc < 0 and next < end: break
        else:
          if inc > 0 and next >= end: break
          elif inc < 0 and next <= end: break
        L.append(next)

    return L

# CB: this function isn't used, as of 2006/06/22
def values_sorted_by_key(d):
    """
    Return the values of dictionary d sorted by key.
    """
    # By Alex Martelli, 2001/04/08
    # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52306
    keys = d.keys()
    keys.sort()
    return map(d.get, keys)

def keys_sorted_by_value(d):
    """
    Return the keys of dictionary d sorted by value.
    """
    # By Daniel Schult, 2004/01/23
    # http://aspn.activestate.com/ASPN/Python/Cookbook/Recipe/52306
    items=d.items()
    backitems=[ [v[1],v[0]] for v in items]
    backitems.sort()
    return [ backitems[i][1] for i in range(0,len(backitems))]

# CEBHACKALERT: this could be improved!
import inspect
from topo.base.parameterizedobject import Parameter, ParameterizedObject
def get_states_of_classes_from_module(module,states_of_classes,processed_modules,exclude=None):
    """
    Recursively search module and get states of classes within it.

    states_of_classes is a dictionary {module.path.and.Classname: state}

    Something is considered a module for our purposes if inspect says it's a module,
    and it defines __all__. We only search through modules listed in __all__.

    Keeps a list of processed modules to avoid looking at the same one
    more than once (since e.g. __main__ contains __main__ contains
    __main__...)

    Packages can be specifically excluded if listed in exclude.
    """
    if not exclude:
        exclude = []

#    print "process",module

    dict_ = module.__dict__
    for (k,v) in dict_.items():
        if '__all__' in dict_ and inspect.ismodule(v) and k not in exclude:
            if k in dict_['__all__'] and v not in processed_modules:
                #print "pickling classes in",k
                get_states_of_classes_from_module(v,states_of_classes,processed_modules,exclude)
            processed_modules.append(v)

        else:
            if isinstance(v,type) and issubclass(v,ParameterizedObject):

                # Note: we take the class name as v.__name__, not k, because
                # k might be just a label for the true class. For example,
                # if Topographica falls back to the unoptimized components,
                # k could be "CFPRF_DotProduct_opt", but v.__name__
                # - and the actual class - is "CFPRF_DotProduct". It
                # is correct to set the attributes on the true class.
                full_class_path = v.__module__+'.'+v.__name__
                states_of_classes[full_class_path] = {}
                # class ALWAYS has __dict__, right? And no P.O. has slots.
                for (name,obj) in v.__dict__.items():
                    if isinstance(obj,Parameter):
                        states_of_classes[full_class_path][name] = obj

def shortclassname(x):
    """
    Returns the class name of x as a string with the leading package information removed.

    E.g. if x is of type "<class 'topo.base.sheet.Sheet'>", returns
    "Sheet"
    """
    return re.sub("'>","",re.sub(".*[.]","",repr(type(x))))

def array_to_ascii(A,outfile):
    """
    Save the given array A to disk as outfile in an ascii format suitable for
    use with Matlab or Octave.

    After using this function to create 'outfile', the following Matlab code
    would create a variable 'outfile' containing the array A:
    load -ascii outfile
    """
    # CB: someone might be concerned with precision; I have taken no notice
    # of it.
    # CEBALERT: there should be a complementary function (for reading in
    # arrays saved by matlab), but the plan is for a proper interface with
    # Matlab...

    # Matlab:
    # - columns separated by whitespace, rows by newline
    # - same number of columns in each row
    # - can't have comments
    # (Octave's more flexible but most people use Matlab.)
    f = open(outfile,'w')
    for row in A:
        for col in row:
            #f.write("%f ")%(col)
            f.write(str(col)+" ")
        f.write("\n")

    f.close()

# CEBHACKALERT: temporary solution for saving savespace state of arrays.
### Python's pickle.Pickler and Numeric's Pickler do not save the
### savespace attribute of arrays, so we implement our own Pickler and
### Unpickler to do this.  These are subclasses of pickle.Pickler and
### pickle.Unpickler, but we override the save_array and load_array
### methods.
###
### These methods are modified ones taken from Numeric's Pickler and
### Unpickler for the moment, but really they should just call
### pickle.Pickler/pickle.Unpickler's own load_array and save_array
### methods, but also save the savespace attribute.

from Numeric import array,multiply,ArrayType,LittleEndian,fromstring,reshape
import string

### CB: code from here until the end mark is almost completely lifted from Numeric.py;
### all modifications are prefaced by 'CB' ###

# These two functions are used in my modified pickle.py so that
# matrices can be pickled.  Notice that matrices are written in
# binary format for efficiency, but that they pay attention to
# byte-order issues for  portability.

def DumpArray(m, fp):
    if m.typecode() == 'O':
        raise TypeError, "Numeric Pickler can't pickle arrays of Objects"
    s = m.shape
    if LittleEndian: endian = "L"
    else: endian = "B"
    # CB: as well as writing typecode, write savespace
    fp.write("A%s%s%d%d " % (m.typecode(), endian, m.spacesaver(), m.itemsize()))
    for d in s:
        fp.write("%d "% d)
    fp.write('\n')
    fp.write(m.tostring())

def LoadArray(fp):
    ln = string.split(fp.readline())
    if ln[0][0] == 'A': ln[0] = ln[0][1:] # Nasty hack showing my ignorance of pickle
    typecode = ln[0][0]
    endian = ln[0][1]
    # CB: read back savespace
    savespace = int(ln[0][2])

    shape = map(lambda x: string.atoi(x), ln[1:])
    itemsize = string.atoi(ln[0][3:])

    sz = reduce(multiply, shape, 1)*itemsize
    data = fp.read(sz)

    m = fromstring(data, typecode)
    m = reshape(m, shape)
    # CB: set savespace
    m.savespace(savespace)

    if (LittleEndian and endian == 'B') or (not LittleEndian and endian == 'L'):
        return m.byteswapped()
    else:
        return m

import pickle, copy
class ExtraUnpickler(pickle.Unpickler):
    def load_array(self):
        self.stack.append(LoadArray(self))

    dispatch = copy.copy(pickle.Unpickler.dispatch)
    dispatch['A'] = load_array

class ExtraPickler(pickle.Pickler):
    def save_array(self, object):
        DumpArray(object, self)

    dispatch = copy.copy(pickle.Pickler.dispatch)
    dispatch[ArrayType] = save_array

### CB: end code from Numeric.py ###
<|endoftext|>


<commit_before>from rlpy.Representations import Tabular
from rlpy.Domains import SystemAdministrator
from rlpy.Agents.TDControlAgent import SARSA
import numpy as np
from rlpy.Tools import __rlpy_location__
import os

def test_seed():
    """ Ensure that providing the same random seed yields same result """
    # [[initialize and run experiment without visual]]
    # [[initialize and run experiment with visual]]
    # [[assert get same results]]

    # TODO get from fiftychain

def test_errs():
    """ Ensure that we can call custom methods without error """

    default_map_dir = os.path.join(
        __rlpy_location__,
        "Domains",
        "SystemAdministratorMaps")
    domain = SystemAdministrator(networkmapname=os.path.join(
                default_map_dir, "20MachTutorial.txt"))

    # loadNetwork() is called by __init__
    # setNeighbors() is run by loadNetwork

def test_transitions():
    """
    Ensure that actions result in expected state transition behavior.
    """
    # [[manually set state, manually turn off stochasticity ie deterministic,
    # and observe transitions, reward, etc.]]
    default_map_dir = os.path.join(
        __rlpy_location__,
        "Domains",
        "SystemAdministratorMaps")
    domain = SystemAdministrator(networkmapname=os.path.join(
                default_map_dir, "5Machines.txt"))
    dummyS = domain.s0()
    up = domain.RUNNING # shorthand
    down = domain.BROKEN # shorthand

    state = np.array([up for dummy in xrange(0, domain.state_space_dims)])
    domain.state = state
    a = 5 # =n on this 5-machine map, ie no action
    ns = state

    # Test that no penalty is applied for a non-reboot action
    r, ns, t, pA = domain.step(a)
    numWorking = len(np.where(ns == up))
    assert r == numWorking

    # Test that penalty is applied for reboot
    r, ns, t, pA = domain.step(0) # restart computer 0
    numWorking = len(np.where(ns == up))
    assert r == numWorking + domain.REBOOT_REWARD

    while np.all(ns == up):
        r, ns, t, pA = domain.step(a)
    # now at least 1 machine has failed

    domain.P_SELF_REPAIR = 0.0
    domain.P_REBOOT_REPAIR = 0.0

    # Test that machine remains down when no reboot taken
    fMachine = np.where(ns == down)[0][0]
    r, ns, t, pA = domain.step(fMachine)
    assert ns[fMachine] == down

    # Test that machine becomes up when reboot taken
    domain.P_REBOOT_REPAIR = 1.0
    r, ns, t, pA = domain.step(fMachine)
    assert ns[fMachine] == up
    <commit_msg>vis randomization test<commit_after>from rlpy.Representations import IncrementalTabular
from rlpy.Domains import SystemAdministrator
from rlpy.Agents.TDControlAgent import SARSA
import numpy as np
from rlpy.Tools import __rlpy_location__
import os

def _make_experiment(exp_id=1, path="./Results/Tmp/test_SystemAdministrator"):
    """
    Each file specifying an experimental setup should contain a
    make_experiment function which returns an instance of the Experiment
    class with everything set up.

    @param id: number used to seed the random number generators
    @param path: output directory where logs and results are stored
    """

    ## Domain:
    domain = SystemAdministrator()

    ## Representation
    # discretization only needed for continuous state spaces, discarded otherwise
    representation  = IncrementalTabular(domain)

    ## Policy
    policy = eGreedy(representation, epsilon=0.2)

    ## Agent
    agent = SARSA(representation=representation, policy=policy,
                  disount_factor=domain.discount_factor,
                       learn_rate=0.1)
    checks_per_policy = 3
    max_steps = 50
    num_policy_checks = 3
    experiment = Experiment(**locals())
    return experiment

def _checkSameExperimentResults(exp1, exp2):
    """ Returns False if experiments gave same results, true if they match. """
    if not np.all(exp1.result["learning_steps"] == exp2.results["learning_steps"]):
        # Same number of steps before failure (where applicable)
        return False
    if not np.all(exp1.result["return"] == exp2.result["return"]):
        # Same return on each test episode
        return False
    if not np.all(exp1.result["steps"] == exp2.result["steps"]):
        # Same number of steps taken on each training episode
        return False
    return True

def test_seed():
    """ Ensure that providing the same random seed yields same result """
    # [[initialize and run experiment without visual]]
    expNoVis = _make_experiment(exp_id=1)
    expNoVis.run(visualize_steps=False,
            visualize_learning=False,
            visualize_performance=0)

    # [[initialize and run experiment with visual]]
    expVis1 = _make_experiment(exp_id=1)
    expVis1.run(visualize_steps=True,
            visualize_learning=False,
            visualize_performance=1)

    expVis2 = _make_experiment(exp_id=1)
    expVis2.run(visualize_steps=False,
            visualize_learning=True,
            visualize_performance=1)

    # [[assert get same results]]
    assert _checkSameExperimentResults(expNoVis, expVis1)
    assert _checkSameExperimentResults(expNoVis, expVis2)

def test_errs():
    """ Ensure that we can call custom methods without error """

    default_map_dir = os.path.join(
        __rlpy_location__,
        "Domains",
        "SystemAdministratorMaps")
    domain = SystemAdministrator(networkmapname=os.path.join(
                default_map_dir, "20MachTutorial.txt"))

    # loadNetwork() is called by __init__
    # setNeighbors() is run by loadNetwork

def test_transitions():
    """
    Ensure that actions result in expected state transition behavior.
    """
    # [[manually set state, manually turn off stochasticity ie deterministic,
    # and observe transitions, reward, etc.]]
    default_map_dir = os.path.join(
        __rlpy_location__,
        "Domains",
        "SystemAdministratorMaps")
    domain = SystemAdministrator(networkmapname=os.path.join(
                default_map_dir, "5Machines.txt"))
    dummyS = domain.s0()
    up = domain.RUNNING # shorthand
    down = domain.BROKEN # shorthand

    state = np.array([up for dummy in xrange(0, domain.state_space_dims)])
    domain.state = state
    a = 5 # =n on this 5-machine map, ie no action
    ns = state

    # Test that no penalty is applied for a non-reboot action
    r, ns, t, pA = domain.step(a)
    numWorking = len(np.where(ns == up))
    assert r == numWorking

    # Test that penalty is applied for reboot
    r, ns, t, pA = domain.step(0) # restart computer 0
    numWorking = len(np.where(ns == up))
    assert r == numWorking + domain.REBOOT_REWARD

    while np.all(ns == up):
        r, ns, t, pA = domain.step(a)
    # now at least 1 machine has failed

    domain.P_SELF_REPAIR = 0.0
    domain.P_REBOOT_REPAIR = 0.0

    # Test that machine remains down when no reboot taken
    fMachine = np.where(ns == down)[0][0]
    r, ns, t, pA = domain.step(fMachine)
    assert ns[fMachine] == down

    # Test that machine becomes up when reboot taken
    domain.P_REBOOT_REPAIR = 1.0
    r, ns, t, pA = domain.step(fMachine)
    assert ns[fMachine] == up
    <|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Notify

In this example we use the bot only for easy notifications to a space.
There is no command in the shell at all, and the bot is not even started.

Multiple questions are adressed in this example:

- How to create a bot and configure it in one line? The simplest approach
  is to set environment variables and then to create the bot. This can be
  done externally, before running the program, for secret variables such
  as the Cisco Spark token (see below). Or variables can be set directly
  from within the script itself, as ``CHAT_ROOM_TITLE`` in this example.

- How to create or to delete a channel? This is done with ``engine.get_bot()`` and
  with ``bot.dispose()`` respectively.

- How to post a notification? Use ``bot.say()`` right after a call
  to ``engine.get_bot()``.

- Why do we not start the bot? There is no call to ``bot.run()`` below
  because there is no need for an active shell. The program updates a channel,
  but is not interactive and cannot answer send to it. Of course, it is easy
  to implement a couple of commands and evolve towards a responsive bot.

To run this script you have to provide a custom configuration, or set
environment variables instead::

- ``CHANNEL_DEFAULT_PARTICIPANTS`` - Mention at least your e-mail address
- ``CISCO_SPARK_BOT_TOKEN`` - Received from Cisco Spark on bot registration
- ``SERVER_URL`` - Public link used by Cisco Spark to reach your server

The token is specific to your run-time, please visit Cisco Spark for
Developers to get more details:

    https://developer.ciscospark.com/

For example, if you run this script under Linux or macOs with support from
ngrok for exposing services to the Internet::

    export CHANNEL_DEFAULT_PARTICIPANTS="alice@acme.com"
    export CISCO_SPARK_BOT_TOKEN="<token id from Cisco Spark for Developers>"
    export SERVER_URL="http://1a107f21.ngrok.io"
    python notify.py

credit: http://pythonfiddle.com/random-sentence-generator/
"""

import os
import random
import time

from shellbot import Engine, ShellBot, Context, Command
Context.set_logger()

# a function to create a random sentence
#
def some_message():

    s_nouns = ["A dude", "My mom", "The king", "Some guy",
               "A cat with rabies", "A sloth", "Your homie",
               "This cool guy my gardener met yesterday", "Superman"]

    s_verbs = ["eats", "kicks", "gives", "treats", "meets with", "creates",
               "hacks", "configures", "spies on", "retards", "meows on",
               "flees from", "tries to automate", "explodes"]

    p_nouns = ["These dudes", "Both of my moms", "All the kings of the world",
               "Some guys", "All of a cattery's cats",
               "The multitude of sloths living under your bed",
               "Your homies", "Like, these, like, all these people", "Supermen"]

    infinitives = ["to make a pie.", "for no apparent reason.",
                   "because the sky is green.", "for a disease.",
                   "to be able to make toast explode.",
                   "to know more about archeology."]

    return u"{} {} {} {}".format(random.choice(s_nouns),
                                 random.choice(s_verbs),
                                 random.choice(p_nouns).lower(),
                                 random.choice(infinitives))

# create a bot and configure it
#
os.environ['CHAT_ROOM_TITLE'] = 'Notifications'
engine = Engine(type='spark', configure=True)

# create a chat channel
#
bot = engine.get_bot(reset=True)  # safe, one single process

# send notifications to the channel
#
for index in range(7):

    bot.say(some_message())
    time.sleep(5)

bot.say(u"Nothing more to say")

# wait for manual interruption
#
print("Press Ctl-C to stop this program")
try:
    while True:
        time.sleep(0.01)
except KeyboardInterrupt:
    pass

# delete the chat channel
#
bot.dispose()
<commit_msg>garden examples/notify - save 10% lines<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Notify

In this example we use the bot only for easy notifications to a space.
There is no command in the shell at all, and the bot is not even started.

Multiple questions are adressed in this example:

- How to create a bot and configure it in one line? The simplest approach
  is to set environment variables and then to create the bot. This can be
  done externally, before running the program, for secret variables such
  as the Cisco Spark token (see below). Or variables can be set directly
  from within the script itself, as ``CHAT_ROOM_TITLE`` in this example.

- How to create or to delete a channel? This is done with ``engine.get_bot()``
  and with ``bot.dispose()`` respectively.

- How to post a notification? Use ``bot.say()`` right after a call
  to ``engine.get_bot()``.

- Why do we not start the bot? There is no call to ``bot.run()`` below
  because there is no need for an active shell. The program updates a channel,
  but is not interactive and cannot answer send to it. Of course, it is easy
  to implement a couple of commands and evolve towards a responsive bot.

To run this script you have to provide a custom configuration, or set
environment variables instead::

- ``CHANNEL_DEFAULT_PARTICIPANTS`` - Mention at least your e-mail address
- ``CISCO_SPARK_BOT_TOKEN`` - Received from Cisco Spark on bot registration
- ``SERVER_URL`` - Public link used by Cisco Spark to reach your server

The token is specific to your run-time, please visit Cisco Spark for
Developers to get more details:

    https://developer.ciscospark.com/

For example, if you run this script under Linux or macOs with support from
ngrok for exposing services to the Internet::

    export CHANNEL_DEFAULT_PARTICIPANTS="alice@acme.com"
    export CISCO_SPARK_BOT_TOKEN="<token id from Cisco Spark for Developers>"
    export SERVER_URL="http://1a107f21.ngrok.io"
    python notify.py

credit: http://pythonfiddle.com/random-sentence-generator/
"""

import os
import random
import time

from shellbot import Engine, Context
Context.set_logger()

def some_message():
    """Creates random sentence"""

    s_nouns = ["A dude", "My mom", "The king", "Some guy",
               "A cat with rabies", "A sloth", "Your homie",
               "This cool guy my gardener met yesterday", "Superman"]

    s_verbs = ["eats", "kicks", "gives", "treats", "meets with", "creates",
               "hacks", "configures", "spies on", "retards", "meows on",
               "flees from", "tries to automate", "explodes"]

    p_nouns = ["These dudes", "Both of my moms", "All the kings of the world",
               "Some guys", "All of a cattery's cats",
               "The multitude of sloths living under your bed",
               "Your homies", "Like, these, like, all these people", "Supermen"]

    infinitives = ["to make a pie.", "for no apparent reason.",
                   "because the sky is green.", "for a disease.",
                   "to be able to make toast explode.",
                   "to know more about archeology."]

    return u"{} {} {} {}".format(random.choice(s_nouns),
                                 random.choice(s_verbs),
                                 random.choice(p_nouns).lower(),
                                 random.choice(infinitives))

os.environ['CHAT_ROOM_TITLE'] = 'Notifications'
engine = Engine(type='spark', configure=True)

bot = engine.get_bot(reset=True)  # create a group channel

for index in range(7):  # send notifications to the channel
    bot.say(some_message())
    time.sleep(5)

bot.say(u"Nothing more to say")

print("Press Ctl-C to stop this program")
try:
    while True:
        time.sleep(0.01)
except KeyboardInterrupt:
    pass

bot.dispose()  # delete the initial group channel
<|endoftext|>


<commit_before>from __future__ import unicode_literals
import array
from collections import defaultdict
import io
import logging
import os
import zipfile

import six
from six.moves.urllib.request import urlretrieve
import torch
from tqdm import tqdm
import tarfile

from .utils import reporthook

logger = logging.getLogger(__name__)

class Vocab(object):
    """Defines a vocabulary object that will be used to numericalize a field.

    Attributes:
        freqs: A collections.Counter object holding the frequencies of tokens
            in the data used to build the Vocab.
        stoi: A collections.defaultdict instance mapping token strings to
            numerical identifiers.
        itos: A list of token strings indexed by their numerical identifiers.
    """
    def __init__(self, counter, max_size=None, min_freq=1, specials=['<pad>'],
                 vectors=None):
        """Create a Vocab object from a collections.Counter.

        Arguments:
            counter: collections.Counter object holding the frequencies of
                each value found in the data.
            max_size: The maximum size of the vocabulary, or None for no
                maximum. Default: None.
            min_freq: The minimum frequency needed to include a token in the
                vocabulary. Values less than 1 will be set to 1. Default: 1.
            specials: The list of special tokens (e.g., padding or eos) that
                will be prepended to the vocabulary in addition to an <unk>
                token. Default: ['<pad>']
            vectors: One of either the available pretrained vectors
                or custom pretrained vectors (see Vocab.load_vectors);
                or a list of aforementioned vectors
        """
        self.freqs = counter.copy()
        min_freq = max(min_freq, 1)
        counter.update(specials)

        self.stoi = defaultdict(_default_unk_index)
        self.stoi.update({tok: i for i, tok in enumerate(specials)})
        self.itos = list(specials)

        counter.subtract({tok: counter[tok] for tok in specials})
        max_size = None if max_size is None else max_size + len(self.itos)

        # sort by frequency, then alphabetically
        words_and_frequencies = sorted(counter.items(), key=lambda tup: tup[0])
        words_and_frequencies.sort(key=lambda tup: tup[1], reverse=True)

        for word, freq in words_and_frequencies:
            if freq < min_freq or len(self.itos) == max_size:
                break
            self.itos.append(word)
            self.stoi[word] = len(self.itos) - 1

        self.vectors = None
        if vectors is not None:
            self.load_vectors(vectors)

    def __eq__(self, other):
        if self.freqs != other.freqs:
            return False
        if self.stoi != other.stoi:
            return False
        if self.itos != other.itos:
            return False
        if self.vectors != other.vectors:
            return False
        return True

    def __len__(self):
        return len(self.itos)

    def extend(self, v, sort=False):
        words = sorted(v.itos) if sort else v.itos
        for w in words:
            if w not in self.stoi:
                self.itos.append(w)
                self.stoi[w] = len(self.itos) - 1

    def load_vectors(self, vectors):
        """
        Arguments:
            vectors: one of or a list containing instantiations of the
                GloVe, CharNGram, or Vectors classes. Alternatively, one
                of or a list of available pretrained vectors:
                    charngram.100d
                    fasttext.en.300d
                    fasttext.simple.300d
                    glove.42B.300d
                    glove.840B.300d
                    glove.twitter.27B.25d
                    glove.twitter.27B.50d
                    glove.twitter.27B.100d
                    glove.twitter.27B.200d
                    glove.6B.50d
                    glove.6B.100d
                    glove.6B.200d
                    glove.6B.300d
        """
        if not isinstance(vectors, list):
            vectors = [vectors]
        for idx, vector in enumerate(vectors):
            if six.PY2 and isinstance(vector, str):
                vector = six.text_type(vector)
            if isinstance(vector, six.string_types):
                # Convert the string pretrained vector identifier
                # to a Vectors object
                if vector not in pretrained_aliases:
                    raise ValueError(
                        "Got string input vector {}, but allowed pretrained "
                        "vectors are {}".format(
                            vector, list(pretrained_aliases.keys())))
                vectors[idx] = pretrained_aliases[vector]()
            elif not isinstance(vector, Vectors):
                raise ValueError(
                    "Got input vectors of type {}, expected str or "
                    "Vectors object".format(type(vector)))

        tot_dim = sum(v.dim for v in vectors)
        self.vectors = torch.Tensor(len(self), tot_dim)
        for i, token in enumerate(self.itos):
            start_dim = 0
            for v in vectors:
                end_dim = start_dim + v.dim
                self.vectors[i][start_dim:end_dim] = v[token.strip()]
                start_dim = end_dim
            assert(start_dim == tot_dim)

    def set_vectors(self, stoi, vectors, dim, unk_init=torch.Tensor.zero_):
        """
        Set the vectors for the Vocab instance from a collection of Tensors.

        Arguments:
            stoi: A dictionary of string to the index of the associated vector
                in the `vectors` input argument.
            vectors: An indexed iterable (or other structure supporting __getitem__) that
                given an input index, returns a FloatTensor representing the vector
                for the token associated with the index. For example,
                vector[stoi["string"]] should return the vector for "string".
            dim: The dimensionality of the vectors.
            unk_init (callback): by default, initialize out-of-vocabulary word vectors
                to zero vectors; can be any function that takes in a Tensor and
                returns a Tensor of the same size. Default: torch.Tensor.zero_
        """
        self.vectors = torch.Tensor(len(self), dim)
        for i, token in enumerate(self.itos):
            wv_index = stoi.get(token, None)
            if wv_index is not None:
                self.vectors[i] = vectors[wv_index]
            else:
                self.vectors[i] = unk_init(self.vectors[i])

class SubwordVocab(Vocab):

    def __init__(self, counter, max_size=None, specials=['<pad>'],
                 vectors=None, unk_init=torch.Tensor.zero_, expand_vocab=False):
        """Create a revtok subword vocabulary from a collections.Counter.

        Arguments:
            counter: collections.Counter object holding the frequencies of
                each word found in the data.
            max_size: The maximum size of the subword vocabulary, or None for no
                maximum. Default: None.
            specials: The list of special tokens (e.g., padding or eos) that
                will be prepended to the vocabulary in addition to an <unk>
                token.
        """
        try:
            import revtok
        except ImportError:
            print("Please install revtok.")
            raise

        self.stoi = defaultdict(_default_unk_index)
        self.stoi.update({tok: i for i, tok in enumerate(specials)})
        self.itos = specials

        self.segment = revtok.SubwordSegmenter(counter, max_size)

        max_size = None if max_size is None else max_size + len(self.itos)

        # sort by frequency/entropy, then alphabetically
        toks = sorted(self.segment.vocab.items(),
                      key=lambda tup: (len(tup[0]) != 1, -tup[1], tup[0]))

        for tok, _ in toks:
            self.itos.append(tok)
            self.stoi[tok] = len(self.itos) - 1

        if vectors is not None:
            self.load_vectors(vectors, unk_init=unk_init, expand_vocab=expand_vocab)

class Vectors(object):

    def __init__(self, name, cache='.vector_cache',
                 url=None, unk_init=torch.Tensor.zero_):
        """Arguments:
               name: name of the file that contains the vectors
               cache: directory for cached vectors
               url: url for download if vectors not found in cache
               unk_init (callback): by default, initalize out-of-vocabulary word vectors
                   to zero vectors; can be any function that takes in a Tensor and
                   returns a Tensor of the same size
         """
        self.unk_init = unk_init
        self.cache(name, cache, url=url)

    def __getitem__(self, token):
        if token in self.stoi:
            return self.vectors[self.stoi[token]]
        else:
            return self.unk_init(torch.Tensor(1, self.dim))

    def cache(self, name, cache, url=None):
        if os.path.isfile(name):
            path = name
            path_pt = os.path.join(cache, os.path.basename(name)) + '.pt'
        else:
            path = os.path.join(cache, name)
            path_pt = path + '.pt'

        if not os.path.isfile(path_pt):
            if not os.path.isfile(path) and url:
                logger.info('Downloading vectors from {}'.format(url))
                if not os.path.exists(cache):
                    os.makedirs(cache)
                dest = os.path.join(cache, os.path.basename(url))
                if not os.path.isfile(dest):
                    with tqdm(unit='B', unit_scale=True, miniters=1, desc=dest) as t:
                        urlretrieve(url, dest, reporthook=reporthook(t))
                logger.info('Extracting vectors into {}'.format(cache))
                ext = os.path.splitext(dest)[1][1:]
                if ext == 'zip':
                    with zipfile.ZipFile(dest, "r") as zf:
                        zf.extractall(cache)
                elif ext == 'gz':
                    with tarfile.open(dest, 'r:gz') as tar:
                        tar.extractall(path=cache)
            if not os.path.isfile(path):
                raise RuntimeError('no vectors found at {}'.format(path))

            # str call is necessary for Python 2/3 compatibility, since
            # argument must be Python 2 str (Python 3 bytes) or
            # Python 3 str (Python 2 unicode)
            itos, vectors, dim = [], array.array(str('d')), None

            # Try to read the whole file with utf-8 encoding.
            binary_lines = False
            try:
                with io.open(path, encoding="utf8") as f:
                    lines = [line for line in f]
            # If there are malformed lines, read in binary mode
            # and manually decode each word from utf-8
            except:
                logger.warning("Could not read {} as UTF8 file, "
                               "reading file as bytes and skipping "
                               "words with malformed UTF8.".format(path))
                with open(path, 'rb') as f:
                    lines = [line for line in f]
                binary_lines = True

            logger.info("Loading vectors from {}".format(path))
            for line in tqdm(lines, total=len(lines)):
                # Explicitly splitting on " " is important, so we don't
                # get rid of Unicode non-breaking spaces in the vectors.
                entries = line.rstrip().split(b" " if binary_lines else " ")

                word, entries = entries[0], entries[1:]
                if dim is None and len(entries) > 1:
                    dim = len(entries)
                elif len(entries) == 1:
                    logger.warning("Skipping token {} with 1-dimensional "
                                   "vector {}; likely a header".format(word, entries))
                    continue
                elif dim != len(entries):
                    raise RuntimeError(
                        "Vector for token {} has {} dimensions, but previously "
                        "read vectors have {} dimensions. All vectors must have "
                        "the same number of dimensions.".format(word, len(entries), dim))

                if binary_lines:
                    try:
                        if isinstance(word, six.binary_type):
                            word = word.decode('utf-8')
                    except:
                        logger.info("Skipping non-UTF8 token {}".format(repr(word)))
                        continue
                vectors.extend(float(x) for x in entries)
                itos.append(word)

            self.itos = itos
            self.stoi = {word: i for i, word in enumerate(itos)}
            self.vectors = torch.Tensor(vectors).view(-1, dim)
            self.dim = dim
            logger.info('Saving vectors to {}'.format(path_pt))
            torch.save((self.itos, self.stoi, self.vectors, self.dim), path_pt)
        else:
            logger.info('Loading vectors from {}'.format(path_pt))
            self.itos, self.stoi, self.vectors, self.dim = torch.load(path_pt)

class GloVe(Vectors):
    url = {
        '42B': 'http://nlp.stanford.edu/data/glove.42B.300d.zip',
        '840B': 'http://nlp.stanford.edu/data/glove.840B.300d.zip',
        'twitter.27B': 'http://nlp.stanford.edu/data/glove.twitter.27B.zip',
        '6B': 'http://nlp.stanford.edu/data/glove.6B.zip',
    }

    def __init__(self, name='840B', dim=300, **kwargs):
        url = self.url[name]
        name = 'glove.{}.{}d.txt'.format(name, str(dim))
        super(GloVe, self).__init__(name, url=url, **kwargs)

class FastText(Vectors):

    url_base = 'https://s3-us-west-1.amazonaws.com/fasttext-vectors/wiki.{}.vec'

    def __init__(self, language="en", **kwargs):
        url = self.url_base.format(language)
        name = os.path.basename(url)
        super(FastText, self).__init__(name, url=url, **kwargs)

class CharNGram(Vectors):

    name = 'charNgram.txt'
    url = ('http://www.logos.t.u-tokyo.ac.jp/~hassy/publications/arxiv2016jmt/'
           'jmt_pre-trained_embeddings.tar.gz')

    def __init__(self, **kwargs):
        super(CharNGram, self).__init__(self.name, url=self.url, **kwargs)

    def __getitem__(self, token):
        vector = torch.Tensor(1, self.dim).zero_()
        if token == "<unk>":
            return self.unk_init(vector)
        # These literals need to be coerced to unicode for Python 2 compatibility
        # when we try to join them with read ngrams from the files.
        chars = ['#BEGIN#'] + list(token) + ['#END#']
        num_vectors = 0
        for n in [2, 3, 4]:
            end = len(chars) - n + 1
            grams = [chars[i:(i + n)] for i in range(end)]
            for gram in grams:
                gram_key = '{}gram-{}'.format(n, ''.join(gram))
                if gram_key in self.stoi:
                    vector += self.vectors[self.stoi[gram_key]]
                    num_vectors += 1
        if num_vectors > 0:
            vector /= num_vectors
        else:
            vector = self.unk_init(vector)
        return vector

def _default_unk_index():
    return 0

pretrained_aliases = {
    "charngram.100d": lambda: CharNGram(),
    "fasttext.en.300d": lambda: FastText(language="en"),
    "fasttext.simple.300d": lambda: FastText(language="simple"),
    "glove.42B.300d": lambda: GloVe(name="42B", dim="300"),
    "glove.840B.300d": lambda: GloVe(name="840B", dim="300"),
    "glove.twitter.27B.25d": lambda: GloVe(name="twitter.27B", dim="25"),
    "glove.twitter.27B.50d": lambda: GloVe(name="twitter.27B", dim="50"),
    "glove.twitter.27B.100d": lambda: GloVe(name="twitter.27B", dim="100"),
    "glove.twitter.27B.200d": lambda: GloVe(name="twitter.27B", dim="200"),
    "glove.6B.50d": lambda: GloVe(name="6B", dim="50"),
    "glove.6B.100d": lambda: GloVe(name="6B", dim="100"),
    "glove.6B.200d": lambda: GloVe(name="6B", dim="200"),
    "glove.6B.300d": lambda: GloVe(name="6B", dim="300")
}
<commit_msg>Fix vocab constructor side effect (#193)<commit_after>from __future__ import unicode_literals
import array
from collections import defaultdict
import io
import logging
import os
import zipfile

import six
from six.moves.urllib.request import urlretrieve
import torch
from tqdm import tqdm
import tarfile

from .utils import reporthook

logger = logging.getLogger(__name__)

class Vocab(object):
    """Defines a vocabulary object that will be used to numericalize a field.

    Attributes:
        freqs: A collections.Counter object holding the frequencies of tokens
            in the data used to build the Vocab.
        stoi: A collections.defaultdict instance mapping token strings to
            numerical identifiers.
        itos: A list of token strings indexed by their numerical identifiers.
    """
    def __init__(self, counter, max_size=None, min_freq=1, specials=['<pad>'],
                 vectors=None):
        """Create a Vocab object from a collections.Counter.

        Arguments:
            counter: collections.Counter object holding the frequencies of
                each value found in the data.
            max_size: The maximum size of the vocabulary, or None for no
                maximum. Default: None.
            min_freq: The minimum frequency needed to include a token in the
                vocabulary. Values less than 1 will be set to 1. Default: 1.
            specials: The list of special tokens (e.g., padding or eos) that
                will be prepended to the vocabulary in addition to an <unk>
                token. Default: ['<pad>']
            vectors: One of either the available pretrained vectors
                or custom pretrained vectors (see Vocab.load_vectors);
                or a list of aforementioned vectors
        """
        self.freqs = counter
        counter = counter.copy()
        min_freq = max(min_freq, 1)
        counter.update(specials)

        self.stoi = defaultdict(_default_unk_index)
        self.stoi.update({tok: i for i, tok in enumerate(specials)})
        self.itos = list(specials)

        counter.subtract({tok: counter[tok] for tok in specials})
        max_size = None if max_size is None else max_size + len(self.itos)

        # sort by frequency, then alphabetically
        words_and_frequencies = sorted(counter.items(), key=lambda tup: tup[0])
        words_and_frequencies.sort(key=lambda tup: tup[1], reverse=True)

        for word, freq in words_and_frequencies:
            if freq < min_freq or len(self.itos) == max_size:
                break
            self.itos.append(word)
            self.stoi[word] = len(self.itos) - 1

        self.vectors = None
        if vectors is not None:
            self.load_vectors(vectors)

    def __eq__(self, other):
        if self.freqs != other.freqs:
            return False
        if self.stoi != other.stoi:
            return False
        if self.itos != other.itos:
            return False
        if self.vectors != other.vectors:
            return False
        return True

    def __len__(self):
        return len(self.itos)

    def extend(self, v, sort=False):
        words = sorted(v.itos) if sort else v.itos
        for w in words:
            if w not in self.stoi:
                self.itos.append(w)
                self.stoi[w] = len(self.itos) - 1

    def load_vectors(self, vectors):
        """
        Arguments:
            vectors: one of or a list containing instantiations of the
                GloVe, CharNGram, or Vectors classes. Alternatively, one
                of or a list of available pretrained vectors:
                    charngram.100d
                    fasttext.en.300d
                    fasttext.simple.300d
                    glove.42B.300d
                    glove.840B.300d
                    glove.twitter.27B.25d
                    glove.twitter.27B.50d
                    glove.twitter.27B.100d
                    glove.twitter.27B.200d
                    glove.6B.50d
                    glove.6B.100d
                    glove.6B.200d
                    glove.6B.300d
        """
        if not isinstance(vectors, list):
            vectors = [vectors]
        for idx, vector in enumerate(vectors):
            if six.PY2 and isinstance(vector, str):
                vector = six.text_type(vector)
            if isinstance(vector, six.string_types):
                # Convert the string pretrained vector identifier
                # to a Vectors object
                if vector not in pretrained_aliases:
                    raise ValueError(
                        "Got string input vector {}, but allowed pretrained "
                        "vectors are {}".format(
                            vector, list(pretrained_aliases.keys())))
                vectors[idx] = pretrained_aliases[vector]()
            elif not isinstance(vector, Vectors):
                raise ValueError(
                    "Got input vectors of type {}, expected str or "
                    "Vectors object".format(type(vector)))

        tot_dim = sum(v.dim for v in vectors)
        self.vectors = torch.Tensor(len(self), tot_dim)
        for i, token in enumerate(self.itos):
            start_dim = 0
            for v in vectors:
                end_dim = start_dim + v.dim
                self.vectors[i][start_dim:end_dim] = v[token.strip()]
                start_dim = end_dim
            assert(start_dim == tot_dim)

    def set_vectors(self, stoi, vectors, dim, unk_init=torch.Tensor.zero_):
        """
        Set the vectors for the Vocab instance from a collection of Tensors.

        Arguments:
            stoi: A dictionary of string to the index of the associated vector
                in the `vectors` input argument.
            vectors: An indexed iterable (or other structure supporting __getitem__) that
                given an input index, returns a FloatTensor representing the vector
                for the token associated with the index. For example,
                vector[stoi["string"]] should return the vector for "string".
            dim: The dimensionality of the vectors.
            unk_init (callback): by default, initialize out-of-vocabulary word vectors
                to zero vectors; can be any function that takes in a Tensor and
                returns a Tensor of the same size. Default: torch.Tensor.zero_
        """
        self.vectors = torch.Tensor(len(self), dim)
        for i, token in enumerate(self.itos):
            wv_index = stoi.get(token, None)
            if wv_index is not None:
                self.vectors[i] = vectors[wv_index]
            else:
                self.vectors[i] = unk_init(self.vectors[i])

class SubwordVocab(Vocab):

    def __init__(self, counter, max_size=None, specials=['<pad>'],
                 vectors=None, unk_init=torch.Tensor.zero_, expand_vocab=False):
        """Create a revtok subword vocabulary from a collections.Counter.

        Arguments:
            counter: collections.Counter object holding the frequencies of
                each word found in the data.
            max_size: The maximum size of the subword vocabulary, or None for no
                maximum. Default: None.
            specials: The list of special tokens (e.g., padding or eos) that
                will be prepended to the vocabulary in addition to an <unk>
                token.
        """
        try:
            import revtok
        except ImportError:
            print("Please install revtok.")
            raise

        self.stoi = defaultdict(_default_unk_index)
        self.stoi.update({tok: i for i, tok in enumerate(specials)})
        self.itos = specials

        self.segment = revtok.SubwordSegmenter(counter, max_size)

        max_size = None if max_size is None else max_size + len(self.itos)

        # sort by frequency/entropy, then alphabetically
        toks = sorted(self.segment.vocab.items(),
                      key=lambda tup: (len(tup[0]) != 1, -tup[1], tup[0]))

        for tok, _ in toks:
            self.itos.append(tok)
            self.stoi[tok] = len(self.itos) - 1

        if vectors is not None:
            self.load_vectors(vectors, unk_init=unk_init, expand_vocab=expand_vocab)

class Vectors(object):

    def __init__(self, name, cache='.vector_cache',
                 url=None, unk_init=torch.Tensor.zero_):
        """Arguments:
               name: name of the file that contains the vectors
               cache: directory for cached vectors
               url: url for download if vectors not found in cache
               unk_init (callback): by default, initalize out-of-vocabulary word vectors
                   to zero vectors; can be any function that takes in a Tensor and
                   returns a Tensor of the same size
         """
        self.unk_init = unk_init
        self.cache(name, cache, url=url)

    def __getitem__(self, token):
        if token in self.stoi:
            return self.vectors[self.stoi[token]]
        else:
            return self.unk_init(torch.Tensor(1, self.dim))

    def cache(self, name, cache, url=None):
        if os.path.isfile(name):
            path = name
            path_pt = os.path.join(cache, os.path.basename(name)) + '.pt'
        else:
            path = os.path.join(cache, name)
            path_pt = path + '.pt'

        if not os.path.isfile(path_pt):
            if not os.path.isfile(path) and url:
                logger.info('Downloading vectors from {}'.format(url))
                if not os.path.exists(cache):
                    os.makedirs(cache)
                dest = os.path.join(cache, os.path.basename(url))
                if not os.path.isfile(dest):
                    with tqdm(unit='B', unit_scale=True, miniters=1, desc=dest) as t:
                        urlretrieve(url, dest, reporthook=reporthook(t))
                logger.info('Extracting vectors into {}'.format(cache))
                ext = os.path.splitext(dest)[1][1:]
                if ext == 'zip':
                    with zipfile.ZipFile(dest, "r") as zf:
                        zf.extractall(cache)
                elif ext == 'gz':
                    with tarfile.open(dest, 'r:gz') as tar:
                        tar.extractall(path=cache)
            if not os.path.isfile(path):
                raise RuntimeError('no vectors found at {}'.format(path))

            # str call is necessary for Python 2/3 compatibility, since
            # argument must be Python 2 str (Python 3 bytes) or
            # Python 3 str (Python 2 unicode)
            itos, vectors, dim = [], array.array(str('d')), None

            # Try to read the whole file with utf-8 encoding.
            binary_lines = False
            try:
                with io.open(path, encoding="utf8") as f:
                    lines = [line for line in f]
            # If there are malformed lines, read in binary mode
            # and manually decode each word from utf-8
            except:
                logger.warning("Could not read {} as UTF8 file, "
                               "reading file as bytes and skipping "
                               "words with malformed UTF8.".format(path))
                with open(path, 'rb') as f:
                    lines = [line for line in f]
                binary_lines = True

            logger.info("Loading vectors from {}".format(path))
            for line in tqdm(lines, total=len(lines)):
                # Explicitly splitting on " " is important, so we don't
                # get rid of Unicode non-breaking spaces in the vectors.
                entries = line.rstrip().split(b" " if binary_lines else " ")

                word, entries = entries[0], entries[1:]
                if dim is None and len(entries) > 1:
                    dim = len(entries)
                elif len(entries) == 1:
                    logger.warning("Skipping token {} with 1-dimensional "
                                   "vector {}; likely a header".format(word, entries))
                    continue
                elif dim != len(entries):
                    raise RuntimeError(
                        "Vector for token {} has {} dimensions, but previously "
                        "read vectors have {} dimensions. All vectors must have "
                        "the same number of dimensions.".format(word, len(entries), dim))

                if binary_lines:
                    try:
                        if isinstance(word, six.binary_type):
                            word = word.decode('utf-8')
                    except:
                        logger.info("Skipping non-UTF8 token {}".format(repr(word)))
                        continue
                vectors.extend(float(x) for x in entries)
                itos.append(word)

            self.itos = itos
            self.stoi = {word: i for i, word in enumerate(itos)}
            self.vectors = torch.Tensor(vectors).view(-1, dim)
            self.dim = dim
            logger.info('Saving vectors to {}'.format(path_pt))
            torch.save((self.itos, self.stoi, self.vectors, self.dim), path_pt)
        else:
            logger.info('Loading vectors from {}'.format(path_pt))
            self.itos, self.stoi, self.vectors, self.dim = torch.load(path_pt)

class GloVe(Vectors):
    url = {
        '42B': 'http://nlp.stanford.edu/data/glove.42B.300d.zip',
        '840B': 'http://nlp.stanford.edu/data/glove.840B.300d.zip',
        'twitter.27B': 'http://nlp.stanford.edu/data/glove.twitter.27B.zip',
        '6B': 'http://nlp.stanford.edu/data/glove.6B.zip',
    }

    def __init__(self, name='840B', dim=300, **kwargs):
        url = self.url[name]
        name = 'glove.{}.{}d.txt'.format(name, str(dim))
        super(GloVe, self).__init__(name, url=url, **kwargs)

class FastText(Vectors):

    url_base = 'https://s3-us-west-1.amazonaws.com/fasttext-vectors/wiki.{}.vec'

    def __init__(self, language="en", **kwargs):
        url = self.url_base.format(language)
        name = os.path.basename(url)
        super(FastText, self).__init__(name, url=url, **kwargs)

class CharNGram(Vectors):

    name = 'charNgram.txt'
    url = ('http://www.logos.t.u-tokyo.ac.jp/~hassy/publications/arxiv2016jmt/'
           'jmt_pre-trained_embeddings.tar.gz')

    def __init__(self, **kwargs):
        super(CharNGram, self).__init__(self.name, url=self.url, **kwargs)

    def __getitem__(self, token):
        vector = torch.Tensor(1, self.dim).zero_()
        if token == "<unk>":
            return self.unk_init(vector)
        # These literals need to be coerced to unicode for Python 2 compatibility
        # when we try to join them with read ngrams from the files.
        chars = ['#BEGIN#'] + list(token) + ['#END#']
        num_vectors = 0
        for n in [2, 3, 4]:
            end = len(chars) - n + 1
            grams = [chars[i:(i + n)] for i in range(end)]
            for gram in grams:
                gram_key = '{}gram-{}'.format(n, ''.join(gram))
                if gram_key in self.stoi:
                    vector += self.vectors[self.stoi[gram_key]]
                    num_vectors += 1
        if num_vectors > 0:
            vector /= num_vectors
        else:
            vector = self.unk_init(vector)
        return vector

def _default_unk_index():
    return 0

pretrained_aliases = {
    "charngram.100d": lambda: CharNGram(),
    "fasttext.en.300d": lambda: FastText(language="en"),
    "fasttext.simple.300d": lambda: FastText(language="simple"),
    "glove.42B.300d": lambda: GloVe(name="42B", dim="300"),
    "glove.840B.300d": lambda: GloVe(name="840B", dim="300"),
    "glove.twitter.27B.25d": lambda: GloVe(name="twitter.27B", dim="25"),
    "glove.twitter.27B.50d": lambda: GloVe(name="twitter.27B", dim="50"),
    "glove.twitter.27B.100d": lambda: GloVe(name="twitter.27B", dim="100"),
    "glove.twitter.27B.200d": lambda: GloVe(name="twitter.27B", dim="200"),
    "glove.6B.50d": lambda: GloVe(name="6B", dim="50"),
    "glove.6B.100d": lambda: GloVe(name="6B", dim="100"),
    "glove.6B.200d": lambda: GloVe(name="6B", dim="200"),
    "glove.6B.300d": lambda: GloVe(name="6B", dim="300")
}
<|endoftext|>


<commit_before># Copyright (c) 2013 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.

import io
import json

import falcon
import six
import testtools

from marconi.queues.transport.wsgi import utils

class TestWSGIutils(testtools.TestCase):

    def test_get_checked_field_missing(self):
        doc = {}

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 'openstack', int)

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 42, int)

        doc = {'openstac': 10}

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 'openstack', int)

    def test_get_checked_field_bad_type(self):
        doc = {'openstack': '10'}

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 'openstack', int)

        doc = {'openstack': 10, 'openstack-mq': 'test'}

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 'openstack', str)

        doc = {'openstack': '[1, 2]'}

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 'openstack', list)

    def test_get_checked_field(self):
        doc = {'hello': 'world', 'the answer': 42, 'question': []}

        value = utils.get_checked_field(doc, 'hello', str)
        self.assertEqual(value, 'world')

        value = utils.get_checked_field(doc, 'the answer', int)
        self.assertEqual(value, 42)

        value = utils.get_checked_field(doc, 'question', list)
        self.assertEqual(value, [])

    def test_filter_missing(self):
        doc = {'body': {'event': 'start_backup'}}
        spec = (('tag', dict),)
        self.assertRaises(falcon.HTTPBadRequest,
                          utils.filter, doc, spec)

    def test_filter_bad_type(self):
        doc = {'ttl': '300', 'bogus': 'yogabbagabba'}
        spec = [('ttl', int)]
        self.assertRaises(falcon.HTTPBadRequest,
                          utils.filter, doc, spec)

    def test_filter(self):
        doc = {'body': {'event': 'start_backup'}}

        def spec():
            yield ('body', dict)

        filtered = utils.filter(doc, spec())
        self.assertEqual(filtered, doc)

        doc = {'ttl': 300, 'bogus': 'yogabbagabba'}
        spec = [('ttl', int)]
        filtered = utils.filter(doc, spec)
        self.assertEqual(filtered, {'ttl': 300})

        doc = {'body': {'event': 'start_backup'}, 'ttl': 300}
        spec = (('body', dict), ('ttl', int))
        filtered = utils.filter(doc, spec)
        self.assertEqual(filtered, doc)

    def test_no_spec(self):
        obj = {u'body': {'event': 'start_backup'}, 'ttl': 300}
        document = six.text_type(json.dumps(obj, ensure_ascii=False))
        doc_stream = io.StringIO(document)

        filtered = utils.filter_stream(doc_stream, len(document), spec=None)
        self.assertEqual(filtered[0], obj)

        # NOTE(kgriffs): Ensure default value for *spec* is None
        doc_stream.seek(0)
        filtered2 = utils.filter_stream(doc_stream, len(document))
        self.assertEqual(filtered2, filtered)

    def test_no_spec_array(self):
        things = [{u'body': {'event': 'start_backup'}, 'ttl': 300}]
        document = six.text_type(json.dumps(things, ensure_ascii=False))
        doc_stream = io.StringIO(document)

        filtered = utils.filter_stream(doc_stream, len(document),
                                       doctype=utils.JSONArray, spec=None)
        self.assertEqual(filtered, things)

    def test_filter_star(self):
        doc = {'ttl': 300, 'body': {'event': 'start_backup'}}

        spec = [('body', '*'), ('ttl', '*')]
        filtered = utils.filter(doc, spec)

        self.assertEqual(filtered, doc)

    def test_filter_stream_expect_obj(self):
        obj = {u'body': {'event': 'start_backup'}, 'id': 'DEADBEEF'}

        document = six.text_type(json.dumps(obj, ensure_ascii=False))
        stream = io.StringIO(document)
        spec = [('body', dict), ('id', basestring)]
        filtered_object, = utils.filter_stream(stream, len(document), spec)

        self.assertEqual(filtered_object, obj)

        stream.seek(0)
        self.assertRaises(falcon.HTTPBadRequest,
                          utils.filter_stream, stream, len(document), spec,
                          doctype=utils.JSONArray)

    def test_filter_stream_expect_array(self):
        array = [{u'body': {u'x': 1}}, {u'body': {u'x': 2}}]

        document = six.text_type(json.dumps(array, ensure_ascii=False))
        stream = io.StringIO(document)
        spec = [('body', dict)]
        filtered_objects = list(utils.filter_stream(
            stream, len(document), spec, doctype=utils.JSONArray))

        self.assertEqual(filtered_objects, array)

        stream.seek(0)
        self.assertRaises(falcon.HTTPBadRequest,
                          utils.filter_stream, stream, len(document), spec,
                          doctype=utils.JSONObject)

    def test_filter_stream_wrong_use(self):
        document = u'3'
        stream = io.StringIO(document)
        spec = None
        self.assertRaises(TypeError,
                          utils.filter_stream, stream, len(document), spec,
                          doctype=int)

    def test_filter_stream_no_reading(self):
        stream = None
        length = None
        self.assertRaises(falcon.HTTPBadRequest,
                          utils.filter_stream, stream, length, None)
<commit_msg>Replace basestring by six for python3 compatability<commit_after># Copyright (c) 2013 Rackspace, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.

import io
import json

import falcon
import six
import testtools

from marconi.queues.transport.wsgi import utils

class TestWSGIutils(testtools.TestCase):

    def test_get_checked_field_missing(self):
        doc = {}

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 'openstack', int)

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 42, int)

        doc = {'openstac': 10}

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 'openstack', int)

    def test_get_checked_field_bad_type(self):
        doc = {'openstack': '10'}

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 'openstack', int)

        doc = {'openstack': 10, 'openstack-mq': 'test'}

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 'openstack', str)

        doc = {'openstack': '[1, 2]'}

        self.assertRaises(falcon.HTTPBadRequest,
                          utils.get_checked_field, doc, 'openstack', list)

    def test_get_checked_field(self):
        doc = {'hello': 'world', 'the answer': 42, 'question': []}

        value = utils.get_checked_field(doc, 'hello', str)
        self.assertEqual(value, 'world')

        value = utils.get_checked_field(doc, 'the answer', int)
        self.assertEqual(value, 42)

        value = utils.get_checked_field(doc, 'question', list)
        self.assertEqual(value, [])

    def test_filter_missing(self):
        doc = {'body': {'event': 'start_backup'}}
        spec = (('tag', dict),)
        self.assertRaises(falcon.HTTPBadRequest,
                          utils.filter, doc, spec)

    def test_filter_bad_type(self):
        doc = {'ttl': '300', 'bogus': 'yogabbagabba'}
        spec = [('ttl', int)]
        self.assertRaises(falcon.HTTPBadRequest,
                          utils.filter, doc, spec)

    def test_filter(self):
        doc = {'body': {'event': 'start_backup'}}

        def spec():
            yield ('body', dict)

        filtered = utils.filter(doc, spec())
        self.assertEqual(filtered, doc)

        doc = {'ttl': 300, 'bogus': 'yogabbagabba'}
        spec = [('ttl', int)]
        filtered = utils.filter(doc, spec)
        self.assertEqual(filtered, {'ttl': 300})

        doc = {'body': {'event': 'start_backup'}, 'ttl': 300}
        spec = (('body', dict), ('ttl', int))
        filtered = utils.filter(doc, spec)
        self.assertEqual(filtered, doc)

    def test_no_spec(self):
        obj = {u'body': {'event': 'start_backup'}, 'ttl': 300}
        document = six.text_type(json.dumps(obj, ensure_ascii=False))
        doc_stream = io.StringIO(document)

        filtered = utils.filter_stream(doc_stream, len(document), spec=None)
        self.assertEqual(filtered[0], obj)

        # NOTE(kgriffs): Ensure default value for *spec* is None
        doc_stream.seek(0)
        filtered2 = utils.filter_stream(doc_stream, len(document))
        self.assertEqual(filtered2, filtered)

    def test_no_spec_array(self):
        things = [{u'body': {'event': 'start_backup'}, 'ttl': 300}]
        document = six.text_type(json.dumps(things, ensure_ascii=False))
        doc_stream = io.StringIO(document)

        filtered = utils.filter_stream(doc_stream, len(document),
                                       doctype=utils.JSONArray, spec=None)
        self.assertEqual(filtered, things)

    def test_filter_star(self):
        doc = {'ttl': 300, 'body': {'event': 'start_backup'}}

        spec = [('body', '*'), ('ttl', '*')]
        filtered = utils.filter(doc, spec)

        self.assertEqual(filtered, doc)

    def test_filter_stream_expect_obj(self):
        obj = {u'body': {'event': 'start_backup'}, 'id': 'DEADBEEF'}

        document = six.text_type(json.dumps(obj, ensure_ascii=False))
        stream = io.StringIO(document)
        spec = [('body', dict), ('id', six.string_types)]
        filtered_object, = utils.filter_stream(stream, len(document), spec)

        self.assertEqual(filtered_object, obj)

        stream.seek(0)
        self.assertRaises(falcon.HTTPBadRequest,
                          utils.filter_stream, stream, len(document), spec,
                          doctype=utils.JSONArray)

    def test_filter_stream_expect_array(self):
        array = [{u'body': {u'x': 1}}, {u'body': {u'x': 2}}]

        document = six.text_type(json.dumps(array, ensure_ascii=False))
        stream = io.StringIO(document)
        spec = [('body', dict)]
        filtered_objects = list(utils.filter_stream(
            stream, len(document), spec, doctype=utils.JSONArray))

        self.assertEqual(filtered_objects, array)

        stream.seek(0)
        self.assertRaises(falcon.HTTPBadRequest,
                          utils.filter_stream, stream, len(document), spec,
                          doctype=utils.JSONObject)

    def test_filter_stream_wrong_use(self):
        document = u'3'
        stream = io.StringIO(document)
        spec = None
        self.assertRaises(TypeError,
                          utils.filter_stream, stream, len(document), spec,
                          doctype=int)

    def test_filter_stream_no_reading(self):
        stream = None
        length = None
        self.assertRaises(falcon.HTTPBadRequest,
                          utils.filter_stream, stream, length, None)
<|endoftext|>