<commit_before>import ast
import helper
import typing

globalVars = {}

# rootNode = ModuleNode(ast.parse(...), None)
class Base:
    def __init__(self, astNode, parent):
        self.ast = astNode
        self.parent = parent

        # Auto-build nodes out of children fields.
        for fieldName in self.ast._fields:
            child = getattr(self.ast, fieldName)
            setattr(self, fieldName, self.resolve(child))
    def resolve(self, element):
        '''
        Returns an string representation of an element if it is primitive,
        or casts an ast.AST object into its abstraction.  Resolves recursively
        for lists.
        '''
        # Handling primitive children
        if typing.isNone(element) or typing.isBoolean(element):
            return element
        elif typing.isPrimitive(element):
            return str(element)
        elif typing.isExpressionContext(element):
            return typing.getClassName(element)
        elif typing.isList(element):
            return [self.resolve(member).compile() for member in element]
        else:
            className = typing.getClassName(element)
            nodeClass = globals()[className]
            return nodeClass(element, self)
    def getChildren(self):
        [getattr(self, fieldName) for fieldName in self.ast._fields]
    def compile(self):
        pass
    def __str__(self):
        return self.compile()

class Module(Base):
    def compile(self):
        return '%s' % Block(self.body, self)

class FunctionDef(Base):
    def compile(self):
        compiledArgs = self.args.compile()
        return 'this.%s = function(%s){%s%s}' % (self.name, compiledArgs[0], compiledArgs[1], self.body)

class Name(Base):
    def compile(self):
        try:
            conversion = {
                'None': 'null',
                'True': 'true',
                'False': 'false'
            }
            self.id = conversion[self.id]
        except KeyError:
            pass

        return '%s' % self.id

class Expr(Base):
    def compile(self):
        return '%s' % self.value

class Compare(Base):
    def compile(self):
        allComparators = [self.left.compile()]
        allComparators.extend(self.comparators[:])
        leftComparators = allComparators[:-1]
        rightComparators = allComparators[1:]
        return ' && '.join('(%s %s %s)' % (left, op, right) for (left, op, right) in zip(leftComparators, self.ops, rightComparators))

class Num(Base):
    def compile(self):
        return self.n

class Str(Base):
    def compile(self):
        stringValue = self.s
        if (stringValue.startswith('"')):
            stringValue = "'%s'" % stringValue
        else:
            stringValue = '"%s"' % stringValue
        return '%s' % stringValue

class Break(Base):
    def compile(self):
        return 'break'

class Continue(Base):
    def compile(self):
        return 'continue'

class Pass(Base):
    def compile(self):
        return 'return void(0)'

class Add(Base):
    def compile(self):
        return '+'

class Sub(Base):
    def compile(self):
        return '-'

class Mult(Base):
    def compile(self):
        return '*'

class Div(Base):
    def compile(self):
        return '/'

class Mod(Base):
    def compile(self):
        return '%'

class LShift(Base):
    def compile(self):
        return '<<'

class RShift(Base):
    def compile(self):
        return '>>'

class BitOr(Base):
    def compile(self):
        return '|'

class BitXor(Base):
    def compile(self):
        return '^'

class BitAnd(Base):
    def compile(self):
        return '&'

class Pow(Base):
    def compile(self):
        return 'Math.pow(%s, %s)' % (self.parent.left, self.parent.right)

class FloorDiv(Base):
    def compile(self):
        return 'Math.floor(%s, %s)' % (self.parent.left, self.parent.right)

class Invert(Base):
    def compile(self):
        return '~'

class Not(Base):
    def compile(self):
        return '!'

class UAdd(Base):
    def compile(self):
        return '+'

class USub(Base):
    def compile(self):
        return '-'

class BinOp(Base):
    def compile(self):
        if typing.getClassName(self.op) in ['Pow', 'FloorDiv']:
            return MathOp(self.ast, self.parent).compile()
        else:
            return '%s %s %s' % (self.left, self.op, self.right)

class MathOp(Base):
    def compile(self):
        return '%s' % self.op

class UnaryOp(Base):
    def compile(self):
        return '%s%s' % (self.op, self.operand)

class IfExp(Base):
    def compile(self):
        condition = self.test.compile()
        ifBody = self.body.compile()
        elseBody = self.orelse.compile()
        return '%s ? %s : %s' % (condition, ifBody, elseBody)

class Return(Base):
    def compile(self):
        return 'return %s' % self.value

class BoolOp(Base):
    def compile(self):
        return  (' %s ' % self.op).join(self.values)

class And(Base):
    def compile(self):
        return '&&'

class Or(Base):
    def compile(self):
        return '||'

# **Consider:** need to handle global var assignments eventually.
class Assign(Base):
    def compile(self):
        return ['var %s = %s' % (target, self.value) for target in self.targets]

class Delete(Base):
    def compile(self):
        return 'delete ' + ', '.join(target for target in self.targets)

class AugAssign(Base):
    def compile(self):
        return '%s %s= %s' % (self.target, self.op, self.value)

class Eq(Base):
    def compile(self):
        return '=='

class NotEq(Base):
    def compile(self):
        return '!='

class Lt(Base):
    def compile(self):
        return '<'

class LtE(Base):
    def compile(self):
        return '<='

class Gt(Base):
    def compile(self):
        return '>'

class GtE(Base):
    def compile(self):
        return '>='

class Is(Base):
    def compile(self):
        return '==='

class IsNot(Base):
    def compile(self):
        return '!=='

# **Consider:** ctx, Ellipsis, ExtSlice form of slice
class Subscript(Base):
    def compile(self):
        return '%s' % self.slice

class Slice(Base):
    def compile(self):
        hasLower = self.lower is not None
        hasUpper = self.upper is not None
        hasStep = self.step is not None and self.step.compile() != 'null'
        slicedArray = None

        if hasLower and hasUpper:
            slicedArray = "%s.slice(%s, %s)" % (self.parent.value, self.lower, self.upper)
        elif hasLower:
            slicedArray = "%s.slice(%s)" % (self.parent.value, self.lower)
        elif hasUpper:
            slicedArray = "%s.slice(0, %s)" % (self.parent.value, self.upper)
        else: #has neither
            slicedArray = "%s.slice(0, %s)" % (self.parent.value, "%s.length" % self.parent.value)
        if hasStep:
            step = int(self.step.compile())
            if step > 0:
                slicedArray += (".filter(" +
                                    "function(element, index, array){" +
                                        "return index %s %s == 0;" +
                                    "}" +
                                ")") % ('%', self.step)
            elif step < 0:
                slicedArray += (".filter(" +
                                    "function(element, index, array){" +
                                        "return (array.length - index - 1) %s %s == 0;" +
                                    "}" +
                                ").reverse()") % ('%', self.step)
            elif step == 0:
                return "ValueError: slice step cannot be zero"
        return slicedArray

class Index(Base):
    def compile(self):
        return '%s[%s]' % (self.parent.value, self.value)

class Dict(Base):
    def compile(self):
            return '{%s}' % (', '.join('%s: %s' % (key, value) for (key, value) in zip(self.keys, self.values)))

# **Consider:** What is nl - newline?
class Print(Base):
    def compile(self):
        return 'console.log(%s)' % ', '.join(value for value in self.values)

class List(Base):
    def compile(self):
        return '[%s]' % ', '.join(value for value in self.elts)

#### Control Flow

class If(Base):
    def compile(self):
        compiled = helper.multiline([
                     'if(%s){',
                     '  %s',
                     '}'
                   ]) % (self.test, helper.formatGroup(self.body))
        # **Consider:** abstract list.isEmpty() ?
        if len(self.orelse) != 0:
            compiled = helper.multiline([
                         compiled,
                         'else{',
                         '  %s',
                         '}'
                       ]) % (helper.formatGroup(self.orelse))
        return compiled

#TODO: Stabilize. Obviously having inlined variable names could result in variable name conflicts
class For(Base):
    def compile(self):
        target = self.target.compile()
        iterable = self.iter.compile()
        return helper.multiline([
                   'var uniqueObj = Object(%s);', # Doing this because strings in python are iterable, but not in javascript
                   'if(Object.keys(uniqueObj).length > 0){',
                   '  var %s;',
                   '  var uniqueCounter = 0;',
                   '  for(uniqueVar in uniqueObj){',
                   '    uniqueCounter++;',
                   '    %s = uniqueObj[uniqueVar];',
                   '    %s',
                   '    if(uniqueCounter >= Object.keys(uniqueObj).length){',
                   '      %s',
                   '      break;',
                   '    }',
                   '  }',
                   '}',
                   'else{',
                   '  %s',
                   '}'
                ]) % (iterable, target, target, helper.formatGroup(self.body), helper.formatGroup(self.orelse), helper.formatGroup(self.orelse))

class While(Base):
    def compile(self):
        return helper.multiline([
                   'if(%s){',
                   '  while(true){',
                   '    %s',
                   '    if(!%s){',
                   '      %s',
                   '      break;',
                   '    }',
                   '  }'
                   '}',
                   'else{',
                   '  %s',
                   '}'
                ]) % (self.test, helper.formatGroup(self.body), self.test, helper.formatGroup(self.orelse), helper.formatGroup(self.orelse))
##################################

#### Functions

# Currently cannot handle variable or keyword arguments.

class arguments(Base):
    def compile(self):
        return ', '.join(self.args)

class Lambda(Base):
    def compile(self):
        return helper.multiline([
            'function(%s){',
            '  return %s;',
            '}'
        ]) % (self.args, self.body)

class FunctionDef(Base):
    def compile(self):
        return helper.multiline([
            'this.%s = function(%s){',
            '  %s',
            '}'
        ]) % (self.name, self.args, helper.formatGroup(self.body))

class Call(Base):
    def compile(self):
        argument = ', '.join(arg for arg in self.args)
        return 'this.%s(%s)' % (self.func, argument)

##################################

#### Error Handling

class ExceptHandler(Base):
    def compile(self):
        return self

# TODO: Incorporate self.inst, trace
class Raise(Base):
    def compile(self):
        exceptionType = self.type
        return 'throw Error("%s")' % exceptionType

class TryExcept(Base):
    def compileCatchStatements(self):
        catchStatements = []
        for handler in self.handlers:
            body = Block(handler.body, self.parent)
            exceptionType = handler.type
            if exceptionType is not None:
                catch = helper.multiline([
                    "if(e.message === '%s'){",
                    "    %s",
                    "}"
                ]) % (exceptionType, body)
            else:
                catch = '%s' % body
            catchStatements.append(catch)
        return '\r\n'.join(catchStatements)

    def compile(self):
        tryBody = Block(self.body, self.parent)
        catchBody = self.compileCatchStatements()
        elseBody = Block(self.orelse, self.parent)
        return helper.multiline([
            'try{',
            '  %s',
            '}',
            'catch(e){',
            '  var caughtException = true;',
            '  %s',
            '}',
            'if(!caughtException){',
            '  %s',
            '}'
        ]) % (tryBody, catchBody, elseBody)

class TryFinally(Base):
    def compile(self):
        tryBody = Block(self.body, self.parent)
        finallyBody = Block(self.finalbody, self.parent)
        return helper.multiline([
            'try{',
            '  %s',
            '}',
            'finally{',
            '  %s',
            '}',
        ]) % (tryBody, finallyBody)
##################################

class Block(Base):
    '''
    An abstraction of a closure.  This occurs in Python whenever we have:
    * module
    * class definition
    * function body
    '''
    def __init__(self, children, parent):
        self.children = children
        self.parent = parent
        self.indent = self.calcIndent()
        self.locals = {}
        # To keep track of global declarations in python.  This causes different
        # assignment statements in JS.
        self.globalRefs = set()

    # Traverse up parents until the nearest Block node, return that indent value
    # plus 1
    def calcIndent(self):
        def finishedTraversal(pointer):
            return pointer != None
        indent = 1
        pointer = self.parent
        while(not finishedTraversal(pointer)):
            if typing.getClassName(pointer) == 'Block':
                indent = pointer.indent + 1
                break
        return indent

    def compile(self):
        # Appends `;` to statements in the body - those passed back in an array
        # each have a `;` attached to their ends.  To avoid this (e.g. in if/
        # else blocks), simply concatenate the code before passing it back
        group = helper.formatGroup(helper.expand(self.children))
        enclosed = helper.closure(group)

        # need to indent the entire block now
        return helper.indent(enclosed, self.indent)
<commit_msg>Removed duplicate FunctionDef class<commit_after>import ast
import helper
import typing

globalVars = {}

# rootNode = ModuleNode(ast.parse(...), None)
class Base:
    def __init__(self, astNode, parent):
        self.ast = astNode
        self.parent = parent

        # Auto-build nodes out of children fields.
        for fieldName in self.ast._fields:
            child = getattr(self.ast, fieldName)
            setattr(self, fieldName, self.resolve(child))
    def resolve(self, element):
        '''
        Returns an string representation of an element if it is primitive,
        or casts an ast.AST object into its abstraction.  Resolves recursively
        for lists.
        '''
        # Handling primitive children
        if typing.isNone(element) or typing.isBoolean(element):
            return element
        elif typing.isPrimitive(element):
            return str(element)
        elif typing.isExpressionContext(element):
            return typing.getClassName(element)
        elif typing.isList(element):
            return [self.resolve(member).compile() for member in element]
        else:
            className = typing.getClassName(element)
            nodeClass = globals()[className]
            return nodeClass(element, self)
    def getChildren(self):
        [getattr(self, fieldName) for fieldName in self.ast._fields]
    def compile(self):
        pass
    def __str__(self):
        return self.compile()

class Module(Base):
    def compile(self):
        return '%s' % Block(self.body, self)

class Name(Base):
    def compile(self):
        try:
            conversion = {
                'None': 'null',
                'True': 'true',
                'False': 'false'
            }
            self.id = conversion[self.id]
        except KeyError:
            pass

        return '%s' % self.id

class Expr(Base):
    def compile(self):
        return '%s' % self.value

class Compare(Base):
    def compile(self):
        allComparators = [self.left.compile()]
        allComparators.extend(self.comparators[:])
        leftComparators = allComparators[:-1]
        rightComparators = allComparators[1:]
        return ' && '.join('(%s %s %s)' % (left, op, right) for (left, op, right) in zip(leftComparators, self.ops, rightComparators))

class Num(Base):
    def compile(self):
        return self.n

class Str(Base):
    def compile(self):
        stringValue = self.s
        if (stringValue.startswith('"')):
            stringValue = "'%s'" % stringValue
        else:
            stringValue = '"%s"' % stringValue
        return '%s' % stringValue

class Break(Base):
    def compile(self):
        return 'break'

class Continue(Base):
    def compile(self):
        return 'continue'

class Pass(Base):
    def compile(self):
        return 'return void(0)'

class Add(Base):
    def compile(self):
        return '+'

class Sub(Base):
    def compile(self):
        return '-'

class Mult(Base):
    def compile(self):
        return '*'

class Div(Base):
    def compile(self):
        return '/'

class Mod(Base):
    def compile(self):
        return '%'

class LShift(Base):
    def compile(self):
        return '<<'

class RShift(Base):
    def compile(self):
        return '>>'

class BitOr(Base):
    def compile(self):
        return '|'

class BitXor(Base):
    def compile(self):
        return '^'

class BitAnd(Base):
    def compile(self):
        return '&'

class Pow(Base):
    def compile(self):
        return 'Math.pow(%s, %s)' % (self.parent.left, self.parent.right)

class FloorDiv(Base):
    def compile(self):
        return 'Math.floor(%s, %s)' % (self.parent.left, self.parent.right)

class Invert(Base):
    def compile(self):
        return '~'

class Not(Base):
    def compile(self):
        return '!'

class UAdd(Base):
    def compile(self):
        return '+'

class USub(Base):
    def compile(self):
        return '-'

class BinOp(Base):
    def compile(self):
        if typing.getClassName(self.op) in ['Pow', 'FloorDiv']:
            return MathOp(self.ast, self.parent).compile()
        else:
            return '%s %s %s' % (self.left, self.op, self.right)

class MathOp(Base):
    def compile(self):
        return '%s' % self.op

class UnaryOp(Base):
    def compile(self):
        return '%s%s' % (self.op, self.operand)

class IfExp(Base):
    def compile(self):
        condition = self.test.compile()
        ifBody = self.body.compile()
        elseBody = self.orelse.compile()
        return '%s ? %s : %s' % (condition, ifBody, elseBody)

class Return(Base):
    def compile(self):
        return 'return %s' % self.value

class BoolOp(Base):
    def compile(self):
        return  (' %s ' % self.op).join(self.values)

class And(Base):
    def compile(self):
        return '&&'

class Or(Base):
    def compile(self):
        return '||'

# **Consider:** need to handle global var assignments eventually.
class Assign(Base):
    def compile(self):
        return ['var %s = %s' % (target, self.value) for target in self.targets]

class Delete(Base):
    def compile(self):
        return 'delete ' + ', '.join(target for target in self.targets)

class AugAssign(Base):
    def compile(self):
        return '%s %s= %s' % (self.target, self.op, self.value)

class Eq(Base):
    def compile(self):
        return '=='

class NotEq(Base):
    def compile(self):
        return '!='

class Lt(Base):
    def compile(self):
        return '<'

class LtE(Base):
    def compile(self):
        return '<='

class Gt(Base):
    def compile(self):
        return '>'

class GtE(Base):
    def compile(self):
        return '>='

class Is(Base):
    def compile(self):
        return '==='

class IsNot(Base):
    def compile(self):
        return '!=='

# **Consider:** ctx, Ellipsis, ExtSlice form of slice
class Subscript(Base):
    def compile(self):
        return '%s' % self.slice

class Slice(Base):
    def compile(self):
        hasLower = self.lower is not None
        hasUpper = self.upper is not None
        hasStep = self.step is not None and self.step.compile() != 'null'
        slicedArray = None

        if hasLower and hasUpper:
            slicedArray = "%s.slice(%s, %s)" % (self.parent.value, self.lower, self.upper)
        elif hasLower:
            slicedArray = "%s.slice(%s)" % (self.parent.value, self.lower)
        elif hasUpper:
            slicedArray = "%s.slice(0, %s)" % (self.parent.value, self.upper)
        else: #has neither
            slicedArray = "%s.slice(0, %s)" % (self.parent.value, "%s.length" % self.parent.value)
        if hasStep:
            step = int(self.step.compile())
            if step > 0:
                slicedArray += (".filter(" +
                                    "function(element, index, array){" +
                                        "return index %s %s == 0;" +
                                    "}" +
                                ")") % ('%', self.step)
            elif step < 0:
                slicedArray += (".filter(" +
                                    "function(element, index, array){" +
                                        "return (array.length - index - 1) %s %s == 0;" +
                                    "}" +
                                ").reverse()") % ('%', self.step)
            elif step == 0:
                return "ValueError: slice step cannot be zero"
        return slicedArray

class Index(Base):
    def compile(self):
        return '%s[%s]' % (self.parent.value, self.value)

class Dict(Base):
    def compile(self):
            return '{%s}' % (', '.join('%s: %s' % (key, value) for (key, value) in zip(self.keys, self.values)))

# **Consider:** What is nl - newline?
class Print(Base):
    def compile(self):
        return 'console.log(%s)' % ', '.join(value for value in self.values)

class List(Base):
    def compile(self):
        return '[%s]' % ', '.join(value for value in self.elts)

#### Control Flow

class If(Base):
    def compile(self):
        compiled = helper.multiline([
                     'if(%s){',
                     '  %s',
                     '}'
                   ]) % (self.test, helper.formatGroup(self.body))
        # **Consider:** abstract list.isEmpty() ?
        if len(self.orelse) != 0:
            compiled = helper.multiline([
                         compiled,
                         'else{',
                         '  %s',
                         '}'
                       ]) % (helper.formatGroup(self.orelse))
        return compiled

#TODO: Stabilize. Obviously having inlined variable names could result in variable name conflicts
class For(Base):
    def compile(self):
        target = self.target.compile()
        iterable = self.iter.compile()
        return helper.multiline([
                   'var uniqueObj = Object(%s);', # Doing this because strings in python are iterable, but not in javascript
                   'if(Object.keys(uniqueObj).length > 0){',
                   '  var %s;',
                   '  var uniqueCounter = 0;',
                   '  for(uniqueVar in uniqueObj){',
                   '    uniqueCounter++;',
                   '    %s = uniqueObj[uniqueVar];',
                   '    %s',
                   '    if(uniqueCounter >= Object.keys(uniqueObj).length){',
                   '      %s',
                   '      break;',
                   '    }',
                   '  }',
                   '}',
                   'else{',
                   '  %s',
                   '}'
                ]) % (iterable, target, target, helper.formatGroup(self.body), helper.formatGroup(self.orelse), helper.formatGroup(self.orelse))

class While(Base):
    def compile(self):
        return helper.multiline([
                   'if(%s){',
                   '  while(true){',
                   '    %s',
                   '    if(!%s){',
                   '      %s',
                   '      break;',
                   '    }',
                   '  }'
                   '}',
                   'else{',
                   '  %s',
                   '}'
                ]) % (self.test, helper.formatGroup(self.body), self.test, helper.formatGroup(self.orelse), helper.formatGroup(self.orelse))
##################################

#### Functions

# Currently cannot handle variable or keyword arguments.

class arguments(Base):
    def compile(self):
        return ', '.join(self.args)

class Lambda(Base):
    def compile(self):
        return helper.multiline([
            'function(%s){',
            '  return %s;',
            '}'
        ]) % (self.args, self.body)

class FunctionDef(Base):
    def compile(self):
        return helper.multiline([
            'this.%s = function(%s){',
            '  %s',
            '}'
        ]) % (self.name, self.args, helper.formatGroup(self.body))

class Call(Base):
    def compile(self):
        argument = ', '.join(arg for arg in self.args)
        return 'this.%s(%s)' % (self.func, argument)

##################################

#### Error Handling

class ExceptHandler(Base):
    def compile(self):
        return self

# TODO: Incorporate self.inst, trace
class Raise(Base):
    def compile(self):
        exceptionType = self.type
        return 'throw Error("%s")' % exceptionType

class TryExcept(Base):
    def compileCatchStatements(self):
        catchStatements = []
        for handler in self.handlers:
            body = Block(handler.body, self.parent)
            exceptionType = handler.type
            if exceptionType is not None:
                catch = helper.multiline([
                    "if(e.message === '%s'){",
                    "    %s",
                    "}"
                ]) % (exceptionType, body)
            else:
                catch = '%s' % body
            catchStatements.append(catch)
        return '\r\n'.join(catchStatements)

    def compile(self):
        tryBody = Block(self.body, self.parent)
        catchBody = self.compileCatchStatements()
        elseBody = Block(self.orelse, self.parent)
        return helper.multiline([
            'try{',
            '  %s',
            '}',
            'catch(e){',
            '  var caughtException = true;',
            '  %s',
            '}',
            'if(!caughtException){',
            '  %s',
            '}'
        ]) % (tryBody, catchBody, elseBody)

class TryFinally(Base):
    def compile(self):
        tryBody = Block(self.body, self.parent)
        finallyBody = Block(self.finalbody, self.parent)
        return helper.multiline([
            'try{',
            '  %s',
            '}',
            'finally{',
            '  %s',
            '}',
        ]) % (tryBody, finallyBody)
##################################

class Block(Base):
    '''
    An abstraction of a closure.  This occurs in Python whenever we have:
    * module
    * class definition
    * function body
    '''
    def __init__(self, children, parent):
        self.children = children
        self.parent = parent
        self.indent = self.calcIndent()
        self.locals = {}
        # To keep track of global declarations in python.  This causes different
        # assignment statements in JS.
        self.globalRefs = set()

    # Traverse up parents until the nearest Block node, return that indent value
    # plus 1
    def calcIndent(self):
        def finishedTraversal(pointer):
            return pointer != None
        indent = 1
        pointer = self.parent
        while(not finishedTraversal(pointer)):
            if typing.getClassName(pointer) == 'Block':
                indent = pointer.indent + 1
                break
        return indent

    def compile(self):
        # Appends `;` to statements in the body - those passed back in an array
        # each have a `;` attached to their ends.  To avoid this (e.g. in if/
        # else blocks), simply concatenate the code before passing it back
        group = helper.formatGroup(helper.expand(self.children))
        enclosed = helper.closure(group)

        # need to indent the entire block now
        return helper.indent(enclosed, self.indent)
<|endoftext|>


<commit_before>import logging
from decimal import Decimal
from django.conf import settings

from exchange.models import Currency, ExchangeRate

from exchange.utils import update_many, insert_many

logger = logging.getLogger(__name__)

class BaseAdapter(object):
    """Base adapter class provides an interface for updating currency and
    exchange rate models

    """

    # If is true, delete currencies which not coming from API
    DELETE_NONEXISTED_CURRENCIES_KEY = "DELETE_NONEXISTED_CURRENCIES"

    def update(self):
        """Actual update process goes here using auxialary ``get_currencies``
        and ``get_exchangerates`` methods. This method creates or updates
        corresponding ``Currency`` and ``ExchangeRate`` models

        """
        currencies = self.get_currencies()

        is_deletable = getattr(settings,
                               BaseAdapter.DELETE_NONEXISTED_CURRENCIES_KEY,
                               False)

        if is_deletable:
            currencies_on_db = list(Currency.objects.all())

            for currency in currencies_on_db:
                if (currency.code, currency.name) not in currencies:
                    currency.delete()

        for code, name in currencies:
            _, created = Currency.objects.get_or_create(
                code=code, defaults={'name': name})
            if created:
                logger.info('currency: %s created', code)

        existing = ExchangeRate.objects.values('source__code',
                                               'target__code',
                                               'id')
        existing = {(d['source__code'], d['target__code']): d['id']
                    for d in existing}
        usd_exchange_rates = dict(self.get_exchangerates('USD'))

        updates = []
        inserts = []
        currencies = list(Currency.objects.all())
        for source in currencies:
            for target in currencies:
                rate = self._get_rate_through_usd(source.code,
                                                  target.code,
                                                  usd_exchange_rates)

                exchange_rate = ExchangeRate(source=source,
                                             target=target,
                                             rate=rate)

                if (source.code, target.code) in existing:
                    exchange_rate.id = existing[(source.code, target.code)]
                    updates.append(exchange_rate)
                    logger.debug('exchange rate updated %s/%s=%s'
                                 % (source, target, rate))
                else:
                    inserts.append(exchange_rate)
                    logger.debug('exchange rate created %s/%s=%s'
                                 % (source, target, rate))

            logger.info('exchange rates updated for %s' % source.code)
        logger.info("Updating %s rows" % len(updates))
        update_many(updates)
        logger.info("Inserting %s rows" % len(inserts))
        insert_many(inserts)
        logger.info('saved rates to db')

    def _get_rate_through_usd(self, source, target, usd_rates):
        # from: https://openexchangerates.org/documentation#how-to-use
        # gbp_hkd = usd_hkd * (1 / usd_gbp)
        usd_source = usd_rates[source]
        usd_target = usd_rates[target]
        rate = usd_target * (Decimal(1.0) / usd_source)
        rate = rate.quantize(Decimal('0.123456'))  # round to 6 decimal places
        return rate

    def get_currencies(self):
        """Subclasses must implement this to provide all currency data

        :returns: currency tuples ``[(currency_code, currency_name),]``
        :rtype: list

        """
        raise NotImplementedError()

    def get_exchangerates(self, base):
        """Subclasses must implement this to provide corresponding exchange
        rates for given base currency

        :returns: exchange rate tuples ``[(currency_code, rate),]``
        :rtype: list

        """
        raise NotImplementedError()
<commit_msg>revert auto remove currencies from exchange table for protect user from cascade deletion on DB<commit_after>import logging
from decimal import Decimal
from django.conf import settings

from exchange.models import Currency, ExchangeRate

from exchange.utils import update_many, insert_many

logger = logging.getLogger(__name__)

class BaseAdapter(object):
    """Base adapter class provides an interface for updating currency and
    exchange rate models

    """

    def update(self):
        """Actual update process goes here using auxialary ``get_currencies``
        and ``get_exchangerates`` methods. This method creates or updates
        corresponding ``Currency`` and ``ExchangeRate`` models

        """
        currencies = self.get_currencies()

        for code, name in currencies:
            _, created = Currency.objects.get_or_create(
                code=code, defaults={'name': name})
            if created:
                logger.info('currency: %s created', code)

        existing = ExchangeRate.objects.values('source__code',
                                               'target__code',
                                               'id')
        existing = {(d['source__code'], d['target__code']): d['id']
                    for d in existing}
        usd_exchange_rates = dict(self.get_exchangerates('USD'))

        updates = []
        inserts = []
        for source in currencies:
            for target in currencies:
                rate = self._get_rate_through_usd(source.code,
                                                  target.code,
                                                  usd_exchange_rates)

                exchange_rate = ExchangeRate(source=source,
                                             target=target,
                                             rate=rate)

                if (source.code, target.code) in existing:
                    exchange_rate.id = existing[(source.code, target.code)]
                    updates.append(exchange_rate)
                    logger.debug('exchange rate updated %s/%s=%s'
                                 % (source, target, rate))
                else:
                    inserts.append(exchange_rate)
                    logger.debug('exchange rate created %s/%s=%s'
                                 % (source, target, rate))

            logger.info('exchange rates updated for %s' % source.code)
        logger.info("Updating %s rows" % len(updates))
        update_many(updates)
        logger.info("Inserting %s rows" % len(inserts))
        insert_many(inserts)
        logger.info('saved rates to db')

    def _get_rate_through_usd(self, source, target, usd_rates):
        # from: https://openexchangerates.org/documentation#how-to-use
        # gbp_hkd = usd_hkd * (1 / usd_gbp)
        usd_source = usd_rates[source]
        usd_target = usd_rates[target]
        rate = usd_target * (Decimal(1.0) / usd_source)
        rate = rate.quantize(Decimal('0.123456'))  # round to 6 decimal places
        return rate

    def get_currencies(self):
        """Subclasses must implement this to provide all currency data

        :returns: currency tuples ``[(currency_code, currency_name),]``
        :rtype: list

        """
        raise NotImplementedError()

    def get_exchangerates(self, base):
        """Subclasses must implement this to provide corresponding exchange
        rates for given base currency

        :returns: exchange rate tuples ``[(currency_code, rate),]``
        :rtype: list

        """
        raise NotImplementedError()
<|endoftext|>


<commit_before>import numpy
import logging
from exp.influence2.MaxInfluence import MaxInfluence

class GraphRanker(object):
    def __init__(self):
        pass

    @staticmethod
    def getNames(computeInfluence=False):
        names = ["Betweenness", "Closeness", "PageRank", "Degree"]

        if computeInfluence:
            names.append("Influence")

        names.append("Shortest path")

        return names

    @staticmethod
    def rankedLists(graph, k=100, p=0.5, numRuns=1000, computeInfluence=False, trainExpertsIdList=None):
        """
        Return a list of ranked lists. The list is: betweenness, pagerank,
        degree and influence.
        """
        outputLists = []

        logging.debug("Computing betweenness")
        scores = graph.betweenness(weights="invWeight")
        rank = numpy.flipud(numpy.argsort(scores))
        outputLists.append(rank)

        logging.debug("Computing closeness")
        scores = graph.closeness(weights="invWeight")
        rank = numpy.flipud(numpy.argsort(scores))
        outputLists.append(rank)

        logging.debug("Computing PageRank")
        scores = graph.pagerank(weights="weight")
        rank = numpy.flipud(numpy.argsort(scores))
        outputLists.append(rank)

        logging.debug("Computing weighted degree distribution")
        #scores = graph.degree(graph.vs)
        scores = graph.strength(weights="weight")
        rank = numpy.flipud(numpy.argsort(scores))
        outputLists.append(rank)

        if computeInfluence:
            logging.debug("Computing influence")
            rank = MaxInfluence.greedyMethod2(graph, k, p=p, numRuns=numRuns)
            outputLists.append(numpy.array(rank))

        """
        logging.debug("Computing shortest path lengths")
        lengths = graph.shortest_paths(trainExpertsIdList, weights="invWeight")
        lengths = numpy.array(lengths)
        lengths[numpy.logical_not(numpy.isfinite(lengths))] = 0
        lengths = numpy.mean(lengths, 0)
        rank = numpy.argsort(lengths)
        outputLists.append(rank)
        """

        logging.debug("Computing hub score")
        scores = graph.hub_score(weights="weight")
        rank = numpy.flipud(numpy.argsort(scores))
        outputLists.append(rank)

        """
        #This is the same as above for undirected graphs
        logging.debug("Computing authority score")
        scores = graph.authority_score(weights="weight")
        rank = numpy.flipud(numpy.argsort(scores))
        outputLists.append(rank)
        """

        return outputLists
<commit_msg>Make OO, restrict ranked items, and include MC2<commit_after>import numpy
import logging
from exp.influence2.MaxInfluence import MaxInfluence
from exp.influence2.RankAggregator import RankAggregator

class GraphRanker(object):
    def __init__(self, k=100, p=0.5, numRuns=1000, computeInfluence=False, trainExpertsIdList=None):
        self.k = k
        self.p = p
        self.numRuns = numRuns
        self.computeInfluence = computeInfluence
        self.trainExpertsIdList = trainExpertsIdList

    def getNames(self):
        names = ["Betweenness", "Closeness", "PageRank", "Degree"]

        if self.computeInfluence:
            names.append("Influence")

        names.append("Hub score")

        return names

    def vertexRankings(self, graph, relevantItems, outputLists=None):
        """
        Return a list of ranked lists. The list is: betweenness, pagerank,
        degree and influence.
        """
        if outputLists == None:
            outputLists = []

        logging.debug("Computing betweenness")
        scores = graph.betweenness(weights="invWeight")
        rank = numpy.flipud(numpy.argsort(scores))
        outputLists.append(self.restrictRankedList(rank, relevantItems))

        logging.debug("Computing closeness")
        scores = graph.closeness(weights="invWeight")
        rank = numpy.flipud(numpy.argsort(scores))
        outputLists.append(self.restrictRankedList(rank, relevantItems))

        logging.debug("Computing PageRank")
        scores = graph.pagerank(weights="weight")
        rank = numpy.flipud(numpy.argsort(scores))
        outputLists.append(self.restrictRankedList(rank, relevantItems))

        logging.debug("Computing weighted degree distribution")
        scores = graph.strength(weights="weight")
        rank = numpy.flipud(numpy.argsort(scores))
        outputLists.append(self.restrictRankedList(rank, relevantItems))

        if self.computeInfluence:
            logging.debug("Computing influence")
            rank = MaxInfluence.greedyMethod2(graph, self.k, p=self.p, numRuns=self.numRuns)
            outputLists.append(numpy.array(self.restrictRankedList(rank, relevantItems)))

        """
        logging.debug("Computing shortest path lengths")
        lengths = graph.shortest_paths(trainExpertsIdList, weights="invWeight")
        lengths = numpy.array(lengths)
        lengths[numpy.logical_not(numpy.isfinite(lengths))] = 0
        lengths = numpy.mean(lengths, 0)
        rank = numpy.argsort(lengths)
        outputLists.append(self.restrictRankedList(rank, relevantItems))
        """

        logging.debug("Computing hub score")
        scores = graph.hub_score(weights="weight")
        rank = numpy.flipud(numpy.argsort(scores))
        outputLists.append(self.restrictRankedList(rank, relevantItems))

        #Now add MC2 aggregated rankings
        logging.debug("Computing MC2 rank aggregation")
        rank = RankAggregator.MC2(outputLists, relevantItems)[0]
        outputLists.append(rank)

        return outputLists

    def restrictRankedList(self, lst, releventList):
        """
        Given an ordered list lst, restrict items to those in releventList,
        retaining the order.
        """
        releventList = set(releventList)
        newList = []
        for item in lst:
            if item in releventList:
                newList.append(item)
        return newList
<|endoftext|>


<commit_before>"""Hardware interfaces for key- and button-presses"""

# Authors: Dan McCloy <drmccloy@uw.edu>
#          Eric Larson <larsoner@uw.edu>
#
# License: BSD (3-clause)

import numpy as np
from psychopy import event
from psychopy import clock as psyclock

from ._utils import wait_secs, psylog

class BaseKeyboard(object):
    """Retrieve presses from various devices.

    Public:
        __init__
        listen_presses
        get_presses
        wait_one_press
        wait_for_presses

    Private:
        _correct_presses
        _init_wait_press

    Requires:
        _get_time_correction
        _clear_events
        _retrieve_events
        """
    def __init__(self, ec):
        self._ec = ec
        self.master_clock = ec._master_clock
        self.check_force_quit = ec._check_force_quit
        self.log_presses = ec._log_presses
        self.force_quit = ec._force_quit_keys
        self.listen_start = None
        self.time_correction = None
        self.time_correction = self._get_time_correction()

    def _clear_events(self):
        """Clear all events from keyboard buffer.
        """
        raise NotImplementedError

    def _retrieve_events(self, live_keys):
        """Get all events since last _clear_events.
        """
        raise NotImplementedError

    def listen_presses(self):
        """Start listening for keypresses.
        """
        self.time_correction = self._get_time_correction()
        self.listen_start = self.master_clock.getTime()
        self._clear_events()

    def get_presses(self, live_keys, timestamp, relative_to):
        """Get the current entire keyboard / button box buffer.
        """
        pressed = []
        if timestamp and relative_to is None:
            if self.listen_start is None:
                raise ValueError('I cannot timestamp: relative_to is None and '
                                 'you have not yet called listen_presses.')
            else:
                relative_to = self.listen_start
        pressed = self._retrieve_events(live_keys)

        print relative_to
        return self._correct_presses(pressed, timestamp, relative_to)

    def wait_one_press(self, max_wait, min_wait, live_keys,
                       timestamp, relative_to):
        """Returns only the first button pressed after min_wait.
        """
        relative_to, start_time = self._init_wait_press(max_wait, min_wait,
                                                        live_keys, timestamp,
                                                        relative_to)
        pressed = []
        while (not len(pressed) and
               self.master_clock.getTime() - start_time < max_wait):
            pressed = self._retrieve_events(live_keys)

        # handle non-presses
        if len(pressed):
            pressed = self._correct_presses(pressed, timestamp, relative_to)[0]
        elif timestamp:
            pressed = (None, None)
        else:
            pressed = None
        return pressed

    def wait_for_presses(self, max_wait, min_wait, live_keys,
                         timestamp, relative_to):
        """Returns all button presses between min_wait and max_wait.
        """
        relative_to, start_time = self._init_wait_press(max_wait, min_wait,
                                                        live_keys, timestamp,
                                                        relative_to)

        pressed = []
        while (self.master_clock.getTime() - start_time < max_wait):
            pressed.extend(self._retrieve_events(live_keys))
        return self._correct_presses(pressed, timestamp, relative_to)

    def _correct_presses(self, pressed, timestamp, relative_to):
        """Correct timing of presses and check for quit press"""
        if len(pressed):
            pressed = [(k, s + self.time_correction) for k, s in pressed]
            self.log_presses(pressed)
            keys = [k for k, _ in pressed]
            self.check_force_quit(keys)
            if timestamp:
                pressed = [(k, s - relative_to) for k, s in pressed]
            else:
                pressed = keys
        return pressed

    def _init_wait_press(self, max_wait, min_wait, live_keys, timestamp,
                         relative_to):
        """Actions common to ``wait_one_press`` and ``wait_for_presses``
        """
        if np.isinf(max_wait) and live_keys == []:
            raise ValueError('max_wait cannot be infinite if there are no live'
                             ' keys.')
        if not min_wait < max_wait:
            raise ValueError('min_wait must be less than max_wait')
        start_time = self.master_clock.getTime()
        if timestamp and relative_to is None:
            relative_to = start_time
        wait_secs(min_wait)
        self.check_force_quit()
        self._clear_events()
        return relative_to, start_time

class PsychKeyboard(BaseKeyboard):
    """Retrieve button presses from keyboard.
    """
    def _get_time_correction(self):
        """Clock correction for psychopy-generated timestamps.

        Notes
        -----
        On Linux, PsychoPy's method for timestamping keypresses from Pyglet
        uses timeit.default_timer, which wraps to time.time and returns seconds
        elapsed since 1970/01/01 at midnight, giving very large numbers. On
        Windows, PsychoPy's timer uses the Windows Query Performance Counter,
        which also returns floats in seconds (apparently measured from when the
        system was last rebooted). Importantly, on either system the units are
        in seconds, and thus can simply be subtracted out.
        """
        other_clock = psyclock.getTime()
        start_time = self.master_clock.getTime()
        time_correction = start_time - other_clock
        if self.time_correction is not None:
            if np.abs(self.time_correction - time_correction) > 0.00001:
                psylog.warn('Expyfun: drift of > 10 microseconds between '
                            'system clock and experiment master clock.')
            psylog.debug('Expyfun: time correction between system clock and '
                         'experiment master clock is {}. This is a change of '
                         '{} from the previous value.'
                         ''.format(time_correction, time_correction -
                                   self.time_correction))
        return time_correction

    def _clear_events(self):
        event.clearEvents('keyboard')

    def _retrieve_events(self, live_keys):
        live_keys = self._add_escape_keys(live_keys)
        return event.getKeys(live_keys, timeStamped=True)

    def _add_escape_keys(self, live_keys):
        """Helper to add force quit keys to button press listener.
        """
        if live_keys is not None:
            live_keys = [str(x) for x in live_keys]  # accept ints
            if len(self.force_quit):  # should always be a list of strings
                live_keys = live_keys + self.force_quit
        return live_keys
<commit_msg>STY: Cleaner<commit_after>"""Hardware interfaces for key- and button-presses"""

# Authors: Dan McCloy <drmccloy@uw.edu>
#          Eric Larson <larsoner@uw.edu>
#
# License: BSD (3-clause)

import numpy as np
from psychopy import event
from psychopy import clock as psyclock

from ._utils import wait_secs, psylog

class BaseKeyboard(object):
    """Retrieve presses from various devices.

    Public metohds:
        __init__
        listen_presses
        get_presses
        wait_one_press
        wait_for_presses

    Requires:
        _get_time_correction
        _clear_events
        _retrieve_events
    """

    def __init__(self, ec):
        self._ec = ec
        self.master_clock = ec._master_clock
        self.check_force_quit = ec._check_force_quit
        self.log_presses = ec._log_presses
        self.force_quit = ec._force_quit_keys
        self.listen_start = None
        self.time_correction = None
        self.time_correction = self._get_time_correction()

    def _clear_events(self):
        """Clear all events from keyboard buffer.
        """
        raise NotImplementedError

    def _retrieve_events(self, live_keys):
        """Get all events since last _clear_events.
        """
        raise NotImplementedError

    def listen_presses(self):
        """Start listening for keypresses.
        """
        self.time_correction = self._get_time_correction()
        self.listen_start = self.master_clock.getTime()
        self._clear_events()

    def get_presses(self, live_keys, timestamp, relative_to):
        """Get the current entire keyboard / button box buffer.
        """
        pressed = []
        if timestamp and relative_to is None:
            if self.listen_start is None:
                raise ValueError('I cannot timestamp: relative_to is None and '
                                 'you have not yet called listen_presses.')
            else:
                relative_to = self.listen_start
        pressed = self._retrieve_events(live_keys)

        print relative_to
        return self._correct_presses(pressed, timestamp, relative_to)

    def wait_one_press(self, max_wait, min_wait, live_keys,
                       timestamp, relative_to):
        """Returns only the first button pressed after min_wait.
        """
        relative_to, start_time = self._init_wait_press(max_wait, min_wait,
                                                        live_keys, timestamp,
                                                        relative_to)
        pressed = []
        while (not len(pressed) and
               self.master_clock.getTime() - start_time < max_wait):
            pressed = self._retrieve_events(live_keys)

        # handle non-presses
        if len(pressed):
            pressed = self._correct_presses(pressed, timestamp, relative_to)[0]
        elif timestamp:
            pressed = (None, None)
        else:
            pressed = None
        return pressed

    def wait_for_presses(self, max_wait, min_wait, live_keys,
                         timestamp, relative_to):
        """Returns all button presses between min_wait and max_wait.
        """
        relative_to, start_time = self._init_wait_press(max_wait, min_wait,
                                                        live_keys, timestamp,
                                                        relative_to)

        pressed = []
        while (self.master_clock.getTime() - start_time < max_wait):
            pressed.extend(self._retrieve_events(live_keys))
        return self._correct_presses(pressed, timestamp, relative_to)

    def _correct_presses(self, pressed, timestamp, relative_to):
        """Correct timing of presses and check for quit press"""
        if len(pressed):
            pressed = [(k, s + self.time_correction) for k, s in pressed]
            self.log_presses(pressed)
            keys = [k for k, _ in pressed]
            self.check_force_quit(keys)
            if timestamp:
                pressed = [(k, s - relative_to) for k, s in pressed]
            else:
                pressed = keys
        return pressed

    def _init_wait_press(self, max_wait, min_wait, live_keys, timestamp,
                         relative_to):
        """Actions common to ``wait_one_press`` and ``wait_for_presses``
        """
        if np.isinf(max_wait) and live_keys == []:
            raise ValueError('max_wait cannot be infinite if there are no live'
                             ' keys.')
        if not min_wait < max_wait:
            raise ValueError('min_wait must be less than max_wait')
        start_time = self.master_clock.getTime()
        if timestamp and relative_to is None:
            relative_to = start_time
        wait_secs(min_wait)
        self.check_force_quit()
        self._clear_events()
        return relative_to, start_time

class PsychKeyboard(BaseKeyboard):
    """Retrieve button presses from keyboard.
    """
    def _get_time_correction(self):
        """Clock correction for psychopy-generated timestamps.

        Notes
        -----
        On Linux, PsychoPy's method for timestamping keypresses from Pyglet
        uses timeit.default_timer, which wraps to time.time and returns seconds
        elapsed since 1970/01/01 at midnight, giving very large numbers. On
        Windows, PsychoPy's timer uses the Windows Query Performance Counter,
        which also returns floats in seconds (apparently measured from when the
        system was last rebooted). Importantly, on either system the units are
        in seconds, and thus can simply be subtracted out.
        """
        other_clock = psyclock.getTime()
        start_time = self.master_clock.getTime()
        time_correction = start_time - other_clock
        if self.time_correction is not None:
            if np.abs(self.time_correction - time_correction) > 0.00001:
                psylog.warn('Expyfun: drift of > 10 microseconds between '
                            'system clock and experiment master clock.')
            psylog.debug('Expyfun: time correction between system clock and '
                         'experiment master clock is {}. This is a change of '
                         '{} from the previous value.'
                         ''.format(time_correction, time_correction -
                                   self.time_correction))
        return time_correction

    def _clear_events(self):
        event.clearEvents('keyboard')

    def _retrieve_events(self, live_keys):
        live_keys = self._add_escape_keys(live_keys)
        return event.getKeys(live_keys, timeStamped=True)

    def _add_escape_keys(self, live_keys):
        """Helper to add force quit keys to button press listener.
        """
        if live_keys is not None:
            live_keys = [str(x) for x in live_keys]  # accept ints
            if len(self.force_quit):  # should always be a list of strings
                live_keys = live_keys + self.force_quit
        return live_keys
<|endoftext|>


<commit_before>from kivy.uix.widget import Widget
from kivy.properties import NumericProperty
from kivy.graphics import Ellipse, Line, Fbo, Rectangle

from shader import circle_shader

class DiceWidget(Widget):
    number = NumericProperty(0)

    # Dots are drawn on a grid dividing the die face into 16 squares
    # and starting from the bottom left. Define coords for each.
    #  _______
    # |_|_|_|_|
    # |_|_|_|_|
    # |_|_|_|_|
    # |_|_|_|_|
    #

    # Locations of dots for each face
    dotmap = {
        0: (),
        1: ((2, 2),),
        2: ((1, 1), (3, 3)),
        3: ((1, 1), (2, 2), (3, 3)),
        4: ((1, 1), (1, 3), (3, 3), (3, 1)),
        5: ((1, 1), (1, 3), (2, 2), (3, 3), (3, 1)),
        6: ((1, 1), (1, 2), (1, 3), (3, 1), (3, 2), (3, 3))
    }

    def __init__(self, **kwargs):
        super(DiceWidget, self).__init__(**kwargs)

        self.bind(size=self.redraw, pos=self.redraw)

        # Components drawn on the canvas
        self.dot_fbo = None
        self.number = 0

        self.redraw()

    def redraw(self, *_args, **_kwargs):
        self.canvas.clear()

        x, y, w, h = self.bounding_box

        # Size of a dot
        dot_w = w / 5
        dot_h = h / 5

        # Render a single dot
        self.dot_fbo = self.render_dot((dot_w, dot_h))

        # Frame buffer objects can be invalidated when apps are minimized,
        # this event rebuilds it.
        self.dot_fbo.add_reload_observer(self.redraw)

        with self.canvas:
            # Draw the outline
            Line(rectangle=(x, y, w, h))

        # Update dots
        self.on_number()

    def on_number(self, *_args):
        x, y, w, h = self.bounding_box

        # Size of a dot
        dot_w = w / 5
        dot_h = h / 5

        # Size of a grid unit
        unit = w / 4

        self.canvas.remove_group('dot')
        with self.canvas:
            # Draw dots
            for grid_x, grid_y in self.dotmap[self.number]:
                Rectangle(
                    pos=(x + (grid_x * unit) - dot_w / 2,
                         y + (grid_y * unit) - dot_h / 2),
                    size=(dot_w, dot_h),
                    texture=self.dot_fbo.texture,
                    group='dot')

    def render_dot(self, size):
        # Render an antialiased dot using the circle shader.
        with self.canvas:
            dot_fbo = Fbo(size=size)

        # Install the shader, and check it compiled.
        original_fs = dot_fbo.shader.fs
        dot_fbo.shader.fs = circle_shader
        if not dot_fbo.shader.success:
            # Shader didn't compile, just render an ellipse.
            dot_fbo.shader.fs = original_fs
            with dot_fbo:
                Ellipse(pos=(2, 2), size=(size[0]-4, size[1]-4))
        else:
            # Render a rectangle with the circle shader.
            border = 3.0
            dot_fbo['centre'] = (size[0] * 0.5, size[1] * 0.5)
            dot_fbo['radius'] = max(size[0] * 0.5 - border, 0.0)
            dot_fbo['border'] = border
            with dot_fbo:
                Rectangle(size=size)

        return dot_fbo

    @property
    def bounding_box(self):
        'Returns a square box (x, y, width, height) centred on this widget.'
        dim = min(self.width, self.height)
        half = dim / 2
        return (self.center_x - half, self.center_y - half, dim, dim)
<commit_msg>Make the dice red<commit_after>from kivy.uix.widget import Widget
from kivy.properties import NumericProperty
from kivy.graphics import Ellipse, Line, Fbo, Rectangle, Color

from shader import circle_shader

class DiceWidget(Widget):
    number = NumericProperty(0)

    # Dots are drawn on a grid dividing the die face into 16 squares
    # and starting from the bottom left. Define coords for each.
    #  _______
    # |_|_|_|_|
    # |_|_|_|_|
    # |_|_|_|_|
    # |_|_|_|_|
    #

    # Locations of dots for each face
    dotmap = {
        0: (),
        1: ((2, 2),),
        2: ((1, 1), (3, 3)),
        3: ((1, 1), (2, 2), (3, 3)),
        4: ((1, 1), (1, 3), (3, 3), (3, 1)),
        5: ((1, 1), (1, 3), (2, 2), (3, 3), (3, 1)),
        6: ((1, 1), (1, 2), (1, 3), (3, 1), (3, 2), (3, 3))
    }

    def __init__(self, **kwargs):
        super(DiceWidget, self).__init__(**kwargs)

        self.bind(size=self.redraw, pos=self.redraw)

        # Components drawn on the canvas
        self.dot_fbo = None
        self.number = 0

        self.redraw()

    def redraw(self, *_args, **_kwargs):
        self.canvas.clear()

        x, y, w, h = self.bounding_box

        # Size of a dot
        dot_w = w / 5
        dot_h = h / 5

        # Render a single dot
        self.dot_fbo = self.render_dot((dot_w, dot_h))

        # Frame buffer objects can be invalidated when apps are minimized,
        # this event rebuilds it.
        self.dot_fbo.add_reload_observer(self.redraw)

        with self.canvas:
            # Draw the outline
            Color(0.9, 0.1, 0.1)
            Rectangle(pos=(x, y), size=(w, h))
            Color(1, 1, 1)
            Line(rectangle=(x, y, w, h), width=(w/200) + 1)

        # Update dots
        self.on_number()

    def on_number(self, *_args):
        x, y, w, h = self.bounding_box

        # Size of a dot
        dot_w = w / 5
        dot_h = h / 5

        # Size of a grid unit
        unit = w / 4

        self.canvas.remove_group('dot')
        with self.canvas:
            # Draw dots
            for grid_x, grid_y in self.dotmap[self.number]:
                Rectangle(
                    pos=(x + (grid_x * unit) - dot_w / 2,
                         y + (grid_y * unit) - dot_h / 2),
                    size=(dot_w, dot_h),
                    texture=self.dot_fbo.texture,
                    group='dot')

    def render_dot(self, size):
        # Render an antialiased dot using the circle shader.
        with self.canvas:
            dot_fbo = Fbo(size=size)

        # Install the shader, and check it compiled.
        original_fs = dot_fbo.shader.fs
        dot_fbo.shader.fs = circle_shader
        if not dot_fbo.shader.success:
            # Shader didn't compile, just render an ellipse.
            dot_fbo.shader.fs = original_fs
            with dot_fbo:
                Ellipse(pos=(2, 2), size=(size[0] - 4, size[1] - 4))
        else:
            # Render a rectangle with the circle shader.
            border = 3.0
            dot_fbo['centre'] = (size[0] * 0.5, size[1] * 0.5)
            dot_fbo['radius'] = max(size[0] * 0.5 - border, 0.0)
            dot_fbo['border'] = border
            with dot_fbo:
                Rectangle(size=size)

        return dot_fbo

    @property
    def bounding_box(self):
        'Returns a square box (x, y, width, height) centred on this widget.'
        dim = min(self.width, self.height)
        half = dim / 2
        return (self.center_x - half, self.center_y - half, dim, dim)
<|endoftext|>


<commit_before># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for TensorForestTrainer."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.contrib.learn.python.learn.datasets import base
from tensorflow.contrib.tensor_forest.client import random_forest
from tensorflow.contrib.tensor_forest.python import tensor_forest
from tensorflow.python.estimator.canned import head as head_lib
from tensorflow.python.estimator.inputs import numpy_io
from tensorflow.python.feature_column import feature_column_lib as core_feature_column
from tensorflow.python.framework import ops
from tensorflow.python.ops.losses import losses
from tensorflow.python.platform import test
from tensorflow.python.training import checkpoint_utils

def _get_classification_input_fns():
  iris = base.load_iris()
  data = iris.data.astype(np.float32)
  labels = iris.target.astype(np.int32)

  train_input_fn = numpy_io.numpy_input_fn(
      x=data, y=labels, batch_size=150, num_epochs=None, shuffle=False)

  predict_input_fn = numpy_io.numpy_input_fn(
      x=data[:1,], y=None, batch_size=1, num_epochs=1, shuffle=False)
  return train_input_fn, predict_input_fn

def _get_regression_input_fns():
  boston = base.load_boston()
  data = boston.data.astype(np.float32)
  labels = boston.target.astype(np.int32)

  train_input_fn = numpy_io.numpy_input_fn(
      x=data, y=labels, batch_size=506, num_epochs=None, shuffle=False)

  predict_input_fn = numpy_io.numpy_input_fn(
      x=data[:1,], y=None, batch_size=1, num_epochs=1, shuffle=False)
  return train_input_fn, predict_input_fn

class TensorForestTrainerTests(test.TestCase):

  def testClassification(self):
    """Tests multi-class classification using matrix data as input."""
    hparams = tensor_forest.ForestHParams(
        num_trees=3,
        max_nodes=1000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)
    classifier = random_forest.TensorForestEstimator(hparams.fill())

    input_fn, predict_input_fn = _get_classification_input_fns()
    classifier.fit(input_fn=input_fn, steps=100)
    res = classifier.evaluate(input_fn=input_fn, steps=10)

    self.assertEqual(1.0, res['accuracy'])
    self.assertAllClose(0.55144483, res['loss'])

    predictions = list(classifier.predict(input_fn=predict_input_fn))
    self.assertAllClose([[0.576117, 0.211942, 0.211942]],
                        [pred['probabilities'] for pred in predictions])

  def testRegression(self):
    """Tests regression using matrix data as input."""

    hparams = tensor_forest.ForestHParams(
        num_trees=5,
        max_nodes=1000,
        num_classes=1,
        num_features=13,
        regression=True,
        split_after_samples=20)

    regressor = random_forest.TensorForestEstimator(hparams.fill())

    input_fn, predict_input_fn = _get_regression_input_fns()

    regressor.fit(input_fn=input_fn, steps=100)
    res = regressor.evaluate(input_fn=input_fn, steps=10)
    self.assertGreaterEqual(0.1, res['loss'])

    predictions = list(regressor.predict(input_fn=predict_input_fn))
    self.assertAllClose([24.], [pred['scores'] for pred in predictions], atol=1)

  def testAdditionalOutputs(self):
    """Tests multi-class classification using matrix data as input."""
    hparams = tensor_forest.ForestHParams(
        num_trees=1,
        max_nodes=100,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)
    classifier = random_forest.TensorForestEstimator(
        hparams.fill(), keys_column='keys', include_all_in_serving=True)

    iris = base.load_iris()
    data = iris.data.astype(np.float32)
    labels = iris.target.astype(np.int32)

    input_fn = numpy_io.numpy_input_fn(
        x={
            'x': data,
            'keys': np.arange(len(iris.data)).reshape(150, 1)
        },
        y=labels,
        batch_size=10,
        num_epochs=1,
        shuffle=False)

    classifier.fit(input_fn=input_fn, steps=100)
    predictions = list(classifier.predict(input_fn=input_fn))
    # Check that there is a key column, tree paths and var.
    for pred in predictions:
      self.assertTrue('keys' in pred)
      self.assertTrue('tree_paths' in pred)
      self.assertTrue('prediction_variance' in pred)

  def _assert_checkpoint(self, model_dir, global_step):
    reader = checkpoint_utils.load_checkpoint(model_dir)
    self.assertEqual(global_step, reader.get_tensor(ops.GraphKeys.GLOBAL_STEP))

  def testEarlyStopping(self):
    """Tests multi-class classification using matrix data as input."""
    hparams = tensor_forest.ForestHParams(
        num_trees=100,
        max_nodes=10000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)
    classifier = random_forest.TensorForestEstimator(
        hparams.fill(),
        # Set a crazy threshold - 30% loss change.
        early_stopping_loss_threshold=0.3,
        early_stopping_rounds=2)

    input_fn, _ = _get_classification_input_fns()
    classifier.fit(input_fn=input_fn, steps=100)

    # We stopped early.
    self._assert_checkpoint(classifier.model_dir, global_step=5)

class CoreTensorForestTests(test.TestCase):

  def testTrainEvaluateInferDoesNotThrowErrorForClassifier(self):
    head_fn = head_lib._multi_class_head_with_softmax_cross_entropy_loss(
        n_classes=3, loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)

    hparams = tensor_forest.ForestHParams(
        num_trees=3,
        max_nodes=1000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)

    est = random_forest.CoreTensorForestEstimator(hparams.fill(), head=head_fn)

    input_fn, predict_input_fn = _get_classification_input_fns()

    est.train(input_fn=input_fn, steps=100)
    res = est.evaluate(input_fn=input_fn, steps=1)

    self.assertEqual(1.0, res['accuracy'])
    self.assertAllClose(0.55144483, res['loss'])

    predictions = list(est.predict(input_fn=predict_input_fn))
    self.assertAllClose([[0.576117, 0.211942, 0.211942]],
                        [pred['probabilities'] for pred in predictions])

  def testRegression(self):
    """Tests regression using matrix data as input."""
    head_fn = head_lib._regression_head(
        label_dimension=1,
        loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)

    hparams = tensor_forest.ForestHParams(
        num_trees=5,
        max_nodes=1000,
        num_classes=1,
        num_features=13,
        regression=True,
        split_after_samples=20)

    regressor = random_forest.CoreTensorForestEstimator(
        hparams.fill(), head=head_fn)

    input_fn, predict_input_fn = _get_regression_input_fns()

    regressor.train(input_fn=input_fn, steps=100)
    res = regressor.evaluate(input_fn=input_fn, steps=10)
    self.assertGreaterEqual(0.1, res['loss'])

    predictions = list(regressor.predict(input_fn=predict_input_fn))
    self.assertAllClose(
        [[24.]], [pred['predictions'] for pred in predictions], atol=1)

  def testWithFeatureColumns(self):
    head_fn = head_lib._multi_class_head_with_softmax_cross_entropy_loss(
        n_classes=3, loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)

    hparams = tensor_forest.ForestHParams(
        num_trees=3,
        max_nodes=1000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)

    est = random_forest.CoreTensorForestEstimator(
        hparams.fill(),
        head=head_fn,
        feature_columns=[core_feature_column.numeric_column('x')])

    iris = base.load_iris()
    data = {'x': iris.data.astype(np.float32)}
    labels = iris.target.astype(np.int32)

    input_fn = numpy_io.numpy_input_fn(
        x=data, y=labels, batch_size=150, num_epochs=None, shuffle=False)

    est.train(input_fn=input_fn, steps=100)
    res = est.evaluate(input_fn=input_fn, steps=1)

    self.assertEqual(1.0, res['accuracy'])
    self.assertAllClose(0.55144483, res['loss'])

  def testAutofillsClassificationHead(self):
    hparams = tensor_forest.ForestHParams(
        num_trees=3,
        max_nodes=1000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)

    est = random_forest.CoreTensorForestEstimator(hparams.fill())

    input_fn, _ = _get_classification_input_fns()

    est.train(input_fn=input_fn, steps=100)
    res = est.evaluate(input_fn=input_fn, steps=1)

    self.assertEqual(1.0, res['accuracy'])
    self.assertAllClose(0.55144483, res['loss'])

  def testAutofillsRegressionHead(self):
    hparams = tensor_forest.ForestHParams(
        num_trees=5,
        max_nodes=1000,
        num_classes=1,
        num_features=13,
        regression=True,
        split_after_samples=20)

    regressor = random_forest.CoreTensorForestEstimator(hparams.fill())

    input_fn, predict_input_fn = _get_regression_input_fns()

    regressor.train(input_fn=input_fn, steps=100)
    res = regressor.evaluate(input_fn=input_fn, steps=10)
    self.assertGreaterEqual(0.1, res['loss'])

    predictions = list(regressor.predict(input_fn=predict_input_fn))
    self.assertAllClose(
        [[24.]], [pred['predictions'] for pred in predictions], atol=1)

  def testAdditionalOutputs(self):
    """Tests multi-class classification using matrix data as input."""
    hparams = tensor_forest.ForestHParams(
        num_trees=1,
        max_nodes=100,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)
    classifier = random_forest.CoreTensorForestEstimator(
        hparams.fill(), keys_column='keys', include_all_in_serving=True)

    iris = base.load_iris()
    data = iris.data.astype(np.float32)
    labels = iris.target.astype(np.int32)

    input_fn = numpy_io.numpy_input_fn(
        x={
            'x': data,
            'keys': np.arange(len(iris.data)).reshape(150, 1)
        },
        y=labels,
        batch_size=10,
        num_epochs=1,
        shuffle=False)

    classifier.train(input_fn=input_fn, steps=100)
    predictions = list(classifier.predict(input_fn=input_fn))
    # Check that there is a key column, tree paths and var.
    for pred in predictions:
      self.assertTrue('keys' in pred)
      self.assertTrue('tree_paths' in pred)
      self.assertTrue('prediction_variance' in pred)

  def _assert_checkpoint(self, model_dir, global_step):
    reader = checkpoint_utils.load_checkpoint(model_dir)
    self.assertEqual(global_step, reader.get_tensor(ops.GraphKeys.GLOBAL_STEP))

  def testEarlyStopping(self):
    head_fn = head_lib._multi_class_head_with_softmax_cross_entropy_loss(
        n_classes=3, loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)

    hparams = tensor_forest.ForestHParams(
        num_trees=3,
        max_nodes=1000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)

    est = random_forest.CoreTensorForestEstimator(
        hparams.fill(),
        head=head_fn,
        # Set a crazy threshold - 30% loss change.
        early_stopping_loss_threshold=0.3,
        early_stopping_rounds=2)

    input_fn, _ = _get_classification_input_fns()
    est.train(input_fn=input_fn, steps=100)
    # We stopped early.
    self._assert_checkpoint(est.model_dir, global_step=5)

if __name__ == "__main__":
  test.main()
<commit_msg>Make early stopping tests less brittle to prevent breaking them when upgrading Eigen.<commit_after># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for TensorForestTrainer."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.contrib.learn.python.learn.datasets import base
from tensorflow.contrib.tensor_forest.client import random_forest
from tensorflow.contrib.tensor_forest.python import tensor_forest
from tensorflow.python.estimator.canned import head as head_lib
from tensorflow.python.estimator.inputs import numpy_io
from tensorflow.python.feature_column import feature_column_lib as core_feature_column
from tensorflow.python.framework import ops
from tensorflow.python.ops.losses import losses
from tensorflow.python.platform import test
from tensorflow.python.training import checkpoint_utils

def _get_classification_input_fns():
  iris = base.load_iris()
  data = iris.data.astype(np.float32)
  labels = iris.target.astype(np.int32)

  train_input_fn = numpy_io.numpy_input_fn(
      x=data, y=labels, batch_size=150, num_epochs=None, shuffle=False)

  predict_input_fn = numpy_io.numpy_input_fn(
      x=data[:1,], y=None, batch_size=1, num_epochs=1, shuffle=False)
  return train_input_fn, predict_input_fn

def _get_regression_input_fns():
  boston = base.load_boston()
  data = boston.data.astype(np.float32)
  labels = boston.target.astype(np.int32)

  train_input_fn = numpy_io.numpy_input_fn(
      x=data, y=labels, batch_size=506, num_epochs=None, shuffle=False)

  predict_input_fn = numpy_io.numpy_input_fn(
      x=data[:1,], y=None, batch_size=1, num_epochs=1, shuffle=False)
  return train_input_fn, predict_input_fn

class TensorForestTrainerTests(test.TestCase):

  def testClassification(self):
    """Tests multi-class classification using matrix data as input."""
    hparams = tensor_forest.ForestHParams(
        num_trees=3,
        max_nodes=1000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)
    classifier = random_forest.TensorForestEstimator(hparams.fill())

    input_fn, predict_input_fn = _get_classification_input_fns()
    classifier.fit(input_fn=input_fn, steps=100)
    res = classifier.evaluate(input_fn=input_fn, steps=10)

    self.assertEqual(1.0, res['accuracy'])
    self.assertAllClose(0.55144483, res['loss'])

    predictions = list(classifier.predict(input_fn=predict_input_fn))
    self.assertAllClose([[0.576117, 0.211942, 0.211942]],
                        [pred['probabilities'] for pred in predictions])

  def testRegression(self):
    """Tests regression using matrix data as input."""

    hparams = tensor_forest.ForestHParams(
        num_trees=5,
        max_nodes=1000,
        num_classes=1,
        num_features=13,
        regression=True,
        split_after_samples=20)

    regressor = random_forest.TensorForestEstimator(hparams.fill())

    input_fn, predict_input_fn = _get_regression_input_fns()

    regressor.fit(input_fn=input_fn, steps=100)
    res = regressor.evaluate(input_fn=input_fn, steps=10)
    self.assertGreaterEqual(0.1, res['loss'])

    predictions = list(regressor.predict(input_fn=predict_input_fn))
    self.assertAllClose([24.], [pred['scores'] for pred in predictions], atol=1)

  def testAdditionalOutputs(self):
    """Tests multi-class classification using matrix data as input."""
    hparams = tensor_forest.ForestHParams(
        num_trees=1,
        max_nodes=100,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)
    classifier = random_forest.TensorForestEstimator(
        hparams.fill(), keys_column='keys', include_all_in_serving=True)

    iris = base.load_iris()
    data = iris.data.astype(np.float32)
    labels = iris.target.astype(np.int32)

    input_fn = numpy_io.numpy_input_fn(
        x={
            'x': data,
            'keys': np.arange(len(iris.data)).reshape(150, 1)
        },
        y=labels,
        batch_size=10,
        num_epochs=1,
        shuffle=False)

    classifier.fit(input_fn=input_fn, steps=100)
    predictions = list(classifier.predict(input_fn=input_fn))
    # Check that there is a key column, tree paths and var.
    for pred in predictions:
      self.assertTrue('keys' in pred)
      self.assertTrue('tree_paths' in pred)
      self.assertTrue('prediction_variance' in pred)

  def _assert_checkpoint(self, model_dir, global_step):
    reader = checkpoint_utils.load_checkpoint(model_dir)
    self.assertLessEqual(
        reader.get_tensor(ops.GraphKeys.GLOBAL_STEP), global_step)

  def testEarlyStopping(self):
    """Tests multi-class classification using matrix data as input."""
    hparams = tensor_forest.ForestHParams(
        num_trees=100,
        max_nodes=10000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)
    classifier = random_forest.TensorForestEstimator(
        hparams.fill(),
        # Set a crazy threshold - 30% loss change.
        early_stopping_loss_threshold=0.3,
        early_stopping_rounds=2)

    input_fn, _ = _get_classification_input_fns()
    classifier.fit(input_fn=input_fn, steps=100)

    # We stopped early.
    self._assert_checkpoint(classifier.model_dir, global_step=5)

class CoreTensorForestTests(test.TestCase):

  def testTrainEvaluateInferDoesNotThrowErrorForClassifier(self):
    head_fn = head_lib._multi_class_head_with_softmax_cross_entropy_loss(
        n_classes=3, loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)

    hparams = tensor_forest.ForestHParams(
        num_trees=3,
        max_nodes=1000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)

    est = random_forest.CoreTensorForestEstimator(hparams.fill(), head=head_fn)

    input_fn, predict_input_fn = _get_classification_input_fns()

    est.train(input_fn=input_fn, steps=100)
    res = est.evaluate(input_fn=input_fn, steps=1)

    self.assertEqual(1.0, res['accuracy'])
    self.assertAllClose(0.55144483, res['loss'])

    predictions = list(est.predict(input_fn=predict_input_fn))
    self.assertAllClose([[0.576117, 0.211942, 0.211942]],
                        [pred['probabilities'] for pred in predictions])

  def testRegression(self):
    """Tests regression using matrix data as input."""
    head_fn = head_lib._regression_head(
        label_dimension=1,
        loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)

    hparams = tensor_forest.ForestHParams(
        num_trees=5,
        max_nodes=1000,
        num_classes=1,
        num_features=13,
        regression=True,
        split_after_samples=20)

    regressor = random_forest.CoreTensorForestEstimator(
        hparams.fill(), head=head_fn)

    input_fn, predict_input_fn = _get_regression_input_fns()

    regressor.train(input_fn=input_fn, steps=100)
    res = regressor.evaluate(input_fn=input_fn, steps=10)
    self.assertGreaterEqual(0.1, res['loss'])

    predictions = list(regressor.predict(input_fn=predict_input_fn))
    self.assertAllClose(
        [[24.]], [pred['predictions'] for pred in predictions], atol=1)

  def testWithFeatureColumns(self):
    head_fn = head_lib._multi_class_head_with_softmax_cross_entropy_loss(
        n_classes=3, loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)

    hparams = tensor_forest.ForestHParams(
        num_trees=3,
        max_nodes=1000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)

    est = random_forest.CoreTensorForestEstimator(
        hparams.fill(),
        head=head_fn,
        feature_columns=[core_feature_column.numeric_column('x')])

    iris = base.load_iris()
    data = {'x': iris.data.astype(np.float32)}
    labels = iris.target.astype(np.int32)

    input_fn = numpy_io.numpy_input_fn(
        x=data, y=labels, batch_size=150, num_epochs=None, shuffle=False)

    est.train(input_fn=input_fn, steps=100)
    res = est.evaluate(input_fn=input_fn, steps=1)

    self.assertEqual(1.0, res['accuracy'])
    self.assertAllClose(0.55144483, res['loss'])

  def testAutofillsClassificationHead(self):
    hparams = tensor_forest.ForestHParams(
        num_trees=3,
        max_nodes=1000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)

    est = random_forest.CoreTensorForestEstimator(hparams.fill())

    input_fn, _ = _get_classification_input_fns()

    est.train(input_fn=input_fn, steps=100)
    res = est.evaluate(input_fn=input_fn, steps=1)

    self.assertEqual(1.0, res['accuracy'])
    self.assertAllClose(0.55144483, res['loss'])

  def testAutofillsRegressionHead(self):
    hparams = tensor_forest.ForestHParams(
        num_trees=5,
        max_nodes=1000,
        num_classes=1,
        num_features=13,
        regression=True,
        split_after_samples=20)

    regressor = random_forest.CoreTensorForestEstimator(hparams.fill())

    input_fn, predict_input_fn = _get_regression_input_fns()

    regressor.train(input_fn=input_fn, steps=100)
    res = regressor.evaluate(input_fn=input_fn, steps=10)
    self.assertGreaterEqual(0.1, res['loss'])

    predictions = list(regressor.predict(input_fn=predict_input_fn))
    self.assertAllClose(
        [[24.]], [pred['predictions'] for pred in predictions], atol=1)

  def testAdditionalOutputs(self):
    """Tests multi-class classification using matrix data as input."""
    hparams = tensor_forest.ForestHParams(
        num_trees=1,
        max_nodes=100,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)
    classifier = random_forest.CoreTensorForestEstimator(
        hparams.fill(), keys_column='keys', include_all_in_serving=True)

    iris = base.load_iris()
    data = iris.data.astype(np.float32)
    labels = iris.target.astype(np.int32)

    input_fn = numpy_io.numpy_input_fn(
        x={
            'x': data,
            'keys': np.arange(len(iris.data)).reshape(150, 1)
        },
        y=labels,
        batch_size=10,
        num_epochs=1,
        shuffle=False)

    classifier.train(input_fn=input_fn, steps=100)
    predictions = list(classifier.predict(input_fn=input_fn))
    # Check that there is a key column, tree paths and var.
    for pred in predictions:
      self.assertTrue('keys' in pred)
      self.assertTrue('tree_paths' in pred)
      self.assertTrue('prediction_variance' in pred)

  def _assert_checkpoint(self, model_dir, global_step):
    reader = checkpoint_utils.load_checkpoint(model_dir)
    self.assertLessEqual(
        reader.get_tensor(ops.GraphKeys.GLOBAL_STEP), global_step)

  def testEarlyStopping(self):
    head_fn = head_lib._multi_class_head_with_softmax_cross_entropy_loss(
        n_classes=3, loss_reduction=losses.Reduction.SUM_OVER_NONZERO_WEIGHTS)

    hparams = tensor_forest.ForestHParams(
        num_trees=3,
        max_nodes=1000,
        num_classes=3,
        num_features=4,
        split_after_samples=20,
        inference_tree_paths=True)

    est = random_forest.CoreTensorForestEstimator(
        hparams.fill(),
        head=head_fn,
        # Set a crazy threshold - 30% loss change.
        early_stopping_loss_threshold=0.3,
        early_stopping_rounds=2)

    input_fn, _ = _get_classification_input_fns()
    est.train(input_fn=input_fn, steps=100)
    # We stopped early.
    self._assert_checkpoint(est.model_dir, global_step=8)

if __name__ == "__main__":
  test.main()
<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Basic tests for TF-TensorRT integration."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.contrib.tensorrt.python import trt_convert
# pylint: disable=unused-import
from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
# pylint: enable=unused-import
from tensorflow.python.client import session
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.platform import test
from tensorflow.python.platform import googletest

class NoDeviceAssignmentTest(googletest.TestCase):

  def testNoDeviceAssignment(self):
    """Test that conversion should succeed when device is not specified."""
    sess = session.Session()  # By default this will consume all the gpu memory.
    used_bytes = 0
    for device in sess.list_devices():
      if 'GPU:0' in device.name:
        used_bytes = device.memory_limit_bytes
    self.assertGreater(used_bytes, 0)

    input_dims = [100, 24, 24, 2]
    g = ops.Graph()
    with g.as_default():
      inp = array_ops.placeholder(
          dtype=dtypes.float32, shape=input_dims, name='input')
      for i in range(2):
        mul = inp * inp
        inp = mul + inp
      array_ops.squeeze(inp, name='output')

    trt_gdef = trt_convert.create_inference_graph(
        input_graph_def=g.as_graph_def(),
        outputs=['output'],
        max_batch_size=input_dims[0],
        # Use half of the allocated memory. It will fail if the converter
        # fallback to use native cudaMalloc(), so here it tests that converter
        # doesn't fallback.
        max_workspace_size_bytes=used_bytes // 4,
        minimum_segment_size=2,
        is_dynamic_op=False)
    self.assertEqual(1,
                     sum([node.op == 'TRTEngineOp' for node in trt_gdef.node]))

if __name__ == '__main__':
  test.main()
<commit_msg>Fix broken test<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Basic tests for TF-TensorRT integration."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.contrib.tensorrt.python import trt_convert
# pylint: disable=unused-import
from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
# pylint: enable=unused-import
from tensorflow.python.client import session
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.platform import test
from tensorflow.python.platform import googletest

class NoDeviceAssignmentTest(googletest.TestCase):

  def testNoDeviceAssignment(self):
    """Test that conversion should succeed when device is not specified."""
    if not trt_convert.is_tensorrt_enabled():
      return
    sess = session.Session()  # By default this will consume all the gpu memory.
    used_bytes = 0
    for device in sess.list_devices():
      if 'GPU:0' in device.name:
        used_bytes = device.memory_limit_bytes
    self.assertGreater(used_bytes, 0)

    input_dims = [100, 24, 24, 2]
    g = ops.Graph()
    with g.as_default():
      inp = array_ops.placeholder(
          dtype=dtypes.float32, shape=input_dims, name='input')
      for i in range(2):
        mul = inp * inp
        inp = mul + inp
      array_ops.squeeze(inp, name='output')

    trt_gdef = trt_convert.create_inference_graph(
        input_graph_def=g.as_graph_def(),
        outputs=['output'],
        max_batch_size=input_dims[0],
        # Use half of the allocated memory. It will fail if the converter
        # fallback to use native cudaMalloc(), so here it tests that converter
        # doesn't fallback.
        max_workspace_size_bytes=used_bytes // 4,
        minimum_segment_size=2,
        is_dynamic_op=False)
    self.assertEqual(1,
                     sum([node.op == 'TRTEngineOp' for node in trt_gdef.node]))

if __name__ == '__main__':
  test.main()
<|endoftext|>


<commit_before># MIT License
# Copyright (c) 2017 MassChallenge, Inc.

import magic

from django.conf import settings
from django.http import HttpResponse
from rest_framework.response import Response
from rest_framework_proxy.views import ProxyView

from impact.permissions import (
    V0APIPermissions
)
from impact.v0.views.utils import encrypt_image_token

class ImageProxyView(ProxyView):
    source = "api/image/"
    verify_ssl = False
    return_raw = True
    permission_classes = (
        V0APIPermissions,
    )

    def create_response(self, response):
        magic.Magic(mime=True)
        ctype = magic.from_buffer(response.content, mime=True)
        return HttpResponse(
            response.content,
            status=response.status_code,
            content_type=ctype)

    def get(self, *args, **kwargs):
        if not self.valid(self.request.GET):
            return Response(status=404, data=self.errors)
        self.request.GET = self.request.GET.copy()
        self._update_get_parameter("Size",
                                   self.request.GET.get("Size", "100x100"))
        self._update_get_parameter("SiteName", settings.V0_SITE_NAME)
        self._update_get_parameter("ImageToken", self._secure_image_token())
        return self.proxy(self.request, *args, **kwargs)

    def valid(self, data):
        self.errors = []
        if "ImageToken" not in data or not data["ImageToken"]:
            self.errors.append("ImageToken not found")
        if "SiteName" in data:
            self.errors.append("SiteName is deprecated")
        if not settings.V0_SITE_NAME:
            self.errors.append("Default SiteName is not set")
        return self.errors == []

    def _update_get_parameter(self, param, value):
        self.request.GET[param] = value
        self.request.query_params._mutable = True
        self.request.query_params[param] = value
        self.request.query_params._mutable = False

    def _secure_image_token(self):
        image_token = encrypt_image_token(self.request.GET["ImageToken"],
                                          settings.V0_SECURITY_KEY)
        return image_token + b"="
<commit_msg>[AC-5382] fix incorrect padding bug:<commit_after># MIT License
# Copyright (c) 2017 MassChallenge, Inc.

import magic
from django.conf import settings
from django.http import HttpResponse
from rest_framework.response import Response
from rest_framework_proxy.views import ProxyView

from impact.permissions import (
    V0APIPermissions
)
from impact.v0.views.utils import encrypt_image_token

class ImageProxyView(ProxyView):
    source = "api/image/"
    verify_ssl = False
    return_raw = True
    permission_classes = (
        V0APIPermissions,
    )

    def create_response(self, response):
        magic.Magic(mime=True)
        ctype = magic.from_buffer(response.content, mime=True)
        return HttpResponse(
            response.content,
            status=response.status_code,
            content_type=ctype)

    def get(self, *args, **kwargs):
        if not self.valid(self.request.GET):
            return Response(status=404, data=self.errors)
        self.request.GET = self.request.GET.copy()
        self._update_get_parameter("Size",
                                   self.request.GET.get("Size", "100x100"))
        self._update_get_parameter("SiteName", settings.V0_SITE_NAME)
        self._update_get_parameter("ImageToken", self._secure_image_token())
        return self.proxy(self.request, *args, **kwargs)

    def valid(self, data):
        self.errors = []
        if "ImageToken" not in data or not data["ImageToken"]:
            self.errors.append("ImageToken not found")
        if "SiteName" in data:
            self.errors.append("SiteName is deprecated")
        if not settings.V0_SITE_NAME:
            self.errors.append("Default SiteName is not set")
        return self.errors == []

    def _update_get_parameter(self, param, value):
        self.request.GET[param] = value
        self.request.query_params._mutable = True
        self.request.query_params[param] = value
        self.request.query_params._mutable = False

    def _secure_image_token(self):
        return encrypt_image_token(self.request.GET["ImageToken"])
<|endoftext|>