<commit_before>#=========================================================================
# SimulationTool.py
#=========================================================================
# Tool for simulating MTL models.
#
# This module contains classes which construct a simulator given a MTL model
# for execution in the python interpreter.

import pprint
import collections
import inspect
import copy
import warnings

from SignalValue import SignalValue
from ast_visitor import DetectLoadsAndStores, get_method_ast

#-------------------------------------------------------------------------
# SimulationTool
#-------------------------------------------------------------------------
# User visible class implementing a tool for simulating MTL models.
#
# This class takes a MTL model instance and creates a simulator for execution
# in the python interpreter.
class SimulationTool( object ):

  #-----------------------------------------------------------------------
  # __init__
  #-----------------------------------------------------------------------
  # Construct a simulator based on a MTL model.
  def __init__( self, model ):

    # Check that the model has been elaborated
    if not model.is_elaborated():
      raise Exception( "cannot initialize {0} tool.\n"
                       "Provided model has not been elaborated yet!!!"
                       "".format( self.__class__.__name__ ) )

    self.model              = model
    self._nets              = []
    self._sequential_blocks = []
    self._register_queue    = []
    self._event_queue       = collections.deque()
    self._svalue_callbacks  = collections.defaultdict(list)
    self.ncycles            = 0

    # TODO: temporary hack
    self._slice_connects    = []

    # Actually construct the simulator
    self._construct_sim()

  #-----------------------------------------------------------------------
  # eval_combinational
  #-----------------------------------------------------------------------
  # Evaluates all combinational logic blocks currently in the event queue.
  def eval_combinational( self ):
    while self._event_queue:
      func = self._event_queue.pop()
      #self.pstats.add_eval_call( func, self.num_cycles )
      try:
        func()
      except TypeError:
        # TODO: can we catch this at static elaboration?
        raise Exception("Concurrent block '{}' must take no parameters!\n"
                        "file: {}\n"
                        "line: {}\n"
                        "".format( func.func_name,
                                   func.func_code.co_filename,
                                   func.func_code.co_firstlineno ) )

  #-----------------------------------------------------------------------
  # cycle
  #-----------------------------------------------------------------------
  # Advance the simulator by a single clock cycle.
  #
  # Executes all sequential @tick and @posedge_clk blocks defined in the
  # design, as well as any @combinational blocks that have been added
  # to the event queue.
  def cycle( self ):
    # Call all events generated by input changes
    self.eval_combinational()

    # TODO: Hacky auto clock generation
    #if self.vcd:
    #  print >> self.o, "#%s" % (10 * self.num_cycles)
    self.model.clk.value = 0

    #if self.vcd:
    #  print >> self.o, "#%s" % ((10 * self.num_cycles) + 5)
    self.model.clk.value = 1

    # Call all rising edge triggered functions
    for func in self._sequential_blocks:
      func()

    # Then flop the shadow state on all registers
    while self._register_queue:
      reg = self._register_queue.pop()
      reg.flop()

    # Call all events generated by synchronous logic
    self.eval_combinational()

    # Increment the simulator cycle count
    self.ncycles += 1

  #-----------------------------------------------------------------------
  # reset
  #-----------------------------------------------------------------------
  # Sets the reset signal high and cycles the simulator.
  def reset( self ):
    self.model.reset.v = 1
    self.cycle()
    self.cycle()
    self.model.reset.v = 0

  #-----------------------------------------------------------------------
  # print_line_trace
  #-----------------------------------------------------------------------
  # Print cycle number and line trace of model.
  def print_line_trace( self ):
    print "{:>3}:".format( self.ncycles ), self.model.line_trace()

  #-----------------------------------------------------------------------
  # add_event
  #-----------------------------------------------------------------------
  # Add an event to the simulator event queue for later execution.
  #
  # This function will check if the written SignalValue instance has any
  # registered events (functions decorated with @combinational), and if
  # so, adds them to the event queue.
  def add_event( self, signal_value ):
    # TODO: debug_event
    #print "    ADDEVENT: VALUE", signal_value.v,
    #print signal_value in self._svalue_callbacks,
    #print [x.fullname for x in signal_value._debug_signals],
    #print self._svalue_callbacks[signal_value]
    if signal_value in self._svalue_callbacks:
      funcs = self._svalue_callbacks[signal_value]
      for func in funcs:
        # TODO: remove this check?  test performance...
        if func not in self._event_queue:
          self._event_queue.appendleft( func )

  #-----------------------------------------------------------------------
  # _construct_sim
  #-----------------------------------------------------------------------
  # Construct a simulator for the provided model.
  def _construct_sim( self ):
    self._create_nets( self.model )
    self._insert_signal_values()
    self._register_decorated_functions( self.model )
    self._create_slice_callbacks()

  #-----------------------------------------------------------------------
  # _create_nets
  #-----------------------------------------------------------------------
  # Generate nets describing structural connections in the model.  Each
  # net describes a set of Signal objects which have been interconnected,
  # either directly or indirectly, by calls to connect().
  def _create_nets( self, model ):

    # DEBUG
    #print 70*'-'
    #print "Model:", model
    #print "Ports:"
    #pprint.pprint( model.get_ports(), indent=3 )
    #print "Submodules:"
    #pprint.pprint( model.get_submodules(), indent=3 )

    #def a_printer( some_set ):
    #  return [ x.parent.name + '.' + x.name for x in some_set ]
    #t = [ a_printer( [ x.src_node, x.dest_node] ) for x in model.get_connections() ]
    #print "Connections:"
    #pprint.pprint( model.get_connections(), indent=3 )
    #pprint.pprint( t, indent=3 )

    # Utility function to collect all the Signal type objects (ports,
    # wires, constants) in the model.
    def collect_signals( model ):
      signals = set( model.get_ports() + model.get_wires() )
      for m in model.get_submodules():
        signals.update( collect_signals( m ) )
      return signals

    signals = collect_signals( model )

    # Utility function to filter only supported connections: ports, and
    # wires.  No slices or Constants.
    def valid_connection( c ):
      if c.src_slice != None or c.dest_slice != None:
        # TEMPORARY HACK, remove slice connections from connections?
        self._slice_connects.append ( c )
        return False
      else:
        return True

    # Iterative Depth-First-Search algorithm, borrowed from Listing 5-5
    # in 'Python Algorithms': http://www.apress.com/9781430232377/
    def iter_dfs( s ):
      S, Q = set(), []
      Q.append( s )
      while Q:
        u = Q.pop()
        if u in S: continue
        S.add( u )
        connected_signals = [ x.other( u ) for x in u.connections
                              if valid_connection( x ) ]
        Q.extend( connected_signals )
        #yield u
      return S

    # Initially signals contains all the Signal type objects in the
    # model.  We perform a depth-first search on the connections of each
    # Signal object, and remove connected objects from the signals set.
    # The result is a collection of nets describing structural
    # connections in the design. Each independent net will later be
    # transformed into a single SignalValue object.
    while signals:
      s = signals.pop()
      net = iter_dfs( s )
      for i in net:
        #if i is not s:
        #  signals.remove( i )
        signals.discard( i )
      self._nets.append( net )

  #-----------------------------------------------------------------------
  # _insert_signal_values
  #-----------------------------------------------------------------------
  # Transform each net into a single SignalValue object. Model attributes
  # currently referencing Signal objects will be modified to reference
  # the SignalValue object of their associated net instead.
  def _insert_signal_values( self ):

    # DEBUG
    #print
    #print "NODE SETS"
    #for set in self._nets:
    #  print '    ', [ x.parent.name + '.' + x.name for x in set ]

    # Utility functions which create SignalValue callbacks.

    def create_comb_update_cb( sim, svalue ):
      def notify_sim_comb_update():
        sim.add_event( svalue )
      return notify_sim_comb_update

    def create_seq_update_cb( sim, svalue ):
      def notify_sim_seq_update():
        sim._register_queue.append( svalue )
      return notify_sim_seq_update

    # Each grouping represents a single SignalValue object. Perform a swap
    # so that all attributes currently pointing to Signal objects in this
    # grouping instead point to the SignalValue.
    for group in self._nets:

      # Get an element out of the set and use it to determine the bitwidth
      # of the net, needed to create a properly sized SignalValue object.
      # TODO: no peek() so have to pop() then reinsert it! Another way?
      # TODO: what about BitStructs?
      temp = group.pop()
      group.add( temp )
      svalue = temp.msg_type
      # TODO: should this be visible to sim?
      svalue._shadow_value = copy.copy( svalue )
      #svalue._debug_signals = group

      # Add a callback to the SignalValue so that the simulator is notified
      # whenever it's value changes.
      # TODO: Currently all SignalValues get both a comb and seq update
      #       callback.  Really should only need one or the other, and
      #       name it notify_sim().
      svalue.notify_sim_comb_update  = create_comb_update_cb( self, svalue )
      svalue.notify_sim_seq_update   = create_seq_update_cb ( self, svalue )
      svalue.notify_sim_slice_update = svalue.notify_sim_comb_update
      svalue._shadow_value.notify_sim_slice_update = svalue.notify_sim_seq_update

      # Modify model attributes currently referencing Signal objects to
      # reference SignalValue objects instead.
      # TODO: hacky based on [idx], fix?
      for x in group:
        # Set the value of the SignalValue object if we encounter a
        # constant (check for Constant object instead?)
        if isinstance( x._signalvalue, int ):
          svalue.write( x._signalvalue )
          svalue.constant = True
        # Handle Lists of Ports
        elif '[' in x.name:
          name, idx = x.name.strip(']').split('[')
          x.parent.__dict__[ name ][ int( idx ) ] = svalue
        # Handle Normal Ports
        else:
          x.parent.__dict__[ x.name ] = svalue

        # Also give signals a pointer to the SignalValue object.
        # (Needed for VCD tracing and slice logic generator).
        x._signalvalue = svalue

  #-----------------------------------------------------------------------
  # _register_decorated_functions
  #-----------------------------------------------------------------------
  # Register all decorated @tick, @posedge_clk, and @combinational
  # functions with the simulator.  Sequential logic blocks get called
  # any time cycle() is called, combinational logic blocks are registered
  # with SignalValue objects and get added to the event queue as values
  # change.
  def _register_decorated_functions( self, model ):

    # Add all cycle driven functions
    self._sequential_blocks.extend( model._tick_blocks )
    self._sequential_blocks.extend( model._posedge_clk_blocks )

    # Utility function to turn attributes/names acquired from the ast
    # into Python objects
    # TODO: should never use eval... but this is easy
    # TODO: how to handle when self is neither 's' nor 'self'?
    # TODO: how to handle temps!
    def name_to_object( name ):
      self = s = model
      if '[?]' in name:
        name, extra = name.split('[?]')
      # TODO: hacky way to account for indexing lists
      elif name.endswith( '.uint' ):
        name = name.rstrip( '.uint' )
      try:
        x = eval( name )
        if isinstance( x, (SignalValue, list) ): return x
        else:                                    raise NameError
      except NameError:
        warnings.warn( "Cannot add variable '{}' to sensitivity list."
                       "".format( name ), Warning )
        return None

    # Get the sensitivity list of each event driven (combinational) block
    # TODO: do before or after we swap value nodes?
    for func in model._combinational_blocks:
      tree = get_method_ast( func )
      loads, stores = DetectLoadsAndStores().enter( tree )
      for x in loads:
        obj = name_to_object( x )
        if   isinstance( obj, list ):
          model._newsenses[ func ].extend( obj )
        elif isinstance( obj, SignalValue ):
          model._newsenses[ func ].append( obj )

    # Iterate through all @combinational decorated function names we
    # detected, retrieve their associated function pointer, then add
    # entries for each item in the function's sensitivity list to
    # svalue_callbacks
    # TODO: merge this code with above to reduce mem of data structures?
    for func_ptr, sensitivity_list in model._newsenses.items():
      for signal_value in sensitivity_list:
        self._svalue_callbacks[ signal_value ].append( func_ptr )
        # Prime the simulation by putting all events on the event_queue
        # This will make sure all nodes come out of reset in a consistent
        # state. TODO: put this in reset() instead?
        if func_ptr not in self._event_queue:
          self._event_queue.appendleft( func_ptr )

    # Recursively perform for submodules
    for m in model.get_submodules():
      self._register_decorated_functions( m )

  #-----------------------------------------------------------------------
  # _create_slice_callbacks
  #-----------------------------------------------------------------------
  # All ConnectionEdges that contain bit slicing need to be turned into
  # combinational blocks.  This significantly simplifies the connection
  # graph update logic.
  def _create_slice_callbacks( self ):

    # Utility function to create our callback
    def create_slice_cb_closure( c ):
      src       = c.src_node._signalvalue
      dest      = c.dest_node._signalvalue
      src_addr  = c.src_slice  if c.src_slice  != None else slice( None )
      dest_addr = c.dest_slice if c.dest_slice != None else slice( None )
      def slice_cb():
        dest.v[ dest_addr ] = src.v[ src_addr ]
      return slice_cb

    for c in self._slice_connects:
      src = c.src_node._signalvalue
      # If slice is connect to a Constant, don't create a callback.
      # Just write the constant value now.
      if isinstance( src, int ):
        dest      = c.dest_node._signalvalue
        dest_addr = c.dest_slice if c.dest_slice != None else slice( None )
        dest.v[ dest_addr ] = src
      # If slice is connected to another Signal, create a callback
      # and put it on the combinational event queue.
      else:
        func_ptr = create_slice_cb_closure( c )
        signal_value = c.src_node._signalvalue
        self._svalue_callbacks[ signal_value ].append( func_ptr )
        self._event_queue.appendleft( func_ptr )

<commit_msg>[new_pymtl] check to prevent duplicate func exec<commit_after>#=========================================================================
# SimulationTool.py
#=========================================================================
# Tool for simulating MTL models.
#
# This module contains classes which construct a simulator given a MTL model
# for execution in the python interpreter.

import pprint
import collections
import inspect
import copy
import warnings

from SignalValue import SignalValue
from ast_visitor import DetectLoadsAndStores, get_method_ast

#-------------------------------------------------------------------------
# SimulationTool
#-------------------------------------------------------------------------
# User visible class implementing a tool for simulating MTL models.
#
# This class takes a MTL model instance and creates a simulator for execution
# in the python interpreter.
class SimulationTool( object ):

  #-----------------------------------------------------------------------
  # __init__
  #-----------------------------------------------------------------------
  # Construct a simulator based on a MTL model.
  def __init__( self, model ):

    # Check that the model has been elaborated
    if not model.is_elaborated():
      raise Exception( "cannot initialize {0} tool.\n"
                       "Provided model has not been elaborated yet!!!"
                       "".format( self.__class__.__name__ ) )

    self.model              = model
    self._nets              = []
    self._sequential_blocks = []
    self._register_queue    = []
    self._event_queue       = collections.deque()
    self._svalue_callbacks  = collections.defaultdict(list)
    self.ncycles            = 0
    self._current_func      = None

    # TODO: temporary hack
    self._slice_connects    = []

    # Actually construct the simulator
    self._construct_sim()

  #-----------------------------------------------------------------------
  # eval_combinational
  #-----------------------------------------------------------------------
  # Evaluates all combinational logic blocks currently in the event queue.
  def eval_combinational( self ):
    while self._event_queue:
      self._current_func = func = self._event_queue.pop()
      #self.pstats.add_eval_call( func, self.num_cycles )
      try:
        func()
        self._current_func = None
      except TypeError:
        # TODO: can we catch this at static elaboration?
        raise Exception("Concurrent block '{}' must take no parameters!\n"
                        "file: {}\n"
                        "line: {}\n"
                        "".format( func.func_name,
                                   func.func_code.co_filename,
                                   func.func_code.co_firstlineno ) )

  #-----------------------------------------------------------------------
  # cycle
  #-----------------------------------------------------------------------
  # Advance the simulator by a single clock cycle.
  #
  # Executes all sequential @tick and @posedge_clk blocks defined in the
  # design, as well as any @combinational blocks that have been added
  # to the event queue.
  def cycle( self ):
    # Call all events generated by input changes
    self.eval_combinational()

    # TODO: Hacky auto clock generation
    #if self.vcd:
    #  print >> self.o, "#%s" % (10 * self.num_cycles)
    self.model.clk.value = 0

    #if self.vcd:
    #  print >> self.o, "#%s" % ((10 * self.num_cycles) + 5)
    self.model.clk.value = 1

    # Call all rising edge triggered functions
    for func in self._sequential_blocks:
      func()

    # Then flop the shadow state on all registers
    while self._register_queue:
      reg = self._register_queue.pop()
      reg.flop()

    # Call all events generated by synchronous logic
    self.eval_combinational()

    # Increment the simulator cycle count
    self.ncycles += 1

  #-----------------------------------------------------------------------
  # reset
  #-----------------------------------------------------------------------
  # Sets the reset signal high and cycles the simulator.
  def reset( self ):
    self.model.reset.v = 1
    self.cycle()
    self.cycle()
    self.model.reset.v = 0

  #-----------------------------------------------------------------------
  # print_line_trace
  #-----------------------------------------------------------------------
  # Print cycle number and line trace of model.
  def print_line_trace( self ):
    print "{:>3}:".format( self.ncycles ), self.model.line_trace()

  #-----------------------------------------------------------------------
  # add_event
  #-----------------------------------------------------------------------
  # Add an event to the simulator event queue for later execution.
  #
  # This function will check if the written SignalValue instance has any
  # registered events (functions decorated with @combinational), and if
  # so, adds them to the event queue.
  def add_event( self, signal_value ):
    # TODO: debug_event
    #print "    ADDEVENT: VALUE", signal_value.v,
    #print signal_value in self._svalue_callbacks,
    #print [x.fullname for x in signal_value._debug_signals],
    #print self._svalue_callbacks[signal_value]
    if signal_value in self._svalue_callbacks:
      funcs = self._svalue_callbacks[signal_value]
      for func in funcs:
        # TODO: remove this check?  test performance...
        if func != self._current_func and func not in self._event_queue:
          self._event_queue.appendleft( func )

  #-----------------------------------------------------------------------
  # _construct_sim
  #-----------------------------------------------------------------------
  # Construct a simulator for the provided model.
  def _construct_sim( self ):
    self._create_nets( self.model )
    self._insert_signal_values()
    self._register_decorated_functions( self.model )
    self._create_slice_callbacks()

  #-----------------------------------------------------------------------
  # _create_nets
  #-----------------------------------------------------------------------
  # Generate nets describing structural connections in the model.  Each
  # net describes a set of Signal objects which have been interconnected,
  # either directly or indirectly, by calls to connect().
  def _create_nets( self, model ):

    # DEBUG
    #print 70*'-'
    #print "Model:", model
    #print "Ports:"
    #pprint.pprint( model.get_ports(), indent=3 )
    #print "Submodules:"
    #pprint.pprint( model.get_submodules(), indent=3 )

    #def a_printer( some_set ):
    #  return [ x.parent.name + '.' + x.name for x in some_set ]
    #t = [ a_printer( [ x.src_node, x.dest_node] ) for x in model.get_connections() ]
    #print "Connections:"
    #pprint.pprint( model.get_connections(), indent=3 )
    #pprint.pprint( t, indent=3 )

    # Utility function to collect all the Signal type objects (ports,
    # wires, constants) in the model.
    def collect_signals( model ):
      signals = set( model.get_ports() + model.get_wires() )
      for m in model.get_submodules():
        signals.update( collect_signals( m ) )
      return signals

    signals = collect_signals( model )

    # Utility function to filter only supported connections: ports, and
    # wires.  No slices or Constants.
    def valid_connection( c ):
      if c.src_slice != None or c.dest_slice != None:
        # TEMPORARY HACK, remove slice connections from connections?
        self._slice_connects.append ( c )
        return False
      else:
        return True

    # Iterative Depth-First-Search algorithm, borrowed from Listing 5-5
    # in 'Python Algorithms': http://www.apress.com/9781430232377/
    def iter_dfs( s ):
      S, Q = set(), []
      Q.append( s )
      while Q:
        u = Q.pop()
        if u in S: continue
        S.add( u )
        connected_signals = [ x.other( u ) for x in u.connections
                              if valid_connection( x ) ]
        Q.extend( connected_signals )
        #yield u
      return S

    # Initially signals contains all the Signal type objects in the
    # model.  We perform a depth-first search on the connections of each
    # Signal object, and remove connected objects from the signals set.
    # The result is a collection of nets describing structural
    # connections in the design. Each independent net will later be
    # transformed into a single SignalValue object.
    while signals:
      s = signals.pop()
      net = iter_dfs( s )
      for i in net:
        #if i is not s:
        #  signals.remove( i )
        signals.discard( i )
      self._nets.append( net )

  #-----------------------------------------------------------------------
  # _insert_signal_values
  #-----------------------------------------------------------------------
  # Transform each net into a single SignalValue object. Model attributes
  # currently referencing Signal objects will be modified to reference
  # the SignalValue object of their associated net instead.
  def _insert_signal_values( self ):

    # DEBUG
    #print
    #print "NODE SETS"
    #for set in self._nets:
    #  print '    ', [ x.parent.name + '.' + x.name for x in set ]

    # Utility functions which create SignalValue callbacks.

    def create_comb_update_cb( sim, svalue ):
      def notify_sim_comb_update():
        sim.add_event( svalue )
      return notify_sim_comb_update

    def create_seq_update_cb( sim, svalue ):
      def notify_sim_seq_update():
        sim._register_queue.append( svalue )
      return notify_sim_seq_update

    # Each grouping represents a single SignalValue object. Perform a swap
    # so that all attributes currently pointing to Signal objects in this
    # grouping instead point to the SignalValue.
    for group in self._nets:

      # Get an element out of the set and use it to determine the bitwidth
      # of the net, needed to create a properly sized SignalValue object.
      # TODO: no peek() so have to pop() then reinsert it! Another way?
      # TODO: what about BitStructs?
      temp = group.pop()
      group.add( temp )
      svalue = temp.msg_type
      # TODO: should this be visible to sim?
      svalue._shadow_value = copy.copy( svalue )
      #svalue._debug_signals = group

      # Add a callback to the SignalValue so that the simulator is notified
      # whenever it's value changes.
      # TODO: Currently all SignalValues get both a comb and seq update
      #       callback.  Really should only need one or the other, and
      #       name it notify_sim().
      svalue.notify_sim_comb_update  = create_comb_update_cb( self, svalue )
      svalue.notify_sim_seq_update   = create_seq_update_cb ( self, svalue )
      svalue.notify_sim_slice_update = svalue.notify_sim_comb_update
      svalue._shadow_value.notify_sim_slice_update = svalue.notify_sim_seq_update

      # Modify model attributes currently referencing Signal objects to
      # reference SignalValue objects instead.
      # TODO: hacky based on [idx], fix?
      for x in group:
        # Set the value of the SignalValue object if we encounter a
        # constant (check for Constant object instead?)
        if isinstance( x._signalvalue, int ):
          svalue.write( x._signalvalue )
          svalue.constant = True
        # Handle Lists of Ports
        elif '[' in x.name:
          name, idx = x.name.strip(']').split('[')
          x.parent.__dict__[ name ][ int( idx ) ] = svalue
        # Handle Normal Ports
        else:
          x.parent.__dict__[ x.name ] = svalue

        # Also give signals a pointer to the SignalValue object.
        # (Needed for VCD tracing and slice logic generator).
        x._signalvalue = svalue

  #-----------------------------------------------------------------------
  # _register_decorated_functions
  #-----------------------------------------------------------------------
  # Register all decorated @tick, @posedge_clk, and @combinational
  # functions with the simulator.  Sequential logic blocks get called
  # any time cycle() is called, combinational logic blocks are registered
  # with SignalValue objects and get added to the event queue as values
  # change.
  def _register_decorated_functions( self, model ):

    # Add all cycle driven functions
    self._sequential_blocks.extend( model._tick_blocks )
    self._sequential_blocks.extend( model._posedge_clk_blocks )

    # Utility function to turn attributes/names acquired from the ast
    # into Python objects
    # TODO: should never use eval... but this is easy
    # TODO: how to handle when self is neither 's' nor 'self'?
    # TODO: how to handle temps!
    def name_to_object( name ):
      self = s = model
      if '[?]' in name:
        name, extra = name.split('[?]')
      # TODO: hacky way to account for indexing lists
      elif name.endswith( '.uint' ):
        name = name.rstrip( '.uint' )
      try:
        x = eval( name )
        if isinstance( x, (SignalValue, list) ): return x
        else:                                    raise NameError
      except NameError:
        warnings.warn( "Cannot add variable '{}' to sensitivity list."
                       "".format( name ), Warning )
        return None

    # Get the sensitivity list of each event driven (combinational) block
    # TODO: do before or after we swap value nodes?
    for func in model._combinational_blocks:
      tree = get_method_ast( func )
      loads, stores = DetectLoadsAndStores().enter( tree )
      for x in loads:
        obj = name_to_object( x )
        if   isinstance( obj, list ):
          model._newsenses[ func ].extend( obj )
        elif isinstance( obj, SignalValue ):
          model._newsenses[ func ].append( obj )

    # Iterate through all @combinational decorated function names we
    # detected, retrieve their associated function pointer, then add
    # entries for each item in the function's sensitivity list to
    # svalue_callbacks
    # TODO: merge this code with above to reduce mem of data structures?
    for func_ptr, sensitivity_list in model._newsenses.items():
      for signal_value in sensitivity_list:
        self._svalue_callbacks[ signal_value ].append( func_ptr )
        # Prime the simulation by putting all events on the event_queue
        # This will make sure all nodes come out of reset in a consistent
        # state. TODO: put this in reset() instead?
        if func_ptr not in self._event_queue:
          self._event_queue.appendleft( func_ptr )

    # Recursively perform for submodules
    for m in model.get_submodules():
      self._register_decorated_functions( m )

  #-----------------------------------------------------------------------
  # _create_slice_callbacks
  #-----------------------------------------------------------------------
  # All ConnectionEdges that contain bit slicing need to be turned into
  # combinational blocks.  This significantly simplifies the connection
  # graph update logic.
  def _create_slice_callbacks( self ):

    # Utility function to create our callback
    def create_slice_cb_closure( c ):
      src       = c.src_node._signalvalue
      dest      = c.dest_node._signalvalue
      src_addr  = c.src_slice  if c.src_slice  != None else slice( None )
      dest_addr = c.dest_slice if c.dest_slice != None else slice( None )
      def slice_cb():
        dest.v[ dest_addr ] = src.v[ src_addr ]
      return slice_cb

    for c in self._slice_connects:
      src = c.src_node._signalvalue
      # If slice is connect to a Constant, don't create a callback.
      # Just write the constant value now.
      if isinstance( src, int ):
        dest      = c.dest_node._signalvalue
        dest_addr = c.dest_slice if c.dest_slice != None else slice( None )
        dest.v[ dest_addr ] = src
      # If slice is connected to another Signal, create a callback
      # and put it on the combinational event queue.
      else:
        func_ptr = create_slice_cb_closure( c )
        signal_value = c.src_node._signalvalue
        self._svalue_callbacks[ signal_value ].append( func_ptr )
        self._event_queue.appendleft( func_ptr )

<|endoftext|>


<commit_before>#
# calculation of natural product-likeness as described in:
#
# Natural Product-likeness Score and Its Application for Prioritization of Compound Libraries
# Peter Ertl, Silvio Roggo, and Ansgar Schuffenhauer
# Journal of Chemical Information and Modeling, 48, 68-74 (2008)
# http://pubs.acs.org/doi/abs/10.1021/ci700286x
#
# for the training of this model only openly available data have been used
# ~50,000 natural products collected from various open databases
# ~1 million drug-like molecules from ZINC as a "non-NP background"
#
# peter ertl, august 2015
#

from __future__ import print_function
from rdkit import Chem
from rdkit.Chem import rdMolDescriptors
import sys, math, gzip, pickle
import os.path

def readNPModel(filename=os.path.join(os.path.dirname(__file__), 'publicnp.model.gz')):
  sys.stderr.write("reading NP model ...\n")
  fscore = pickle.load(gzip.open(filename))
  sys.stderr.write("model in\n")
  return fscore

def scoreMol(mol, fscore):
  if mol is None:
    raise ValueError('invalid molecule')
  fp = rdMolDescriptors.GetMorganFingerprint(mol, 2)
  bits = fp.GetNonzeroElements()

  # calculating the score
  score = 0.
  for bit in bits:
    score += fscore.get(bit, 0)
  score /= float(mol.GetNumAtoms())

  # preventing score explosion for exotic molecules
  if score > 4:
    score = 4. + math.log10(score - 4. + 1.)
  if score < -4:
    score = -4. - math.log10(-4. - score + 1.)
  return score

def processMols(fscore, suppl):
  sys.stderr.write("calculating ...\n")
  count = {}
  n = 0
  for i, m in enumerate(suppl):
    if m is None:
      continue

    n += 1
    score = "%.3f" % scoreMol(m, fscore)

    smiles = Chem.MolToSmiles(m, True)
    name = m.GetProp('_Name')
    print(smiles + "\t" + name + "\t" + score)

  sys.stderr.write("finished, " + str(n) + " molecules processed\n")

if __name__ == '__main__':

  fscore = readNPModel()  # fills fscore

  suppl = Chem.SmilesMolSupplier(sys.argv[1], smilesColumn=0, nameColumn=1, titleLine=False)
  processMols(fscore, suppl)

#
# Copyright (c) 2015, Novartis Institutes for BioMedical Research Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Novartis Institutes for BioMedical Research Inc.
#       nor the names of its contributors may be used to endorse or promote
#       products derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
<commit_msg>NP Likeness with confidence value (#1608)<commit_after>#
# calculation of natural product-likeness as described in:
#
# Natural Product-likeness Score and Its Application for Prioritization of Compound Libraries
# Peter Ertl, Silvio Roggo, and Ansgar Schuffenhauer
# Journal of Chemical Information and Modeling, 48, 68-74 (2008)
# http://pubs.acs.org/doi/abs/10.1021/ci700286x
#
# for the training of this model only openly available data have been used
# ~50,000 natural products collected from various open databases
# ~1 million drug-like molecules from ZINC as a "non-NP background"
#
# peter ertl, august 2015
#

from __future__ import print_function
from rdkit import Chem
from rdkit.Chem import rdMolDescriptors
import sys, math, gzip, pickle
import os.path
from collections import namedtuple

def readNPModel(filename=os.path.join(os.path.dirname(__file__), 'publicnp.model.gz')):
  """Reads and returns the scoring model,
  which has to be passed to the scoring functions."""
  print("reading NP model ...", file=sys.stderr)
  fscore = pickle.load(gzip.open(filename))
  print("model in", file=sys.stderr)
  return fscore

def scoreMolWConfidence(mol, fscore):
  """Next to the NP Likeness Score, this function outputs a confidence value
  between 0..1 that descibes how many fragments of the tested molecule
  were found in the model data set (1: all fragments were found).

  Returns namedtuple NPLikeness(nplikeness, confidence)"""

  if mol is None:
    raise ValueError('invalid molecule')
  fp = rdMolDescriptors.GetMorganFingerprint(mol, 2)
  bits = fp.GetNonzeroElements()

  # calculating the score
  score = 0.0
  bits_found = 0
  for bit in bits:
    if bit in fscore:
      bits_found += 1
      score += fscore[bit]

  score /= float(mol.GetNumAtoms())
  confidence = float(bits_found / len(bits))

  # preventing score explosion for exotic molecules
  if score > 4:
    score = 4. + math.log10(score - 4. + 1.)
  elif score < -4:
    score = -4. - math.log10(-4. - score + 1.)
  NPLikeness = namedtuple("NPLikeness", "nplikeness,confidence")
  return NPLikeness(score, confidence)

def scoreMol(mol, fscore):
  """Calculates the Natural Product Likeness of a molecule.

  Returns the score as float in the range -5..5."""
  return scoreMolWConfidence(mol, fscore).nplikeness

def processMols(fscore, suppl):
  print("calculating ...", file=sys.stderr)
  count = {}
  n = 0
  for i, m in enumerate(suppl):
    if m is None:
      continue

    n += 1
    score = "%.3f" % scoreMol(m, fscore)

    smiles = Chem.MolToSmiles(m, True)
    name = m.GetProp('_Name')
    print(smiles + "\t" + name + "\t" + score)

  print("finished, " + str(n) + " molecules processed", file=sys.stderr)

if __name__ == '__main__':

  fscore = readNPModel()  # fills fscore

  suppl = Chem.SmilesMolSupplier(sys.argv[1], smilesColumn=0, nameColumn=1, titleLine=False)
  processMols(fscore, suppl)

#
# Copyright (c) 2015, Novartis Institutes for BioMedical Research Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Novartis Institutes for BioMedical Research Inc.
#       nor the names of its contributors may be used to endorse or promote
#       products derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
<|endoftext|>


<commit_before>#
# calculation of synthetic accessibility score as described in:
#
# Estimation of Synthetic Accessibility Score of Drug-like Molecules based on Molecular Complexity and Fragment Contributions
# Peter Ertl and Ansgar Schuffenhauer
# Journal of Cheminformatics 1:8 (2009)
# http://www.jcheminf.com/content/1/1/8
#
# several small modifications to the original paper are included
# particularly slightly different formula for marocyclic penalty
# and taking into account also molecule symmetry (fingerprint density)
#
# for a set of 10k diverse molecules the agreement between the original method
# as implemented in PipelinePilot and this implementation is r2 = 0.97
#
# peter ertl & greg landrum, september 2013
#
from __future__ import print_function

from rdkit import Chem
from rdkit.Chem import rdMolDescriptors
from rdkit.six.moves import cPickle
from rdkit.six import iteritems

import math
from collections import defaultdict

import os.path as op

_fscores = None
def readFragmentScores(name='fpscores'):
    import gzip
    global _fscores
    # generate the full path filename:
    if name == "fpscores":
        name = op.join(op.dirname(__file__), name)
    _fscores = cPickle.load(gzip.open('%s.pkl.gz'%name))
    outDict = {}
    for i in _fscores:
        for j in range(1,len(i)):
            outDict[i[j]] = float(i[0])
    _fscores = outDict

def numBridgeheadsAndSpiro(mol,ri=None):
  if ri is None:
    ri=mol.GetRingInfo()
  arings = [set(x) for x in ri.AtomRings()]
  spiros=set()
  for i,ari in enumerate(arings):
    for j in range(i+1,len(arings)):
      shared=ari&arings[j]
      if len(shared)==1:
        spiros.update(shared)
  nSpiro=len(spiros)

  # find bonds that are shared between rings that share at least 2 bonds:
  brings = [set(x) for x in ri.BondRings()]
  bridges=set()
  for i,bri in enumerate(brings):
    for j in range(i+1,len(brings)):
      shared=bri&brings[j]
      if len(shared)>1:
        atomCounts=defaultdict(int)
        for bi in shared:
          bond = mol.GetBondWithIdx(bi)
          atomCounts[bond.GetBeginAtomIdx()]+=1
          atomCounts[bond.GetEndAtomIdx()]+=1
        tmp=0
        for ai,cnt in atomCounts.items():
          if cnt==1:
            tmp+=1
            bridges.add(ai)
        #if tmp!=2: # no need to stress the users
          #print 'huh:',tmp
  return len(bridges),nSpiro

def calculateScore(m):
  if _fscores is None: readFragmentScores()

  # fragment score
  fp = rdMolDescriptors.GetMorganFingerprint(m,2)  #<- 2 is the *radius* of the circular fingerprint
  fps = fp.GetNonzeroElements()
  score1 = 0.
  nf = 0
  for bitId,v in iteritems(fps):
    nf += v
    sfp = bitId
    score1 += _fscores.get(sfp,-4)*v
  score1 /= nf

  # features score
  nAtoms = m.GetNumAtoms()
  nChiralCenters = len(Chem.FindMolChiralCenters(m,includeUnassigned=True))
  ri = m.GetRingInfo()
  nBridgeheads,nSpiro=numBridgeheadsAndSpiro(m,ri)
  nMacrocycles=0
  for x in ri.AtomRings():
    if len(x)>8: nMacrocycles+=1

  sizePenalty = nAtoms**1.005 - nAtoms
  stereoPenalty = math.log10(nChiralCenters+1)
  spiroPenalty = math.log10(nSpiro+1)
  bridgePenalty = math.log10(nBridgeheads+1)
  macrocyclePenalty = 0.
  # ---------------------------------------
  # This differs from the paper, which defines:
  #  macrocyclePenalty = math.log10(nMacrocycles+1)
  # This form generates better results when 2 or more macrocycles are present
  if nMacrocycles > 0: macrocyclePenalty = math.log10(2)

  score2 = 0. -sizePenalty -stereoPenalty -spiroPenalty -bridgePenalty -macrocyclePenalty

  # correction for the fingerprint density
  # not in the original publication, added in version 1.1
  # to make highly symmetrical molecules easier to synthetise
  score3 = 0.
  if nAtoms > len(fps):
    score3 = math.log(float(nAtoms) / len(fps)) * .5

  sascore = score1 + score2 + score3

  # need to transform "raw" value into scale between 1 and 10
  min = -4.0
  max = 2.5
  sascore = 11. - (sascore - min + 1) / (max - min) * 9.
  # smooth the 10-end
  if sascore > 8.: sascore = 8. + math.log(sascore+1.-9.)
  if sascore > 10.: sascore = 10.0
  elif sascore < 1.: sascore = 1.0

  return sascore

def processMols(mols,outf):

  print('smiles\tName\tsa_score')
  for i,m in enumerate(mols):
    if m is None:
      continue

    s = calculateScore(m)

    smiles = Chem.MolToSmiles(m)
    print(smiles+"\t"+m.GetProp('_Name') + "\t%3f"%s)

if __name__=='__main__':
  import sys,gzip,time

  outf = None

  t1=time.time()
  readFragmentScores("fpscores")
  t2=time.time()

  suppl = Chem.SmilesMolSupplier(sys.argv[1])
  t3=time.time()
  processMols(suppl,outf)
  t4=time.time()

  print('Reading took %.2f seconds. Calculating took %.2f seconds'%((t2-t1),(t4-t3)), file=sys.stderr)

#
#  Copyright (c) 2013, Novartis Institutes for BioMedical Research Inc.
#  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Novartis Institutes for BioMedical Research Inc.
#       nor the names of its contributors may be used to endorse or promote
#       products derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
<commit_msg>remove the outf parameter<commit_after>#
# calculation of synthetic accessibility score as described in:
#
# Estimation of Synthetic Accessibility Score of Drug-like Molecules based on Molecular Complexity and Fragment Contributions
# Peter Ertl and Ansgar Schuffenhauer
# Journal of Cheminformatics 1:8 (2009)
# http://www.jcheminf.com/content/1/1/8
#
# several small modifications to the original paper are included
# particularly slightly different formula for marocyclic penalty
# and taking into account also molecule symmetry (fingerprint density)
#
# for a set of 10k diverse molecules the agreement between the original method
# as implemented in PipelinePilot and this implementation is r2 = 0.97
#
# peter ertl & greg landrum, september 2013
#
from __future__ import print_function

from rdkit import Chem
from rdkit.Chem import rdMolDescriptors
from rdkit.six.moves import cPickle
from rdkit.six import iteritems

import math
from collections import defaultdict

import os.path as op

_fscores = None
def readFragmentScores(name='fpscores'):
    import gzip
    global _fscores
    # generate the full path filename:
    if name == "fpscores":
        name = op.join(op.dirname(__file__), name)
    _fscores = cPickle.load(gzip.open('%s.pkl.gz'%name))
    outDict = {}
    for i in _fscores:
        for j in range(1,len(i)):
            outDict[i[j]] = float(i[0])
    _fscores = outDict

def numBridgeheadsAndSpiro(mol,ri=None):
  if ri is None:
    ri=mol.GetRingInfo()
  arings = [set(x) for x in ri.AtomRings()]
  spiros=set()
  for i,ari in enumerate(arings):
    for j in range(i+1,len(arings)):
      shared=ari&arings[j]
      if len(shared)==1:
        spiros.update(shared)
  nSpiro=len(spiros)

  # find bonds that are shared between rings that share at least 2 bonds:
  brings = [set(x) for x in ri.BondRings()]
  bridges=set()
  for i,bri in enumerate(brings):
    for j in range(i+1,len(brings)):
      shared=bri&brings[j]
      if len(shared)>1:
        atomCounts=defaultdict(int)
        for bi in shared:
          bond = mol.GetBondWithIdx(bi)
          atomCounts[bond.GetBeginAtomIdx()]+=1
          atomCounts[bond.GetEndAtomIdx()]+=1
        tmp=0
        for ai,cnt in atomCounts.items():
          if cnt==1:
            tmp+=1
            bridges.add(ai)
        #if tmp!=2: # no need to stress the users
          #print 'huh:',tmp
  return len(bridges),nSpiro

def calculateScore(m):
  if _fscores is None: readFragmentScores()

  # fragment score
  fp = rdMolDescriptors.GetMorganFingerprint(m,2)  #<- 2 is the *radius* of the circular fingerprint
  fps = fp.GetNonzeroElements()
  score1 = 0.
  nf = 0
  for bitId,v in iteritems(fps):
    nf += v
    sfp = bitId
    score1 += _fscores.get(sfp,-4)*v
  score1 /= nf

  # features score
  nAtoms = m.GetNumAtoms()
  nChiralCenters = len(Chem.FindMolChiralCenters(m,includeUnassigned=True))
  ri = m.GetRingInfo()
  nBridgeheads,nSpiro=numBridgeheadsAndSpiro(m,ri)
  nMacrocycles=0
  for x in ri.AtomRings():
    if len(x)>8: nMacrocycles+=1

  sizePenalty = nAtoms**1.005 - nAtoms
  stereoPenalty = math.log10(nChiralCenters+1)
  spiroPenalty = math.log10(nSpiro+1)
  bridgePenalty = math.log10(nBridgeheads+1)
  macrocyclePenalty = 0.
  # ---------------------------------------
  # This differs from the paper, which defines:
  #  macrocyclePenalty = math.log10(nMacrocycles+1)
  # This form generates better results when 2 or more macrocycles are present
  if nMacrocycles > 0: macrocyclePenalty = math.log10(2)

  score2 = 0. -sizePenalty -stereoPenalty -spiroPenalty -bridgePenalty -macrocyclePenalty

  # correction for the fingerprint density
  # not in the original publication, added in version 1.1
  # to make highly symmetrical molecules easier to synthetise
  score3 = 0.
  if nAtoms > len(fps):
    score3 = math.log(float(nAtoms) / len(fps)) * .5

  sascore = score1 + score2 + score3

  # need to transform "raw" value into scale between 1 and 10
  min = -4.0
  max = 2.5
  sascore = 11. - (sascore - min + 1) / (max - min) * 9.
  # smooth the 10-end
  if sascore > 8.: sascore = 8. + math.log(sascore+1.-9.)
  if sascore > 10.: sascore = 10.0
  elif sascore < 1.: sascore = 1.0

  return sascore

def processMols(mols):
  print('smiles\tName\tsa_score')
  for i,m in enumerate(mols):
    if m is None:
      continue

    s = calculateScore(m)

    smiles = Chem.MolToSmiles(m)
    print(smiles+"\t"+m.GetProp('_Name') + "\t%3f"%s)

if __name__=='__main__':
  import sys,time

  t1=time.time()
  readFragmentScores("fpscores")
  t2=time.time()

  suppl = Chem.SmilesMolSupplier(sys.argv[1])
  t3=time.time()
  processMols(suppl)
  t4=time.time()

  print('Reading took %.2f seconds. Calculating took %.2f seconds'%((t2-t1),(t4-t3)), file=sys.stderr)

#
#  Copyright (c) 2013, Novartis Institutes for BioMedical Research Inc.
#  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Novartis Institutes for BioMedical Research Inc.
#       nor the names of its contributors may be used to endorse or promote
#       products derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
<|endoftext|>


<commit_before>import logging

from django.core.management.base import BaseCommand

import amo
from mkt.constants.regions import (ALL_REGIONS_WITH_CONTENT_RATINGS,
                                   REGIONS_CHOICES_ID_DICT)

log = logging.getLogger('z.task')

class Command(BaseCommand):
    """
    Backfill Webapp Geodata by inferring regional popularity from
    AddonExcludedRegion objects (or lack thereof).
    Remove AddonExcludedRegion objects for free apps (except unrated games).
    """

    def handle(self, *args, **options):
        from mkt.webapps.models import Webapp

        paid_types = amo.ADDON_PREMIUMS + (amo.ADDON_FREE_INAPP,)

        games_cat = Webapp.category('games')
        content_region_ids = [x.id for x in ALL_REGIONS_WITH_CONTENT_RATINGS()]

        apps = Webapp.objects.all()
        for app in apps:
            geodata = {}

            # If this app was excluded in every region except one,
            # let's consider it regionally popular in that particular region.
            region_ids = app.get_region_ids()
            if len(region_ids) == 1:
                geodata['popular_region'] = (
                    REGIONS_CHOICES_ID_DICT[region_ids[0]].slug
                )

            if app.premium_type in paid_types:
                geodata['restricted'] = True
            else:
                for exclusion in app.addonexcludedregion.all():
                    # Do not delete region exclusions meant for unrated games.
                    region = REGIONS_CHOICES_ID_DICT[exclusion.region]
                    if (games_cat and region.id in content_region_ids and
                        app.listed_in(category='games') and
                        not app.content_ratings_in(region)):
                        continue

                    log.info('[App %s - %s] Removed exclusion: %s'
                             % (app.pk, app.slug, exclusion))

                    # Remove all other existing exclusions, since all apps
                    # are public in every region by default. If developers
                    # want to hard-restrict their apps they can now do that.
                    exclusion.delete()

            app.geodata.update(**geodata)
<commit_msg>update migrate_geodata to ignore already georestricted apps<commit_after>import logging

from django.core.management.base import BaseCommand

import amo
from mkt.constants.regions import (ALL_REGIONS_WITH_CONTENT_RATINGS,
                                   REGIONS_CHOICES_ID_DICT)

log = logging.getLogger('z.task')

class Command(BaseCommand):
    """
    Backfill Webapp Geodata by inferring regional popularity from
    AddonExcludedRegion objects (or lack thereof).
    Remove AddonExcludedRegion objects for free apps (except unrated games).
    """

    def handle(self, *args, **options):
        from mkt.webapps.models import Webapp

        paid_types = amo.ADDON_PREMIUMS + (amo.ADDON_FREE_INAPP,)

        games_cat = Webapp.category('games')
        content_region_ids = [x.id for x in ALL_REGIONS_WITH_CONTENT_RATINGS()]

        apps = Webapp.objects.all()
        for app in apps:
            # If it's already restricted, don't bother.
            if app.geodata.restricted:
                continue

            geodata = {}

            # If this app was excluded in every region except one,
            # let's consider it regionally popular in that particular region.
            region_ids = app.get_region_ids()
            if len(region_ids) == 1:
                geodata['popular_region'] = (
                    REGIONS_CHOICES_ID_DICT[region_ids[0]].slug
                )

            if app.premium_type in paid_types:
                geodata['restricted'] = True
            else:
                for exclusion in app.addonexcludedregion.all():
                    # Do not delete region exclusions meant for unrated games.
                    region = REGIONS_CHOICES_ID_DICT[exclusion.region]
                    if (games_cat and region.id in content_region_ids and
                        app.listed_in(category='games') and
                        not app.content_ratings_in(region)):
                        continue

                    log.info('[App %s - %s] Removed exclusion: %s'
                             % (app.pk, app.slug, exclusion))

                    # Remove all other existing exclusions, since all apps
                    # are public in every region by default. If developers
                    # want to hard-restrict their apps they can now do that.
                    exclusion.delete()

            app.geodata.update(**geodata)
<|endoftext|>


<commit_before>#
# component_manager.py <Peter.Bienstman@UGent.be>
#

class ComponentManager(object):

    """Manages the different components. Each component belongs to a type
    (database, scheduler, card_type, card_type_widget, ...).

    The component manager can also store relationships between components,
    e.g. a card_type_widget is used for a certain card_type.

    For certain components, many can be active at the same time (card types,
    filters, function hooks, ...). For others, there can be only on active
    at the same time, like schedule, database ... The idea is that the last
    one registered takes preference. This means that e.g. the default
    scheduler needs to be registered first.

    Managed components:

       ======================   ===============================
       "config"                 config instance
       "database"               database instance
       "scheduler"              scheduler instance
       "filter"                 filter instance
       "card_type"              card_type instance
       "card_type_widget"       card_type_widget class,
                                used_for card_type class name
       "renderer"               renderer instance,
                                used_for card_type class name
       "ui_controller_review"   ui_controller_review instance
       "review_widget"          review_widget class
       ======================   ===============================

    Note: for widgets we store the class name as opposed to an instance,
    since creating widgets can be time consuming.

    TODO: plugin, function hook

    """

    def __init__(self):
        self.components = {} # { used_for : {type : [component]} }
        self.card_type_by_id = {}

    def register(self, type, component, used_for=None):
        if not self.components.has_key(used_for):
            self.components[used_for] = {}
        if not self.components[used_for].has_key(type):
            self.components[used_for][type] = [component]
        else:
            if component not in self.components[used_for][type]:
                self.components[used_for][type].append(component)
        if type == "card_type":
            self.card_type_by_id[component.id] = component

    def unregister(self, type, component, used_for=None):
        self.components[used_for][type].remove(component)

    def get_all(self, type, used_for=None):

        """For components for which there can be many active at once."""

        if type in self.components[used_for]:
            return self.components[used_for][type]
        else:
            return []

    def get_current(self, type, used_for=None):

        """For component for which there can be only one active at one time."""

        if type in self.components[used_for]:
            return self.components[used_for][type][-1]
        else:
            return None

# The component manager needs to be accessed by many different parts of the
# library, so we hold it in a global variable.

component_manager = ComponentManager()

# Convenience functions, for easier access from the outside world.

def config():
    return component_manager.get_current("config")

def database():
    return component_manager.get_current("database")

def scheduler():
    return component_manager.get_current("scheduler")

def ui_controller_main():
    return component_manager.get_current("ui_controller_main")

def ui_controller_review():
    return component_manager.get_current("ui_controller_review")

def card_types():
    return component_manager.get_all("card_type")

def card_type_by_id(id):
    return component_manager.card_type_by_id[id]

def filters():
    return component_manager.get_all("filter")

<commit_msg>Doc string and comments on components<commit_after>#
# component_manager.py <Peter.Bienstman@UGent.be>
#

class ComponentManager(object):

    """Manages the different components. Each component belongs to a type
    (database, scheduler, card_type, card_type_widget, ...).

    The component manager can also store relationships between components,
    e.g. a card_type_widget is used for a certain card_type.

    For certain components, many can be active at the same time (card types,
    filters, function hooks, ...). For others, there can be only on active
    at the same time, like schedule, database ... The idea is that the last
    one registered takes preference. This means that e.g. the default
    scheduler needs to be registered first.

    Managed components:

       ======================   ===============================
       "config"                 config instance
       "database"               database instance
       "scheduler"              scheduler instance
       "filter"                 filter instance
       "card_type"              card_type instance
       "card_type_widget"       card_type_widget class,
                                used_for card_type class name
       "renderer"               renderer instance,
                                used_for card_type class name
       "ui_controller_review"   ui_controller_review instance
       "review_widget"          review_widget class
       ======================   ===============================

    Note: for widgets we store the class name as opposed to an instance,
    since creating widgets can be time consuming.

    TODO: plugin, function hook

    """

    def __init__(self):
        self.components = {} # { used_for : {type : [component]} }
        self.card_type_by_id = {}

    def register(self, type, component, used_for=None):
        """
        type:      String describing the type.
                   (XXX: make these registerable to prevent typos)
        component: Instance of Component.
        used_for:  Context for which a component is registered.
                   This is used for example to register certain
                   UI elements for specific card types.
        XXX: When handling used_for cases we should make use of class
             inheritance, TODO KW
        """
        if not self.components.has_key(used_for):
            self.components[used_for] = {}
        if not self.components[used_for].has_key(type):
            self.components[used_for][type] = [component]
        else:
            if component not in self.components[used_for][type]:
                self.components[used_for][type].append(component)
        if type == "card_type":
            self.card_type_by_id[component.id] = component

    def unregister(self, type, component, used_for=None):
        self.components[used_for][type].remove(component)

    def get_all(self, type, used_for=None):

        """For components for which there can be many active at once."""

        if type in self.components[used_for]:
            return self.components[used_for][type]
        else:
            return []

    def get_current(self, type, used_for=None):

        """For component for which there can be only one active at one time."""

        if type in self.components[used_for]:
            return self.components[used_for][type][-1]
        else:
            return None

# The component manager needs to be accessed by many different parts of the
# library, so we hold it in a global variable.

component_manager = ComponentManager()

# Convenience functions, for easier access from the outside world.

def config():
    return component_manager.get_current("config")

def database():
    return component_manager.get_current("database")

def scheduler():
    return component_manager.get_current("scheduler")

def ui_controller_main():
    return component_manager.get_current("ui_controller_main")

def ui_controller_review():
    return component_manager.get_current("ui_controller_review")

def card_types():
    return component_manager.get_all("card_type")

def card_type_by_id(id):
    # XXX Can be solved by using named components, TODO KW
    return component_manager.card_type_by_id[id]

def filters():
    return component_manager.get_all("filter")

<|endoftext|>


<commit_before>## Converting SI units to/from specific units
# Every units are given as a ratio to the SI units (m/kg/s)
#
# For each conversion a local unit can be given
# OR
# variables local_time/local_length/local_mass are used by default
# (these variables can be set by the user to define local units)
#
# Note that Poisson ratio and strain have no units

# TODO have a look to units for stress and elasticity parameters in 2D and 1D

import math

### some CONSTANTS are defined as user utilities
# ratio to the SI units

# LENGTH
length_hm  = 1e2
length_dam = 1e1
length_m   = 1.0
length_dm  = 1e-1
length_cm  = 1e-2
length_mm  = 1e-3
imperial_length_ft = 0.3048                  # foot
imperial_length_in = imperial_length_ft/12.0 # inch
imperial_length_yd = imperial_length_ft*3.0  # yard
imperial_length_mi = imperial_length_ft*5.28 # mile

# MASS
mass_g   = 1e-3
mass_kg  = 1.0
mass_ton = 1e3
imperial_mass_lb  = 0.45359237               # pound
imperial_mass_oz  = imperial_mass_lb/16.0    # ounce
imperial_mass_ton = imperial_mass_lb*2240.0 # imperial ton

# TIME
time_h   = 3600.0
time_min = 60.0
time_s   = 1.0
time_ms  = 1e-3

### LOCAL UNITS
# user variables to set to describe a set of local units

local_time = time_s
local_length = length_m
local_mass = mass_kg

### conversion methods

# BASE

# s
def time_from_SI( t, time_unit=None ):
    time_unit = time_unit or local_time
    return t / time_unit

def time_to_SI( t, time_unit=None ):
    time_unit = time_unit or local_time
    return t * time_unit

# m
def length_from_SI( l, length_unit=None ):
    length_unit = length_unit or local_length
    return l / length_unit

def length_to_SI( l, length_unit=None ):
    length_unit = length_unit or local_length
    return l * length_unit

# kg
def mass_from_SI( m, mass_unit=None ):
    mass_unit = mass_unit or local_mass
    return m / mass_unit

def mass_to_SI( m, mass_unit=None ):
    mass_unit = mass_unit or local_mass
    return m * mass_unit

# DERIVED

# m2
def area_from_SI( a, length_unit=None ):
    length_unit = length_unit or local_length
    return a / length_unit / length_unit

def area_to_SI( a, length_unit=None ):
    length_unit = length_unit or local_length
    return a * length_unit * length_unit

# m3
def volume_from_SI( v, length_unit=None ):
    length_unit = length_unit or local_length
    return v / length_unit / length_unit / length_unit

def volume_to_SI( v, length_unit=None ):
    length_unit = length_unit or local_length
    return v * length_unit * length_unit * length_unit

# m/s
def velocity_from_SI( v, length_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    time_unit = time_unit or local_time
    return v / length_unit * time_unit

def velocity_to_SI( v, length_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    time_unit = time_unit or local_time
    return v * length_unit / time_unit

# m/s2
def acceleration_from_SI( v, length_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    time_unit = time_unit or local_time
    return v / length_unit * time_unit * time_unit

def acceleration_to_SI( v, length_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    time_unit = time_unit or local_time
    return v * length_unit / time_unit / time_unit

# N = kg.m/s2
def force_from_SI( f, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return f / length_unit / mass_unit * time_unit * time_unit

def force_to_SI( f, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return f * length_unit * mass_unit / time_unit / time_unit

# Pa = N/m2 = kg/(m.s2)
def pressure_from_SI( p, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return p / mass_unit * length_unit * time_unit * time_unit

def pressure_to_SI( p, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return p * mass_unit / length_unit / time_unit / time_unit

# J = N.m = kg.m2/s2
def energy_from_SI( e, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return e / length_unit / length_unit / mass_unit * time_unit * time_unit

def energy_to_SI( e, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return e * length_unit * length_unit * mass_unit / time_unit / time_unit

# MATERIAL

# Pa
def elasticity_from_SI( e, length_unit=None, mass_unit=None, time_unit=None ):
    ## works for Young/bulk/shear modulus and Lame parameters
    return pressure_from_SI( e, length_unit, mass_unit, time_unit )

def elasticity_to_SI( e, length_unit=None, mass_unit=None, time_unit=None ):
    ## works for Young/bulk/shear modulus and Lame parameters
    return pressure_to_SI( e, length_unit, mass_unit, time_unit )

# kg/m3
def density_from_SI( d, length_unit=None, mass_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    return d * length_unit * length_unit * length_unit / mass_unit

def density_to_SI( d, length_unit=None, mass_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    return d / length_unit / length_unit / length_unit * mass_unit

# w/o
def poissonRatio_from_SI( pr ):
    return pr

def poissonRatio_to_SI( pr ):
    return pr

# MECHANICS

# Pa
def stress_from_SI( s, length_unit=None, mass_unit=None, time_unit=None ):
    return pressure_from_SI( s, length_unit, mass_unit, time_unit )

def stress_to_SI( s, length_unit=None, mass_unit=None, time_unit=None ):
    return pressure_to_SI( s, length_unit, mass_unit, time_unit )

# w/o
def strain_from_SI( s ):
    return s

def strain_to_SI( s ):
    return s

# m
def deformation_from_SI( d, length_unit=None ):
    return length_from_SI( d, length_unit )

def deformation_to_SI( d, length_unit=None ):
    return length_to_SI( d, length_unit )
<commit_msg>update on the units transformation: Torque, Inertia<commit_after>## Converting SI units to/from specific units
# Every units are given as a ratio to the SI units (m/kg/s)
#
# For each conversion a local unit can be given
# OR
# variables local_time/local_length/local_mass are used by default
# (these variables can be set by the user to define local units)
#
# Note that Poisson ratio and strain have no units

# TODO have a look to units for stress and elasticity parameters in 2D and 1D

import math

### some CONSTANTS are defined as user utilities
# ratio to the SI units

# LENGTH
length_hm  = 1e2
length_dam = 1e1
length_m   = 1.0
length_dm  = 1e-1
length_cm  = 1e-2
length_mm  = 1e-3
imperial_length_ft = 0.3048                  # foot
imperial_length_in = imperial_length_ft/12.0 # inch
imperial_length_yd = imperial_length_ft*3.0  # yard
imperial_length_mi = imperial_length_ft*5.28 # mile

# MASS
mass_g   = 1e-3
mass_kg  = 1.0
mass_ton = 1e3
imperial_mass_lb  = 0.45359237               # pound
imperial_mass_oz  = imperial_mass_lb/16.0    # ounce
imperial_mass_ton = imperial_mass_lb*2240.0 # imperial ton

# TIME
time_h   = 3600.0
time_min = 60.0
time_s   = 1.0
time_ms  = 1e-3

### LOCAL UNITS
# user variables to set to describe a set of local units

local_time = time_s
local_length = length_m
local_mass = mass_kg

### conversion methods

# BASE

# s
def time_from_SI( t, time_unit=None ):
    time_unit = time_unit or local_time
    return t / time_unit

def time_to_SI( t, time_unit=None ):
    time_unit = time_unit or local_time
    return t * time_unit

# m
def length_from_SI( l, length_unit=None ):
    length_unit = length_unit or local_length
    return l / length_unit

def length_to_SI( l, length_unit=None ):
    length_unit = length_unit or local_length
    return l * length_unit

# kg
def mass_from_SI( m, mass_unit=None ):
    mass_unit = mass_unit or local_mass
    return m / mass_unit

def mass_to_SI( m, mass_unit=None ):
    mass_unit = mass_unit or local_mass
    return m * mass_unit

# DERIVED

# m2
def area_from_SI( a, length_unit=None ):
    length_unit = length_unit or local_length
    return a / length_unit / length_unit

def area_to_SI( a, length_unit=None ):
    length_unit = length_unit or local_length
    return a * length_unit * length_unit

# m3
def volume_from_SI( v, length_unit=None ):
    length_unit = length_unit or local_length
    return v / length_unit / length_unit / length_unit

def volume_to_SI( v, length_unit=None ):
    length_unit = length_unit or local_length
    return v * length_unit * length_unit * length_unit

# m/s
def velocity_from_SI( v, length_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    time_unit = time_unit or local_time
    return v / length_unit * time_unit

def velocity_to_SI( v, length_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    time_unit = time_unit or local_time
    return v * length_unit / time_unit

# m/s2
def acceleration_from_SI( v, length_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    time_unit = time_unit or local_time
    return v / length_unit * time_unit * time_unit

def acceleration_to_SI( v, length_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    time_unit = time_unit or local_time
    return v * length_unit / time_unit / time_unit

# N = kg.m/s2
def force_from_SI( f, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return f / length_unit / mass_unit * time_unit * time_unit

def force_to_SI( f, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return f * length_unit * mass_unit / time_unit / time_unit

# C = N.m = kg.m2/s2
def torque_from_SI( c, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return c / length_unit / length_unit / mass_unit * time_unit * time_unit

def torque_to_SI( c, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return c * length_unit * length_unit * mass_unit / time_unit / time_unit

# Pa = N/m2 = kg/(m.s2)
def pressure_from_SI( p, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return p / mass_unit * length_unit * time_unit * time_unit

def pressure_to_SI( p, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return p * mass_unit / length_unit / time_unit / time_unit

# J = N.m = kg.m2/s2
def energy_from_SI( e, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return e / length_unit / length_unit / mass_unit * time_unit * time_unit

def energy_to_SI( e, length_unit=None, mass_unit=None, time_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    time_unit = time_unit or local_time
    return e * length_unit * length_unit * mass_unit / time_unit / time_unit

# I = kg.m2
def inertia_from_SI( i, length_unit=None, mass_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    return i / length_unit / length_unit / mass_unit

def inertia_to_SI( i, length_unit=None, mass_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    return i * length_unit * length_unit * mass_unit

# MATERIAL

# Pa
def elasticity_from_SI( e, length_unit=None, mass_unit=None, time_unit=None ):
    ## works for Young/bulk/shear modulus and Lame parameters
    return pressure_from_SI( e, length_unit, mass_unit, time_unit )

def elasticity_to_SI( e, length_unit=None, mass_unit=None, time_unit=None ):
    ## works for Young/bulk/shear modulus and Lame parameters
    return pressure_to_SI( e, length_unit, mass_unit, time_unit )

# kg/m3
def density_from_SI( d, length_unit=None, mass_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    return d * length_unit * length_unit * length_unit / mass_unit

def density_to_SI( d, length_unit=None, mass_unit=None ):
    length_unit = length_unit or local_length
    mass_unit = mass_unit or local_mass
    return d / length_unit / length_unit / length_unit * mass_unit

# w/o
def poissonRatio_from_SI( pr ):
    return pr

def poissonRatio_to_SI( pr ):
    return pr

# MECHANICS

# Pa
def stress_from_SI( s, length_unit=None, mass_unit=None, time_unit=None ):
    return pressure_from_SI( s, length_unit, mass_unit, time_unit )

def stress_to_SI( s, length_unit=None, mass_unit=None, time_unit=None ):
    return pressure_to_SI( s, length_unit, mass_unit, time_unit )

# w/o
def strain_from_SI( s ):
    return s

def strain_to_SI( s ):
    return s

# m
def deformation_from_SI( d, length_unit=None ):
    return length_from_SI( d, length_unit )

def deformation_to_SI( d, length_unit=None ):
    return length_to_SI( d, length_unit )
<|endoftext|>


<commit_before><commit_msg>Oracleaddress objects mismatch<commit_after><|endoftext|>


<commit_before>#! /usr/bin/env python

import optparse

import numpy as np
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.collections

import vdj
import vdj.analysis

option_parser = optparse.OptionParser()
option_parser.add_option('-t','--times')
option_parser.add_option('-q','--quantify',choices=['clone','junction','v','j'])
option_parser.add_option('-r','--threshold',type='float')
(options,args) = option_parser.parse_args()

if len(args) == 1:
    inhandle = open(args[0],'r')
else:
    raise ValueError, "Must give a single argument to vdjxml file"

# determine mapping between barcodes and times
timedict = {}
ip = open(options.times,'r')
for line in ip:
    timedict[line.split()[0]] = float(line.split()[1])
ip.close()

features = [options.quantify,'barcode']
(uniq_feature_values,countdict) = vdj.analysis.vdjxml2countdict(inhandle,features)
times = np.array([timedict[bc] for bc in uniq_feature_values['barcode']])
timesort = np.argsort(times)
times = times[timesort]
countmatrix = vdj.analysis.countdict2matrix(features,uniq_feature_values,countdict)
countmatrix = countmatrix[:,timesort]  # sort in increasing times
freqmatrix = np.float_(countmatrix) / countmatrix.sum(axis=0)

# define which idxs to plot
if options.threshold:
    idxs = np.sum(freqmatrix>=options.threshold,axis=1)>0 # breaks threshold at least once
else:
    idxs = [True]*freqmatrix.shape[0]
# idxs = np.sum(time_series_freqs>0,axis=1)>2 # seen at least twice
# idxs_bool = np.logical_and(idxs_bool_1,idxs_bool_2)
# idxs_bool = np.array([False]*len(reference_clones))
print "Number of lines plotted: %i" % np.sum(idxs)

# make the plots
outbasename = '.'.join(args[0].split('.')[:-1])

random_color = lambda: '#%02x%02x%02x' % tuple(np.random.randint(0,256,3))

segments = [zip(times,freqs) for freqs in freqmatrix[idxs]]
colors = [random_color() for i in xrange(len(segments))]
lines = mpl.collections.LineCollection(segments,colors=colors,linewidths=0.5)
lines.set_alpha(0.75)

fig = plt.figure()
ax = fig.add_subplot(111)
ax.add_collection(lines)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['bottom'].set_position(('outward',5))
ax.spines['left'].set_position(('outward',5))
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_major_locator(mpl.ticker.FixedLocator(times))
ax.set_xlim([times.min(),times.max()])
ax.autoscale_view(scalex=False,scaley=True)
# ax.set_yscale('log')
ax.set_xlabel('time')
ax.set_ylabel(options.quantify+' frequency')
# fig.show()
fig.savefig(outbasename+'.%stimeseries.png' % options.quantify)
fig.savefig(outbasename+'.%stimeseries.pdf' % options.quantify)

# # only stuff that's positive everywhere
segments = [np.asarray(zip(times,freqs)) for freqs in freqmatrix[idxs]]
segments = [segment[segment[:,1]>0] for segment in segments if segment[:,1].sum()>0]
lines = mpl.collections.LineCollection(segments,colors=colors,linewidths=0.5)
lines.set_alpha(0.75)

figlog = plt.figure()
ax = figlog.add_subplot(111)
ax.add_collection(lines)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['bottom'].set_position(('outward',5))
ax.spines['left'].set_position(('outward',5))
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_major_locator(mpl.ticker.FixedLocator(times))
ax.set_yscale('log')
ax.set_xlim([times.min(),times.max()])
ax.set_xlabel('time')
ax.set_ylabel(options.quantify+' frequency')
# fig.show()
figlog.savefig(outbasename+'.%stimeseries.log.png' % options.quantify)
figlog.savefig(outbasename+'.%stimeseries.log.pdf' % options.quantify)
<commit_msg>Changed vdjxml2timeseries to only output time series data file.  Plotting handled separately<commit_after>#! /usr/bin/env python

import optparse
import sys

import numpy as np
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.collections

import vdj
import vdj.analysis

option_parser = optparse.OptionParser()
option_parser.add_option('-t','--times')
option_parser.add_option('-q','--quantify',choices=['clone','junction','v','j','vj','vdj'])
(options,args) = option_parser.parse_args()

if len(args) == 2:
    inhandle = open(args[0],'r')
    outhandle = open(args[1],'w')
elif len(args) == 1:
    inhandle = open(args[0],'r')
    outhandle = sys.stdout
else:
    raise ValueError, "Must give a single argument to vdjxml file"

# determine mapping between barcodes and times
timedict = {}
ip = open(options.times,'r')
for line in ip:
    timedict[line.split()[0]] = float(line.split()[1])
ip.close()

# compute counts
features = [options.quantify,'barcode']
(uniq_feature_values,countdict) = vdj.analysis.vdjxml2countdict(inhandle,features)
times = np.array([timedict[bc] for bc in uniq_feature_values['barcode']])
timesort = np.argsort(times)
times = times[timesort]
countmatrix = vdj.analysis.countdict2matrix(features,uniq_feature_values,countdict)
countmatrix = countmatrix[:,timesort]  # sort in increasing times

# write time series data to file
print >>outhandle, '#times ' + ' '.join(map(str,times))
for (feature_value,timeseries) in zip(uniq_feature_values[options.quantify],countmatrix):
    print >>outhandle, ' '.join([feature_value]+timeseries)
<|endoftext|>