<commit_before># Copyright 2011 James McCauley
#
# This file is part of POX.
#
# POX is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# POX is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with POX.  If not, see <http://www.gnu.org/licenses/>.

"""
Various utility functions
"""

import traceback
import struct
import sys
import os

#FIXME: ugh, why can't I make importing pox.core work here?
import logging
log = logging.getLogger("util")

class DirtyList (list):
  #TODO: right now the callback may be called more often than needed
  #      and it may not be called with good names/parameters.
  #      All you can really rely on is that it will be called in
  #      some way if something may have changed.
  def __init__ (self, *args, **kw):
    list.__init__(self, *args, **kw)
    self.dirty = False
    self.callback = None

  def __setslice__ (self, k, v):
    #TODO: actually check for change
    self._smudge('__setslice__', k, v)
    list.__setslice__(self, k, v)

  def __delslice__ (self, k):
    #TODO: actually check for change
    self._smudge('__delslice__', k, None)
    list.__delslice__(self, k)

  def append (self, v):
    self._smudge('append', None, v)
    list.append(self, v)

  def extend (self, v):
    self._smudge('extend', None, v)
    list.extend(self, v)

  def insert (self, i, v):
    self._smudge('insert', k, v)
    list.extend(self, v)

  def pop (self, i=-1):
    self._smudge('pop', i, None)
    list.pop(self, i)

  def remove (self, v):
    if v in self:
      self._smudge('remove', None, v)
    list.remove(self, v)

  def reverse (self):
    if len(self):
      self._smudge('reverse', None, None)
    list.reverse(self)

  def sort (self, *arg, **kw):
    #TODO: check for changes?
    self._smudge('sort', None, None)
    list.sort(self, *arg, **kw)

  def __setitem__ (self, k, v):
    if isinstance(k, slice):
      #TODO: actually check for change
      self._smudge('__setitem__slice',k,v)
    elif self[k] != v:
      self._smudge('__setitem__',k,v)
    list.__setitem__(self, k, v)
    assert good

  def __delitem__ (self, k):
    list.__delitem__(self, k)
    if isinstance(k, slice):
      #TODO: actually check for change
      self._smudge('__delitem__slice',k,v)
    else:
      self._smudge('__delitem__', k, None)

  def _smudge (self, reason, k, v):
    if self.callback:
      if self.callback(reason, k, v) is not True:
        self.dirty = True
    else:
      self.dirty = True

class DirtyDict (dict):
  """
  A dict that tracks whether values have been changed shallowly.
  If you set a callback, it will be called when the value changes, and
  passed three values: "add"/"modify"/"delete", key, value
  """
  def __init__ (self, *args, **kw):
    dict.__init__(self, *args, **kw)
    self.dirty = False
    self.callback = None

  def _smudge (self, reason, k, v):
    if self.callback:
      if self.callback(reason, k, v) is not True:
        self.dirty = True
    else:
      self.dirty = True

  def __setitem__ (self, k, v):
    if k not in self:
      self._smudge('__setitem__add',k,v)
    elif self[k] != v:
      self._smudge('__setitem__modify',k,v)
    dict.__setitem__(self, k, v)

  def __delitem__ (self, k):
    self._smudge('__delitem__', k, None)
    dict.__delitem__(self, k)

def set_extend (l, index, item, emptyValue = None):
  """
  Adds item to the list l at position index.  If index is beyond the end
  of the list, it will pad the list out until it's large enough, using
  emptyValue for the new entries.
  """
  if index >= len(l):
    l += ([emptyValue] * (index - len(self) + 1))
  l[index] = item

def strToDPID (s):
  """
  Convert a DPID in the canonical string form into a long int.
  """
  s = s.replace("-", "").split("|", 2)
  a = int(s[0], 16)
  b = 0
  if len(s) == 2:
    b = int(s[1])
  return a | (b << 48)

def dpidToStr (dpid, alwaysLong = False):
  """
  Convert a DPID from a long into into the canonical string form.
  """
  """ In flux. """
  if type(dpid) is long or type(dpid) is int:
    # Not sure if this is right
    dpid = struct.pack('!Q', dpid)

  assert len(dpid) == 8

  r = '-'.join(['%02x' % (ord(x),) for x in dpid[2:]])

  if alwaysLong or dpid[0:2] != (b'\x00'*2):
    r += '|' + str(struct.unpack('!H', dpid[0:2])[0])

  return r

def assert_type(name, obj, types, none_ok=True):
  """
  Assert that a parameter is of a given type.
  Raise an Assertion Error with a descriptive error msg if not.

  name: name of the parameter for error messages
  obj: parameter value to be checked
  types: type or list or tuple of types that is acceptable
  none_ok: whether 'None' is an ok value
  """
  if obj is None:
    if none_ok:
      return True
    else:
      raise AssertionError("%s may not be None" % name)

  if not isinstance(types, (tuple, list)):
    types = [ types ]

  for cls in types:
    if isinstance(obj, cls):
      return True
  allowed_types = "|".join(map(lambda x: str(x), types))
  stack = traceback.extract_stack()
  stack_msg = "Function call %s() in %s:%d" % (stack[-2][2], stack[-3][0], stack[-3][1])
  type_msg = "%s must be instance of %s (but is %s)" % (name, allowed_types , str(type(obj)))

  raise AssertionError(stack_msg + ": " + type_msg)

def initHelper (obj, kw):
  """
  Inside a class's __init__, this will copy keyword arguments to fields
  of the same name.  See libopenflow for an example.
  """
  for k,v in kw.iteritems():
    if not hasattr(obj, k):
      raise TypeError(obj.__class__.__name__ + " constructor got "
      + "unexpected keyword argument '" + k + "'")
    setattr(obj, k, v)

def makePinger ():
  """
  A pinger is basically a thing to let you wake a select().
  On Unix systems, this makes a pipe pair.  But on Windows, select() only
  works with sockets, so it makes a pair of connected sockets.
  """

  class PipePinger (object):
    def __init__ (self, pair):
      self._w = pair[1]
      self._r = pair[0]
      assert os is not None

    def ping (self):
      if os is None: return #TODO: Is there a better fix for this?
      os.write(self._w, ' ')

    def fileno (self):
      return self._r

    def pongAll (self):
      #TODO: make this actually read all
      os.read(self._r, 1024)

    def pong (self):
      os.read(self._r, 1)

    def __del__ (self):
      try:
        os.close(self._w)
      except:
        pass
      try:
        os.close(self._r)
      except:
        pass

  class SocketPinger (object):
    def __init__ (self, pair):
      self._w = pair[1]
      self._r = pair[0]
    def ping (self):
      self._w.send(' ')
    def pong (self):
      self._r.recv(1)
    def pongAll (self):
      #TODO: make this actually read all
      self._r.recv(1024)
    def fileno (self):
      return self._r.fileno()

  #return PipePinger((os.pipe()[0],os.pipe()[1]))  # To test failure case

  if os.name == "posix":
    return PipePinger(os.pipe())

  #TODO: clean up sockets?
  localaddress = '127.127.127.127'
  startPort = 10000

  import socket
  import select

  def tryConnect ():
    l = socket.socket()
    l.setblocking(0)

    port = startPort
    while True:
      try:
        l.bind( (localaddress, port) )
        break
      except:
        port += 1
        if port - startPort > 1000:
          raise RuntimeError("Could not find a free socket")
    l.listen(0)

    r = socket.socket()

    try:
      r.connect((localaddress, port))
    except:
      import traceback
      ei = sys.exc_info()
      ei = traceback.format_exception_only(ei[0], ei[1])
      ei = ''.join(ei).strip()
      log.warning("makePinger: connect exception:\n" + ei)
      return False

    rlist, wlist,elist = select.select([l], [], [l], 2)
    if len(elist):
      log.warning("makePinger: socket error in select()")
      return False
    if len(rlist) == 0:
      log.warning("makePinger: socket didn't connect")
      return False

    try:
      w, addr = l.accept()
    except:
      return False

    #w.setblocking(0)
    if addr != r.getsockname():
      log.info("makePinger: pair didn't connect to each other!")
      return False

    r.setblocking(1)

    # Turn off Nagle
    r.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
    w.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)

    return (r, w)

  # Try a few times
  for i in range(0, 3):
    result = tryConnect()
    if result is not False:
      return SocketPinger(result)

  raise RuntimeError("Could not allocate a local socket pair")

def str_to_bool (s):
  """
  Given a string, parses out whether it is meant to be True or not
  """
  s = str(s).lower() # Make sure
  if s in ['true', 't', 'yes', 'y', 'on', 'enable', 'enabled', 'ok',
           'okay', '1', 'allow', 'allowed']:
    return True
  try:
    r = 10
    if s.startswith("0x"):
      s = s[2:]
      r = 16
    i = int(s, r)
    if i != 0:
      return True
  except:
    pass
  return False

def hexdump (data):
  if isinstance(data, str):
    data = [ord(c) for c in data]
  o = ""
  def chunks (data, length):
    return (data[i:i+length] for i in xrange(0, len(data), length))
  def filt (c):
    if c >= 32 and c <= 126: return chr(c)
    return '.'

  for i,chunk in enumerate(chunks(data,16)):
    if i > 0: o += "\n"
    o += "%04x: " % (i * 16,)
    l = ' '.join("%02x" % (c,) for  c in chunk)
    l = "%-48s" % (l,)
    l = l[:3*8-1] + "  " + l[3*8:]
    t = ''.join([filt(x) for x in chunk])
    l += '  %-16s' % (t,)
    o += l
  return o

if __name__ == "__main__":
  def cb (t,k,v): print v
  l = DirtyList([10,20,30,40,50])
  l.callback = cb

  l.append(3)

  print l

<commit_msg>lib.util: adding in connect_socket_with_backoff()<commit_after># Copyright 2011 James McCauley
#
# This file is part of POX.
#
# POX is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# POX is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with POX.  If not, see <http://www.gnu.org/licenses/>.

"""
Various utility functions
"""

import traceback
import struct
import sys
import os
import time
import socket

#FIXME: ugh, why can't I make importing pox.core work here?
import logging
log = logging.getLogger("util")

class DirtyList (list):
  #TODO: right now the callback may be called more often than needed
  #      and it may not be called with good names/parameters.
  #      All you can really rely on is that it will be called in
  #      some way if something may have changed.
  def __init__ (self, *args, **kw):
    list.__init__(self, *args, **kw)
    self.dirty = False
    self.callback = None

  def __setslice__ (self, k, v):
    #TODO: actually check for change
    self._smudge('__setslice__', k, v)
    list.__setslice__(self, k, v)

  def __delslice__ (self, k):
    #TODO: actually check for change
    self._smudge('__delslice__', k, None)
    list.__delslice__(self, k)

  def append (self, v):
    self._smudge('append', None, v)
    list.append(self, v)

  def extend (self, v):
    self._smudge('extend', None, v)
    list.extend(self, v)

  def insert (self, i, v):
    self._smudge('insert', k, v)
    list.extend(self, v)

  def pop (self, i=-1):
    self._smudge('pop', i, None)
    list.pop(self, i)

  def remove (self, v):
    if v in self:
      self._smudge('remove', None, v)
    list.remove(self, v)

  def reverse (self):
    if len(self):
      self._smudge('reverse', None, None)
    list.reverse(self)

  def sort (self, *arg, **kw):
    #TODO: check for changes?
    self._smudge('sort', None, None)
    list.sort(self, *arg, **kw)

  def __setitem__ (self, k, v):
    if isinstance(k, slice):
      #TODO: actually check for change
      self._smudge('__setitem__slice',k,v)
    elif self[k] != v:
      self._smudge('__setitem__',k,v)
    list.__setitem__(self, k, v)
    assert good

  def __delitem__ (self, k):
    list.__delitem__(self, k)
    if isinstance(k, slice):
      #TODO: actually check for change
      self._smudge('__delitem__slice',k,v)
    else:
      self._smudge('__delitem__', k, None)

  def _smudge (self, reason, k, v):
    if self.callback:
      if self.callback(reason, k, v) is not True:
        self.dirty = True
    else:
      self.dirty = True

class DirtyDict (dict):
  """
  A dict that tracks whether values have been changed shallowly.
  If you set a callback, it will be called when the value changes, and
  passed three values: "add"/"modify"/"delete", key, value
  """
  def __init__ (self, *args, **kw):
    dict.__init__(self, *args, **kw)
    self.dirty = False
    self.callback = None

  def _smudge (self, reason, k, v):
    if self.callback:
      if self.callback(reason, k, v) is not True:
        self.dirty = True
    else:
      self.dirty = True

  def __setitem__ (self, k, v):
    if k not in self:
      self._smudge('__setitem__add',k,v)
    elif self[k] != v:
      self._smudge('__setitem__modify',k,v)
    dict.__setitem__(self, k, v)

  def __delitem__ (self, k):
    self._smudge('__delitem__', k, None)
    dict.__delitem__(self, k)

def set_extend (l, index, item, emptyValue = None):
  """
  Adds item to the list l at position index.  If index is beyond the end
  of the list, it will pad the list out until it's large enough, using
  emptyValue for the new entries.
  """
  if index >= len(l):
    l += ([emptyValue] * (index - len(self) + 1))
  l[index] = item

def strToDPID (s):
  """
  Convert a DPID in the canonical string form into a long int.
  """
  s = s.replace("-", "").split("|", 2)
  a = int(s[0], 16)
  b = 0
  if len(s) == 2:
    b = int(s[1])
  return a | (b << 48)

def dpidToStr (dpid, alwaysLong = False):
  """
  Convert a DPID from a long into into the canonical string form.
  """
  """ In flux. """
  if type(dpid) is long or type(dpid) is int:
    # Not sure if this is right
    dpid = struct.pack('!Q', dpid)

  assert len(dpid) == 8

  r = '-'.join(['%02x' % (ord(x),) for x in dpid[2:]])

  if alwaysLong or dpid[0:2] != (b'\x00'*2):
    r += '|' + str(struct.unpack('!H', dpid[0:2])[0])

  return r

def assert_type(name, obj, types, none_ok=True):
  """
  Assert that a parameter is of a given type.
  Raise an Assertion Error with a descriptive error msg if not.

  name: name of the parameter for error messages
  obj: parameter value to be checked
  types: type or list or tuple of types that is acceptable
  none_ok: whether 'None' is an ok value
  """
  if obj is None:
    if none_ok:
      return True
    else:
      raise AssertionError("%s may not be None" % name)

  if not isinstance(types, (tuple, list)):
    types = [ types ]

  for cls in types:
    if isinstance(obj, cls):
      return True
  allowed_types = "|".join(map(lambda x: str(x), types))
  stack = traceback.extract_stack()
  stack_msg = "Function call %s() in %s:%d" % (stack[-2][2], stack[-3][0], stack[-3][1])
  type_msg = "%s must be instance of %s (but is %s)" % (name, allowed_types , str(type(obj)))

  raise AssertionError(stack_msg + ": " + type_msg)

def initHelper (obj, kw):
  """
  Inside a class's __init__, this will copy keyword arguments to fields
  of the same name.  See libopenflow for an example.
  """
  for k,v in kw.iteritems():
    if not hasattr(obj, k):
      raise TypeError(obj.__class__.__name__ + " constructor got "
      + "unexpected keyword argument '" + k + "'")
    setattr(obj, k, v)

def makePinger ():
  """
  A pinger is basically a thing to let you wake a select().
  On Unix systems, this makes a pipe pair.  But on Windows, select() only
  works with sockets, so it makes a pair of connected sockets.
  """

  class PipePinger (object):
    def __init__ (self, pair):
      self._w = pair[1]
      self._r = pair[0]
      assert os is not None

    def ping (self):
      if os is None: return #TODO: Is there a better fix for this?
      os.write(self._w, ' ')

    def fileno (self):
      return self._r

    def pongAll (self):
      #TODO: make this actually read all
      os.read(self._r, 1024)

    def pong (self):
      os.read(self._r, 1)

    def __del__ (self):
      try:
        os.close(self._w)
      except:
        pass
      try:
        os.close(self._r)
      except:
        pass

  class SocketPinger (object):
    def __init__ (self, pair):
      self._w = pair[1]
      self._r = pair[0]
    def ping (self):
      self._w.send(' ')
    def pong (self):
      self._r.recv(1)
    def pongAll (self):
      #TODO: make this actually read all
      self._r.recv(1024)
    def fileno (self):
      return self._r.fileno()

  #return PipePinger((os.pipe()[0],os.pipe()[1]))  # To test failure case

  if os.name == "posix":
    return PipePinger(os.pipe())

  #TODO: clean up sockets?
  localaddress = '127.127.127.127'
  startPort = 10000

  import socket
  import select

  def tryConnect ():
    l = socket.socket()
    l.setblocking(0)

    port = startPort
    while True:
      try:
        l.bind( (localaddress, port) )
        break
      except:
        port += 1
        if port - startPort > 1000:
          raise RuntimeError("Could not find a free socket")
    l.listen(0)

    r = socket.socket()

    try:
      r.connect((localaddress, port))
    except:
      import traceback
      ei = sys.exc_info()
      ei = traceback.format_exception_only(ei[0], ei[1])
      ei = ''.join(ei).strip()
      log.warning("makePinger: connect exception:\n" + ei)
      return False

    rlist, wlist,elist = select.select([l], [], [l], 2)
    if len(elist):
      log.warning("makePinger: socket error in select()")
      return False
    if len(rlist) == 0:
      log.warning("makePinger: socket didn't connect")
      return False

    try:
      w, addr = l.accept()
    except:
      return False

    #w.setblocking(0)
    if addr != r.getsockname():
      log.info("makePinger: pair didn't connect to each other!")
      return False

    r.setblocking(1)

    # Turn off Nagle
    r.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
    w.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)

    return (r, w)

  # Try a few times
  for i in range(0, 3):
    result = tryConnect()
    if result is not False:
      return SocketPinger(result)

  raise RuntimeError("Could not allocate a local socket pair")

def str_to_bool (s):
  """
  Given a string, parses out whether it is meant to be True or not
  """
  s = str(s).lower() # Make sure
  if s in ['true', 't', 'yes', 'y', 'on', 'enable', 'enabled', 'ok',
           'okay', '1', 'allow', 'allowed']:
    return True
  try:
    r = 10
    if s.startswith("0x"):
      s = s[2:]
      r = 16
    i = int(s, r)
    if i != 0:
      return True
  except:
    pass
  return False

def hexdump (data):
  if isinstance(data, str):
    data = [ord(c) for c in data]
  o = ""
  def chunks (data, length):
    return (data[i:i+length] for i in xrange(0, len(data), length))
  def filt (c):
    if c >= 32 and c <= 126: return chr(c)
    return '.'

  for i,chunk in enumerate(chunks(data,16)):
    if i > 0: o += "\n"
    o += "%04x: " % (i * 16,)
    l = ' '.join("%02x" % (c,) for  c in chunk)
    l = "%-48s" % (l,)
    l = l[:3*8-1] + "  " + l[3*8:]
    t = ''.join([filt(x) for x in chunk])
    l += '  %-16s' % (t,)
    o += l
  return o

def connect_socket_with_backoff(address, port, max_backoff=32):
  '''
  Connect to the given address and port. If the connection attempt fails,
  exponentially back off, up to the max backoff

  return the connected socket, or raise an exception if the connection was unsuccessful
  '''
  backoff_seconds = 1
  socket = None
  while True:
    try:
      socket = socket.socket()
      socket.connect( (address, port) )
      break
    except socket.error as e:
      print >>sys.stderr, "%s. Backing off %d seconds ..." % (str(e), backoff_seconds)
      if backoff_seconds >= 64:
        raise RuntimeError("Could not connect to controller %s:%d" % (address, port))
      else:
        time.sleep(backoff_seconds)
      backoff_seconds <<= 1
  return socket

if __name__ == "__main__":
  def cb (t,k,v): print v
  l = DirtyList([10,20,30,40,50])
  l.callback = cb

  l.append(3)

  print l

<|endoftext|>


<commit_before>import pytz

import logging
import requests
from dateutil.parser import parse
from datetime import datetime, timedelta

from django.db.models import Q
from website.app import init_app
from scripts.analytics.base import SummaryAnalytics

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

LOG_THRESHOLD = 11

class PreprintSummary(SummaryAnalytics):

    @property
    def collection_name(self):
        return 'preprint_summary'

    def get_events(self, date):
        super(PreprintSummary, self).get_events(date)
        from osf.models import PreprintService, PreprintProvider

        # Convert to a datetime at midnight for queries and the timestamp
        timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
        query_datetime = timestamp_datetime + timedelta(1)

        elastic_query = {
            "query": {
                "bool": {
                    "must": {
                        "match": {
                            "type": "preprints"
                        },
                        "match": {
                            "sources": None
                        }
                    },
                    'filter': [
                        {
                            "range": {
                                "date": {
                                    "lte": "{}||/d".format(query_datetime.strftime('%Y-%m-%d'))
                                }
                            }
                        }
                    ]
                }
            }
        }

        counts = []
        for preprint_provider in PreprintProvider.objects.all():
            name = preprint_provider.name if  preprint_provider.name != 'Open Science Framework' else 'OSF'
            elastic_query['query']['bool']['must']['match']['sources'] = name
            resp = requests.post('https://share.osf.io/api/v2/search/creativeworks/_search', json=elastic_query).json()
            counts.append({
                'keen': {
                    'timestamp': timestamp_datetime.isoformat()
                },
                'provider': {
                    'name': preprint_provider.name,
                    'total': resp['hits']['total'],
                },
            })

        return counts

def get_class():
    return PreprintSummary

if __name__ == '__main__':
    init_app()

    preprint_summary = PreprintSummary()
    args = preprint_summary.parse_args()
    yesterday = args.yesterday
    if yesterday:
        date = (datetime.today() - timedelta(1)).date()
    else:
        date = parse(args.date).date() if args.date else None
    events = preprint_summary.get_events(date)
    preprint_summary.send_events(events)
<commit_msg>Filter out unnamed preprint provider and fix double space typo<commit_after>import pytz

import logging
import requests
from dateutil.parser import parse
from datetime import datetime, timedelta

from django.db.models import Q
from website.app import init_app
from scripts.analytics.base import SummaryAnalytics

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

LOG_THRESHOLD = 11

class PreprintSummary(SummaryAnalytics):

    @property
    def collection_name(self):
        return 'preprint_summary'

    def get_events(self, date):
        super(PreprintSummary, self).get_events(date)
        from osf.models import PreprintService, PreprintProvider

        # Convert to a datetime at midnight for queries and the timestamp
        timestamp_datetime = datetime(date.year, date.month, date.day).replace(tzinfo=pytz.UTC)
        query_datetime = timestamp_datetime + timedelta(1)

        elastic_query = {
            "query": {
                "bool": {
                    "must": {
                        "match": {
                            "type": "preprints"
                        },
                        "match": {
                            "sources": None
                        }
                    },
                    'filter': [
                        {
                            "range": {
                                "date": {
                                    "lte": "{}||/d".format(query_datetime.strftime('%Y-%m-%d'))
                                }
                            }
                        }
                    ]
                }
            }
        }

        counts = []
        for preprint_provider in PreprintProvider.objects.all():
            name = preprint_provider.name if preprint_provider.name != 'Open Science Framework' else 'OSF'
            if name == '???':
                continue
            elastic_query['query']['bool']['must']['match']['sources'] = name
            resp = requests.post('https://share.osf.io/api/v2/search/creativeworks/_search', json=elastic_query).json()
            counts.append({
                'keen': {
                    'timestamp': timestamp_datetime.isoformat()
                },
                'provider': {
                    'name': preprint_provider.name,
                    'total': resp['hits']['total'],
                },
            })

        return counts

def get_class():
    return PreprintSummary

if __name__ == '__main__':
    init_app()

    preprint_summary = PreprintSummary()
    args = preprint_summary.parse_args()
    yesterday = args.yesterday
    if yesterday:
        date = (datetime.today() - timedelta(1)).date()
    else:
        date = parse(args.date).date() if args.date else None
    events = preprint_summary.get_events(date)
    preprint_summary.send_events(events)
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2020 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Wiki40B: A clean Wikipedia dataset for 40+ languages."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import logging
import tensorflow.compat.v2 as tf
import tensorflow_datasets.public_api as tfds

_CITATION = """
"""

_DESCRIPTION = """
Clean-up text for 40+ Wikipedia languages editions of pages
correspond to entities. The datasets have train/dev/test splits per language.
The dataset is cleaned up by page filtering to remove disambiguation pages,
redirect pages, deleted pages, and non-entity pages. Each example contains the
wikidata id of the entity, and the full Wikipedia article after page processing
that removes non-content sections and structured objects.
"""

_LICENSE = """
This work is licensed under the Creative Commons Attribution-ShareAlike
3.0 Unported License. To view a copy of this license, visit
http://creativecommons.org/licenses/by-sa/3.0/ or send a letter to
Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
"""

_URL = "https://research.google/pubs/pub49029/"

_DATA_DIRECTORY = "/bigstore/tfds-data/downloads/wiki40b/tfrecord_prod"

WIKIPEDIA_LANGUAGES = [
    "en", "ar", "zh-cn", "zh-tw", "nl", "fr", "de", "it", "ja", "ko", "pl",
    "pt", "ru", "es", "th", "tr", "bg", "ca", "cs", "da", "el", "et", "fa",
    "fi", "he", "hi", "hr", "hu", "id", "lt", "lv", "ms", "no", "ro", "sk",
    "sl", "sr", "sv", "tl", "uk", "vi"]

class Wiki40bConfig(tfds.core.BuilderConfig):
  """BuilderConfig for Wiki40B."""

  @tfds.core.disallow_positional_args
  def __init__(self, language=None, **kwargs):
    """BuilderConfig for Wiki40B.

    Args:
      language: string, the language code for the Wiki40B dataset to use.
      **kwargs: keyword arguments forwarded to super.
    """
    super(Wiki40bConfig, self).__init__(
        name="Wiki40B.{0}".format(language),
        description="Wiki40B dataset for {0}.".format(language),
        **kwargs)
    self.language = language

_VERSION = tfds.core.Version("1.0.0")

class Wiki40b(tfds.core.BeamBasedBuilder):
  """Wiki40B: A Clean Wikipedia Dataset for Mutlilingual Language Modeling."""

  BUILDER_CONFIGS = [
      Wiki40bConfig(  # pylint:disable=g-complex-comprehension
          version=_VERSION,
          language=lang,
      ) for lang in WIKIPEDIA_LANGUAGES
  ]

  def _info(self):
    return tfds.core.DatasetInfo(
        builder=self,
        description=_DESCRIPTION,
        features=tfds.features.FeaturesDict({
            "wikidata_id":
                tfds.features.Text(),
            "text":
                tfds.features.Text(),
        }),
        supervised_keys=None,
        homepage=_URL,
        citation=_CITATION,
        redistribution_info={"license": _LICENSE},
    )

  def _split_generators(self, dl_manager):
    """Returns SplitGenerators."""

    del dl_manager  # Unused

    lang = self._builder_config.language

    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            gen_kwargs={
                "filepaths": "%s/train/%s_examples-*" % (_DATA_DIRECTORY,
                                                         lang)},
        ),
        tfds.core.SplitGenerator(
            name=tfds.Split.VALIDATION,
            gen_kwargs={"filepaths": "%s/dev/%s_examples-*" % (_DATA_DIRECTORY,
                                                               lang)},
        ),
        tfds.core.SplitGenerator(
            name=tfds.Split.TEST,
            gen_kwargs={"filepaths": "%s/test/%s_examples-*" % (_DATA_DIRECTORY,
                                                                lang)},
        ),
    ]

  def _build_pcollection(self, pipeline, filepaths):
    """Build PCollection of examples."""
    beam = tfds.core.lazy_imports.apache_beam
    logging.info("generating examples from = %s", filepaths)

    def _extract_content(example):
      """Extracts content from a TFExample."""
      wikidata_id = example.features.feature[
          "wikidata_id"].bytes_list.value[0].decode("utf-8")
      text = example.features.feature[
          "text"].bytes_list.value[0].decode("utf-8")

      # wikidata_id could be duplicated with different texts.
      yield wikidata_id + text, {"wikidata_id": wikidata_id, "text": text,}

    return (
        pipeline
        | beam.io.ReadFromTFRecord(
            filepaths, coder=beam.coders.ProtoCoder(tf.train.Example))
        | beam.FlatMap(_extract_content))
<commit_msg>Internal<commit_after># coding=utf-8
# Copyright 2020 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Wiki40B: A clean Wikipedia dataset for 40+ languages."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import logging
import tensorflow.compat.v2 as tf
import tensorflow_datasets.public_api as tfds

_CITATION = """
"""

_DESCRIPTION = """
Clean-up text for 40+ Wikipedia languages editions of pages
correspond to entities. The datasets have train/dev/test splits per language.
The dataset is cleaned up by page filtering to remove disambiguation pages,
redirect pages, deleted pages, and non-entity pages. Each example contains the
wikidata id of the entity, and the full Wikipedia article after page processing
that removes non-content sections and structured objects.
"""

_LICENSE = """
This work is licensed under the Creative Commons Attribution-ShareAlike
3.0 Unported License. To view a copy of this license, visit
http://creativecommons.org/licenses/by-sa/3.0/ or send a letter to
Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.
"""

_URL = "https://research.google/pubs/pub49029/"

_DATA_DIRECTORY = "/bigstore/tfds-data/downloads/wiki40b/tfrecord_prod"

WIKIPEDIA_LANGUAGES = [
    "en", "ar", "zh-cn", "zh-tw", "nl", "fr", "de", "it", "ja", "ko", "pl",
    "pt", "ru", "es", "th", "tr", "bg", "ca", "cs", "da", "el", "et", "fa",
    "fi", "he", "hi", "hr", "hu", "id", "lt", "lv", "ms", "no", "ro", "sk",
    "sl", "sr", "sv", "tl", "uk", "vi"]

class Wiki40bConfig(tfds.core.BuilderConfig):
  """BuilderConfig for Wiki40B."""

  @tfds.core.disallow_positional_args
  def __init__(self, language=None, **kwargs):
    """BuilderConfig for Wiki40B.

    Args:
      language: string, the language code for the Wiki40B dataset to use.
      **kwargs: keyword arguments forwarded to super.
    """
    super(Wiki40bConfig, self).__init__(
        name="Wiki40B.{0}".format(language),
        description="Wiki40B dataset for {0}.".format(language),
        **kwargs)
    self.language = language

_VERSION = tfds.core.Version("1.1.0")

class Wiki40b(tfds.core.BeamBasedBuilder):
  """Wiki40B: A Clean Wikipedia Dataset for Mutlilingual Language Modeling."""

  BUILDER_CONFIGS = [
      Wiki40bConfig(  # pylint:disable=g-complex-comprehension
          version=_VERSION,
          language=lang,
      ) for lang in WIKIPEDIA_LANGUAGES
  ]

  def _info(self):
    return tfds.core.DatasetInfo(
        builder=self,
        description=_DESCRIPTION,
        features=tfds.features.FeaturesDict({
            "wikidata_id":
                tfds.features.Text(),
            "text":
                tfds.features.Text(),
            "version_id":
                tfds.features.Text(),
        }),
        supervised_keys=None,
        homepage=_URL,
        citation=_CITATION,
        redistribution_info={"license": _LICENSE},
    )

  def _split_generators(self, dl_manager):
    """Returns SplitGenerators."""

    del dl_manager  # Unused

    lang = self._builder_config.language

    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            gen_kwargs={
                "filepaths": "%s/train/%s_examples-*" % (_DATA_DIRECTORY,
                                                         lang)},
        ),
        tfds.core.SplitGenerator(
            name=tfds.Split.VALIDATION,
            gen_kwargs={"filepaths": "%s/dev/%s_examples-*" % (_DATA_DIRECTORY,
                                                               lang)},
        ),
        tfds.core.SplitGenerator(
            name=tfds.Split.TEST,
            gen_kwargs={"filepaths": "%s/test/%s_examples-*" % (_DATA_DIRECTORY,
                                                                lang)},
        ),
    ]

  def _build_pcollection(self, pipeline, filepaths):
    """Build PCollection of examples."""
    beam = tfds.core.lazy_imports.apache_beam
    logging.info("generating examples from = %s", filepaths)

    def _extract_content(example):
      """Extracts content from a TFExample."""
      wikidata_id = example.features.feature[
          "wikidata_id"].bytes_list.value[0].decode("utf-8")
      text = example.features.feature[
          "text"].bytes_list.value[0].decode("utf-8")
      version_id = example.features.feature[
          "version_id"].bytes_list.value[0].decode("utf-8")

      # wikidata_id could be duplicated with different texts.
      yield wikidata_id + text, {"wikidata_id": wikidata_id,
                                 "text": text,
                                 "version_id": version_id,}

    return (
        pipeline
        | beam.io.ReadFromTFRecord(
            filepaths, coder=beam.coders.ProtoCoder(tf.train.Example))
        | beam.FlatMap(_extract_content))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2010-2013 Associazione OpenERP Italia
#    (<http://www.openerp-italia.org>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    'name': 'Italian Localisation - Account Didotech',
    'version': '2.12.23.22',
    'category': 'Localisation/Italy',
    'description': """This module customizes OpenERP in order to fit italian laws and mores - Account version

Functionalities:

- Fiscal code computation for partner
- Check invoice date consistency
- CIG on invoice

""",
    'author': 'OpenERP Italian Community, Didotech srl',
    'website': 'http://www.openerp-italia.org, http://www.didotech.com',
    'license': 'AGPL-3',
    "depends": [
        'account',
        'base_vat',
        'account_chart',
        'base_iban',
        'l10n_it_base',
        'account_voucher',
        'sale_order_confirm',
        #'account_invoice_entry_date', not possible for use of a field defined here invoice_supplier_number
    ],
    "data": [
        'account/partner_view.xml',
        'account/fiscal_position_view.xml',
        'account/account_sequence.xml',
        'account/invoice_view.xml',
        'account/voucher_view.xml',
        'account/payment_type_view.xml',
        'wizard/select_fiscal_position_view.xml',
        'data/bank_iban_data.xml',
        'account/account_move.xml',
        'account/res_bank_view.xml',
        'account/account_tax_view.xml',
        'account/res_company_view.xml',
    ],
    "demo": [],
    "active": False,
    "installable": True
}
<commit_msg>[FIX] tranlation<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2010-2013 Associazione OpenERP Italia
#    (<http://www.openerp-italia.org>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

{
    'name': 'Italian Localisation - Account Didotech',
    'version': '2.13.23.24',
    'category': 'Localisation/Italy',
    'description': """This module customizes OpenERP in order to fit italian laws and mores - Account version

Functionalities:

- Fiscal code computation for partner
- Check invoice date consistency
- CIG on invoice

""",
    'author': 'OpenERP Italian Community, Didotech srl',
    'website': 'http://www.openerp-italia.org, http://www.didotech.com',
    'license': 'AGPL-3',
    "depends": [
        'account',
        'base_vat',
        'account_chart',
        'base_iban',
        'l10n_it_base',
        'account_voucher',
        'sale_order_confirm',
        #'account_invoice_entry_date', not possible for use of a field defined here invoice_supplier_number
    ],
    "data": [
        'account/partner_view.xml',
        'account/fiscal_position_view.xml',
        'account/account_sequence.xml',
        'account/invoice_view.xml',
        'account/voucher_view.xml',
        'account/payment_type_view.xml',
        'wizard/select_fiscal_position_view.xml',
        'data/bank_iban_data.xml',
        'account/account_move.xml',
        'account/res_bank_view.xml',
        'account/account_tax_view.xml',
        'account/res_company_view.xml',
    ],
    "demo": [],
    "active": False,
    "installable": True
}
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand, CommandError
from django.utils.timezone import now
from info_internet_connection.models import Internet
import datetime
import huawei_b593_status
import json
import redis

#{'WIFI': 'off', 'SIG': '5', 'Mode': '4g', 'Roam': 'home', 'SIM': 'normal', 'Connect': 'connected'}

class Command(BaseCommand):
    args = ''
    help = 'Fetches home 4g router status information'

    def handle(self, *args, **options):
        r = redis.StrictRedis()
        status = huawei_b593_status.HuaweiStatus()
        data = status.read()
        age_threshold = datetime.timedelta(minutes=2)

        try:
            latest_data = Internet.objects.latest()
            if now() - latest_data.timestamp > age_threshold:
                latest_data = Internet()
        except Internet.DoesNotExist:
            latest_data = Internet()

        latest_data.wifi = data["WIFI"] != "off"
        latest_data.signal = int(data["SIG"])
        latest_data.mode = data["Mode"]
        latest_data.sim = data["SIM"]
        latest_data.connect_status = data["Connect"]
        latest_data.update_timestamp = now()
        latest_data.save()
        r.publish("home:broadcast:generic", json.dumps({"key": "internet", "content": "updated"}) # TODO: send all information
<commit_msg>Add missing )<commit_after>from django.core.management.base import BaseCommand, CommandError
from django.utils.timezone import now
from info_internet_connection.models import Internet
import datetime
import huawei_b593_status
import json
import redis

#{'WIFI': 'off', 'SIG': '5', 'Mode': '4g', 'Roam': 'home', 'SIM': 'normal', 'Connect': 'connected'}

class Command(BaseCommand):
    args = ''
    help = 'Fetches home 4g router status information'

    def handle(self, *args, **options):
        r = redis.StrictRedis()
        status = huawei_b593_status.HuaweiStatus()
        data = status.read()
        age_threshold = datetime.timedelta(minutes=2)

        try:
            latest_data = Internet.objects.latest()
            if now() - latest_data.timestamp > age_threshold:
                latest_data = Internet()
        except Internet.DoesNotExist:
            latest_data = Internet()

        latest_data.wifi = data["WIFI"] != "off"
        latest_data.signal = int(data["SIG"])
        latest_data.mode = data["Mode"]
        latest_data.sim = data["SIM"]
        latest_data.connect_status = data["Connect"]
        latest_data.update_timestamp = now()
        latest_data.save()
        r.publish("home:broadcast:generic", json.dumps({"key": "internet", "content": "updated"})) # TODO: send all information
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2018-2020 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import numpy as np
import scipy.stats
from collections import deque
from .callback import Callback
from tqdm import tqdm
from scipy.signal import convolve

from typing import TYPE_CHECKING

AUTOLR_MIN = 0.000001
AUTOLR_MAX = 10
AUTOLR_EPOCHS = 8
AUTOLR_BETA = 0.98

MOMENTUM_MAX = 0.95
MOMENTUM_MIN = 0.85

if False:
    from .trainer import Trainer

def decreasing_probability(values: np.ndarray) -> float:
    """Compute probability that a sequence is decreasing

    Parameters
    ----------
    values : np.ndarray
        Sequence of values

    Returns
    -------
    probability : float
        Probability that sequence of values is decreasing

    Reference
    ---------
    Davis King. "Automatic Learning Rate Scheduling That Really Works".
    http://blog.dlib.net/2018/02/automatic-learning-rate-scheduling-that.html
    """
    n_steps = len(values)
    steps = np.arange(n_steps)

    A = np.vstack([steps, np.ones(n_steps)]).T
    loc, shift = np.linalg.lstsq(A, values, rcond=None)[0]

    values_ = loc * steps + shift
    sigma2 = np.sum((values - values_) ** 2) / (n_steps - 2)

    scale = np.sqrt(12 * sigma2 / (n_steps ** 3 - n_steps))
    return scipy.stats.norm.cdf(0.0, loc=loc, scale=scale)

def steps_without_decrease(values: np.ndarray, robust: bool = False) -> int:
    """Count number of steps without decrease

    Parameters
    ----------
    values : np.ndarray
        Sequence of values
    robust : bool
        Remove 10% highest values before counting steps.

    Returns
    -------
    n_steps : int
        Number of steps

    Reference
    ---------
    Davis King. "Automatic Learning Rate Scheduling That Really Works".
    http://blog.dlib.net/2018/02/automatic-learning-rate-scheduling-that.html
    """

    if robust:
        values = values[values < np.percentile(values, 90)]

    steps_without_decrease = 0
    n_steps = len(values)
    for i in reversed(range(n_steps - 2)):
        p = decreasing_probability(values[i:])
        if p < 0.51:
            steps_without_decrease = n_steps - i
    return steps_without_decrease

class BaseSchedulerCallback(Callback):
    """Base scheduler with support for AutoLR

    Reference
    ---------
    Leslie N. Smith. "Cyclical Learning Rates for Training Neural Networks"
    IEEE Winter Conference on Applications of Computer Vision (WACV, 2017).
    """

    def on_train_start(self, trainer: "Trainer") -> None:
        self.optimizer_ = trainer.optimizer
        if trainer.base_learning_rate_ == "auto":
            trainer.base_learning_rate_ = self.auto_lr(trainer)
        self.learning_rate = trainer.base_learning_rate_

    def on_epoch_start(self, trainer: "Trainer") -> None:
        """Log learning rate to tensorboard"""

        trainer.tensorboard_.add_scalar(
            f"train/lr", self.learning_rate, global_step=trainer.epoch_
        )

    @property
    def learning_rate(self) -> float:
        return self._learning_rate

    @learning_rate.setter
    def learning_rate(self, learning_rate: float) -> None:
        for g in self.optimizer_.param_groups:
            g["lr"] = learning_rate
            g["momentum"] = MOMENTUM_MAX
        self._learning_rate = learning_rate

    @staticmethod
    def _choose_lr(lrs: np.ndarray, losses: np.ndarray) -> float:
        """Choose learning rate upper bound

        This is done by selecting the [0.1 x lr, lr] range that leads to the
        largest decrease in terms of loss value.

        Parameters
        ----------
        lrs : np.ndarray
            Sequence of learning rates
        losses : np.ndarray
            Corresponding sequence of loss values

        Returns
        -------
        max_lr : float
            Return learning rate upper bound
        """

        min_lr, max_lr = np.min(lrs), np.max(lrs)
        n_batches = len(lrs)

        # `factor` by which the learning rate is multiplied after every batch,
        # to get from `min_lr` to `max_lr` in `n_batches` step.
        factor = (max_lr / min_lr) ** (1 / n_batches)

        # `K` batches to increase the learning rate by one order of magnitude
        K = int(np.log(10) / np.log(factor))

        # loss improvement on each [0.1 x lr, lr] range
        improvement = losses[:-K] - losses[K:]

        # return lr such that [0.1 x lr, lr] leads to the best improvement
        return lrs[K + np.argmax(improvement)]

    def auto_lr(self, trainer: "Trainer") -> float:
        """Find optimal learning rate automatically

        Parameters
        ----------
        trainer : 'Trainer'

        Returns
        -------
        learning_rate : float
            Optimal learning rate

        Reference
        ---------
        Leslie N. Smith. "Cyclical Learning Rates for Training Neural Networks"
        IEEE Winter Conference on Applications of Computer Vision (WACV, 2017).

            There is a simple way to estimate reasonable minimum and maximum
            boundary values with one training run of the network for a few
            epochs. It is a "LR range test"; run your model for several epochs
            while letting the learning rate increase linearly between low and
            high LR values. This test is enormously valuable whenever you are
            facing a new architecture or dataset. Next, plot the accuracy
            versus learning rate. Note the learning rate value when the
            accuracy starts to increase and when the accuracy slows, becomes
            ragged, or starts to fall. These two learning rates are good
            choices for bounds; that is, set base_lr to the first value and
            set max_lr to the latter value. Alternatively, one can use the rule
            of thumb that the optimum learning rate is usually within a factor
            of two of the largest one that converges and set base_lr to 1/3 or
            1/4 of max_lr. [...] Whenever one is starting with a new
            architecture or dataset, a single LR range test provides both a
            good LR value and a good range. Then one should compare runs with a
            fixed LR versus CLR with this range. Whichever wins can be used
            with confidence for the rest of one’s experiments.
        """
        # save states to disk
        trainer.save_state()

        # initialize optimizer with a low learning rate
        self.learning_rate = AUTOLR_MIN

        # `factor` by which the learning rate is multiplied after every batch,
        # to get from `min_lr` to `max_lr` in `n_batches` step.
        n_batches: int = AUTOLR_EPOCHS * trainer.batches_per_epoch
        factor = (AUTOLR_MAX / AUTOLR_MIN) ** (1 / n_batches)

        # progress bar
        pbar = tqdm(
            desc="AutoLR",
            total=n_batches,
            leave=False,
            ncols=80,
            unit="batch",
            position=1,
        )

        loss_moving_avg = 0.0
        losses, losses_smoothened, lrs = [], [], []

        # loop on n_batches batches
        for i in range(n_batches):

            batch = trainer.get_new_batch()
            loss = trainer.batch_loss(batch)
            loss["loss"].backward()
            trainer.optimizer_.step()
            trainer.optimizer_.zero_grad()

            l = loss["loss"].item()
            if np.isnan(l):
                break

            losses.append(l)
            lrs.append(self.learning_rate)

            loss_moving_avg = (
                AUTOLR_BETA * loss_moving_avg + (1 - AUTOLR_BETA) * losses[-1]
            )
            losses_smoothened.append(loss_moving_avg / (1 - AUTOLR_BETA ** (i + 1)))

            # update progress bar
            pbar.update(1)
            pbar.set_postfix(
                ordered_dict={"loss": losses_smoothened[-1], "lr": self.learning_rate}
            )

            # increase learning rate
            self.learning_rate = factor * self.learning_rate

            # stop AutoLR early when loss starts to explode
            if i > 1 and losses_smoothened[-1] > 100 * np.nanmin(losses_smoothened):
                break

        # reload model using its initial state
        trainer.load_state()

        # choose learning rate based on loss = f(learning_rate) curve
        auto_lr = self._choose_lr(np.array(lrs), np.array(losses_smoothened))

        # log curve and auto_lr to tensorboard as an image
        try:
            # import matplotlib with headless backend
            import matplotlib

            matplotlib.use("Agg")
            import matplotlib.pyplot as plt

            # create AutoLR loss = f(learning_rate) curve
            fig, ax = plt.subplots()
            ax.semilogx(lrs, losses, ".", alpha=0.3, label="Raw loss")
            ax.semilogx(lrs, losses_smoothened, linewidth=2, label="Smoothened loss")
            ax.set_xlabel("Learning rate")
            ax.set_ylabel("Loss")
            ax.legend()

            # indicate selected learning rate by a vertical line
            ax.plot(
                [auto_lr, auto_lr],
                [np.nanmin(losses_smoothened), np.nanmax(losses_smoothened)],
                linewidth=3,
            )

            # zoom on meaningful part of the curve
            m = np.nanmin(losses_smoothened)
            M = 1.1 * losses_smoothened[10]
            ax.set_ylim(m, M)

            # indicate selected learning rate in the figure title
            ax.set_title(f"AutoLR = {auto_lr:g}")

            # send matplotlib figure to Tensorboard
            trainer.tensorboard_.add_figure(
                "train/auto_lr", fig, global_step=trainer.epoch_, close=True
            )

        except ImportError as e:
            msg = (
                f"Something went wrong when trying to send AutoLR figure "
                f"to Tensorboard. Did you install matplotlib?\n\n{e}\n\n"
            )
            print(msg)
            print(e)

        except Exception as e:
            msg = (
                f"Something went wrong when trying to send AutoLR figure "
                f"to Tensorboard. It is OK but you might want to have a "
                f"look at why this happened.\n\n{e}\n\n"
            )
            print(msg)

        return auto_lr

class ConstantScheduler(BaseSchedulerCallback):
    """Constant learning rate"""

    pass

class DavisKingScheduler(BaseSchedulerCallback):
    """Automatic Learning Rate Scheduling That Really Works

    http://blog.dlib.net/2018/02/automatic-learning-rate-scheduling-that.html

    Parameters
    ----------
    factor : float, optional
        Factor by which the learning rate will be reduced.
        new_lr = old_lr * factor. Defaults to 0.5
    patience : int, optional
        Number of epochs with no improvement after which learning rate will
        be reduced. Defaults to 10.
    """

    def __init__(self, factor: float = 0.5, patience: int = 10):
        super().__init__()
        self.factor = factor
        self.patience = patience

    def on_train_start(self, trainer: "Trainer") -> None:
        super().on_train_start(trainer)
        maxlen = 2 * self.patience * trainer.batches_per_epoch
        self.losses_ = deque([], maxlen=maxlen)

    def on_epoch_end(self, trainer: "Trainer") -> None:

        # compute statistics on batch loss trend
        count = steps_without_decrease(np.array(self.losses_))
        count_robust = steps_without_decrease(np.array(self.losses_), robust=True)

        # if batch loss hasn't been decreasing for a while
        patience = self.patience * trainer.batches_per_epoch
        if count > patience and count_robust > patience:
            self.learning_rate = self.factor * self.learning_rate
            self.losses_.clear()

    def on_batch_end(self, trainer, batch_loss):
        super().on_batch_end(trainer, batch_loss)

        # store current batch loss
        self.losses_.append(batch_loss["loss"].item())

class CyclicScheduler(BaseSchedulerCallback):
    """Cyclic learning rate (and momentum)

    Parameters
    ----------
    epochs_per_cycle : int, optional
        Number of epochs per cycle. Defaults to 10.
    decay : {float, 'auto'}, optional
        Update base learning rate at the end of each cycle:
            - when `float`, multiply base learning rate by this amount;
            - when 'auto', apply AutoLR;
            - defaults to doing nothing.

    Reference
    ---------
    Leslie N. Smith. "Cyclical Learning Rates for Training Neural Networks"
    IEEE Winter Conference on Applications of Computer Vision (WACV, 2017).
    """

    def __init__(self, epochs_per_cycle: int = 10, decay=None):
        super().__init__()
        self.epochs_per_cycle = epochs_per_cycle
        self.decay = decay

    @property
    def momentum(self) -> float:
        return self._momentum

    @momentum.setter
    def momentum(self, momentum: float) -> None:
        for g in self.optimizer_.param_groups:
            g["momentum"] = momentum
        self._momentum = momentum

    def on_train_start(self, trainer: "Trainer") -> None:
        """Initialize batch/epoch counters"""

        super().on_train_start(trainer)
        self.batches_per_cycle_ = self.epochs_per_cycle * trainer.batches_per_epoch
        self.n_batches_ = 0
        self.n_epochs_ = 0

        self.learning_rate = trainer.base_learning_rate_ * 0.1

    def on_epoch_end(self, trainer: "Trainer") -> None:
        """Update base learning rate at the end of cycle"""

        super().on_epoch_end(trainer)

        # reached end of cycle?
        self.n_epochs_ += 1
        if self.n_epochs_ % self.epochs_per_cycle == 0:

            # apply AutoLR
            if self.decay == "auto":
                trainer.base_learning_rate_ = self.auto_lr(trainer)

            # decay base learning rate
            elif self.decay is not None:
                trainer.base_learning_rate_ *= self.decay

            # reset epoch/batch counters
            self.n_epochs_ = 0
            self.n_batches_ = 0

    def on_batch_start(self, trainer: "Trainer", batch: dict) -> dict:
        """Update learning rate & momentum according to position in cycle"""

        super().on_batch_start(trainer, batch)

        # position within current cycle (reversed V)
        rho = 1.0 - abs(2 * (self.n_batches_ / self.batches_per_cycle_ - 0.5))

        self.learning_rate = trainer.base_learning_rate_ * (0.1 + 0.9 * rho)
        self.momentum = MOMENTUM_MAX - (MOMENTUM_MAX - MOMENTUM_MIN) * rho

        self.n_batches_ += 1

        return batch
<commit_msg>chore: return half of previous AutoLR result<commit_after>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2018-2020 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import numpy as np
import scipy.stats
from collections import deque
from .callback import Callback
from tqdm import tqdm
from scipy.signal import convolve

from typing import TYPE_CHECKING

AUTOLR_MIN = 0.000001
AUTOLR_MAX = 10
AUTOLR_EPOCHS = 8
AUTOLR_BETA = 0.98

MOMENTUM_MAX = 0.95
MOMENTUM_MIN = 0.85

if False:
    from .trainer import Trainer

def decreasing_probability(values: np.ndarray) -> float:
    """Compute probability that a sequence is decreasing

    Parameters
    ----------
    values : np.ndarray
        Sequence of values

    Returns
    -------
    probability : float
        Probability that sequence of values is decreasing

    Reference
    ---------
    Davis King. "Automatic Learning Rate Scheduling That Really Works".
    http://blog.dlib.net/2018/02/automatic-learning-rate-scheduling-that.html
    """
    n_steps = len(values)
    steps = np.arange(n_steps)

    A = np.vstack([steps, np.ones(n_steps)]).T
    loc, shift = np.linalg.lstsq(A, values, rcond=None)[0]

    values_ = loc * steps + shift
    sigma2 = np.sum((values - values_) ** 2) / (n_steps - 2)

    scale = np.sqrt(12 * sigma2 / (n_steps ** 3 - n_steps))
    return scipy.stats.norm.cdf(0.0, loc=loc, scale=scale)

def steps_without_decrease(values: np.ndarray, robust: bool = False) -> int:
    """Count number of steps without decrease

    Parameters
    ----------
    values : np.ndarray
        Sequence of values
    robust : bool
        Remove 10% highest values before counting steps.

    Returns
    -------
    n_steps : int
        Number of steps

    Reference
    ---------
    Davis King. "Automatic Learning Rate Scheduling That Really Works".
    http://blog.dlib.net/2018/02/automatic-learning-rate-scheduling-that.html
    """

    if robust:
        values = values[values < np.percentile(values, 90)]

    steps_without_decrease = 0
    n_steps = len(values)
    for i in reversed(range(n_steps - 2)):
        p = decreasing_probability(values[i:])
        if p < 0.51:
            steps_without_decrease = n_steps - i
    return steps_without_decrease

class BaseSchedulerCallback(Callback):
    """Base scheduler with support for AutoLR

    Reference
    ---------
    Leslie N. Smith. "Cyclical Learning Rates for Training Neural Networks"
    IEEE Winter Conference on Applications of Computer Vision (WACV, 2017).
    """

    def on_train_start(self, trainer: "Trainer") -> None:
        self.optimizer_ = trainer.optimizer
        if trainer.base_learning_rate_ == "auto":
            trainer.base_learning_rate_ = self.auto_lr(trainer)
        self.learning_rate = trainer.base_learning_rate_

    def on_epoch_start(self, trainer: "Trainer") -> None:
        """Log learning rate to tensorboard"""

        trainer.tensorboard_.add_scalar(
            f"train/lr", self.learning_rate, global_step=trainer.epoch_
        )

    @property
    def learning_rate(self) -> float:
        return self._learning_rate

    @learning_rate.setter
    def learning_rate(self, learning_rate: float) -> None:
        for g in self.optimizer_.param_groups:
            g["lr"] = learning_rate
            g["momentum"] = MOMENTUM_MAX
        self._learning_rate = learning_rate

    @staticmethod
    def _choose_lr(lrs: np.ndarray, losses: np.ndarray) -> float:
        """Choose learning rate upper bound

        This is done by selecting the [0.1 x lr, lr] range that leads to the
        largest decrease in terms of loss value.

        Parameters
        ----------
        lrs : np.ndarray
            Sequence of learning rates
        losses : np.ndarray
            Corresponding sequence of loss values

        Returns
        -------
        max_lr : float
            Return learning rate upper bound
        """

        min_lr, max_lr = np.min(lrs), np.max(lrs)
        n_batches = len(lrs)

        # `factor` by which the learning rate is multiplied after every batch,
        # to get from `min_lr` to `max_lr` in `n_batches` step.
        factor = (max_lr / min_lr) ** (1 / n_batches)

        # K batches to increase the learning rate by one order of magnitude
        K = int(np.log(10) / np.log(factor))

        # loss improvement on each [0.1 x lr, lr] range
        improvement = losses[:-K] - losses[K:]

        # return half LR such that [0.1 x lr, lr] leads to the best improvement
        return 0.5 * lrs[K + np.argmax(improvement)]

    def auto_lr(self, trainer: "Trainer") -> float:
        """Find optimal learning rate automatically

        Parameters
        ----------
        trainer : 'Trainer'

        Returns
        -------
        learning_rate : float
            Optimal learning rate

        Reference
        ---------
        Leslie N. Smith. "Cyclical Learning Rates for Training Neural Networks"
        IEEE Winter Conference on Applications of Computer Vision (WACV, 2017).

            There is a simple way to estimate reasonable minimum and maximum
            boundary values with one training run of the network for a few
            epochs. It is a "LR range test"; run your model for several epochs
            while letting the learning rate increase linearly between low and
            high LR values. This test is enormously valuable whenever you are
            facing a new architecture or dataset. Next, plot the accuracy
            versus learning rate. Note the learning rate value when the
            accuracy starts to increase and when the accuracy slows, becomes
            ragged, or starts to fall. These two learning rates are good
            choices for bounds; that is, set base_lr to the first value and
            set max_lr to the latter value. Alternatively, one can use the rule
            of thumb that the optimum learning rate is usually within a factor
            of two of the largest one that converges and set base_lr to 1/3 or
            1/4 of max_lr. [...] Whenever one is starting with a new
            architecture or dataset, a single LR range test provides both a
            good LR value and a good range. Then one should compare runs with a
            fixed LR versus CLR with this range. Whichever wins can be used
            with confidence for the rest of one’s experiments.
        """
        # save states to disk
        trainer.save_state()

        # initialize optimizer with a low learning rate
        self.learning_rate = AUTOLR_MIN

        # `factor` by which the learning rate is multiplied after every batch,
        # to get from `min_lr` to `max_lr` in `n_batches` step.
        n_batches: int = AUTOLR_EPOCHS * trainer.batches_per_epoch
        factor = (AUTOLR_MAX / AUTOLR_MIN) ** (1 / n_batches)

        # progress bar
        pbar = tqdm(
            desc="AutoLR",
            total=n_batches,
            leave=False,
            ncols=80,
            unit="batch",
            position=1,
        )

        loss_moving_avg = 0.0
        losses, losses_smoothened, lrs = [], [], []

        # loop on n_batches batches
        for i in range(n_batches):

            batch = trainer.get_new_batch()
            loss = trainer.batch_loss(batch)
            loss["loss"].backward()
            trainer.optimizer_.step()
            trainer.optimizer_.zero_grad()

            l = loss["loss"].item()
            if np.isnan(l):
                break

            losses.append(l)
            lrs.append(self.learning_rate)

            loss_moving_avg = (
                AUTOLR_BETA * loss_moving_avg + (1 - AUTOLR_BETA) * losses[-1]
            )
            losses_smoothened.append(loss_moving_avg / (1 - AUTOLR_BETA ** (i + 1)))

            # update progress bar
            pbar.update(1)
            pbar.set_postfix(
                ordered_dict={"loss": losses_smoothened[-1], "lr": self.learning_rate}
            )

            # increase learning rate
            self.learning_rate = factor * self.learning_rate

            # stop AutoLR early when loss starts to explode
            if i > 1 and losses_smoothened[-1] > 100 * np.nanmin(losses_smoothened):
                break

        # reload model using its initial state
        trainer.load_state()

        # choose learning rate based on loss = f(learning_rate) curve
        auto_lr = self._choose_lr(np.array(lrs), np.array(losses_smoothened))

        # log curve and auto_lr to tensorboard as an image
        try:
            # import matplotlib with headless backend
            import matplotlib

            matplotlib.use("Agg")
            import matplotlib.pyplot as plt

            # create AutoLR loss = f(learning_rate) curve
            fig, ax = plt.subplots()
            ax.semilogx(lrs, losses, ".", alpha=0.3, label="Raw loss")
            ax.semilogx(lrs, losses_smoothened, linewidth=2, label="Smoothened loss")
            ax.set_xlabel("Learning rate")
            ax.set_ylabel("Loss")
            ax.legend()

            # indicate selected learning rate by a vertical line
            ax.plot(
                [auto_lr, auto_lr],
                [np.nanmin(losses_smoothened), np.nanmax(losses_smoothened)],
                linewidth=3,
            )

            # zoom on meaningful part of the curve
            m = np.nanmin(losses_smoothened)
            M = 1.1 * losses_smoothened[10]
            ax.set_ylim(m, M)

            # indicate selected learning rate in the figure title
            ax.set_title(f"AutoLR = {auto_lr:g}")

            # send matplotlib figure to Tensorboard
            trainer.tensorboard_.add_figure(
                "train/auto_lr", fig, global_step=trainer.epoch_, close=True
            )

        except ImportError as e:
            msg = (
                f"Something went wrong when trying to send AutoLR figure "
                f"to Tensorboard. Did you install matplotlib?\n\n{e}\n\n"
            )
            print(msg)
            print(e)

        except Exception as e:
            msg = (
                f"Something went wrong when trying to send AutoLR figure "
                f"to Tensorboard. It is OK but you might want to have a "
                f"look at why this happened.\n\n{e}\n\n"
            )
            print(msg)

        return auto_lr

class ConstantScheduler(BaseSchedulerCallback):
    """Constant learning rate"""

    pass

class DavisKingScheduler(BaseSchedulerCallback):
    """Automatic Learning Rate Scheduling That Really Works

    http://blog.dlib.net/2018/02/automatic-learning-rate-scheduling-that.html

    Parameters
    ----------
    factor : float, optional
        Factor by which the learning rate will be reduced.
        new_lr = old_lr * factor. Defaults to 0.5
    patience : int, optional
        Number of epochs with no improvement after which learning rate will
        be reduced. Defaults to 10.
    """

    def __init__(self, factor: float = 0.5, patience: int = 10):
        super().__init__()
        self.factor = factor
        self.patience = patience

    def on_train_start(self, trainer: "Trainer") -> None:
        super().on_train_start(trainer)
        maxlen = 2 * self.patience * trainer.batches_per_epoch
        self.losses_ = deque([], maxlen=maxlen)

    def on_epoch_end(self, trainer: "Trainer") -> None:

        # compute statistics on batch loss trend
        count = steps_without_decrease(np.array(self.losses_))
        count_robust = steps_without_decrease(np.array(self.losses_), robust=True)

        # if batch loss hasn't been decreasing for a while
        patience = self.patience * trainer.batches_per_epoch
        if count > patience and count_robust > patience:
            self.learning_rate = self.factor * self.learning_rate
            self.losses_.clear()

    def on_batch_end(self, trainer, batch_loss):
        super().on_batch_end(trainer, batch_loss)

        # store current batch loss
        self.losses_.append(batch_loss["loss"].item())

class CyclicScheduler(BaseSchedulerCallback):
    """Cyclic learning rate (and momentum)

    Parameters
    ----------
    epochs_per_cycle : int, optional
        Number of epochs per cycle. Defaults to 10.
    decay : {float, 'auto'}, optional
        Update base learning rate at the end of each cycle:
            - when `float`, multiply base learning rate by this amount;
            - when 'auto', apply AutoLR;
            - defaults to doing nothing.

    Reference
    ---------
    Leslie N. Smith. "Cyclical Learning Rates for Training Neural Networks"
    IEEE Winter Conference on Applications of Computer Vision (WACV, 2017).
    """

    def __init__(self, epochs_per_cycle: int = 10, decay=None):
        super().__init__()
        self.epochs_per_cycle = epochs_per_cycle
        self.decay = decay

    @property
    def momentum(self) -> float:
        return self._momentum

    @momentum.setter
    def momentum(self, momentum: float) -> None:
        for g in self.optimizer_.param_groups:
            g["momentum"] = momentum
        self._momentum = momentum

    def on_train_start(self, trainer: "Trainer") -> None:
        """Initialize batch/epoch counters"""

        super().on_train_start(trainer)
        self.batches_per_cycle_ = self.epochs_per_cycle * trainer.batches_per_epoch
        self.n_batches_ = 0
        self.n_epochs_ = 0

        self.learning_rate = trainer.base_learning_rate_ * 0.1

    def on_epoch_end(self, trainer: "Trainer") -> None:
        """Update base learning rate at the end of cycle"""

        super().on_epoch_end(trainer)

        # reached end of cycle?
        self.n_epochs_ += 1
        if self.n_epochs_ % self.epochs_per_cycle == 0:

            # apply AutoLR
            if self.decay == "auto":
                trainer.base_learning_rate_ = self.auto_lr(trainer)

            # decay base learning rate
            elif self.decay is not None:
                trainer.base_learning_rate_ *= self.decay

            # reset epoch/batch counters
            self.n_epochs_ = 0
            self.n_batches_ = 0

    def on_batch_start(self, trainer: "Trainer", batch: dict) -> dict:
        """Update learning rate & momentum according to position in cycle"""

        super().on_batch_start(trainer, batch)

        # position within current cycle (reversed V)
        rho = 1.0 - abs(2 * (self.n_batches_ / self.batches_per_cycle_ - 0.5))

        self.learning_rate = trainer.base_learning_rate_ * (0.1 + 0.9 * rho)
        self.momentum = MOMENTUM_MAX - (MOMENTUM_MAX - MOMENTUM_MIN) * rho

        self.n_batches_ += 1

        return batch
<|endoftext|>


<commit_before># Copyright 2014 The ALIVe authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from language import *

class BoolPred:
  def fixupTypes(self, types):
    for attr in dir(self):
      a = getattr(self, attr)
      if isinstance(a, (Type, Value, BoolPred)):
        a.fixupTypes(types)

################################
class TruePred(BoolPred):
  def __repr__(self):
    return 'true'

  def getTypeConstraints(self):
    return BoolVal(True)

  def toSMT(self, state):
    return BoolVal(True)

################################
class PredNot(BoolPred):
  def __init__(self, v):
    assert isinstance(v, BoolPred)
    self.v = v

  def __repr__(self):
    return '!%s' % self.v

  def getTypeConstraints(self):
    return self.v.getTypeConstraints()

  def toSMT(self, state):
    return Not(self.v.toSMT(state))

################################
class PredAnd(BoolPred):
  def __init__(self, v1, v2):
    assert isinstance(v1, BoolPred)
    assert isinstance(v2, BoolPred)
    self.v1 = v1
    self.v2 = v2

  def __repr__(self):
    return '(%s && %s)' % (self.v1, self.v2)

  def getTypeConstraints(self):
    return mk_and([self.v1.getTypeConstraints(),
                   self.v2.getTypeConstraints()])

  def toSMT(self, state):
    return And(self.v1.toSMT(state), self.v2.toSMT(state))

################################
class PredOr(BoolPred):
  def __init__(self, v1, v2):
    assert isinstance(v1, BoolPred)
    assert isinstance(v2, BoolPred)
    self.v1 = v1
    self.v2 = v2

  def __repr__(self):
    return '(%s || %s)' % (self.v1, self.v2)

  def getTypeConstraints(self):
    return mk_and([self.v1.getTypeConstraints(),
                   self.v2.getTypeConstraints()])

  def toSMT(self, state):
    return Or(self.v1.toSMT(state), self.v2.toSMT(state))

################################
class BinaryBoolPred(BoolPred):
  EQ, NE, Last = range(3)

  opnames = ['==', '!=']

  def __init__(self, op, v1, v2):
    assert 0 <= op < self.Last
    assert isinstance(v1, (Constant, Input))
    assert isinstance(v2, (Constant, Input))
    self.op = op
    self.v1 = v1
    self.v2 = v2

  def __repr__(self):
    return '(%s %s %s)' % (self.v1, self.opnames[self.op], self.v2)

  @staticmethod
  def getOpId(name):
    for i in range(BinaryBoolPred.Last):
      if BinaryBoolPred.opnames[i] == name:
        return i
    assert False

  def getTypeConstraints(self):
    return mk_and([self.v1.type == self.v2.type,
                   self.v1.getTypeConstraints(),
                   self.v2.getTypeConstraints()])

  def toSMT(self, state):
    v1 = self.v1.toSMT([], state, [])
    v2 = self.v2.toSMT([], state, [])
    return {
      self.EQ: lambda a,b: a == b,
      self.NE: lambda a,b: a != b,
    }[self.op](v1, v2)

################################
class LLVMBoolPred(BoolPred):
  isSignBit, NSWAdd, Last = range(3)

  opnames = {
    isSignBit: 'isSignBit',
    NSWAdd:    'WillNotOverflowSignedAdd',
  }
  opids = {v:k for k, v in opnames.items()}

  num_args = {
    isSignBit: 1,
    NSWAdd:    2,
  }

  def __init__(self, op, args):
    assert 0 <= op < self.Last
    for i in range(len(args)):
      assert self.argAccepts(op, i+1, args[i])[0]
    self.op = op
    self.args = args
    if self.num_args[op] != len(args):
      raise ParseError('Wrong number of arguments (got %d, expected %d)' %\
                       (len(args), self.num_args[op]))

  def __repr__(self):
    args = [str(a) for a in self.args]
    return '%s(%s)' % (self.opnames[self.op], ', '.join(args))

  def getOpName(self):
    return self.opnames[self.op]

  @staticmethod
  def getOpId(name):
    try:
      return LLVMBoolPred.opids[name]
    except:
      raise ParseError('Unknown boolean predicate')

  arg_types = {
    isSignBit: 'const',
    NSWAdd:    'input',
  }

  @staticmethod
  def argAccepts(op, arg, val):
    kind = LLVMBoolPred.arg_types[op]
    if kind == 'input':
      return (isinstance(val, (Input, Constant)), 'constant or input var')
    if kind == 'const':
      if isinstance(val, Input):
        ok = val.getName()[0] == 'C'
      else:
        ok = isinstance(val, Constant)
      return (ok, 'constant')
    assert False

  argConstraints = {
    isSignBit: lambda a: [a.type.typevar == Type.Int],
    NSWAdd:    lambda a,b: [a.type.typevar == Type.Int,
                            b.type.typevar == Type.Int],
  }

  def getTypeConstraints(self):
    c = self.argConstraints[self.op](*self.args)
    c += [v.getTypeConstraints() for v in self.args]

    # For now, assume all operands must have the same type
    for i in range(1, len(self.args)):
      c += [self.args[0].type == self.args[i].type]
    return mk_and(c)

  def toSMT(self, state):
    args = [v.toSMT([], state, []) for v in self.args]
    return {
      self.isSignBit: lambda a: a == (1 << (a.sort().size()-1)),
      self.NSWAdd:    lambda a,b: SignExt(1,a)+SignExt(1,b) == SignExt(1, a+b)
    }[self.op](*args)
<commit_msg>Add predicate MaskedValueIsZero<commit_after># Copyright 2014 The ALIVe authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from language import *

class BoolPred:
  def fixupTypes(self, types):
    for attr in dir(self):
      a = getattr(self, attr)
      if isinstance(a, (Type, Value, BoolPred)):
        a.fixupTypes(types)

################################
class TruePred(BoolPred):
  def __repr__(self):
    return 'true'

  def getTypeConstraints(self):
    return BoolVal(True)

  def toSMT(self, state):
    return BoolVal(True)

################################
class PredNot(BoolPred):
  def __init__(self, v):
    assert isinstance(v, BoolPred)
    self.v = v

  def __repr__(self):
    return '!%s' % self.v

  def getTypeConstraints(self):
    return self.v.getTypeConstraints()

  def toSMT(self, state):
    return Not(self.v.toSMT(state))

################################
class PredAnd(BoolPred):
  def __init__(self, v1, v2):
    assert isinstance(v1, BoolPred)
    assert isinstance(v2, BoolPred)
    self.v1 = v1
    self.v2 = v2

  def __repr__(self):
    return '(%s && %s)' % (self.v1, self.v2)

  def getTypeConstraints(self):
    return mk_and([self.v1.getTypeConstraints(),
                   self.v2.getTypeConstraints()])

  def toSMT(self, state):
    return And(self.v1.toSMT(state), self.v2.toSMT(state))

################################
class PredOr(BoolPred):
  def __init__(self, v1, v2):
    assert isinstance(v1, BoolPred)
    assert isinstance(v2, BoolPred)
    self.v1 = v1
    self.v2 = v2

  def __repr__(self):
    return '(%s || %s)' % (self.v1, self.v2)

  def getTypeConstraints(self):
    return mk_and([self.v1.getTypeConstraints(),
                   self.v2.getTypeConstraints()])

  def toSMT(self, state):
    return Or(self.v1.toSMT(state), self.v2.toSMT(state))

################################
class BinaryBoolPred(BoolPred):
  EQ, NE, Last = range(3)

  opnames = ['==', '!=']

  def __init__(self, op, v1, v2):
    assert 0 <= op < self.Last
    assert isinstance(v1, (Constant, Input))
    assert isinstance(v2, (Constant, Input))
    self.op = op
    self.v1 = v1
    self.v2 = v2

  def __repr__(self):
    return '(%s %s %s)' % (self.v1, self.opnames[self.op], self.v2)

  @staticmethod
  def getOpId(name):
    for i in range(BinaryBoolPred.Last):
      if BinaryBoolPred.opnames[i] == name:
        return i
    assert False

  def getTypeConstraints(self):
    return mk_and([self.v1.type == self.v2.type,
                   self.v1.getTypeConstraints(),
                   self.v2.getTypeConstraints()])

  def toSMT(self, state):
    v1 = self.v1.toSMT([], state, [])
    v2 = self.v2.toSMT([], state, [])
    return {
      self.EQ: lambda a,b: a == b,
      self.NE: lambda a,b: a != b,
    }[self.op](v1, v2)

################################
class LLVMBoolPred(BoolPred):
  isSignBit, NSWAdd, maskZero, Last = range(4)

  opnames = {
    isSignBit: 'isSignBit',
    NSWAdd:    'WillNotOverflowSignedAdd',
    maskZero:  'MaskedValueIsZero',
  }
  opids = {v:k for k, v in opnames.items()}

  num_args = {
    isSignBit: 1,
    NSWAdd:    2,
    maskZero:  2,
  }

  def __init__(self, op, args):
    assert 0 <= op < self.Last
    for i in range(len(args)):
      assert self.argAccepts(op, i+1, args[i])[0]
    self.op = op
    self.args = args
    if self.num_args[op] != len(args):
      raise ParseError('Wrong number of arguments (got %d, expected %d)' %\
                       (len(args), self.num_args[op]))

  def __repr__(self):
    args = [str(a) for a in self.args]
    return '%s(%s)' % (self.opnames[self.op], ', '.join(args))

  def getOpName(self):
    return self.opnames[self.op]

  @staticmethod
  def getOpId(name):
    try:
      return LLVMBoolPred.opids[name]
    except:
      raise ParseError('Unknown boolean predicate')

  arg_types = {
    isSignBit: 'const',
    NSWAdd:    'input',
    maskZero:  'input',  # FIXME: first is input, second is const
  }

  @staticmethod
  def argAccepts(op, arg, val):
    kind = LLVMBoolPred.arg_types[op]
    if kind == 'input':
      return (isinstance(val, (Input, Constant)), 'constant or input var')
    if kind == 'const':
      if isinstance(val, Input):
        ok = val.getName()[0] == 'C'
      else:
        ok = isinstance(val, Constant)
      return (ok, 'constant')
    assert False

  argConstraints = {
    isSignBit: lambda a: [a.type.typevar == Type.Int],
    NSWAdd:    lambda a,b: [a.type.typevar == Type.Int,
                            b.type.typevar == Type.Int],
    maskZero:  lambda a,b: []
  }

  def getTypeConstraints(self):
    c = self.argConstraints[self.op](*self.args)
    c += [v.getTypeConstraints() for v in self.args]

    # For now, assume all operands must have the same type
    for i in range(1, len(self.args)):
      c += [self.args[0].type == self.args[i].type]
    return mk_and(c)

  def toSMT(self, state):
    args = [v.toSMT([], state, []) for v in self.args]
    return {
      self.isSignBit: lambda a: a == (1 << (a.sort().size()-1)),
      self.NSWAdd:    lambda a,b: SignExt(1,a)+SignExt(1,b) == SignExt(1, a+b),
      self.maskZero:  lambda a,b: a & b == 0,
    }[self.op](*args)
<|endoftext|>