<commit_before>
"""
Medtronic - openaps driver for Medtronic
"""
from openaps.uses.use import Use
from openaps.uses.registry import Registry
from openaps.configurable import Configurable
import decocare
import argparse
import json
from decocare import stick, session, link, commands, history
from datetime import datetime
from dateutil import relativedelta
from dateutil.parser import parse

def configure_use_app (app, parser):
  pass
  # parser.add_argument('foobar', help="LOOK AT ME")

def configure_add_app (app, parser):
  parser.add_argument('serial')

def configure_app (app, parser):
  if app.parent.name == 'add':
    """
    print "CONFIG INNER", app, app.parent.name, app.name
    """
def configure_parser (parser):
  pass
def main (args, app):
  """
  print "MEDTRONIC", args, app
  print "app commands", app.selected.name
  """

use = Registry( )

@use( )
class scan (Use):
  """ scan for usb stick """
  def configure_app (self, app, parser):
    pass
    # print "hahaha"
  def scanner (self):
    from decocare.scan import scan
    return scan( )
  def main (self, args, app):
    return self.scanner( )

import logging
import logging.handlers
class MedtronicTask (scan):
  requires_session = True
  save_session = True
  record_stats = True

  def before_main (self, args, app):
    self.setup_medtronic( )
    if self.requires_session:
      self.check_session(app)
    else:
      self.pump.setModel(number=self.device.fields.get('model', ''))

  def after_main (self, args, app):
    if self.save_session:
      self.device.store(app.config)
      app.config.save( )

  def get_session_info (self):
    expires = self.device.fields.get('expires', None)
    now = datetime.now( )
    out = dict(device=self.device.name
      , vendor=__name__
      , used=now
      )
    if expires is None or parse(expires) < now:
      fields = self.create_session( )
      out.update(**self.update_session_info(fields))
    else:
      out['expires'] = parse(expires)
      out['model'] = self.get_model( )
    return out

  def update_session_info (self, fields):
    out = { }
    self.device.add_option('expires', fields['expires'].isoformat( ))
    self.device.add_option('model', fields['model'])
    out['expires'] = fields['expires']
    out['model'] = fields['model']
    return out

  def create_session (self):
    minutes = int(self.device.fields.get('minutes', 10))
    now = datetime.now( )
    self.pump.power_control(minutes=minutes)
    model = self.get_model( )
    offset = relativedelta.relativedelta(minutes=minutes)
    out = dict(device=self.device.name
      , model=model
      , vendor=__name__
      , created_at=now
      , started=now
      , expires=now + offset
      )
    return out
  def check_session (self, app):
    self.session = self.get_session_info( )
    self.device.add_option('model', self.device.fields.get('model', self.get_model( )))
  def get_model (self):
    model = self.pump.read_model( ).getData( )
    return model
  def setup_medtronic (self):
    log = logging.getLogger(decocare.__name__)
    log.setLevel(logging.INFO)
    log.addHandler(logging.handlers.SysLogHandler(address='/dev/log'))
    self.uart = stick.Stick(link.Link(self.scanner( )))
    self.uart.open( )
    serial = self.device.fields['serial']
    self.pump = session.Pump(self.uart, serial)
    stats = self.uart.interface_stats( )
  def main (self, args, app):
    return self.scanner( )

class Session (MedtronicTask):
  """ session for pump
  """
  requires_session = False
  def configure_parser (self, parser):
    parser.add_argument('--minutes', type=int, default='10')
  def setup_application (self):
    """
    """
  def main (self, args, app):
    info = self.create_session( )
    info.update(**self.update_session_info(info))
    return info

@use( )
class model (MedtronicTask):
  """ Get model number
  """
  def configure_app (self, app, parser):
    pass
  def main (self, args, app):
    model = self.pump.read_model( ).getData( )
    return model

@use( )
class read_status (MedtronicTask):
  """ Get pump status
  """
  def main (self, args, app):
    return self.pump.model.read_status( )

@use( )
class status (read_status):
  """ Get pump status (alias for read_status)
  """

@use( )
class reservoir (MedtronicTask):
  """ Get pump remaining insulin
  """
  def main (self, args, app):
    return self.pump.model.read_reservoir( )

@use( )
class settings (MedtronicTask):
  """ Get pump settings
  """
  def main (self, args, app):
    return self.pump.model.read_settings( )

@use( )
class mytest (MedtronicTask):
  """ Testing read_settings
  """
  requires_session = False
  def main (self, args, app):
    return self.pump.model.my_read_settings( )

@use( )
class read_clock (MedtronicTask):
  """ Read date/time of pump
  """
  def main (self, args, app):
    return self.pump.model.read_clock( )

class SameNameCommand (MedtronicTask):
  def main (self, args, app):
    name = self.__class__.__name__.split('.').pop( )
    return getattr(self.pump.model, name)(**self.get_params(args))

@use( )
class read_temp_basal (SameNameCommand):
  """ Read temporary basal rates. """

@use( )
class read_settings (SameNameCommand):
  """ Read settings. """

@use( )
class read_carb_ratios (SameNameCommand):
  """ Read carb_ratios. """

@use( )
class read_basal_profile_std (SameNameCommand):
  """ Read default basal profile. """

@use( )
class read_basal_profile_A (SameNameCommand):
  """ Read basal profile A. """

@use( )
class read_basal_profile_B (SameNameCommand):
  """ Read basal profile B. """

@use( )
class read_selected_basal_profile (SameNameCommand):
  """ Fetch the currently selected basal profile. """

@use( )
class read_current_glucose_pages (SameNameCommand):
  """ Read current glucose pages. """

@use( )
class read_current_history_pages (SameNameCommand):
  """ Read current history pages. """

@use( )
class suspend_pump (SameNameCommand):
  """ Suspend pumping. """
  def main (self, args, app):
    result = super(suspend_pump, self).main(args, app)
    result.update(enacted_at=datetime.now( ))
    return result

@use( )
class resume_pump (suspend_pump):
  """ resume pumping. """

@use( )
class read_battery_status (SameNameCommand):
  """ Check battery status. """

@use( )
class read_bg_targets (SameNameCommand):
  """ Read bg targets. """

@use( )
class read_insulin_sensitivies (SameNameCommand):
  """ Read insulin sensitivies. """

@use( )
class read_glucose_data (SameNameCommand):
  """ Read pump glucose page
  """
  def configure_app (self, app, parser):
    parser.add_argument('page', type=int, default=0)

  def get_params (self, args):
    return dict(page=int(args.page))

@use( )
class set_temp_basal (MedtronicTask):
  """ Set temporary basal rates.
  """
  def get_params (self, args):
    return dict(input=args.input)
  def configure_app (self, app, parser):
    parser.add_argument('input', default='-')
  def main (self, args, app):
    params = self.get_params(args)
    program = json.load(argparse.FileType('r')(params.get('input')))
    program.update(timestamp=datetime.now( ), **self.pump.model.set_temp_basal(**program))
    return program

@use( )
class bolus (MedtronicTask):
  """ Send bolus.
  """
  def get_params (self, args):
    return dict(input=args.input)
  def configure_app (self, app, parser):
    parser.add_argument('input', default='-')
  def main (self, args, app):
    params = self.get_params(args)
    program = json.load(argparse.FileType('r')(params.get('input')))
    program.update(timestamp=datetime.now( ), **self.pump.model.bolus(**program))
    return program

@use( )
class filter_glucose_date (SameNameCommand):
  """ Search for glucose pages including begin and end dates (iso 8601).
  """
  def get_params (self, args):
    return dict(begin=args.begin, end=args.end)
  def configure_app (self, app, parser):
    parser.add_argument('begin', default='')
    parser.add_argument('end', default='')

@use( )
class filter_isig_date (filter_glucose_date):
  """ Search for isig pages including begin and end dates (iso 8601).
  """

@use( )
class read_history_data (MedtronicTask):
  """ Read pump history page
  """
  def get_params (self, args):
    return dict(page=int(args.page))
  def configure_app (self, app, parser):
    parser.add_argument('page', type=int, default=0)

  def main (self, args, app):
    history = self.pump.model.read_history_data(**self.get_params(args))
    return history

@use( )
class iter_glucose (MedtronicTask):
  """ Read latest 100 glucose records
  """
  def get_params (self, args):
    return dict(count=int(args.count))
  def configure_app (self, app, parser):
    parser.add_argument('count', type=int, default=99)
  def range (self):
    return self.pump.model.iter_glucose_pages( )
  maxCount = 99
  def main (self, args, app):
    self.maxCount = self.get_params(args).get('count', self.maxCount)
    num = 0
    records = [ ]
    for rec in self.range( ):
      records.append(rec)
      num = num + 1
      if num > self.maxCount:
        break
    return records

@use( )
class iter_pump (iter_glucose):
  """ Read latest 100 pump records
  """
  def range (self):
    return self.pump.model.iter_history_pages( )

def set_config (args, device):
  device.add_option('serial', args.serial)

def display_device (device):
  return ''

known_uses = [
  Session,
]
def get_uses (device, config):
  all_uses = known_uses[:] + use.get_uses(device, config)
  all_uses.sort(key=lambda usage: getattr(usage, 'sortOrder', usage.__name__))
  return all_uses

<commit_msg>simplify "program uploading"<commit_after>
"""
Medtronic - openaps driver for Medtronic
"""
from openaps.uses.use import Use
from openaps.uses.registry import Registry
from openaps.configurable import Configurable
import decocare
import argparse
import json
from decocare import stick, session, link, commands, history
from datetime import datetime
from dateutil import relativedelta
from dateutil.parser import parse

def configure_use_app (app, parser):
  pass
  # parser.add_argument('foobar', help="LOOK AT ME")

def configure_add_app (app, parser):
  parser.add_argument('serial')

def configure_app (app, parser):
  if app.parent.name == 'add':
    """
    print "CONFIG INNER", app, app.parent.name, app.name
    """
def configure_parser (parser):
  pass
def main (args, app):
  """
  print "MEDTRONIC", args, app
  print "app commands", app.selected.name
  """

use = Registry( )

@use( )
class scan (Use):
  """ scan for usb stick """
  def configure_app (self, app, parser):
    pass
    # print "hahaha"
  def scanner (self):
    from decocare.scan import scan
    return scan( )
  def main (self, args, app):
    return self.scanner( )

import logging
import logging.handlers
class MedtronicTask (scan):
  requires_session = True
  save_session = True
  record_stats = True

  def before_main (self, args, app):
    self.setup_medtronic( )
    if self.requires_session:
      self.check_session(app)
    else:
      self.pump.setModel(number=self.device.fields.get('model', ''))

  def after_main (self, args, app):
    if self.save_session:
      self.device.store(app.config)
      app.config.save( )

  def get_session_info (self):
    expires = self.device.fields.get('expires', None)
    now = datetime.now( )
    out = dict(device=self.device.name
      , vendor=__name__
      , used=now
      )
    if expires is None or parse(expires) < now:
      fields = self.create_session( )
      out.update(**self.update_session_info(fields))
    else:
      out['expires'] = parse(expires)
      out['model'] = self.get_model( )
    return out

  def update_session_info (self, fields):
    out = { }
    self.device.add_option('expires', fields['expires'].isoformat( ))
    self.device.add_option('model', fields['model'])
    out['expires'] = fields['expires']
    out['model'] = fields['model']
    return out

  def create_session (self):
    minutes = int(self.device.fields.get('minutes', 10))
    now = datetime.now( )
    self.pump.power_control(minutes=minutes)
    model = self.get_model( )
    offset = relativedelta.relativedelta(minutes=minutes)
    out = dict(device=self.device.name
      , model=model
      , vendor=__name__
      , created_at=now
      , started=now
      , expires=now + offset
      )
    return out
  def check_session (self, app):
    self.session = self.get_session_info( )
    self.device.add_option('model', self.device.fields.get('model', self.get_model( )))
  def get_model (self):
    model = self.pump.read_model( ).getData( )
    return model
  def setup_medtronic (self):
    log = logging.getLogger(decocare.__name__)
    log.setLevel(logging.INFO)
    log.addHandler(logging.handlers.SysLogHandler(address='/dev/log'))
    self.uart = stick.Stick(link.Link(self.scanner( )))
    self.uart.open( )
    serial = self.device.fields['serial']
    self.pump = session.Pump(self.uart, serial)
    stats = self.uart.interface_stats( )
  def main (self, args, app):
    return self.scanner( )

class Session (MedtronicTask):
  """ session for pump
  """
  requires_session = False
  def configure_parser (self, parser):
    parser.add_argument('--minutes', type=int, default='10')
  def setup_application (self):
    """
    """
  def main (self, args, app):
    info = self.create_session( )
    info.update(**self.update_session_info(info))
    return info

@use( )
class model (MedtronicTask):
  """ Get model number
  """
  def configure_app (self, app, parser):
    pass
  def main (self, args, app):
    model = self.pump.read_model( ).getData( )
    return model

@use( )
class read_status (MedtronicTask):
  """ Get pump status
  """
  def main (self, args, app):
    return self.pump.model.read_status( )

@use( )
class status (read_status):
  """ Get pump status (alias for read_status)
  """

@use( )
class reservoir (MedtronicTask):
  """ Get pump remaining insulin
  """
  def main (self, args, app):
    return self.pump.model.read_reservoir( )

@use( )
class settings (MedtronicTask):
  """ Get pump settings
  """
  def main (self, args, app):
    return self.pump.model.read_settings( )

@use( )
class mytest (MedtronicTask):
  """ Testing read_settings
  """
  requires_session = False
  def main (self, args, app):
    return self.pump.model.my_read_settings( )

@use( )
class read_clock (MedtronicTask):
  """ Read date/time of pump
  """
  def main (self, args, app):
    return self.pump.model.read_clock( )

class SameNameCommand (MedtronicTask):
  def main (self, args, app):
    name = self.__class__.__name__.split('.').pop( )
    return getattr(self.pump.model, name)(**self.get_params(args))

@use( )
class read_temp_basal (SameNameCommand):
  """ Read temporary basal rates. """

@use( )
class read_settings (SameNameCommand):
  """ Read settings. """

@use( )
class read_carb_ratios (SameNameCommand):
  """ Read carb_ratios. """

@use( )
class read_basal_profile_std (SameNameCommand):
  """ Read default basal profile. """

@use( )
class read_basal_profile_A (SameNameCommand):
  """ Read basal profile A. """

@use( )
class read_basal_profile_B (SameNameCommand):
  """ Read basal profile B. """

@use( )
class read_selected_basal_profile (SameNameCommand):
  """ Fetch the currently selected basal profile. """

@use( )
class read_current_glucose_pages (SameNameCommand):
  """ Read current glucose pages. """

@use( )
class read_current_history_pages (SameNameCommand):
  """ Read current history pages. """

@use( )
class suspend_pump (SameNameCommand):
  """ Suspend pumping. """
  def main (self, args, app):
    result = super(suspend_pump, self).main(args, app)
    result.update(enacted_at=datetime.now( ))
    return result

@use( )
class resume_pump (suspend_pump):
  """ resume pumping. """

@use( )
class read_battery_status (SameNameCommand):
  """ Check battery status. """

@use( )
class read_bg_targets (SameNameCommand):
  """ Read bg targets. """

@use( )
class read_insulin_sensitivies (SameNameCommand):
  """ Read insulin sensitivies. """

@use( )
class read_glucose_data (SameNameCommand):
  """ Read pump glucose page
  """
  def configure_app (self, app, parser):
    parser.add_argument('page', type=int, default=0)

  def get_params (self, args):
    return dict(page=int(args.page))

class InputProgramRequired (MedtronicTask):
  def upload_program (self, program):
    raise NotImplementedError( )
  def get_params (self, args):
    return dict(input=args.input)
  def configure_app (self, app, parser):
    parser.add_argument('input', default='-')
  def get_program (self, args):
    params = self.get_params(args)
    program = json.load(argparse.FileType('r')(params.get('input')))
    return program
  def main (self, args, app):
    program = self.get_program(args)
    results = self.upload_program(program)
    program.update(timestamp=datetime.now( ), **results)
    return program

@use( )
class set_temp_basal (InputProgramRequired):
  """ Set temporary basal rates.
  """
  def upload_program (self, program):
    return self.pump.model.set_temp_basal(**program)

@use( )
class bolus (InputProgramRequired):
  """ Send bolus.
  """
  def upload_program (self, program):
    return self.pump.model.bolus(**program)

@use( )
class filter_glucose_date (SameNameCommand):
  """ Search for glucose pages including begin and end dates (iso 8601).
  """
  def get_params (self, args):
    return dict(begin=args.begin, end=args.end)
  def configure_app (self, app, parser):
    parser.add_argument('begin', default='')
    parser.add_argument('end', default='')

@use( )
class filter_isig_date (filter_glucose_date):
  """ Search for isig pages including begin and end dates (iso 8601).
  """

@use( )
class read_history_data (MedtronicTask):
  """ Read pump history page
  """
  def get_params (self, args):
    return dict(page=int(args.page))
  def configure_app (self, app, parser):
    parser.add_argument('page', type=int, default=0)

  def main (self, args, app):
    history = self.pump.model.read_history_data(**self.get_params(args))
    return history

@use( )
class iter_glucose (MedtronicTask):
  """ Read latest 100 glucose records
  """
  def get_params (self, args):
    return dict(count=int(args.count))
  def configure_app (self, app, parser):
    parser.add_argument('count', type=int, default=99)
  def range (self):
    return self.pump.model.iter_glucose_pages( )
  maxCount = 99
  def main (self, args, app):
    self.maxCount = self.get_params(args).get('count', self.maxCount)
    num = 0
    records = [ ]
    for rec in self.range( ):
      records.append(rec)
      num = num + 1
      if num > self.maxCount:
        break
    return records

@use( )
class iter_pump (iter_glucose):
  """ Read latest 100 pump records
  """
  def range (self):
    return self.pump.model.iter_history_pages( )

def set_config (args, device):
  device.add_option('serial', args.serial)

def display_device (device):
  return ''

known_uses = [
  Session,
]
def get_uses (device, config):
  all_uses = known_uses[:] + use.get_uses(device, config)
  all_uses.sort(key=lambda usage: getattr(usage, 'sortOrder', usage.__name__))
  return all_uses

<|endoftext|>


<commit_before>#-*- coding:utf-8 -*-
import logging
logging.basicConfig(level=logging.ERROR)
from cache_abstract import CacheAbstractDriver
try:
	import redis
except ImportError:
	logging.error("Can't import 'redis' module")
	exit(-1)
try:
	import aioredis
except ImportError:
	logging.error("Can't import 'aioredis' module")
	exit(-1)
class AsyncRedisCacheClient(object):
	def __init__(self,host,port,db,*args,*kwargs):
		assert isinstance(host,str)
		assert isinstance(port,int)
		assert isinstance(db,int) and 0<=self.__db<16
		self.__host=host
		self.__port=port
		self.__db=db
		self.__connection=None
		self.__key_type_map={
				"1":"string",
				"2":"hash",
				"3":"list"
		}
		self.__key_type_hash="_key_type"
	@asyncio.coroutine
	def get_connection(self,loop=None):
		self.__connection=yield from aioredis.create_redis((self.__host,self.__port),db=self.__db,loop=loop)
		return self.__connection
	@asyncio.coroutine
	def set(self,key,val=None,expires,key_prefix):
		pass

class RedisCacheClient(object):
	def __init__(self,host,port,db,*args,**kwargs):
		assert isinstance(host,str)
		assert isinstance(port,int)
		assert isinstance(db,int) and 0<=self.__db<16
		self.__host=host
		self.__port=port
		self.__db=db
		self._connection=redis.StrictRedis(self.__host,self.__port,self.__db)
		self.__key_type_map={
			"1":"string",
			"2":"hash",
			"3":"list"
		}
		self.__key_type_hash="_key_type"
	def set(self,key,value=None,expires,key_prefix):
		if not value and isinstance(key,dict):
			pipe=self._connection.pipeline()
			for k,item in key:
				pipe.hset(self.__key_type_hash,k,1)
				pipe.set(k,item)
				if expires>0:
					pipe.expire(k,expires)
			return pipe.execute()
		else:
			key=key_prefix+key
			if isinstance(value,str):
				pipe=self._connection.pipeline()
				pipe.hset(self.__key_type_hash,key,1)
				pipe.set(key,value)
				if expires>0:
					pipe.expire(key,expires)
				return pipe.execute()
			elif isinstance(value,dict):
				pipe=self._connection.pipeline()
				pipe.hset(self.__key_type_hash,key,2)
				pipe.hmset(key,value)
				if expires>0:
					pipe.expire(key,expires)
				return pipe.execute()
			elif isinstance(value,(list,tuple)):
				pipe=self._connection.pipeline()
				pipe.hset(self.__key_type_hash,key,3)
				pipe.lpush(key,*value)
				if expires>0:
					pipe.expire(key,expires)
				return pipe.execute()
	def get(self,key,key_prefix):
		key_type=self.exists(key,key_prefix)
		key=key_prefix+key
		if key_type=="string":
			return self._connection.get(key) or ""
		elif key_type=="hash":
			return self._connection.hgetall(key) or {}
		elif key_type=="list":
			return self._connection.lrange(key,0,-1) or []
		else:
			return None
	def delete(self,key,key_prefix):
		key_type=self.exists(key,key_prefix)
		result=""
		if key_type=="string":
			result=self._connection.delete(key_prefix+key)
		elif key_type=="hash":
			result=self._connection.hdel(key,*self._connection.hkeys(key_prefix+key))
		elif key_type=="list":
			length=self._connection.llen(key_prefix+key)
			if length:
				result=self._connection.ltrim(key_prefix+key,length+1,length+1)
		else:
			result=None
		self.delete_key(key,key_prefix)
		return result
	def inc(self,key,delta,key_prefix):
		key_type=self.exists(key,key_prefix)
		key=key_prefix+key
		if key_type=="string":
			return self._connection.incrby(key,delta)
		else:
			raise TypeError("can't  increment  '%s'"%key)
	def dec(self,key,delta,key_prefix):
		key_type=self.exists(key,key_prefix)
		key=key_prefix+key
		if key_type=="string":
			return self._connection.incrby(key,delta)
		else:
			raise TypeError("can't decrement '%s'"%key)
	def exists(self,key,key_prefix):
		key=key_prefix+key
		key_type=self._connection.hget(self.__key_type_hash,key)
		if key_type:
			return self.__key_type_map.get(key_type.decode("utf-8"))
		return None
	def delete_key(self,key,key_prefix):
		return self._connection.hdel(self.__key_type_hash,key_prefix+key)
class RedisCache(CacheAbstractDriver):
	def __init__(self,host,port,cache_db,*args,**kwargs):
		self.__client=RedisCacheClient(host,port,cache_db,*args,**kwargs)
	def __str__(self):
		return str(self.__client)
	def __repr__(self):
		return str(self.__client)
	def put(self,key,value,expires=0,key_prefix=""):
		self.__client.set(key,value,expires,key_prefix)
	def delete(self,key,key_prefix=""):
		return self.__client.delete(key,key_prefix)
	def get(self,key,key_prefix=""):
		return self.__client.get(key,key_prefix)
	def get_delete(self,key,key_prefix=""):
		re=self.__client.get(key,key_prefix)
		self.__client.delete(key,key_prefix)
		return re
	def exists(self,key,key_prefix=""):
		return self.__client.exists(key,key_prefix)
	def increment(self,key,delta=1,key_prefix=""):
		return self.__client.inc(key,delta,key_prefix)
	def decrement(self,key,delta=1,key_prefix=""):
		return self.__client.dec(key,delta,key_prefix)
	def update(self,key,value,expires=0,key_prefix=""):
		return self.__client.set(key,value,expires,key_prefix)

if __name__=='__main__':
	pass
	r'''
	#re=RedisCache('localhost',6379,1)
	#re.put("1","hello",key_prefix="user:name:")
	#re.put("2",[21,32,32,121,42],key_prefix="user:list:")
	#re.put("3",{"name":"huangbiao","age":22,"email":"19941222hb@gmail.com"},key_prefix="user:info:")
	#print(re.get_delete("3",key_prefix="user:info:"))
	#print(re.get_delete("2",key_prefix="user:list:"))
	#print(re.get_delete("1",key_prefix="user:name:"))
	#print(re.get("user:3"))
	'''

<commit_msg>implement set(**) method in AsyncRedisCacheClient<commit_after>#-*- coding:utf-8 -*-
import logging
logging.basicConfig(level=logging.ERROR)
from cache_abstract import CacheAbstractDriver
try:
	import redis
except ImportError:
	logging.error("Can't import 'redis' module")
	exit(-1)
try:
	import aioredis
except ImportError:
	logging.error("Can't import 'aioredis' module")
	exit(-1)
class AsyncRedisCacheClient(object):
	def __init__(self,host,port,db,*args,*kwargs):
		assert isinstance(host,str)
		assert isinstance(port,int)
		assert isinstance(db,int) and 0<=self.__db<16
		self.__host=host
		self.__port=port
		self.__db=db
		self.__connection=None
		self.__key_type_map={
				"1":"string",
				"2":"hash",
				"3":"list"
		}
		self.__key_type_hash="_key_type"
	@asyncio.coroutine
	def get_connection(self,loop=None):
		self.__connection=yield from aioredis.create_redis((self.__host,self.__port),db=self.__db,loop=loop)
		return self.__connection
	@asyncio.coroutine
	def set(self,key,value=None,expires,key_prefix):
		if not self.__connection:
			yield from self.get_connection()
		if not value and isinstance(key,dict):
			pipe=self.__connection.pipeline()
			for k,item in keys.items():
				pipe.hset(self.__key_type_hash,key_prefix+k,1)
				pipe.set(key_prefix+k,item)
				if expires>0:
					pipe.expire(key_prefix+k,expires)
			return (yield from pipe.execute())
		else:
			key=key_prefix+key
			if isinstance(value,str):
				pipe=self.__connection.pipeline()
				pipe.hset(self.__key_type_hash,key,1)
				pipe.set(key,value)
				if expires>0:
					pipe.expire(key,expires)
				return (yield from pipe.execute())
			elif isinstance(value,dict):
				pipe=self.__connection.pipeline()
				pipe.hset(self.__key_type_hash,key,2)
				pipe.hmset(key,value)
				if expires>0:
					pipe.expire(key,expires)
				return (yield from pipe.execute())
			elif isinstance(value,(list,tuple)):
				pipe=self.__connection.pipeline()
				pipe.hset(self.__key_type_hash,key,3)
				value=list(value)
				pipe.lpush(key,*value)
				if expires>0:
					pipe.expire(key,expires)
				return (yield from pipe.execute())

class RedisCacheClient(object):
	def __init__(self,host,port,db,*args,**kwargs):
		assert isinstance(host,str)
		assert isinstance(port,int)
		assert isinstance(db,int) and 0<=self.__db<16
		self.__host=host
		self.__port=port
		self.__db=db
		self._connection=redis.StrictRedis(self.__host,self.__port,self.__db)
		self.__key_type_map={
			"1":"string",
			"2":"hash",
			"3":"list"
		}
		self.__key_type_hash="_key_type"
	def set(self,key,value=None,expires,key_prefix):
		if not value and isinstance(key,dict):
			pipe=self._connection.pipeline()
			for k,item in key.items():
				pipe.hset(self.__key_type_hash,key_prefix+k,1)
				pipe.set(key_prefix+k,item)
				if expires>0:
					pipe.expire(key_prefix+k,expires)
			return pipe.execute()
		else:
			key=key_prefix+key
			if isinstance(value,str):
				pipe=self._connection.pipeline()
				pipe.hset(self.__key_type_hash,key,1)
				pipe.set(key,value)
				if expires>0:
					pipe.expire(key,expires)
				return pipe.execute()
			elif isinstance(value,dict):
				pipe=self._connection.pipeline()
				pipe.hset(self.__key_type_hash,key,2)
				pipe.hmset(key,value)
				if expires>0:
					pipe.expire(key,expires)
				return pipe.execute()
			elif isinstance(value,(list,tuple)):
				pipe=self._connection.pipeline()
				pipe.hset(self.__key_type_hash,key,3)
				pipe.lpush(key,*value)
				if expires>0:
					pipe.expire(key,expires)
				return pipe.execute()
	def get(self,key,key_prefix):
		key_type=self.exists(key,key_prefix)
		key=key_prefix+key
		if key_type=="string":
			return self._connection.get(key) or ""
		elif key_type=="hash":
			return self._connection.hgetall(key) or {}
		elif key_type=="list":
			return self._connection.lrange(key,0,-1) or []
		else:
			return None
	def delete(self,key,key_prefix):
		key_type=self.exists(key,key_prefix)
		result=""
		if key_type=="string":
			result=self._connection.delete(key_prefix+key)
		elif key_type=="hash":
			result=self._connection.hdel(key,*self._connection.hkeys(key_prefix+key))
		elif key_type=="list":
			length=self._connection.llen(key_prefix+key)
			if length:
				result=self._connection.ltrim(key_prefix+key,length+1,length+1)
		else:
			result=None
		self.delete_key(key,key_prefix)
		return result
	def inc(self,key,delta,key_prefix):
		key_type=self.exists(key,key_prefix)
		key=key_prefix+key
		if key_type=="string":
			return self._connection.incrby(key,delta)
		else:
			raise TypeError("can't  increment  '%s'"%key)
	def dec(self,key,delta,key_prefix):
		key_type=self.exists(key,key_prefix)
		key=key_prefix+key
		if key_type=="string":
			return self._connection.incrby(key,delta)
		else:
			raise TypeError("can't decrement '%s'"%key)
	def exists(self,key,key_prefix):
		key=key_prefix+key
		key_type=self._connection.hget(self.__key_type_hash,key)
		if key_type:
			return self.__key_type_map.get(key_type.decode("utf-8"))
		return None
	def delete_key(self,key,key_prefix):
		return self._connection.hdel(self.__key_type_hash,key_prefix+key)
class RedisCache(CacheAbstractDriver):
	def __init__(self,host,port,cache_db,*args,**kwargs):
		self.__client=RedisCacheClient(host,port,cache_db,*args,**kwargs)
	def __str__(self):
		return str(self.__client)
	def __repr__(self):
		return str(self.__client)
	def put(self,key,value,expires=0,key_prefix=""):
		self.__client.set(key,value,expires,key_prefix)
	def delete(self,key,key_prefix=""):
		return self.__client.delete(key,key_prefix)
	def get(self,key,key_prefix=""):
		return self.__client.get(key,key_prefix)
	def get_delete(self,key,key_prefix=""):
		re=self.__client.get(key,key_prefix)
		self.__client.delete(key,key_prefix)
		return re
	def exists(self,key,key_prefix=""):
		return self.__client.exists(key,key_prefix)
	def increment(self,key,delta=1,key_prefix=""):
		return self.__client.inc(key,delta,key_prefix)
	def decrement(self,key,delta=1,key_prefix=""):
		return self.__client.dec(key,delta,key_prefix)
	def update(self,key,value,expires=0,key_prefix=""):
		return self.__client.set(key,value,expires,key_prefix)

if __name__=='__main__':
	pass
	r'''
	#re=RedisCache('localhost',6379,1)
	#re.put("1","hello",key_prefix="user:name:")
	#re.put("2",[21,32,32,121,42],key_prefix="user:list:")
	#re.put("3",{"name":"huangbiao","age":22,"email":"19941222hb@gmail.com"},key_prefix="user:info:")
	#print(re.get_delete("3",key_prefix="user:info:"))
	#print(re.get_delete("2",key_prefix="user:list:"))
	#print(re.get_delete("1",key_prefix="user:name:"))
	#print(re.get("user:3"))
	'''

<|endoftext|>


<commit_before>"""Functions for finding the foreground in an image with a clean
background (i.e. mostly white or black)"""

from __future__ import division
from collections import namedtuple
import numpy as np
from crash_kiss.config import BLACK, WHITE, FULL_DEPTH
import crash_kiss.util as util

def find_foreground(img, background, threshold):
    """Find the foreground of the image by subracting each RGB element
    in the image by the background. If the background has been reduced
    to a simple int or float, we'll try to avoid calling `np.abs`
    by checking to see if the background value is near 0 or 255."""
    if len(img.shape) == 2:
        return _compare_pixels(img, background, threshold)
    rgb_views = [img[:, :, idx] for idx in range(img.shape[-1])]

    # the foreground is a 2D array where 1==foreground 0==background
    fg = _compare_pixels(rgb_views[0], background, threshold)
    for view in rgb_views[1:]:
        bg = fg == 0
        new_data = _compare_pixels(view[bg], background, threshold)
        fg[bg] = new_data
    return fg

def _compare_pixels(img, background, threshold):
    """Compare a 2-D or 3-D image array
    to a background value given a certain threshold"""
    is_num = isinstance(background, int)
    if background - BLACK <= 5:
        diff = img - background > threshold
    elif WHITE - background <= 5:
        diff = background - img > threshold
    else:
        diff = np.abs(img - background) > threshold
    if len(diff.shape) == 3:
        diff = np.any(diff, axis=2)  # we're using a 3D array
    return diff

def simplify_background(background, config):
    """See if the background's RGB elements are similar.
    If each element in the background is similar enough, we can do a simple
    array - int operation instead of the more expensive array - [R, G, B]
    or the even pricier array - <array of shape (NROWS, 1, 3)> operation."""
    while isinstance(background, np.ndarray):
        bg_change_tolerance = config["bg_change_tolerance"]
        bmax, bmin = background.max(axis=0), background.min(axis=0)
        diff = (bmax - bmin) < bg_change_tolerance
        if len(background.shape) >= 2:
            diff = np.all(diff)
        if diff:
            background = np.median(background, axis=0)
        else:
            break  # we've simplified the background as much as we can
    return background

_MID_FG = 0xFFFF

@profile
def center_smash(img, fg, bounds):
    """Move the rows of each subject together until they touch.
    Write over the vacated space with whatever the row's negative space
    is (probably white or transparent pixels)."""
    start, stop, fg_mid = bounds
    max_depth = fg.shape[1] // 4
    fg_l = fg_mid - max_depth
    fg_r = fg_mid + max_depth
    rlen = img.shape[1] - stop
    mid_left = start + max_depth
    center = start + 2*max_depth
    mid_right = center + max_depth
    side_len = fg.shape[1] // 2

    lfg = fg[:, :bounds.fg_mid]
    lfg = util.invert_horizontal(lfg)
    rfg = fg[:, bounds.fg_mid:]
    lstart = np.argmax(lfg, axis=1)
    lstart[lfg[:, 0] == 1] = _MID_FG
    rstart = np.argmax(rfg, axis=1)
    rstart[rfg[:, 0] == 1] = _MID_FG

    def mov_to_center(irow, frow):
        """Smash a row whose foreground intersect the center line"""
        lextra = frow[:fg_mid][::-1].argmin()
        rextra = frow[fg_mid:].argmin()
        lidx = start + max_depth - lextra
        lidx_fg = max_depth - lextra
        lsubj = irow[lidx: center][frow[lidx_fg: fg_mid]].copy()
        ridx = center + max_depth + rextra
        ridx_fg = fg_mid + max_depth + rextra
        rsubj = irow[center: ridx][frow[fg_mid: ridx_fg]].copy()
        llen = len(lsubj)
        rlen = len(rsubj)
        lmov = max_depth + lextra - llen
        rmov = max_depth + rextra - rlen
        irow[lmov: center] = irow[:center - lmov]
        irow[center - llen: center] = lsubj
        irow[center: -rmov] = irow[center + rmov:]
        irow[center: center + rlen] = rsubj
        return lmov, rmov

    def mov_empty_fg(irow):
        """Smash a row with an empty foreground area"""
        irow[center + rmov: -rmov] = irow[stop:]
        irow[lmov: start+lmov] = irow[:start]

    def mov_no_collision(irow, frow):
        """Smash a row whose foreground area will not touch"""
        irow[center: -max_depth] = irow[center + max_depth:]
        irow[max_depth: center] = irow[:start + max_depth]

    def mov_left_overshoot(irow, frow, left_of_center):
        """Smash a row where the left side overshoots the center line"""
        irow[center + max_depth: -max_depth] = irow[stop:]  # no RHS FG
        irow[max_depth: center + max_depth] = irow[:center]

    def mov_right_overshoot(irow, frow, right_of_center):
        """Smash a row where the right side overshoots the center line"""
        irow[max_depth: center - max_depth] = irow[:start]  # no LHS FG
        irow[center - max_depth: -max_depth] = irow[center:]

    @profile
    def smash(irow, frow, ls, rs):
        offs = rs - ls
        dist = rs + ls - 1
        ledge_mov = dist // 2  # TRUNCATION less on left
        redge_mov = dist - ledge_mov
        fg_l_stop = fg_r_start = -ls + ledge_mov - 1
        bg_mask = frow[fg_mid + fg_l_stop - max_depth: fg_mid + fg_r_start + max_depth]
        l_bg_mask = bg_mask[:max_depth]
        r_bg_mask = bg_mask[max_depth:]
        llen = np.count_nonzero(l_bg_mask)
        rlen = np.count_nonzero(r_bg_mask)
        lsquash = len(l_bg_mask) - llen - ledge_mov
        rsquash = len(r_bg_mask) - rlen - redge_mov
        lmov = ledge_mov + lsquash
        rmov = redge_mov + rsquash
        subj = irow[center + fg_l_stop - max_depth: center + fg_r_start + max_depth]
        subj = subj[bg_mask]
        irow[center + fg_l_stop - llen: center + fg_r_start + rlen] = subj
        irow[lmov: center + fg_l_stop - llen] = (
            irow[:center + fg_l_stop - llen - lmov])
        irow[center + fg_r_start + rlen: -rmov] = (
            irow[center + fg_r_start + rlen + rmov:])
        return lmov, rmov

    def mov_near_collision(irow, frow, ls, rs):
        irow[lmov: center - ls + lmov] = irow[: center - ls]
        irow[center + rs - rmov: -rmov] = irow[center + rs:]

    for irow, ls, rs, frow in zip(img, lstart, rstart, fg):
        lmov = rmov = max_depth
        if (rs < max_depth) and (ls < max_depth):
            lmov, rmov = smash(irow, frow, ls, rs)
        elif rs + ls <= side_len:
            lmov, rmov = smash(irow, frow, ls, rs)
        elif (ls < max_depth) or (rs < max_depth):
            mov_near_collision(irow, frow, ls, rs)
        elif rs == _MID_FG or ls == _MID_FG:
        #    lmov, rmov = mov_to_center(irow, frow)
            lmov, rmov = smash(irow, frow, rs, ls)
        elif not ls and not rs:
            mov_empty_fg(irow)
        elif ls and not rs:
            mov_left_overshoot(irow, frow, ls)
        elif rs and not ls:
            mov_right_overshoot(irow, frow, rs)
        elif (ls >= max_depth) and (rs >= max_depth):
            mov_no_collision(irow, frow)
        else:
            raise Exception("This is most unexpected!")
        irow[:lmov] = WHITE
        irow[-rmov:] = WHITE

def get_foreground_area(img, max_depth):
    bounds = _get_fg_bounds(img.shape, max_depth)
    return img[:, bounds.start:bounds.stop], bounds

_fg_bounds = namedtuple("fg_bounds", "start stop fg_mid")

def _get_fg_bounds(img_shape, max_depth):
    """Returns start, stop idx of the 'smashable foreground'
    area in the middle of an image.

    Indexing the image with `img[:, start:stop]` will successfully
    select the foreground columns."""
    width = img_shape[1]
    half = width // 2
    if max_depth >= half or max_depth == FULL_DEPTH:
        max_depth = width // 4
    start = (width // 2) - (max_depth * 2)
    stop = start + max_depth * 4
    assert stop - start == max_depth * 4
    fg_mid = (stop - start) // 2
    return _fg_bounds(start, stop, fg_mid)

PURPLE = [118, 0, 118]
TEAL = [60, 120, 160]
GREEN = [0, 255, 0]
YELLOW = [255, 255, 0]
PINK = [255, 0, 255]

def reveal_foreground(img, foreground, bounds):
    """Paints the foreground of the foreground selection area
    1) purple if the pixel is in an OUTER quadrant
    or 2) black if the pixel is in an INNER quadrant"""
    start, stop, fg_mid = bounds
    max_depth = (stop - start) // 4
    critical_fg = foreground[:, max_depth: -max_depth]
    img[:, start: stop][foreground] = PURPLE
    img[:, start + max_depth: stop - max_depth][critical_fg] = BLACK

def reveal_quadrants(img, bounds):
    """Places vertical lines to represent the "quadrants" that
    crash_kiss uses to determine the "smashable area" of the image"""
    start, stop, fg_mid = bounds
    max_depth = (stop - start) // 4
    width = 4
    width = img.shape[1] // 2000 + 1
    lmid = start + max_depth
    mid = start + max_depth * 2
    rmid = stop - max_depth
    img[:, start - width: start + width] = GREEN
    img[:, stop - width: stop + width] = GREEN
    img[:, mid - width: mid + width] = YELLOW
    img[:, lmid - width: lmid + width] = PINK
    img[:, rmid - width: rmid + width] = PINK

def reveal_background(img, foreground, bounds):
    """Paints the background of the foreground selection area teal"""
    img[:, bounds.start: bounds.stop][foreground == 0] = TEAL

<commit_msg>Fixed broken ordering of if-elif in smash loop<commit_after>"""Functions for finding the foreground in an image with a clean
background (i.e. mostly white or black)"""

from __future__ import division
from collections import namedtuple
import numpy as np
from crash_kiss.config import BLACK, WHITE, FULL_DEPTH
import crash_kiss.util as util

def find_foreground(img, background, threshold):
    """Find the foreground of the image by subracting each RGB element
    in the image by the background. If the background has been reduced
    to a simple int or float, we'll try to avoid calling `np.abs`
    by checking to see if the background value is near 0 or 255."""
    if len(img.shape) == 2:
        return _compare_pixels(img, background, threshold)
    rgb_views = [img[:, :, idx] for idx in range(img.shape[-1])]

    # the foreground is a 2D array where 1==foreground 0==background
    fg = _compare_pixels(rgb_views[0], background, threshold)
    for view in rgb_views[1:]:
        bg = fg == 0
        new_data = _compare_pixels(view[bg], background, threshold)
        fg[bg] = new_data
    return fg

def _compare_pixels(img, background, threshold):
    """Compare a 2-D or 3-D image array
    to a background value given a certain threshold"""
    is_num = isinstance(background, int)
    if background - BLACK <= 5:
        diff = img - background > threshold
    elif WHITE - background <= 5:
        diff = background - img > threshold
    else:
        diff = np.abs(img - background) > threshold
    if len(diff.shape) == 3:
        diff = np.any(diff, axis=2)  # we're using a 3D array
    return diff

def simplify_background(background, config):
    """See if the background's RGB elements are similar.
    If each element in the background is similar enough, we can do a simple
    array - int operation instead of the more expensive array - [R, G, B]
    or the even pricier array - <array of shape (NROWS, 1, 3)> operation."""
    while isinstance(background, np.ndarray):
        bg_change_tolerance = config["bg_change_tolerance"]
        bmax, bmin = background.max(axis=0), background.min(axis=0)
        diff = (bmax - bmin) < bg_change_tolerance
        if len(background.shape) >= 2:
            diff = np.all(diff)
        if diff:
            background = np.median(background, axis=0)
        else:
            break  # we've simplified the background as much as we can
    return background

_MID_FG = 0xFFFF

@profile
def center_smash(img, fg, bounds):
    """Move the rows of each subject together until they touch.
    Write over the vacated space with whatever the row's negative space
    is (probably white or transparent pixels)."""
    start, stop, fg_mid = bounds
    max_depth = fg.shape[1] // 4
    fg_l = fg_mid - max_depth
    fg_r = fg_mid + max_depth
    rlen = img.shape[1] - stop
    mid_left = start + max_depth
    center = start + 2*max_depth
    mid_right = center + max_depth
    side_len = fg.shape[1] // 2

    lfg = fg[:, :bounds.fg_mid]
    lfg = util.invert_horizontal(lfg)
    rfg = fg[:, bounds.fg_mid:]
    lstart = np.argmax(lfg, axis=1)
    lstart[lfg[:, 0] == 1] = _MID_FG
    rstart = np.argmax(rfg, axis=1)
    rstart[rfg[:, 0] == 1] = _MID_FG

    def mov_to_center(irow, frow):
        """Smash a row whose foreground intersect the center line"""
        lextra = frow[:fg_mid][::-1].argmin()
        rextra = frow[fg_mid:].argmin()
        lidx = start + max_depth - lextra
        lidx_fg = max_depth - lextra
        lsubj = irow[lidx: center][frow[lidx_fg: fg_mid]].copy()
        ridx = center + max_depth + rextra
        ridx_fg = fg_mid + max_depth + rextra
        rsubj = irow[center: ridx][frow[fg_mid: ridx_fg]].copy()
        llen = len(lsubj)
        rlen = len(rsubj)
        lmov = max_depth + lextra - llen
        rmov = max_depth + rextra - rlen
        irow[lmov: center] = irow[:center - lmov]
        irow[center - llen: center] = lsubj
        irow[center: -rmov] = irow[center + rmov:]
        irow[center: center + rlen] = rsubj
        return lmov, rmov

    def mov_empty_fg(irow):
        """Smash a row with an empty foreground area"""
        irow[center + rmov: -rmov] = irow[stop:]
        irow[lmov: start+lmov] = irow[:start]

    def mov_no_collision(irow, frow):
        """Smash a row whose foreground area will not touch"""
        irow[center: -max_depth] = irow[center + max_depth:]
        irow[max_depth: center] = irow[:start + max_depth]

    def mov_left_overshoot(irow, frow, ls):
        """Smash a row where the left side overshoots the center line"""
        irow[mid_right: -max_depth] = irow[stop:]  # no RHS FG
        irow[max_depth: mid_right] = irow[:center]

    def mov_right_overshoot(irow, frow, rs):
        """Smash a row where the right side overshoots the center line"""
        irow[max_depth: mid_left] = irow[:start]  # no LHS FG
        irow[mid_left: -max_depth] = irow[center:]

    @profile
    def smash(irow, frow, ls, rs):
        lextra = rextra = 0
        if ls == _MID_FG:
            lextra = frow[:fg_mid][::-1].argmin()
        if rs == _MID_FG:
            rextra = frow[fg_mid:].argmin()
            #irow[:] = BLACK
        offs = rs - ls
        dist = rs + ls - 1
        ledge_mov = dist // 2  # TRUNCATION less on left
        redge_mov = dist - ledge_mov
        fg_l_stop = fg_r_start = -ls + ledge_mov - 1
        bg_mask = frow[fg_mid + fg_l_stop - max_depth: fg_mid + fg_r_start + max_depth]
        l_bg_mask = bg_mask[:max_depth]
        r_bg_mask = bg_mask[max_depth:]
        llen = np.count_nonzero(l_bg_mask)
        rlen = np.count_nonzero(r_bg_mask)
        lsquash = len(l_bg_mask) - llen - ledge_mov
        rsquash = len(r_bg_mask) - rlen - redge_mov
        lmov = ledge_mov + lsquash + lextra
        rmov = redge_mov + rsquash + rextra
        subj = irow[center + fg_l_stop - max_depth: center + fg_r_start + max_depth]
        subj = subj[bg_mask]
        irow[center + fg_l_stop - llen: center + fg_r_start + rlen] = subj
        irow[lmov: center + fg_l_stop - llen] = (
            irow[:center + fg_l_stop - llen - lmov])
        irow[center + fg_r_start + rlen: -rmov] = (
            irow[center + fg_r_start + rlen + rmov:])
        return lmov, rmov

    def mov_near_collision(irow, frow, ls, rs):
        irow[lmov: center - ls + lmov] = irow[: center - ls]
        irow[center + rs - rmov: -rmov] = irow[center + rs:]

    for irow, ls, rs, frow in zip(img, lstart, rstart, fg):
        lmov = rmov = max_depth
        if not ls and not rs:
            mov_empty_fg(irow)
        elif ls and not rs:
            mov_left_overshoot(irow, frow, ls)
        elif rs and not ls:
            mov_right_overshoot(irow, frow, rs)
        elif rs == _MID_FG or ls == _MID_FG:
            lmov, rmov = smash(irow, frow, rs, ls)
        elif (rs < max_depth) and (ls < max_depth):
            lmov, rmov = smash(irow, frow, ls, rs)
        elif rs + ls <= side_len:
            lmov, rmov = smash(irow, frow, ls, rs)
        elif (ls < max_depth) or (rs < max_depth):
            mov_near_collision(irow, frow, ls, rs)
        else:
            mov_no_collision(irow, frow)
        irow[:lmov] = WHITE
        irow[-rmov:] = WHITE

def get_foreground_area(img, max_depth):
    bounds = _get_fg_bounds(img.shape, max_depth)
    return img[:, bounds.start:bounds.stop], bounds

_fg_bounds = namedtuple("fg_bounds", "start stop fg_mid")

def _get_fg_bounds(img_shape, max_depth):
    """Returns start, stop idx of the 'smashable foreground'
    area in the middle of an image.

    Indexing the image with `img[:, start:stop]` will successfully
    select the foreground columns."""
    width = img_shape[1]
    half = width // 2
    if max_depth >= half or max_depth == FULL_DEPTH:
        max_depth = width // 4
    start = (width // 2) - (max_depth * 2)
    stop = start + max_depth * 4
    assert stop - start == max_depth * 4
    fg_mid = (stop - start) // 2
    return _fg_bounds(start, stop, fg_mid)

PURPLE = [118, 0, 118]
TEAL = [60, 120, 160]
GREEN = [0, 255, 0]
YELLOW = [255, 255, 0]
PINK = [255, 0, 255]

def reveal_foreground(img, foreground, bounds):
    """Paints the foreground of the foreground selection area
    1) purple if the pixel is in an OUTER quadrant
    or 2) black if the pixel is in an INNER quadrant"""
    start, stop, fg_mid = bounds
    max_depth = (stop - start) // 4
    critical_fg = foreground[:, max_depth: -max_depth]
    img[:, start: stop][foreground] = PURPLE
    img[:, start + max_depth: stop - max_depth][critical_fg] = BLACK

def reveal_quadrants(img, bounds):
    """Places vertical lines to represent the "quadrants" that
    crash_kiss uses to determine the "smashable area" of the image"""
    start, stop, fg_mid = bounds
    max_depth = (stop - start) // 4
    width = 4
    width = img.shape[1] // 2000 + 1
    lmid = start + max_depth
    mid = start + max_depth * 2
    rmid = stop - max_depth
    img[:, start - width: start + width] = GREEN
    img[:, stop - width: stop + width] = GREEN
    img[:, mid - width: mid + width] = YELLOW
    img[:, lmid - width: lmid + width] = PINK
    img[:, rmid - width: rmid + width] = PINK

def reveal_background(img, foreground, bounds):
    """Paints the background of the foreground selection area teal"""
    img[:, bounds.start: bounds.stop][foreground == 0] = TEAL

<|endoftext|>


<commit_before>"""
Standardize names of data files from the Pennsylvania Secretary of State.

The state offers CSV files containing precinct-level results for regularly scheduled primary and general elections; these were split from a single
zip file into election-specific files and thus have no `raw_url` attribute. Special elections are pre-processed CSV files from HTML files. All files
are available in the https://github.com/openelections/openelections-data-pa repository.
"""
from os.path import join
import json
import unicodecsv
import urlparse
import requests

from openelex import PROJECT_ROOT
from openelex.base.datasource import BaseDatasource
from openelex.lib import build_github_url

class Datasource(BaseDatasource):

    # PUBLIC INTERFACE
    def mappings(self, year=None):
        """Return array of dicts containing source url and
        standardized filename for raw results file, along
        with other pieces of metadata
        """
        mappings = []
        for yr, elecs in self.elections(year).items():
            mappings.extend(self._build_metadata(yr, elecs))
        return mappings

    def target_urls(self, year=None):
        "Get list of source data urls, optionally filtered by year"
        return [item['raw_url'] for item in self.mappings(year)]

    def filename_url_pairs(self, year=None):
        return [(item['generated_filename'], self._url_for_fetch(item))
                for item in self.mappings(year)]

    # PRIVATE METHODS

    def _build_metadata(self, year, elections):
        meta = []
        year_int = int(year)
        for election in elections:
            if election['special']:
                results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True]
            else:
                results = [x for x in self._url_paths() if x['date'] == election['start_date']]
            for result in results:
                if election['direct_links']:
                    raw_url = election['direct_links'][0]
                else:
                    raw_url = None
                generated_filename = self._generate_filename(election['start_date'], election['race_type'], result)
                meta.append({
                    "generated_filename": generated_filename,
                    "raw_url": raw_url,
                    "pre_processed_url": build_github_url(self.state, generated_filename),
                    "ocd_id": 'ocd-division/country:us/state:pa',
                    "name": 'Pennsylvania',
                    "election": election['slug']
                })
        return meta

    def _generate_filename(self, start_date, election_type, result):
        if result['district'] == '':
            office = result['office']
        else:
            office = result['office'] + '__' + result['district']
        if result['special']:
            election_type = 'special__' + election_type
        else:
            election_type = election_type+'__precinct'
        bits = [
            start_date.replace('-',''),
            self.state.lower(),
            election_type,
            office
        ]
        if office == '':
            bits.remove(office)
        name = "__".join(bits) + '.csv'
        return name

    def _jurisdictions(self):
        """Pennsylvania counties"""
        m = self.jurisdiction_mappings()
        mappings = [x for x in m if x['county'] != ""]
        return mappings

    def _url_for_fetch(self, mapping):
        if mapping['pre_processed_url']:
            return mapping['pre_processed_url']
        else:
            return mapping['raw_url']
<commit_msg>fixed special election handling<commit_after>"""
Standardize names of data files from the Pennsylvania Secretary of State.

The state offers CSV files containing precinct-level results for regularly scheduled primary and general elections; these were split from a single
zip file into election-specific files and thus have no `raw_url` attribute. Special elections are pre-processed CSV files from HTML files. All files
are available in the https://github.com/openelections/openelections-data-pa repository.
"""
from os.path import join
import json
import unicodecsv
import urlparse
import requests

from openelex import PROJECT_ROOT
from openelex.base.datasource import BaseDatasource
from openelex.lib import build_github_url

class Datasource(BaseDatasource):

    # PUBLIC INTERFACE
    def mappings(self, year=None):
        """Return array of dicts containing source url and
        standardized filename for raw results file, along
        with other pieces of metadata
        """
        mappings = []
        for yr, elecs in self.elections(year).items():
            mappings.extend(self._build_metadata(yr, elecs))
        return mappings

    def target_urls(self, year=None):
        "Get list of source data urls, optionally filtered by year"
        return [item['raw_url'] for item in self.mappings(year)]

    def filename_url_pairs(self, year=None):
        return [(item['generated_filename'], self._url_for_fetch(item))
                for item in self.mappings(year)]

    # PRIVATE METHODS

    def _build_metadata(self, year, elections):
        meta = []
        year_int = int(year)
        for election in elections:
            if election['special']:
                results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == True]
            else:
                results = [x for x in self._url_paths() if x['date'] == election['start_date'] and x['special'] == False]
            for result in results:
                if election['direct_links']:
                    raw_url = election['direct_links'][0]
                else:
                    raw_url = None
                generated_filename = self._generate_filename(election['start_date'], election['race_type'], result)
                meta.append({
                    "generated_filename": generated_filename,
                    "raw_url": raw_url,
                    "pre_processed_url": build_github_url(self.state, generated_filename),
                    "ocd_id": 'ocd-division/country:us/state:pa',
                    "name": 'Pennsylvania',
                    "election": election['slug']
                })
        return meta

    def _generate_filename(self, start_date, election_type, result):
        if result['district'] == '':
            office = result['office']
        else:
            office = result['office'] + '__' + result['district']
        if result['special']:
            election_type = 'special__' + election_type
        else:
            election_type = election_type+'__precinct'
        bits = [
            start_date.replace('-',''),
            self.state.lower(),
            election_type,
            office
        ]
        if office == '':
            bits.remove(office)
        name = "__".join(bits) + '.csv'
        return name

    def _jurisdictions(self):
        """Pennsylvania counties"""
        m = self.jurisdiction_mappings()
        mappings = [x for x in m if x['county'] != ""]
        return mappings

    def _url_for_fetch(self, mapping):
        if mapping['pre_processed_url']:
            return mapping['pre_processed_url']
        else:
            return mapping['raw_url']
<|endoftext|>


<commit_before>#!/usr/bin/env python
############################################################################
#                                                                          #
# Copyright 2014 Prelert Ltd                                               #
#                                                                          #
# Licensed under the Apache License, Version 2.0 (the "License");          #
# you may not use this file except in compliance with the License.         #
# You may obtain a copy of the License at                                  #
#                                                                          #
#    http://www.apache.org/licenses/LICENSE-2.0                            #
#                                                                          #
# Unless required by applicable law or agreed to in writing, software      #
# distributed under the License is distributed on an "AS IS" BASIS,        #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and      #
# limitations under the License.                                           #
#                                                                          #
############################################################################

'''
Script to pull metric data from AWS CloudWatch and analyze it in
the Prelert Engine API. There are 2 modes of operation: historical
where stored metric data is extracted between 2 dates and a continous
realtime mode where the preceeding few minutes of data is queried in
a loop.

The path to a configuration file containing the AWS connection parameters
must be passed to the script the file should have the following propteries:

    region=REGION
    aws_access_key_id=YOUR_ACCESS_ID
    aws_secret_access_key=YOUR_SECRET_KEY

Where REGION is one of us-east-1, eu-west-1, etc

If the --start-date parameter is set then this will query historical data
from CloudWatch until --end-date or the current time if --end-date is not
set. Otherwise the script will run in an infinite loop pulling realtime
data, use Ctrl-C to quit the realtime mode as the script will catch
the interrupt and handle the exit gracefully.

Only EC2 metrics are monitored and only those belonging to an instance.
Aggregated metrics by instance type and AMI metrics are ignored.
'''

import argparse
import ConfigParser
from datetime import datetime, timedelta, tzinfo
import json
import StringIO
import time

import boto.ec2
import boto.ec2.cloudwatch
from boto.exception import BotoServerError

from prelert.engineApiClient import EngineApiClient

# Prelert Engine API default connection prarams
API_HOST = 'localhost'
API_PORT = 8080
API_BASE_URL = 'engine/v1'

''' Interval between query new data from CloudWatch (seconds)'''
UPDATE_INTERVAL=300

''' In realtime mode run this many seconds behind realtime '''
DELAY=600

'''
    Prelert Engine job configuration.
    The detector is configured to analyze the mean of the field named
    'Average' by the field 'metric_name' where the value of 'metric_name'
    is one of the AWS metrics e.g. CPUUtilization, DiskWriteOps, etc.
    The analysis is partition by AWS instance ID.
    bucketSpan is set the same as the CloudWatch reporting interval.
'''
JOB_CONFIG = '{"analysisConfig" : {\
                    "bucketSpan":' + str(UPDATE_INTERVAL) + ',\
                    "detectors" :[{"function":"mean","fieldName":"Average","byFieldName":"metric_name","partitionFieldName":"instance"}] \
                },\
                "dataDescription" : {"format":"JSON","timeField":"timestamp","timeFormat":"yyyy-MM-dd\'T\'HH:mm:ssX"} \
            }'

class MetricRecord:
    '''
    Simple holder class for the CloudWatch metrics.
    toJsonStr returns the metric in a format for the job
    configuration above.
    '''
    def __init__(self, timestamp, instance, metric_name, metric_value):
        self.timestamp = timestamp
        self.instance = instance
        self.metric_name = metric_name
        self.metric_value = metric_value

    def toJsonStr(self):
        result = '{"timestamp":"' + self.timestamp.isoformat() + \
            '", "instance":"' + self.instance + '", "metric_name":"' + \
            self.metric_name + '", "Average":' + str(self.metric_value) + '}'

        return result

class UTC(tzinfo):
    '''
    UTC timezone class
    '''

    def utcoffset(self, dt):
        return timedelta(0)

    def tzname(self, dt):
        return "UTC"

    def dst(self, dt):
        return timedelta(0)

def parseArguments():
    parser = argparse.ArgumentParser()

    parser.add_argument("config", help="The AWS connection parameters.")

    parser.add_argument("--api-host", help="The Prelert Engine API host, defaults to "
        + API_HOST, default=API_HOST, dest="api_host")
    parser.add_argument("--api-port", help="The Prelert Engine API port, defaults to "
        + str(API_PORT), default=API_PORT, dest="api_port")

    parser.add_argument("--job-id", help="Send data to this job. If not set a \
        new job will be created.", default=None, dest="job_id")

    parser.add_argument("--start-date", help="Request data from this date. If not \
        set then run in realtime mode. Dates must be in YYYY-MM-DD format",
        default=None, dest="start_date")
    parser.add_argument("--end-date", help="if --start-date is set then pull \
        and analyze only the metric data between those dates. \
        If --start-date is not set this argument has no meaning. \
        Dates must be in YYYY-MM-DD format",
        default=None, dest="end_date")

    return parser.parse_args()

def runHistorical(job_id, start_date, end_date, cloudwatch_conn, engine_client):
    '''
    Query and analyze the CloudWatch metrics from start_date to end_date.
    If end_date == None then run until the time now.
    '''

    end = start_date
    delta = timedelta(seconds=UPDATE_INTERVAL)

    while True:

        start = end
        end = start + delta

        end_condition = end_date if end_date != None else datetime.utcnow()
        if end > end_condition:
            break

        print "Querying metrics starting at time " + str(start.isoformat())

        try:
            metrics = cloudwatch_conn.list_metrics(namespace='AWS/EC2')
            metric_records = queryMetricRecords(metrics, start, end, reporting_interval = 60)

            data = ''
            for mr in metric_records:
                data += mr.toJsonStr() + '\n'

            (http_status, response) = engine_client.upload(job_id, data)
            if http_status != 202:
                print "Error uploading metric data to the Engine"
                print http_status, json.dumps(response)

        except BotoServerError as error:
            print "Error querying CloudWatch"
            print error

def queryMetricRecords(metrics, start, end, reporting_interval):
    metric_records = []
    for m in metrics:
        if 'InstanceId' not in m.dimensions:
            continue
        instance = m.dimensions['InstanceId'][0]

        datapoints = m.query(start, end, 'Average', period=reporting_interval)
        for dp in datapoints:
            # annoyingly Boto does not return datetimes with a timezone
            utc_time = replaceTimezoneWithUtc(dp['Timestamp'])
            mr = MetricRecord(utc_time, instance, m.name, dp['Average'])
            metric_records.append(mr)

        metric_records.sort(key=lambda r : r.timestamp)

    return metric_records

def runRealtime(job_id, cloudwatch_conn, engine_client):
    '''
    Query the previous 5 minutes of metric data every 5 minutes
    then upload to the Prelert Engine.

    This function runs in an infinite loop but will catch the
    keyboard interrupt (Ctrl C) and exit gracefully
    '''
    try:
        delay = timedelta(seconds=DELAY)
        end = datetime.utcnow() - delay - timedelta(seconds=UPDATE_INTERVAL)
        end = replaceTimezoneWithUtc(end)

        while True:

            start = end
            end = datetime.utcnow() - delay
            end = replaceTimezoneWithUtc(end)

            print "Querying metrics from " + str(start.isoformat())  + " to " + end.isoformat()

            try:
                metrics = cloudwatch_conn.list_metrics(namespace='AWS/EC2')
                metric_records = queryMetricRecords(metrics, start, end, reporting_interval = UPDATE_INTERVAL)

                data = ''
                for mr in metric_records:
                    data += mr.toJsonStr() + '\n'

                (http_status, response) = engine_client.upload(job_id, data)
                if http_status != 202:
                    print "Error uploading metric data to the Engine"
                    print http_status, json.dumps(response)

            except BotoServerError as error:
                print "Error querying CloudWatch"
                print error

            duration = datetime.utcnow() - delay - end
            sleep_time = max(UPDATE_INTERVAL - duration.seconds, 0)
            print "sleeping for " + str(sleep_time) + " seconds"
            if sleep_time > 0:
                time.sleep(sleep_time)

    except KeyboardInterrupt:
        print "Interrupt caught... terminating real time queries"
        return

def main():

    args = parseArguments()

    # read the config file
    config = ConfigParser.RawConfigParser()
    try:
        # insert a section header into the config so
        # ConfigParser will read it without complaint
        with open(args.config, "r") as config_file:
            ini_str = '[root]\n' + config_file.read()
            ini_fp = StringIO.StringIO(ini_str)
            config.readfp(ini_fp)
    except IOError:
        print "Error opening file " + args.config
        return

    try:
        region = config.get('root', 'region')
        access_id = config.get('root', 'aws_access_key_id')
        secret_key = config.get('root', 'aws_secret_access_key')
    except ConfigParser.NoOptionError as e:
        print e
        return

    # AWS CloudWatch connection
    cloudwatch_conn = boto.ec2.cloudwatch.connect_to_region(region,
                 aws_access_key_id=access_id,
                 aws_secret_access_key=secret_key)

    # The Prelert REST API client
    engine_client = EngineApiClient(args.api_host, API_BASE_URL, args.api_port)

    # If no job ID is supplied create a new job
    job_id = args.job_id
    if job_id == None:
        (http_status, response) = engine_client.createJob(JOB_CONFIG)
        if http_status != 201:
            print "Error creating job"
            print response
            return

        job_id = response['id']
        print "Created job with ID " + str(job_id)
    else:
        print "Using job ID " + job_id

    # default start date is None meaning run realtime
    start_date = None
    if args.start_date != None:
        start_date = datetime.strptime(args.start_date, "%Y-%m-%d")
        start_date = replaceTimezoneWithUtc(start_date)

    if start_date == None:
        runRealtime(job_id, cloudwatch_conn, engine_client)
    else:
        # historical mode, check for an end date
        end_date = None
        if args.end_date != None:
            end_date = datetime.strptime(args.end_date, "%Y-%m-%d")
            end_date = replaceTimezoneWithUtc(end_date)

        runHistorical(job_id, start_date, end_date, cloudwatch_conn, engine_client)

    print "Closing job..."
    engine_client.close(job_id)

def replaceTimezoneWithUtc(date):
    return date.replace(tzinfo=UTC())

if __name__ == "__main__":
    main()
<commit_msg>Introduce and use constant for the reporting interval<commit_after>#!/usr/bin/env python
############################################################################
#                                                                          #
# Copyright 2014 Prelert Ltd                                               #
#                                                                          #
# Licensed under the Apache License, Version 2.0 (the "License");          #
# you may not use this file except in compliance with the License.         #
# You may obtain a copy of the License at                                  #
#                                                                          #
#    http://www.apache.org/licenses/LICENSE-2.0                            #
#                                                                          #
# Unless required by applicable law or agreed to in writing, software      #
# distributed under the License is distributed on an "AS IS" BASIS,        #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and      #
# limitations under the License.                                           #
#                                                                          #
############################################################################

'''
Script to pull metric data from AWS CloudWatch and analyze it in
the Prelert Engine API. There are 2 modes of operation: historical
where stored metric data is extracted between 2 dates and a continous
realtime mode where the preceeding few minutes of data is queried in
a loop.

The path to a configuration file containing the AWS connection parameters
must be passed to the script the file should have the following propteries:

    region=REGION
    aws_access_key_id=YOUR_ACCESS_ID
    aws_secret_access_key=YOUR_SECRET_KEY

Where REGION is one of us-east-1, eu-west-1, etc

If the --start-date parameter is set then this will query historical data
from CloudWatch until --end-date or the current time if --end-date is not
set. Otherwise the script will run in an infinite loop pulling realtime
data, use Ctrl-C to quit the realtime mode as the script will catch
the interrupt and handle the exit gracefully.

Only EC2 metrics are monitored and only those belonging to an instance.
Aggregated metrics by instance type and AMI metrics are ignored.
'''

import argparse
import ConfigParser
from datetime import datetime, timedelta, tzinfo
import json
import StringIO
import time

import boto.ec2
import boto.ec2.cloudwatch
from boto.exception import BotoServerError

from prelert.engineApiClient import EngineApiClient

# Prelert Engine API default connection prarams
API_HOST = 'localhost'
API_PORT = 8080
API_BASE_URL = 'engine/v1'

''' Interval between query new data from CloudWatch (seconds)'''
UPDATE_INTERVAL=300

''' Interval between data points that are being fetched from CloudWatch (seconds)'''
REPORTING_INTERVAL=60

''' In realtime mode run this many seconds behind realtime '''
DELAY=600

'''
    Prelert Engine job configuration.
    The detector is configured to analyze the mean of the field named
    'Average' by the field 'metric_name' where the value of 'metric_name'
    is one of the AWS metrics e.g. CPUUtilization, DiskWriteOps, etc.
    The analysis is partition by AWS instance ID.
    bucketSpan is set the same as the CloudWatch reporting interval.
'''
JOB_CONFIG = '{"analysisConfig" : {\
                    "bucketSpan":' + str(UPDATE_INTERVAL) + ',\
                    "detectors" :[{"function":"mean","fieldName":"Average","byFieldName":"metric_name","partitionFieldName":"instance"}] \
                },\
                "dataDescription" : {"format":"JSON","timeField":"timestamp","timeFormat":"yyyy-MM-dd\'T\'HH:mm:ssX"} \
            }'

class MetricRecord:
    '''
    Simple holder class for the CloudWatch metrics.
    toJsonStr returns the metric in a format for the job
    configuration above.
    '''
    def __init__(self, timestamp, instance, metric_name, metric_value):
        self.timestamp = timestamp
        self.instance = instance
        self.metric_name = metric_name
        self.metric_value = metric_value

    def toJsonStr(self):
        result = '{"timestamp":"' + self.timestamp.isoformat() + \
            '", "instance":"' + self.instance + '", "metric_name":"' + \
            self.metric_name + '", "Average":' + str(self.metric_value) + '}'

        return result

class UTC(tzinfo):
    '''
    UTC timezone class
    '''

    def utcoffset(self, dt):
        return timedelta(0)

    def tzname(self, dt):
        return "UTC"

    def dst(self, dt):
        return timedelta(0)

def parseArguments():
    parser = argparse.ArgumentParser()

    parser.add_argument("config", help="The AWS connection parameters.")

    parser.add_argument("--api-host", help="The Prelert Engine API host, defaults to "
        + API_HOST, default=API_HOST, dest="api_host")
    parser.add_argument("--api-port", help="The Prelert Engine API port, defaults to "
        + str(API_PORT), default=API_PORT, dest="api_port")

    parser.add_argument("--job-id", help="Send data to this job. If not set a \
        new job will be created.", default=None, dest="job_id")

    parser.add_argument("--start-date", help="Request data from this date. If not \
        set then run in realtime mode. Dates must be in YYYY-MM-DD format",
        default=None, dest="start_date")
    parser.add_argument("--end-date", help="if --start-date is set then pull \
        and analyze only the metric data between those dates. \
        If --start-date is not set this argument has no meaning. \
        Dates must be in YYYY-MM-DD format",
        default=None, dest="end_date")

    return parser.parse_args()

def runHistorical(job_id, start_date, end_date, cloudwatch_conn, engine_client):
    '''
    Query and analyze the CloudWatch metrics from start_date to end_date.
    If end_date == None then run until the time now.
    '''

    end = start_date
    delta = timedelta(seconds=UPDATE_INTERVAL)

    while True:

        start = end
        end = start + delta

        end_condition = end_date if end_date != None else datetime.utcnow()
        if end > end_condition:
            break

        print "Querying metrics starting at time " + str(start.isoformat())

        try:
            metrics = cloudwatch_conn.list_metrics(namespace='AWS/EC2')
            metric_records = queryMetricRecords(metrics, start, end, reporting_interval = REPORTING_INTERVAL)

            data = ''
            for mr in metric_records:
                data += mr.toJsonStr() + '\n'

            (http_status, response) = engine_client.upload(job_id, data)
            if http_status != 202:
                print "Error uploading metric data to the Engine"
                print http_status, json.dumps(response)

        except BotoServerError as error:
            print "Error querying CloudWatch"
            print error

def queryMetricRecords(metrics, start, end, reporting_interval):
    metric_records = []
    for m in metrics:
        if 'InstanceId' not in m.dimensions:
            continue
        instance = m.dimensions['InstanceId'][0]

        datapoints = m.query(start, end, 'Average', period=reporting_interval)
        for dp in datapoints:
            # annoyingly Boto does not return datetimes with a timezone
            utc_time = replaceTimezoneWithUtc(dp['Timestamp'])
            mr = MetricRecord(utc_time, instance, m.name, dp['Average'])
            metric_records.append(mr)

        metric_records.sort(key=lambda r : r.timestamp)

    return metric_records

def runRealtime(job_id, cloudwatch_conn, engine_client):
    '''
    Query the previous 5 minutes of metric data every 5 minutes
    then upload to the Prelert Engine.

    This function runs in an infinite loop but will catch the
    keyboard interrupt (Ctrl C) and exit gracefully
    '''
    try:
        delay = timedelta(seconds=DELAY)
        end = datetime.utcnow() - delay - timedelta(seconds=UPDATE_INTERVAL)
        end = replaceTimezoneWithUtc(end)

        while True:

            start = end
            end = datetime.utcnow() - delay
            end = replaceTimezoneWithUtc(end)

            print "Querying metrics from " + str(start.isoformat())  + " to " + end.isoformat()

            try:
                metrics = cloudwatch_conn.list_metrics(namespace='AWS/EC2')
                metric_records = queryMetricRecords(metrics, start, end, reporting_interval = REPORTING_INTERVAL)

                data = ''
                for mr in metric_records:
                    data += mr.toJsonStr() + '\n'

                (http_status, response) = engine_client.upload(job_id, data)
                if http_status != 202:
                    print "Error uploading metric data to the Engine"
                    print http_status, json.dumps(response)

            except BotoServerError as error:
                print "Error querying CloudWatch"
                print error

            duration = datetime.utcnow() - delay - end
            sleep_time = max(UPDATE_INTERVAL - duration.seconds, 0)
            print "sleeping for " + str(sleep_time) + " seconds"
            if sleep_time > 0:
                time.sleep(sleep_time)

    except KeyboardInterrupt:
        print "Interrupt caught... terminating real time queries"
        return

def main():

    args = parseArguments()

    # read the config file
    config = ConfigParser.RawConfigParser()
    try:
        # insert a section header into the config so
        # ConfigParser will read it without complaint
        with open(args.config, "r") as config_file:
            ini_str = '[root]\n' + config_file.read()
            ini_fp = StringIO.StringIO(ini_str)
            config.readfp(ini_fp)
    except IOError:
        print "Error opening file " + args.config
        return

    try:
        region = config.get('root', 'region')
        access_id = config.get('root', 'aws_access_key_id')
        secret_key = config.get('root', 'aws_secret_access_key')
    except ConfigParser.NoOptionError as e:
        print e
        return

    # AWS CloudWatch connection
    cloudwatch_conn = boto.ec2.cloudwatch.connect_to_region(region,
                 aws_access_key_id=access_id,
                 aws_secret_access_key=secret_key)

    # The Prelert REST API client
    engine_client = EngineApiClient(args.api_host, API_BASE_URL, args.api_port)

    # If no job ID is supplied create a new job
    job_id = args.job_id
    if job_id == None:
        (http_status, response) = engine_client.createJob(JOB_CONFIG)
        if http_status != 201:
            print "Error creating job"
            print response
            return

        job_id = response['id']
        print "Created job with ID " + str(job_id)
    else:
        print "Using job ID " + job_id

    # default start date is None meaning run realtime
    start_date = None
    if args.start_date != None:
        start_date = datetime.strptime(args.start_date, "%Y-%m-%d")
        start_date = replaceTimezoneWithUtc(start_date)

    if start_date == None:
        runRealtime(job_id, cloudwatch_conn, engine_client)
    else:
        # historical mode, check for an end date
        end_date = None
        if args.end_date != None:
            end_date = datetime.strptime(args.end_date, "%Y-%m-%d")
            end_date = replaceTimezoneWithUtc(end_date)

        runHistorical(job_id, start_date, end_date, cloudwatch_conn, engine_client)

    print "Closing job..."
    engine_client.close(job_id)

def replaceTimezoneWithUtc(date):
    return date.replace(tzinfo=UTC())

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""Functions for finding the edges of the foreground in an
image-as-a-numpy-array"""

from __future__ import division
from collections import namedtuple
from itertools import tee
import numpy as np
from crash_kiss import util

def config(**kw_overrides):
    invalid = set(kw_overrides) - set(_config_defaults)
    if invalid:
        raise Exception("Invalid config keys: {0}".format(invalid))
    conf = dict(_config_defaults)
    conf.update(kw_overrides)
    return conf

_side_names = "left", "right", "up", "down"

_config_defaults = dict(
    neg_sample_size=5,
    threshold=7,
    bg_change_tolerance=3,
    relative_sides=("left", "right"),
    chunksize=100,
)

_orientors = dict(
    left=util.orient_left_to_right,
    right=util.orient_right_to_left,
    up=util.orient_up_to_down,
    down=util.orient_down_to_up,
)

class Subject(object):
    """Container of `Side` instances, which contain arrays that define the
    edges of the foreground of an image"""

    def __init__(self, img=None, config=config()):
        self._config = config
        self._img = img
        used_sides = config["relative_sides"]
        self._sides = tuple(self._make_side(side)
                            if side in used_sides else None
                            for side in _side_names)
        self._active_sides = filter(None, self._sides)
        self.left, self.right, self.up, self.down = self._sides
        self.img = img

    def _make_side(self, orientation):
        return Side(orientation, img=self.img, config=self._config)

    @property
    def img(self):
        return self._img

    @img.setter
    def img(self, img):
        self._img = img
        for side in self:
            side.img = img

    @property
    def background(self):
        return np.array([side.background for side in self])

    @background.setter
    def background(self, precomputed_background):
        for side in self:
            side.background = precomputed_background

    @property
    def edges(self):
        return np.array([side.edge for side in self])

    def __iter__(self):
        for side in self._active_sides:
            yield side

class Side(object):
    """A view of an image's subject from a certain perspective
    (left to right, up to down, etc)"""

    def __init__(self, orientation, img=None, config=config()):
        self._edge = self._view = self._background = self._img = None
        self._relative_side = orientation
        self._orient = _orientors[orientation]
        self.img = img
        self._config = config

    @property
    def img(self):
        return self._img

    @img.setter
    def img(self, new_img):
        if self._img is not None:
            self._view = None
        self._img = new_img

    @property
    def view(self):
        if self._view is None:
            self._view = self._orient(self.img)
        return self._view

    @property
    def edge(self):
        if self._edge is None:
            bg = self.background
            self._edge = get_edge(self.view, bg, self._config)
        return self._edge

    @property
    def background(self):
        if self._background is None:
            s_size = self._config["neg_sample_size"]
            self._background = get_background(self.view, s_size)
        return self._background

    @background.setter
    def background(self, precomupted_background):
        self._background = precomputed_background

    def __iter__(self):
        return iter(self.edge)

def get_background(img, sample_size):
    """Returns an array of median RGB values for the background of the image
    based on the values along the image's edges."""
    bg = np.median(img[::, :sample_size:], axis=1)
    bg = bg.reshape((bg.shape[0], 1, bg.shape[1]))
    return bg

@profile
def get_edge(img, background, config):
    """Finds the 'edge' of the subject of an image based on a background
    value or an array of background values. Returns an array of indices that
    represent the first non-background pixel moving from left to right in
    the image. Since it finds edges from left to right, it's up to the
    caller to pass in an appropriately inverted or rotated view of the image
    to account for this."""
    # The expensive things here are
    # 1) subtraction / comparison of the whole ndarray
    # 2) calling np.abs across the whole ndarray
    # 3) calling np.all across the whole ndarray
    # 4) repeatedly creating slice views of the whole ndarray
    # argmax is relatively cheap
    bg = _simplify_background(background, config)
    bg_is_array = isinstance(bg, np.ndarray)
    edge = np.zeros(img.shape[0], dtype=np.uint16)
    chunks = _column_blocks(img, config["chunksize"])
    for img_chunk, prev_idx in chunks:
        for img_slice, start, stop in _row_slices(img_chunk, edge):
            if bg_is_array:
                bg = background[start: stop]
            fg = _find_foreground(img_slice, bg, config)
            sub_edge = np.argmax(fg, axis=1)
            nz_sub_edge = sub_edge != 0
            sub_edge[nz_sub_edge] += prev_idx
            edge[start: stop] = sub_edge
    return edge

def _column_blocks(img, chunksize):
    n_cols = img.shape[1]
    chunksize = min(chunksize, n_cols)
    n_chunks = ((n_cols // chunksize) // 2) + 1
    chunks = [chunksize * n for n in range(1, n_chunks)]
    if not chunks or chunks[-1] < n_cols:
        chunks.append(n_cols - 1)  # n_cols - 1 + 1 == n_cols
    prev_idx = 0
    for idx in chunks:
        yield img[::, prev_idx: idx + 1], prev_idx
        prev_idx = idx

def _row_slices(img, edge):
    z_edge = edge == 0
    stop = 0
    n_rows = img.shape[0]
    while stop != n_rows:
        img_slice, start, stop = _get_contiguous_slice(img, z_edge, stop)
        yield img_slice, start, stop

def _get_contiguous_slice(img, z_edge, offset):
    z_edge = z_edge[offset:]
    start = np.argmax(z_edge)
    if not start and not z_edge[start]:
        raise StopIteration
    stop = np.argmin(z_edge[start:])
    start += offset
    if not stop:
        if z_edge[-1]:
            stop = img.shape[0]
        else:
            stop = start + 1
    else:
        stop += offset
    return img[start: stop], start, stop

@profile
def _find_foreground(img, background, config):
    """Find the foreground of the image by subracting each RGB element
    in the image by the background. If the background has been reduced
    to a simple int or float, we'll try to avoid calling `np.abs`
    by checking to see if the background value is near 0 or 255."""
    threshold = config["threshold"]
    is_num = isinstance(background, (float, int))
    if is_num and background < 50:
        diff = img - background > threshold
    elif is_num and background > 200:
        diff = background - img > threshold
    else:
        diff = np.abs((img - background) > threshold)
    return np.all(diff, axis=2)

def _simplify_background(background, config):
    """Here, we see if the background's RGB elements are similar.
    This way we can do a simple
    array - int operation instead of the more expensive array - [R, G, B]
    or the even pricier array - <array of shape (NROWS, 1, 3)> operation."""
    while isinstance(background, np.ndarray):
        bg_change_tolerance = config["bg_change_tolerance"]
        bmax, bmin = background.max(axis=0), background.min(axis=0)
        diff = (bmax - bmin) < bg_change_tolerance
        if len(background.shape) >= 2:
            diff = np.all(diff)
        if diff:
            background = np.median(background, axis=0)
        else:
            break  # we've simplified the background as much as we can
    return background

def _sliding_window(iterable, size):
    it = iter(iterable)
    win = deque((next(it, None) for _ in xrange(size)), maxlen=size)
    yield win
    append = win.append
    for e in it:
        append(e)
        yield win

<commit_msg>removed line profiler decorators<commit_after>"""Functions for finding the edges of the foreground in an
image-as-a-numpy-array"""

from __future__ import division
from collections import namedtuple
from itertools import tee
import numpy as np
from crash_kiss import util

def config(**kw_overrides):
    invalid = set(kw_overrides) - set(_config_defaults)
    if invalid:
        raise Exception("Invalid config keys: {0}".format(invalid))
    conf = dict(_config_defaults)
    conf.update(kw_overrides)
    return conf

_side_names = "left", "right", "up", "down"

_config_defaults = dict(
    neg_sample_size=5,
    threshold=7,
    bg_change_tolerance=3,
    relative_sides=("left", "right"),
    chunksize=100,
)

_orientors = dict(
    left=util.orient_left_to_right,
    right=util.orient_right_to_left,
    up=util.orient_up_to_down,
    down=util.orient_down_to_up,
)

class Subject(object):
    """Container of `Side` instances, which contain arrays that define the
    edges of the foreground of an image"""

    def __init__(self, img=None, config=config()):
        self._config = config
        self._img = img
        used_sides = config["relative_sides"]
        self._sides = tuple(self._make_side(side)
                            if side in used_sides else None
                            for side in _side_names)
        self._active_sides = filter(None, self._sides)
        self.left, self.right, self.up, self.down = self._sides
        self.img = img

    def _make_side(self, orientation):
        return Side(orientation, img=self.img, config=self._config)

    @property
    def img(self):
        return self._img

    @img.setter
    def img(self, img):
        self._img = img
        for side in self:
            side.img = img

    @property
    def background(self):
        return np.array([side.background for side in self])

    @background.setter
    def background(self, precomputed_background):
        for side in self:
            side.background = precomputed_background

    @property
    def edges(self):
        return np.array([side.edge for side in self])

    def __iter__(self):
        for side in self._active_sides:
            yield side

class Side(object):
    """A view of an image's subject from a certain perspective
    (left to right, up to down, etc)"""

    def __init__(self, orientation, img=None, config=config()):
        self._edge = self._view = self._background = self._img = None
        self._relative_side = orientation
        self._orient = _orientors[orientation]
        self.img = img
        self._config = config

    @property
    def img(self):
        return self._img

    @img.setter
    def img(self, new_img):
        if self._img is not None:
            self._view = None
        self._img = new_img

    @property
    def view(self):
        if self._view is None:
            self._view = self._orient(self.img)
        return self._view

    @property
    def edge(self):
        if self._edge is None:
            bg = self.background
            self._edge = get_edge(self.view, bg, self._config)
        return self._edge

    @property
    def background(self):
        if self._background is None:
            s_size = self._config["neg_sample_size"]
            self._background = get_background(self.view, s_size)
        return self._background

    @background.setter
    def background(self, precomupted_background):
        self._background = precomputed_background

    def __iter__(self):
        return iter(self.edge)

def get_background(img, sample_size):
    """Returns an array of median RGB values for the background of the image
    based on the values along the image's edges."""
    bg = np.median(img[::, :sample_size:], axis=1)
    bg = bg.reshape((bg.shape[0], 1, bg.shape[1]))
    return bg

def get_edge(img, background, config):
    """Finds the 'edge' of the subject of an image based on a background
    value or an array of background values. Returns an array of indices that
    represent the first non-background pixel moving from left to right in
    the image. Since it finds edges from left to right, it's up to the
    caller to pass in an appropriately inverted or rotated view of the image
    to account for this."""
    # The expensive things here are
    # 1) subtraction / comparison of the whole ndarray
    # 2) calling np.abs across the whole ndarray
    # 3) calling np.all across the whole ndarray
    # 4) repeatedly creating slice views of the whole ndarray
    # argmax is relatively cheap
    bg = _simplify_background(background, config)
    bg_is_array = isinstance(bg, np.ndarray)
    edge = np.zeros(img.shape[0], dtype=np.uint16)
    chunks = _column_blocks(img, config["chunksize"])
    for img_chunk, prev_idx in chunks:
        for img_slice, start, stop in _row_slices(img_chunk, edge):
            if bg_is_array:
                bg = background[start: stop]
            fg = _find_foreground(img_slice, bg, config)
            sub_edge = np.argmax(fg, axis=1)
            nz_sub_edge = sub_edge != 0
            sub_edge[nz_sub_edge] += prev_idx
            edge[start: stop] = sub_edge
    return edge

def _column_blocks(img, chunksize):
    n_cols = img.shape[1]
    chunksize = min(chunksize, n_cols)
    n_chunks = ((n_cols // chunksize) // 2) + 1
    chunks = [chunksize * n for n in range(1, n_chunks)]
    if not chunks or chunks[-1] < n_cols:
        chunks.append(n_cols - 1)  # n_cols - 1 + 1 == n_cols
    prev_idx = 0
    for idx in chunks:
        yield img[::, prev_idx: idx + 1], prev_idx
        prev_idx = idx

def _row_slices(img, edge):
    z_edge = edge == 0
    stop = 0
    n_rows = img.shape[0]
    while stop != n_rows:
        img_slice, start, stop = _get_contiguous_slice(img, z_edge, stop)
        yield img_slice, start, stop

def _get_contiguous_slice(img, z_edge, offset):
    z_edge = z_edge[offset:]
    start = np.argmax(z_edge)
    if not start and not z_edge[start]:
        raise StopIteration
    stop = np.argmin(z_edge[start:])
    start += offset
    if not stop:
        if z_edge[-1]:
            stop = img.shape[0]
        else:
            stop = start + 1
    else:
        stop += offset
    return img[start: stop], start, stop

def _find_foreground(img, background, config):
    """Find the foreground of the image by subracting each RGB element
    in the image by the background. If the background has been reduced
    to a simple int or float, we'll try to avoid calling `np.abs`
    by checking to see if the background value is near 0 or 255."""
    threshold = config["threshold"]
    is_num = isinstance(background, (float, int))
    if is_num and background < 50:
        diff = img - background > threshold
    elif is_num and background > 200:
        diff = background - img > threshold
    else:
        diff = np.abs((img - background) > threshold)
    return np.all(diff, axis=2)

def _simplify_background(background, config):
    """Here, we see if the background's RGB elements are similar.
    This way we can do a simple
    array - int operation instead of the more expensive array - [R, G, B]
    or the even pricier array - <array of shape (NROWS, 1, 3)> operation."""
    while isinstance(background, np.ndarray):
        bg_change_tolerance = config["bg_change_tolerance"]
        bmax, bmin = background.max(axis=0), background.min(axis=0)
        diff = (bmax - bmin) < bg_change_tolerance
        if len(background.shape) >= 2:
            diff = np.all(diff)
        if diff:
            background = np.median(background, axis=0)
        else:
            break  # we've simplified the background as much as we can
    return background

def _sliding_window(iterable, size):
    it = iter(iterable)
    win = deque((next(it, None) for _ in xrange(size)), maxlen=size)
    yield win
    append = win.append
    for e in it:
        append(e)
        yield win

<|endoftext|>