<commit_before># -*- coding: utf-8 -*-

"""
Kay application.

:Copyright: (c) 2009 Accense Technology, Inc. All rights reserved.
:license: BSD, see LICENSE for more details.
"""

import sys
import os
import logging

from werkzeug import (
  Request, ClosingIterator, DispatcherMiddleware,
)
from werkzeug.exceptions import (
  HTTPException, InternalServerError, NotFound
)
from werkzeug import (
  Response, redirect
)
from jinja2 import (
  Environment, Undefined,
)
from werkzeug.routing import Submount
from google.appengine.runtime.apiproxy_errors import CapabilityDisabledError

import kay
from kay.utils import (
  local, local_manager, reverse, render_to_string,
)
from kay.utils.importlib import import_module
from kay import (
  utils, exceptions, mail,
)
from kay.utils.filters import nl2br

from kay.conf import settings, _settings, LazySettings

translations_cache = {}

def model_name_from_key(key):
  return key.path().element_list()[0].type()

def db_hook(service, call, request, response):
  if call == 'Put':
    from kay.utils.db_hook import execute_hooks
    for key, entity in zip(response.key_list(), request.entity_list()):
      kind = model_name_from_key(key)
      execute_hooks(kind, key, entity)
  elif call == 'Commit':
    from kay.utils.db_hook import execute_reserved_hooks
    execute_reserved_hooks()
  elif call == 'Rollback' or call == 'BeginTransaction':
    from kay.utils.db_hook import clear_reserved_hooks
    clear_reserved_hooks()

def get_application(settings=_settings):
  application = KayApp(settings)
  submount_apps = {}
  for app_name in settings.SUBMOUNT_APPS:
    app = KayApp(LazySettings('%s.settings' % app_name))
    submount_apps['/%s' % app_name] = app
  application = DispatcherMiddleware(application, submount_apps)
  return application

class NullUndefined(Undefined):
  """
  Do nothing except for logging when the specified variable doesn't exist.
  """
  __slots__ = ()
  def __int__(self):
    return 0
  def __getattr__(self, value):
    return u''
  def __html__(self):
    self.debug_log()
    return u''
  def debug_log(self):
    f = sys._getframe(1)
    while not 'templates' in f.f_code.co_filename:
      f = f.f_back
    logging.warn("%s: %s is undefined." %
                 (f.f_code.co_filename, self._undefined_name))

def get_app_tailname(app):
  dot = app.rfind('.')
  if dot >= 0:
    return app[dot+1:]
  else:
    return app

class KayApp(object):

  def __init__(self, app_settings):
    self.app_settings = app_settings
    self.url_map = None
    self.views = None
    self._request_middleware = self._response_middleware = \
        self._view_middleware = self._exception_middleware = None
    self.auth_backend = None
    self.init_jinja2_environ()

  def get_mount_point(self, app):
    if app == 'kay._internal':
      return '/_kay'
    return self.app_settings.APP_MOUNT_POINTS.get(
      app, "/%s" % get_app_tailname(app))

  def get_installed_apps(self):
    return self.app_settings.INSTALLED_APPS+['kay._internal']

  def init_url_map(self):
    self.has_error_on_init_url_map = False
    mod = import_module(self.app_settings.ROOT_URL_MODULE)

    make_url = getattr(mod, 'make_url')
    all_views = getattr(mod, 'all_views')
    self.views = all_views
    self.url_map = make_url()
    for app in self.get_installed_apps():
      try:
        url_mod = import_module("%s.urls" % app)
      except ImportError:
        logging.warning("Failed to import app '%s.urls', skipped." % app)
        logging.debug("Reason:\n%s" % self._get_traceback(sys.exc_info()))
        continue
      mountpoint = self.get_mount_point(app)
      make_rules = getattr(url_mod, 'make_rules', None)
      if make_rules:
        self.url_map.add(Submount(mountpoint, make_rules()))
      all_views = getattr(url_mod, 'all_views', None)
      if all_views:
        self.views.update(all_views)
    if 'kay.auth.middleware.AuthenticationMiddleware' in \
          self.app_settings.MIDDLEWARE_CLASSES:
      try:
        dot = self.app_settings.AUTH_USER_BACKEND.rindex('.')
      except ValueError:
        raise exceptions.ImproperlyConfigured, \
            'Error importing auth backend %s: "%s"' % \
            (self.app_settings.AUTH_USER_BACKEND, e)
      backend_module, backend_classname = \
          self.app_settings.AUTH_USER_BACKEND[:dot], \
          self.app_settings.AUTH_USER_BACKEND[dot+1:]
      try:
        mod = import_module(backend_module)
      except ImportError, e:
        raise exceptions.ImproperlyConfigured, \
            'Error importing auth backend %s: "%s"' % (backend_module, e)
      try:
        klass = getattr(mod, backend_classname)
      except AttributeError:
        raise exceptions.ImproperlyConfigured, \
            'Auth backend module "%s" does not define a "%s" class' % \
            (backend_module, backend_classname)
      self.auth_backend = klass()

  def init_jinja2_environ(self):
    """
    Initialize the environment for jinja2.
    TODO: Capability to disable i18n stuff.
    TODO: Pluggable utility mechanism.
    """
    if 'SERVER_SOFTWARE' in os.environ and \
          os.environ['SERVER_SOFTWARE'].startswith('Dev'):
      from jinja2 import (FileSystemLoader, ChoiceLoader, PrefixLoader,)
      template_dirname = "templates"
    else:
      from kay.utils.jinja2utils.code_loaders import FileSystemCodeLoader as \
          FileSystemLoader
      from kay.utils.jinja2utils.code_loaders import ChoiceCodeLoader as \
          ChoiceLoader
      from kay.utils.jinja2utils.code_loaders import PrefixCodeLoader as \
          PrefixLoader
      template_dirname = "templates_compiled"
    per_app_loaders = {}
    for app in self.get_installed_apps():
      try:
        mod = import_module(app)
      except ImportError:
        logging.warning("Failed to import app '%s', skipped." % app)
        continue
      try:
        app_key = getattr(mod, 'template_loader_key')
      except AttributeError:
        app_key = get_app_tailname(app)
      per_app_loaders[app_key] = FileSystemLoader(
        os.path.join(os.path.dirname(mod.__file__), template_dirname))
    loader = PrefixLoader(per_app_loaders)
    if self.app_settings.TEMPLATE_DIRS:
      target = [d.replace("templates", template_dirname)
                for d in self.app_settings.TEMPLATE_DIRS]
      import kay
      base_loader = FileSystemLoader(
        [os.path.join(kay.PROJECT_DIR, d) for d in target])
      loader = ChoiceLoader([base_loader, loader])
    env_dict = dict(
      loader = loader,
      autoescape=True,
      undefined=NullUndefined,
      extensions=['jinja2.ext.i18n'],
    )
    self.jinja2_env = Environment(**env_dict)
    for key, filter_str in self.app_settings.JINJA2_FILTERS.iteritems():
      try:
        dot = filter_str.rindex('.')
      except ValueError:
        logging.warn('%s is invalid for JINJA2_FILTERS.' % filter_str)
        continue
      filter_mod, filter_func = filter_str[:dot], filter_str[dot+1:]
      try:
        mod = import_module(filter_mod)
      except ImportError, e:
        logging.warn('Error importing auth backend %s: "%s"' % (filter_mod, e))
        continue
      try:
        func = getattr(mod, filter_func)
      except AttributeError:
        logging.warn('Module "%s" does not define a "%s" func' %
                     (mod, filter_func))
        continue
      if self.jinja2_env.filters.has_key(key):
        logging.warn('Key "%s" has already defined, skipped.' % key)
        continue
      self.jinja2_env.filters[key] = func

  def init_lang(self, lang):
    """
    Initialize translations with specified language.
    """
    from kay.i18n import load_translations
    global translations_cache
    if self.app_settings.USE_I18N:
      translations = translations_cache.get("%s:%s" %
                                            (self.app_settings.APP_NAME, lang),
                                            None)
      if translations is None:
        translations = load_translations(lang)
        translations_cache["trans:%s:%s" %
                     (self.app_settings.APP_NAME, lang)] = translations
      self.active_translations = translations
      self.jinja2_env.install_gettext_translations(translations)
    else:
      from kay.i18n import KayNullTranslations
      self.active_translations = KayNullTranslations()
      self.jinja2_env.install_null_translations()

  def load_middleware(self):
    self._response_middleware = []
    self._view_middleware = []
    self._exception_middleware = []
    request_middleware = []
    for mw_path in self.app_settings.MIDDLEWARE_CLASSES:
      try:
        dot = mw_path.rindex('.')
      except ValueError:
        raise exceptions.ImproperlyConfigured, \
            '%s isn\'t a middleware module' % mw_path
      mw_module, mw_classname = mw_path[:dot], mw_path[dot+1:]
      try:
        mod = import_module(mw_module)
      except ImportError, e:
        raise exceptions.ImproperlyConfigured, \
            'Error importing middleware %s: "%s"' % (mw_module, e)
      try:
        mw_class = getattr(mod, mw_classname)
      except AttributeError:
        raise exceptions.ImproperlyConfigured, \
            'Middleware module "%s" does not define a "%s" class' % \
            (mw_module, mw_classname)
      try:
        mw_instance = mw_class()
      except exceptions.MiddlewareNotUsed:
        continue

      if hasattr(mw_instance, 'process_request'):
        request_middleware.append(mw_instance.process_request)
      if hasattr(mw_instance, 'process_view'):
        self._view_middleware.append(mw_instance.process_view)
      if hasattr(mw_instance, 'process_response'):
        self._response_middleware.insert(0, mw_instance.process_response)
      if hasattr(mw_instance, 'process_exception'):
        self._exception_middleware.insert(0, mw_instance.process_exception)

    # We only assign to this when initialization is complete as it is used
    # as a flag for initialization being complete.
    self._request_middleware = request_middleware

  def get_response(self, request):
    if self._request_middleware is None:
      self.load_middleware()
    # apply request middleware
    for mw_method in self._request_middleware:
      response = mw_method(request)
      if response:
        return response

    lang = (request.accept_languages.best or
            self.app_settings.DEFAULT_LANG)
    pos = lang.find('-')
    if pos >= 0:
      lang = lang[:pos].lower()+'_'+lang[pos+1:].upper()
    else:
      lang = lang.lower()
    self.init_lang(lang)

    try:
      endpoint, values = local.url_adapter.match()
      view_func = self.views.get(endpoint, None)
      if view_func is None:
        raise NotFound
      for mw_method in self._view_middleware:
        response = mw_method(request, view_func, **values)
        if response:
          return response
      try:
        response = view_func(request, **values)
      except Exception, e:
        # If the view raised an exception, run it through exception
        # middleware, and if the exception middleware returns a
        # response, use that. Otherwise, reraise the exception.
        for middleware_method in self._exception_middleware:
          response = middleware_method(request, e)
          if response:
            return response
        raise
    except HTTPException, e:
      logging.warning(e)
      response = e
    except SystemExit:
      # Allow sys.exit() to actually exit.
      raise
    except CapabilityDisabledError, e:
      logging.error(e)
      # Saving session will also fail.
      if hasattr(request, 'session'):
        del(request.session)
      return Response(
        render_to_string(
          "_internal/maintenance.html",
          {"message": _('Appengine might be under maintenance.')}),
        status=503)
    except: # Handle everything else, including SuspiciousOperation, etc.
      # Get the exception info now, in case another exception is thrown later.
      exc_info = sys.exc_info()
      return self.handle_uncaught_exception(request, exc_info)
    return response

  def handle_uncaught_exception(self, request, exc_info):
    import os
    if 'SERVER_SOFTWARE' in os.environ and \
          os.environ['SERVER_SOFTWARE'].startswith('Dev'):
      raise
    else:
      subject = 'Error %s: %s' % (request.remote_addr, request.path)
      try:
        from kay.utils import repr
        request_repr = repr.dump(request)
      except Exception, e:
        request_repr = "Request repr() unavailable"
      message = "%s\n\n%s" % (self._get_traceback(exc_info), request_repr)
      logging.error(message)
      mail.mail_admins(subject, message, fail_silently=True)
      # TODO: Return an HttpResponse that displays a friendly error message.
      return InternalServerError()

  def _get_traceback(self, exc_info):
    "Helper function to return the traceback as a string"
    import traceback
    return '\n'.join(traceback.format_exception(*(exc_info or sys.exc_info())))

  def __call__(self, environ, start_response):
    kay.setup_syspath()
    if _settings.USE_DB_HOOK:
      from google.appengine.api import apiproxy_stub_map
      apiproxy_stub_map.apiproxy.GetPostCallHooks().Clear()
      apiproxy_stub_map.apiproxy.GetPostCallHooks().Append(
        'db_hook', db_hook, 'datastore_v3')
    local.app = self
    local.request = request = Request(environ)
    if self.url_map is None or self.has_error_on_init_url_map:
      try:
        self.init_url_map()
      except (StandardError, exceptions.ImproperlyConfigured):
        self.has_error_on_init_url_map = True
        raise
    local.url_adapter = self.url_map.bind_to_environ(environ)

    response = self.get_response(request)

    for middleware_method in self._response_middleware:
      response = middleware_method(request, response)

    return ClosingIterator(response(environ, start_response),
        [local_manager.cleanup])

<commit_msg>Display tracebacks when DEBUG=True.<commit_after># -*- coding: utf-8 -*-

"""
Kay application.

:Copyright: (c) 2009 Accense Technology, Inc. All rights reserved.
:license: BSD, see LICENSE for more details.
"""

import sys
import os
import logging

from werkzeug import (
  Request, ClosingIterator, DispatcherMiddleware,
)
from werkzeug.exceptions import (
  HTTPException, InternalServerError, NotFound
)
from werkzeug import (
  Response, redirect
)
from jinja2 import (
  Environment, Undefined,
)
from werkzeug.routing import Submount
from google.appengine.runtime.apiproxy_errors import CapabilityDisabledError

import kay
from kay.utils import (
  local, local_manager, reverse, render_to_string,
)
from kay.utils.importlib import import_module
from kay import (
  utils, exceptions, mail,
)
from kay.utils.filters import nl2br

from kay.conf import settings, _settings, LazySettings

translations_cache = {}

def model_name_from_key(key):
  return key.path().element_list()[0].type()

def db_hook(service, call, request, response):
  if call == 'Put':
    from kay.utils.db_hook import execute_hooks
    for key, entity in zip(response.key_list(), request.entity_list()):
      kind = model_name_from_key(key)
      execute_hooks(kind, key, entity)
  elif call == 'Commit':
    from kay.utils.db_hook import execute_reserved_hooks
    execute_reserved_hooks()
  elif call == 'Rollback' or call == 'BeginTransaction':
    from kay.utils.db_hook import clear_reserved_hooks
    clear_reserved_hooks()

def get_application(settings=_settings):
  application = KayApp(settings)
  submount_apps = {}
  for app_name in settings.SUBMOUNT_APPS:
    app = KayApp(LazySettings('%s.settings' % app_name))
    submount_apps['/%s' % app_name] = app
  application = DispatcherMiddleware(application, submount_apps)
  return application

class NullUndefined(Undefined):
  """
  Do nothing except for logging when the specified variable doesn't exist.
  """
  __slots__ = ()
  def __int__(self):
    return 0
  def __getattr__(self, value):
    return u''
  def __html__(self):
    self.debug_log()
    return u''
  def debug_log(self):
    f = sys._getframe(1)
    while not 'templates' in f.f_code.co_filename:
      f = f.f_back
    logging.warn("%s: %s is undefined." %
                 (f.f_code.co_filename, self._undefined_name))

def get_app_tailname(app):
  dot = app.rfind('.')
  if dot >= 0:
    return app[dot+1:]
  else:
    return app

class KayApp(object):

  def __init__(self, app_settings):
    self.app_settings = app_settings
    self.url_map = None
    self.views = None
    self._request_middleware = self._response_middleware = \
        self._view_middleware = self._exception_middleware = None
    self.auth_backend = None
    self.init_jinja2_environ()

  def get_mount_point(self, app):
    if app == 'kay._internal':
      return '/_kay'
    return self.app_settings.APP_MOUNT_POINTS.get(
      app, "/%s" % get_app_tailname(app))

  def get_installed_apps(self):
    return self.app_settings.INSTALLED_APPS+['kay._internal']

  def init_url_map(self):
    self.has_error_on_init_url_map = False
    mod = import_module(self.app_settings.ROOT_URL_MODULE)

    make_url = getattr(mod, 'make_url')
    all_views = getattr(mod, 'all_views')
    self.views = all_views
    self.url_map = make_url()
    for app in self.get_installed_apps():
      try:
        url_mod = import_module("%s.urls" % app)
      except ImportError:
        logging.warning("Failed to import app '%s.urls', skipped." % app)
        logging.debug("Reason:\n%s" % self._get_traceback(sys.exc_info()))
        continue
      mountpoint = self.get_mount_point(app)
      make_rules = getattr(url_mod, 'make_rules', None)
      if make_rules:
        self.url_map.add(Submount(mountpoint, make_rules()))
      all_views = getattr(url_mod, 'all_views', None)
      if all_views:
        self.views.update(all_views)
    if 'kay.auth.middleware.AuthenticationMiddleware' in \
          self.app_settings.MIDDLEWARE_CLASSES:
      try:
        dot = self.app_settings.AUTH_USER_BACKEND.rindex('.')
      except ValueError:
        raise exceptions.ImproperlyConfigured, \
            'Error importing auth backend %s: "%s"' % \
            (self.app_settings.AUTH_USER_BACKEND, e)
      backend_module, backend_classname = \
          self.app_settings.AUTH_USER_BACKEND[:dot], \
          self.app_settings.AUTH_USER_BACKEND[dot+1:]
      try:
        mod = import_module(backend_module)
      except ImportError, e:
        raise exceptions.ImproperlyConfigured, \
            'Error importing auth backend %s: "%s"' % (backend_module, e)
      try:
        klass = getattr(mod, backend_classname)
      except AttributeError:
        raise exceptions.ImproperlyConfigured, \
            'Auth backend module "%s" does not define a "%s" class' % \
            (backend_module, backend_classname)
      self.auth_backend = klass()

  def init_jinja2_environ(self):
    """
    Initialize the environment for jinja2.
    TODO: Capability to disable i18n stuff.
    TODO: Pluggable utility mechanism.
    """
    if 'SERVER_SOFTWARE' in os.environ and \
          os.environ['SERVER_SOFTWARE'].startswith('Dev'):
      from jinja2 import (FileSystemLoader, ChoiceLoader, PrefixLoader,)
      template_dirname = "templates"
    else:
      from kay.utils.jinja2utils.code_loaders import FileSystemCodeLoader as \
          FileSystemLoader
      from kay.utils.jinja2utils.code_loaders import ChoiceCodeLoader as \
          ChoiceLoader
      from kay.utils.jinja2utils.code_loaders import PrefixCodeLoader as \
          PrefixLoader
      template_dirname = "templates_compiled"
    per_app_loaders = {}
    for app in self.get_installed_apps():
      try:
        mod = import_module(app)
      except ImportError:
        logging.warning("Failed to import app '%s', skipped." % app)
        continue
      try:
        app_key = getattr(mod, 'template_loader_key')
      except AttributeError:
        app_key = get_app_tailname(app)
      per_app_loaders[app_key] = FileSystemLoader(
        os.path.join(os.path.dirname(mod.__file__), template_dirname))
    loader = PrefixLoader(per_app_loaders)
    if self.app_settings.TEMPLATE_DIRS:
      target = [d.replace("templates", template_dirname)
                for d in self.app_settings.TEMPLATE_DIRS]
      import kay
      base_loader = FileSystemLoader(
        [os.path.join(kay.PROJECT_DIR, d) for d in target])
      loader = ChoiceLoader([base_loader, loader])
    env_dict = dict(
      loader = loader,
      autoescape=True,
      undefined=NullUndefined,
      extensions=['jinja2.ext.i18n'],
    )
    self.jinja2_env = Environment(**env_dict)
    for key, filter_str in self.app_settings.JINJA2_FILTERS.iteritems():
      try:
        dot = filter_str.rindex('.')
      except ValueError:
        logging.warn('%s is invalid for JINJA2_FILTERS.' % filter_str)
        continue
      filter_mod, filter_func = filter_str[:dot], filter_str[dot+1:]
      try:
        mod = import_module(filter_mod)
      except ImportError, e:
        logging.warn('Error importing auth backend %s: "%s"' % (filter_mod, e))
        continue
      try:
        func = getattr(mod, filter_func)
      except AttributeError:
        logging.warn('Module "%s" does not define a "%s" func' %
                     (mod, filter_func))
        continue
      if self.jinja2_env.filters.has_key(key):
        logging.warn('Key "%s" has already defined, skipped.' % key)
        continue
      self.jinja2_env.filters[key] = func

  def init_lang(self, lang):
    """
    Initialize translations with specified language.
    """
    from kay.i18n import load_translations
    global translations_cache
    if self.app_settings.USE_I18N:
      translations = translations_cache.get("%s:%s" %
                                            (self.app_settings.APP_NAME, lang),
                                            None)
      if translations is None:
        translations = load_translations(lang)
        translations_cache["trans:%s:%s" %
                     (self.app_settings.APP_NAME, lang)] = translations
      self.active_translations = translations
      self.jinja2_env.install_gettext_translations(translations)
    else:
      from kay.i18n import KayNullTranslations
      self.active_translations = KayNullTranslations()
      self.jinja2_env.install_null_translations()

  def load_middleware(self):
    self._response_middleware = []
    self._view_middleware = []
    self._exception_middleware = []
    request_middleware = []
    for mw_path in self.app_settings.MIDDLEWARE_CLASSES:
      try:
        dot = mw_path.rindex('.')
      except ValueError:
        raise exceptions.ImproperlyConfigured, \
            '%s isn\'t a middleware module' % mw_path
      mw_module, mw_classname = mw_path[:dot], mw_path[dot+1:]
      try:
        mod = import_module(mw_module)
      except ImportError, e:
        raise exceptions.ImproperlyConfigured, \
            'Error importing middleware %s: "%s"' % (mw_module, e)
      try:
        mw_class = getattr(mod, mw_classname)
      except AttributeError:
        raise exceptions.ImproperlyConfigured, \
            'Middleware module "%s" does not define a "%s" class' % \
            (mw_module, mw_classname)
      try:
        mw_instance = mw_class()
      except exceptions.MiddlewareNotUsed:
        continue

      if hasattr(mw_instance, 'process_request'):
        request_middleware.append(mw_instance.process_request)
      if hasattr(mw_instance, 'process_view'):
        self._view_middleware.append(mw_instance.process_view)
      if hasattr(mw_instance, 'process_response'):
        self._response_middleware.insert(0, mw_instance.process_response)
      if hasattr(mw_instance, 'process_exception'):
        self._exception_middleware.insert(0, mw_instance.process_exception)

    # We only assign to this when initialization is complete as it is used
    # as a flag for initialization being complete.
    self._request_middleware = request_middleware

  def get_response(self, request):
    if self._request_middleware is None:
      self.load_middleware()
    # apply request middleware
    for mw_method in self._request_middleware:
      response = mw_method(request)
      if response:
        return response

    lang = (request.accept_languages.best or
            self.app_settings.DEFAULT_LANG)
    pos = lang.find('-')
    if pos >= 0:
      lang = lang[:pos].lower()+'_'+lang[pos+1:].upper()
    else:
      lang = lang.lower()
    self.init_lang(lang)

    try:
      endpoint, values = local.url_adapter.match()
      view_func = self.views.get(endpoint, None)
      if view_func is None:
        raise NotFound
      for mw_method in self._view_middleware:
        response = mw_method(request, view_func, **values)
        if response:
          return response
      try:
        response = view_func(request, **values)
      except Exception, e:
        # If the view raised an exception, run it through exception
        # middleware, and if the exception middleware returns a
        # response, use that. Otherwise, reraise the exception.
        for middleware_method in self._exception_middleware:
          response = middleware_method(request, e)
          if response:
            return response
        raise
    except HTTPException, e:
      logging.warning(e)
      response = e
    except SystemExit:
      # Allow sys.exit() to actually exit.
      raise
    except CapabilityDisabledError, e:
      logging.error(e)
      # Saving session will also fail.
      if hasattr(request, 'session'):
        del(request.session)
      return Response(
        render_to_string(
          "_internal/maintenance.html",
          {"message": _('Appengine might be under maintenance.')}),
        status=503)
    except: # Handle everything else, including SuspiciousOperation, etc.
      # Get the exception info now, in case another exception is thrown later.
      exc_info = sys.exc_info()
      return self.handle_uncaught_exception(request, exc_info)
    return response

  def handle_uncaught_exception(self, request, exc_info):
    import os
    if 'SERVER_SOFTWARE' in os.environ and \
          os.environ['SERVER_SOFTWARE'].startswith('Dev'):
      raise
    else:
      subject = 'Error %s: %s' % (request.remote_addr, request.path)
      try:
        from kay.utils import repr
        request_repr = repr.dump(request)
      except Exception, e:
        request_repr = "Request repr() unavailable"
      message = "%s\n\n%s" % (self._get_traceback(exc_info), request_repr)
      logging.error(message)
      if self.app_settings.DEBUG:
        return InternalServerError(message.replace("\n", "<br/>\n"))
      else:
        mail.mail_admins(subject, message, fail_silently=True)
        # TODO: Return an HttpResponse that displays a friendly error message.
        return InternalServerError()

  def _get_traceback(self, exc_info):
    "Helper function to return the traceback as a string"
    import traceback
    return '\n'.join(traceback.format_exception(*(exc_info or sys.exc_info())))

  def __call__(self, environ, start_response):
    kay.setup_syspath()
    if _settings.USE_DB_HOOK:
      from google.appengine.api import apiproxy_stub_map
      apiproxy_stub_map.apiproxy.GetPostCallHooks().Clear()
      apiproxy_stub_map.apiproxy.GetPostCallHooks().Append(
        'db_hook', db_hook, 'datastore_v3')
    local.app = self
    local.request = request = Request(environ)
    if self.url_map is None or self.has_error_on_init_url_map:
      try:
        self.init_url_map()
      except (StandardError, exceptions.ImproperlyConfigured):
        self.has_error_on_init_url_map = True
        raise
    local.url_adapter = self.url_map.bind_to_environ(environ)

    response = self.get_response(request)

    for middleware_method in self._response_middleware:
      response = middleware_method(request, response)

    return ClosingIterator(response(environ, start_response),
        [local_manager.cleanup])

<|endoftext|>


<commit_before># Module:   env
# Date:     11th September 2008
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""env - System Environment

System environment that acts as a container of objects in the system for
easier access by other parts of the system including plugins.
Every plugin is passed an instnace of this environment.
"""

import os
import sys
import socket
import inspect
import weakref
from time import time
from traceback import format_exc

from pymills.utils import safe__import__

from circuits.lib import config
from circuits.lib.env import Environment
from circuits import Event, Component, Bridge, Debugger

from circuits.lib.log import (
        Info as LogInfo,
        Debug as LogDebug,
        Error as LogError,
        Warning as LogWarning,
        Exception as LogException)

import defaults
from bot import Bot
from plugin import BasePlugin

from db import (
        Databases,
        Load as LoadDatabases,
        Create as CreateDatabases)

from __init__ import (
        __name__ as systemName,
        __description__ as systemDesc,
        __version__ as systemVersion)

class SystemEnvironment(Environment):

    version = 1
    name = systemName

    def created(self):
        for section in defaults.CONFIG:
            if not self.config.has_section(section):
                self.config.add_section(section)
            for option, value in defaults.CONFIG[section].iteritems():
                if type(value) == str:
                    value = value % {"name": self.name}
                self.config.set(section, option, value)
        self.send(config.Save(), "save", "config")

        self.db = Databases(self)
        self.manager += self.db
        self.send(CreateDatabases(), "create", "db")

    def loaded(self):
        self.db = Databases(self)
        self.manager += self.db
        self.send(LoadDatabases(), "load", "db")

        self.debug = self.config.getboolean("logging", "debug", False)
        self.verbose = self.config.getboolean("logging", "verbose", False)

        self.plugins = weakref.WeakValueDictionary()

        self.errors = 0
        self.events = 0
        self.sTime = time()

        port = self.config.getint("server", "port", 80)
        address = self.config.get("server", "address", "0.0.0.0")
        ssl = self.config.getboolean("server", "ssl", False)
        bind = self.config.get("server", "bind", None)

        auth = {
                "host": socket.gethostname(),
                "server": address,
                "nick": self.config.get("bot", "nick", systemName),
                "ident": self.config.get("bot", "ident", systemName),
                "name": self.config.get("bot", "name", systemDesc)
        }
        if self.config.has_option("server", "password"):
            auth["password"] = self.config.get("server", "password")

        self.bot = Bot(self, port, address, ssl, bind, auth)
        self.manager += self.bot

        port = self.config.getint("bridge", "port", 8000)
        address = self.config.get("bridge", "address", "0.0.0.0")
        self.bridge = Bridge(port, address, channel="bridge")
        self.manager += self.bridge

        if self.debug:
            self.manager += Debugger(self.verbose, True, self.log)

    def loadPlugin(self, plugin):
        """E.loadPlugin(plugin) -> None

        Load a single plugin given by plugin.
        If this plugin is already laoded, it'll be
        replaced.
        """

        if plugin in self.plugins:
            self.push(
                    LogWarning("Not loading plugin '%s' - Already loaded!" % plugin),
                    "warning", "log")
            return False

        try:
            fqplugin = "kdb.plugins.%s" % plugin
            if sys.modules.has_key(fqplugin):
                try:
                    reload(sys.modules[fqplugin])
                except Exception, err:
                    self.push(
                            LogError("Problem reloading plugin '%s'" % plugins),
                            "error", "log")
                    self.push(LogException(err), "exception", "log")
                    self.push(LogDebug(format_exc()), "debug", "log")
                    return False

            m = safe__import__("plugins.%s" % plugin,
                    globals(), locals(), "kdb")

            classes = inspect.getmembers(m,
                    lambda x: inspect.isclass(x) and
                    issubclass(x, BasePlugin) and
                    not x == BasePlugin)
            for name, c in classes:
                o = c(self, self.bot)
                self.manager += o
                self.plugins[plugin] = o
            self.push(LogInfo("Loaded plugin: %s" % plugin), "info", "log")
            return True
        except Exception, err:
            self.push(
                    LogError("Problem loading plugin '%s'" % plugin),
                    "error", "log")
            self.push(LogException(err), "exception", "log")
            self.push(LogDebug(format_exc()), "debug", "log")
            return False

    def unloadPlugin(self, plugin):
        """E.unloadPlugin(plugin) -> None

        Unload the specified plugin if it has been loaded.
        """

        if self.plugins.has_key(plugin):
            o = self.plugins[plugin]
            o.unregister()
            if hasattr(o, "cleanup"):
                o.cleanup()
            del self.plugins[plugin]
            self.push(LogInfo("Unloaded plugin: %s" % plugin), "info", "log")

    def loadPlugins(self):
        """E.loadPlugins() -> None

        Load any available plugins loading the global/default
        ones first, then loading the ones found in the
        environment. Plugins found in the environment
        may override existing plugins already loaded.
        """

        plugins = list(defaults.PLUGINS)

        if self.config.has_section("plugins"):
            for name, value in self.config.items("plugins"):
                plugin, comp = name.split(".")
                if value.lower() == "enabled":
                    if plugin not in plugins:
                        plugins.append(plugin)

        for plugin in plugins:
            try:
                self.loadPlugin(plugin)
            except Exception, err:
                self.errors += 1
                self.push(
                        LogError("Failed to load plugin: %s" % plugin),
                        "error", "log")
                self.push(LogException(err), "exception", "log")
                self.push(LogDebug(format_exc()), "debug", "log")

    def unloadPlugins(self):
        """E.unloadPlugins() -> None

        Unload all loaded plugins calling their cleanup
        methods if they exist.
        """

        for plugin in self.plugins.copy():
            self.unloadPlugin(plugin)
<commit_msg>env: Fixed logger argument to Debugger<commit_after># Module:   env
# Date:     11th September 2008
# Author:   James Mills, prologic at shortcircuit dot net dot au

"""env - System Environment

System environment that acts as a container of objects in the system for
easier access by other parts of the system including plugins.
Every plugin is passed an instnace of this environment.
"""

import os
import sys
import socket
import inspect
import weakref
from time import time
from traceback import format_exc

from pymills.utils import safe__import__

from circuits.lib import config
from circuits.lib.env import Environment
from circuits import Event, Component, Bridge, Debugger

from circuits.lib.log import (
        Info as LogInfo,
        Debug as LogDebug,
        Error as LogError,
        Warning as LogWarning,
        Exception as LogException)

import defaults
from bot import Bot
from plugin import BasePlugin

from db import (
        Databases,
        Load as LoadDatabases,
        Create as CreateDatabases)

from __init__ import (
        __name__ as systemName,
        __description__ as systemDesc,
        __version__ as systemVersion)

class SystemEnvironment(Environment):

    version = 1
    name = systemName

    def created(self):
        for section in defaults.CONFIG:
            if not self.config.has_section(section):
                self.config.add_section(section)
            for option, value in defaults.CONFIG[section].iteritems():
                if type(value) == str:
                    value = value % {"name": self.name}
                self.config.set(section, option, value)
        self.send(config.Save(), "save", "config")

        self.db = Databases(self)
        self.manager += self.db
        self.send(CreateDatabases(), "create", "db")

    def loaded(self):
        self.db = Databases(self)
        self.manager += self.db
        self.send(LoadDatabases(), "load", "db")

        self.debug = self.config.getboolean("logging", "debug", False)
        self.verbose = self.config.getboolean("logging", "verbose", False)

        self.plugins = weakref.WeakValueDictionary()

        self.errors = 0
        self.events = 0
        self.sTime = time()

        port = self.config.getint("server", "port", 80)
        address = self.config.get("server", "address", "0.0.0.0")
        ssl = self.config.getboolean("server", "ssl", False)
        bind = self.config.get("server", "bind", None)

        auth = {
                "host": socket.gethostname(),
                "server": address,
                "nick": self.config.get("bot", "nick", systemName),
                "ident": self.config.get("bot", "ident", systemName),
                "name": self.config.get("bot", "name", systemDesc)
        }
        if self.config.has_option("server", "password"):
            auth["password"] = self.config.get("server", "password")

        self.bot = Bot(self, port, address, ssl, bind, auth)
        self.manager += self.bot

        port = self.config.getint("bridge", "port", 8000)
        address = self.config.get("bridge", "address", "0.0.0.0")
        self.bridge = Bridge(port, address, channel="bridge")
        self.manager += self.bridge

        if self.debug:
            self.manager += Debugger(events=self.verbose, logger=self.log)

    def loadPlugin(self, plugin):
        """E.loadPlugin(plugin) -> None

        Load a single plugin given by plugin.
        If this plugin is already laoded, it'll be
        replaced.
        """

        if plugin in self.plugins:
            self.push(
                    LogWarning("Not loading plugin '%s' - Already loaded!" % plugin),
                    "warning", "log")
            return False

        try:
            fqplugin = "kdb.plugins.%s" % plugin
            if sys.modules.has_key(fqplugin):
                try:
                    reload(sys.modules[fqplugin])
                except Exception, err:
                    self.push(
                            LogError("Problem reloading plugin '%s'" % plugins),
                            "error", "log")
                    self.push(LogException(err), "exception", "log")
                    self.push(LogDebug(format_exc()), "debug", "log")
                    return False

            m = safe__import__("plugins.%s" % plugin,
                    globals(), locals(), "kdb")

            classes = inspect.getmembers(m,
                    lambda x: inspect.isclass(x) and
                    issubclass(x, BasePlugin) and
                    not x == BasePlugin)
            for name, c in classes:
                o = c(self, self.bot)
                self.manager += o
                self.plugins[plugin] = o
            self.push(LogInfo("Loaded plugin: %s" % plugin), "info", "log")
            return True
        except Exception, err:
            self.push(
                    LogError("Problem loading plugin '%s'" % plugin),
                    "error", "log")
            self.push(LogException(err), "exception", "log")
            self.push(LogDebug(format_exc()), "debug", "log")
            return False

    def unloadPlugin(self, plugin):
        """E.unloadPlugin(plugin) -> None

        Unload the specified plugin if it has been loaded.
        """

        if self.plugins.has_key(plugin):
            o = self.plugins[plugin]
            o.unregister()
            if hasattr(o, "cleanup"):
                o.cleanup()
            del self.plugins[plugin]
            self.push(LogInfo("Unloaded plugin: %s" % plugin), "info", "log")

    def loadPlugins(self):
        """E.loadPlugins() -> None

        Load any available plugins loading the global/default
        ones first, then loading the ones found in the
        environment. Plugins found in the environment
        may override existing plugins already loaded.
        """

        plugins = list(defaults.PLUGINS)

        if self.config.has_section("plugins"):
            for name, value in self.config.items("plugins"):
                plugin, comp = name.split(".")
                if value.lower() == "enabled":
                    if plugin not in plugins:
                        plugins.append(plugin)

        for plugin in plugins:
            try:
                self.loadPlugin(plugin)
            except Exception, err:
                self.errors += 1
                self.push(
                        LogError("Failed to load plugin: %s" % plugin),
                        "error", "log")
                self.push(LogException(err), "exception", "log")
                self.push(LogDebug(format_exc()), "debug", "log")

    def unloadPlugins(self):
        """E.unloadPlugins() -> None

        Unload all loaded plugins calling their cleanup
        methods if they exist.
        """

        for plugin in self.plugins.copy():
            self.unloadPlugin(plugin)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# labview.py
"""
Thin wrapper so that labview can perform phase retrieval.

Copyright (c) 2018, David Hoffman
"""

from pyOTF.phaseretrieval import retrieve_phase
from pyOTF.utils import prep_data_for_PR

def labview(data, wl, na, ni, res, zres, max_iters=200, pupil_tol=1e-8, mse_tol=1e-8, phase_only=False):
    """Generate a PSF object

    Parameters
    ----------
    data : ndarray (3 dim)
        The experimentally measured PSF of a subdiffractive source
    wl : numeric
        Emission wavelength of the simulation
    na : numeric
        Numerical aperature of the simulation
    ni : numeric
        index of refraction for the media
    res : numeric
        x/y resolution of the simulation, must have same units as wl
    zres : numeric
        z resolution of simuation, must have same units a wl
    max_iters : int
        The maximum number of iterations to run, default is 200
    pupil_tol : float
        the tolerance in percent change in change in pupil, default is 1e-8
    mse_tol : float
        the tolerance in percent change for the mean squared error between
        data and simulated data, default is 1e-8
    phase_only : bool
        True means only the phase of the back pupil is retrieved while the
        amplitude is not.
    """

    params = dict(wl=wl, na=na, ni=ni, res=res, zres=zres)

    data_prepped = prep_data_for_PR(data)

    pr_result = retrieve_phase(data_prepped, params, max_iters, pupil_tol, mse_tol, phase_only)

    return pr_result.phase

if __name__ == "__main__":
    # phase retrieve a pupil
    import os
    import time
    import logging
    from skimage.external import tifffile as tif
    from matplotlib import pyplot as plt

    logging.basicConfig()
    logging.getLogger().setLevel(logging.INFO)
    # read in data from fixtures
    data = tif.imread(os.path.split(__file__)[0] + "/fixtures/psf_wl520nm_z300nm_x130nm_na0.85_n1.0.tif")
    # set up model params
    params = dict(
        wl=520,
        na=0.85,
        ni=1.0,
        res=130,
        zres=300
    )
    # retrieve the phase
    pr_start = time.time()
    print("Starting phase retrieval with data of size {}".format(data.shape))
    phase = labview(data, **params, pupil_tol=1e-6)
    print("It took {} seconds to retrieve the pupil function".format(
        time.time() - pr_start))
    # plot
    max_val = abs(phase).max()
    plt.matshow(phase, cmap="seismic", vmin=-max_val, vmax=max_val)
    plt.colorbar()
    # show
    plt.show()
<commit_msg>Update labview.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# labview.py
"""
Thin wrapper so that labview can perform phase retrieval.

Copyright (c) 2018, David Hoffman
"""

from pyOTF.phaseretrieval import retrieve_phase
from pyOTF.utils import prep_data_for_PR

def labview(data, wl, na, ni, res, zres, max_iters=200, pupil_tol=1e-8, mse_tol=1e-8, phase_only=False):
    """Generate a PSF object

    Parameters
    ----------
    data : ndarray (3 dim)
        The experimentally measured PSF of a subdiffractive source
    wl : numeric
        Emission wavelength of the simulation
    na : numeric
        Numerical aperature of the simulation
    ni : numeric
        index of refraction for the media
    res : numeric
        x/y resolution of the simulation, must have same units as wl
    zres : numeric
        z resolution of simuation, must have same units a wl
    max_iters : int
        The maximum number of iterations to run, default is 200
    pupil_tol : float
        the tolerance in percent change in change in pupil, default is 1e-8
    mse_tol : float
        the tolerance in percent change for the mean squared error between
        data and simulated data, default is 1e-8
    phase_only : bool
        True means only the phase of the back pupil is retrieved while the
        amplitude is not.
    """

    params = dict(wl=wl, na=na, ni=ni, res=res, zres=zres)

    data_prepped = prep_data_for_PR(data)

    pr_result = retrieve_phase(data_prepped, params, max_iters, pupil_tol, mse_tol, phase_only)

    return pr_result.phase

if __name__ == "__main__":
    # phase retrieve a pupil
    import os
    import time
    import logging
    from skimage.external import tifffile as tif
    from matplotlib import pyplot as plt

    logging.basicConfig()
    logging.getLogger().setLevel(logging.INFO)
    # read in data from fixtures
    data = tif.imread(os.path.split(__file__)[0] + "/fixtures/psf_wl520nm_z300nm_x130nm_na0.85_n1.0.tif")
    # set up model params
    params = dict(
        wl=520,
        na=0.85,
        ni=1.0,
        res=130,
        zres=300
    )
    # retrieve the phase
    pr_start = time.time()
    print("Starting phase retrieval with data of size {}".format(data.shape))
    phase = labview(data[6:-5], **params, pupil_tol=1e-6)
    print("It took {} seconds to retrieve the pupil function".format(
        time.time() - pr_start))
    # plot
    max_val = abs(phase).max()
    plt.matshow(phase, cmap="seismic", vmin=-max_val, vmax=max_val)
    plt.colorbar()
    # show
    plt.show()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#coding:utf-8
"""
  Author: Meska
  Purpose: Lampone telegram bot, chat to @LamponeBot
  Created: 07/12/15

  commands
  start - Starts!
  stop - stops ?
  groupmode - Set groupchat mode

"""
import os,sys
from wrapper import Bot
from megahal import MegaHAL
from threading import Timer
from shutil import copy2
from datetime import datetime,timedelta
from configparser import ConfigParser
from random import randrange

class Lampone(Bot):

    def __init__(self, token,admins=""):
        super().__init__(token) # init classe principale
        self.admins = [ int(x) for x in admins.split(",") ]
        # init megaHal
        self.lastbackup = None
        self.listening = []
        self.brainfile_name = os.path.join(os.path.split(__file__)[0],"lampone.brain")
        # TODO in some os shelve appends .db to filename
        self.brainfile_name_real = os.path.join(os.path.split(__file__)[0],"lampone.brain.db")
        self.groupmode = {}

        if os.path.exists(self.brainfile_name) or os.path.exists(self.brainfile_name_real):
            # need learning
            self.need_learn = False
        else:
            self.need_learn = True

        try:
            self.megahal = MegaHAL(brainfile=self.brainfile_name)
        except Exception as e:
            print("Database Error: %s"% e)
            # starts with new db
            try:
                os.unlink(self.brainfile_name)
            except:
                pass
            try:
                os.unlink(self.brainfile_name_real)
            except:
                pass

    def log(self,msg):
        # meglio non loggare nulla va
        return
        #try:
            #with open(os.path.join(os.path.split(__file__)[0],"lampone.log"),"a") as logfile:
                #logfile.write("%s\n" % msg)
        #except Exception as e:
            #print(e)

    def log_learn(self,msg):
        # loggo solo le frasi in caso il db si smerdi
        try:
            with open(os.path.join(os.path.split(__file__)[0],"lampone_learn.txt"),"a") as logfile:
                logfile.write("%s\n" % msg)
        except Exception as e:
            print(e)

    def learn(self,message):
        lines = message['text'].splitlines()[1:]
        for l in lines:
            print("learning: %s" % l)
            self.megahal.learn(l)
            self.log_learn(l)
        self.megahal.sync()

    def autolearn(self):
        # learns from previous chats
        if os.path.exists(os.path.join(os.path.split(__file__)[0],"lampone_learn.txt")):
            with open(os.path.join(os.path.split(__file__)[0],"lampone_learn.txt"),"rb") as learnfile:
                for r in learnfile.readlines():
                    l = r.decode('utf8').strip()
                    try:
                        print("learning: %s" % l)
                        self.megahal.learn(l)
                    except:
                        pass
            self.megahal.sync()

    def backupBrain(self):
        # backup brain file in case of crash
        print("Brain Backup! %s" % datetime.now().hour)
        self.megahal.sync()
        self.megahal._MegaHAL__brain.db.close()

        # check for brainfile ext
        if not os.path.exists(self.brainfile_name_real):
            self.brainfile_name_real = self.brainfile_name

        if os.path.exists("%s.%s" % (self.brainfile_name_real,datetime.now().hour)):
            os.unlink("%s.%s" % (self.brainfile_name_real,datetime.now().hour))
        copy2(self.brainfile_name_real, "%s.%s" % (self.brainfile_name_real,datetime.now().hour))

        self.megahal = MegaHAL(brainfile=self.brainfile_name)

    def parsedocument(self,chat_id,message):
        print("Documento ricevuto da %s" % message['from'] )

    def parsemessage(self,chat_id,message):
        print("Messaggio ricevuto da %s" % message['from'] )
        print(message['text'])
        if self.stop:
            # stopping, ignore messages
            return

        if self.lastbackup != datetime.now().hour:
            self.lastbackup = datetime.now().hour
            self.backupBrain()

        if message['text'].startswith('/learn') and message['from']['id'] in self.admins:
            self.learn(message)
            return

        if message['text'].startswith('/update') and message['from']['id'] in self.admins:
            """
            Bot updates from git and then quits,
            to reload automatically use a cron like this:
            */1 * * * * [ `ps aux | grep python3 | grep lampone | grep -v grep | wc -l` -eq 0 ] && /usr/bin/python3 /home/pi/lampone/lampone.py  > /dev/null 2>&1
            """
            self.sendMessage(chat_id,"Updating...")
            res = os.popen("cd %s && git fetch --all && git reset --hard origin" % os.path.join(os.path.split(__file__)[0])).read()
            self.sendMessage(chat_id,res)
            self.sendMessage(chat_id,"Restarting...")
            self.megahal.sync()
            self.megahal._MegaHAL__brain.db.close()
            self.stop = True
            return

        if message['text'].startswith('/f') and message['from']['id'] in self.admins:
            ## fortune auto learning, needs fortune-mod installed
            ## check for params
            count = 1
            if len(message['text'].split()) > 1:
                param = message['text'].split()[1]
                if param.isdecimal():
                    count = int(param)
                    if count > 100:
                        count = 100
            self.sendMessage(chat_id,"Learning %s fortunes"%count)
            for x in range(count):
                txt = os.popen('fortune | grep -v "\-\-\s.*" | grep -v ".*:$" | grep -v ".*http://"').read()
                if txt:
                    self.megahal.learn(txt)
            self.sendMessage(chat_id,"Done")
            return

        if message['text'] == "/start":
            self.sendMessage(chat_id,"Welcome to Lampone Bot")
            return

        if message['text'] == "/groupmode":
            if chat_id > 0:
                self.sendMessage(chat_id,"This is not a group.")
            else:
                self.sendMessage(chat_id,"Select Group Mode:",reply_markup={'keyboard':[['/g1 Respond all messages'],['/g2 Respond some messages'],['/g3 Respond only for Lampone'],]})
            return

        if message['text'].startswith("/g1"):
            if chat_id > 0:
                self.sendMessage(chat_id,"This is not a group.",reply_markup={'hide_keyboard':True})
            else:
                self.groupmode[chat_id] = 1
                self.sendMessage(chat_id,"Groupmode 1 enabled",reply_markup={'hide_keyboard':True})
            return

        if message['text'].startswith("/g2"):
            if chat_id > 0:
                self.sendMessage(chat_id,"This is not a group.",reply_markup={'hide_keyboard':True})
            else:
                self.groupmode[chat_id] = 2
                self.sendMessage(chat_id,"Groupmode 2 enabled",reply_markup={'hide_keyboard':True})
            return

        if message['text'].startswith("/g3"):
            if chat_id > 0:
                self.sendMessage(chat_id,"This is not a group.",reply_markup={'hide_keyboard':True})
            else:
                self.groupmode[chat_id] = 3
                self.sendMessage(chat_id,"Groupmode 3 enabled",reply_markup={'hide_keyboard':True})
            return

        if message['text'] == "/help":
            self.sendMessage(chat_id,"This is a simple AI bot, just talk to him or invite to your group and he will learn and respond\nTry /groupmode for limit group interaction")
            return

        if message['text'] == "/stop":
            self.sendMessage(chat_id,"Command not needed, just close the chat :)")
            return

        if message['text'] == "/s" and message['from']['id'] in self.admins:
            self.megahal.sync()
            self.megahal._MegaHAL__brain.db.close()
            self.megahal = MegaHAL(brainfile=self.brainfile_name)
            self.sendMessage(chat_id,"Sync db")
            return

        if message['text'] == "/autolearn" and message['from']['id'] in self.admins:
            self.autolearn()
            return

        if message['text'] == "/listen" and message['from']['id'] in self.admins:
            self.sendMessage(chat_id,"Listening enabled, stop with /stoplisten")
            if not chat_id in self.listening:
                self.listening.append(chat_id)
            return

        if message['text'] == "/stoplisten" and message['from']['id'] in self.admins:
            self.sendMessage(chat_id,"Listening stopped")
            if chat_id in self.listening:
                self.listening.pop(self.listening.index(chat_id))
            return

        if not message['text'].startswith('/'):
            for ll in self.listening:
                self.sendMessage(ll,"<-- %s" % message['text'])

            #self.log('%s --- MSG FROM:%s --- %s' % (datetime.now(),message['from'],message['text']))

            if len(message['text'].split()) < 2:
                # nothing to learn
                return

            if len(message['text'].split()) > 50:
                # spam received ignore
                return

            if 'http' in message['text']:
                # spam received ignore
                return

            self.log_learn(message['text'])

            rispondi = True
            if chat_id in self.groupmode:
                gm = self.groupmode[chat_id]
                if gm == 2:
                    if not 'lampone' in message['text'].lower():
                        if len(message['text'].split()) > 3:
                            rispondi = True if randrange(0,3) == 0 else False
                        else:
                            rispondi = True if randrange(0,7) == 0 else False
                if gm == 3:
                    if not 'lampone' in message['text'].lower():
                        rispondi = False
            else:
                # default set group mode to 2
                if chat_id < 0:
                    #self.sendMessage(chat_id,'Lampone is here!\ndefault group mode is 2, use /groupmode to switch')
                    self.groupmode[chat_id] = 2
                    rispondi = False

            if rispondi:
                self.action_typing(chat_id)
                reply = self.megahal.get_reply(message['text'])
                #self.log('%s --- MSG TO:%s --- %s' % (datetime.now(),message['from'],reply))
                self.sendMessage(chat_id,reply)

                for ll in self.listening:
                    self.sendMessage(ll,"--> %s" % reply)
            else:
                # learn always
                self.megahal.learn(message['text'])

if __name__ == '__main__':

    # load config
    cf = ConfigParser()
    if os.path.exists(os.path.join(os.path.split(__file__)[0],"lampone.conf")):
        cf.read(os.path.join(os.path.split(__file__)[0],"lampone.conf"))
    else:
        # set defaults
        cf.add_section("telegram")
        cf.set("telegram","token","YOUR TOKEN HERE")
        cf.set("telegram","admins","12345,12345,1232")
        with open(os.path.join(os.path.split(__file__)[0],"lampone.conf"),"w") as cf_file:
            cf.write(cf_file)

    if cf['telegram']['token'] == "YOUR TOKEN HERE":
        print("Token not defined, check config!")
    else:
        l = Lampone(cf['telegram']['token'],admins=cf['telegram']['admins'])
        #l.clearWebHook()
        print(l.get('getMe'))
        if l.need_learn:
            l.autolearn()
        for admin in l.admins:
            # notify admins when online
            l.action_typing(admin)
            l.sendMessage(admin,"Lampone is Online!")
        l.getUpdates()
<commit_msg>ignore some more messages<commit_after>#!/usr/bin/env python
#coding:utf-8
"""
  Author: Meska
  Purpose: Lampone telegram bot, chat to @LamponeBot
  Created: 07/12/15

  commands
  start - Starts!
  stop - stops ?
  groupmode - Set groupchat mode

"""
import os,sys
from wrapper import Bot
from megahal import MegaHAL
from threading import Timer
from shutil import copy2
from datetime import datetime,timedelta
from configparser import ConfigParser
from random import randrange

class Lampone(Bot):

    def __init__(self, token,admins=""):
        super().__init__(token) # init classe principale
        self.admins = [ int(x) for x in admins.split(",") ]
        # init megaHal
        self.lastbackup = None
        self.listening = []
        self.brainfile_name = os.path.join(os.path.split(__file__)[0],"lampone.brain")
        # TODO in some os shelve appends .db to filename
        self.brainfile_name_real = os.path.join(os.path.split(__file__)[0],"lampone.brain.db")
        self.groupmode = {}

        if os.path.exists(self.brainfile_name) or os.path.exists(self.brainfile_name_real):
            # need learning
            self.need_learn = False
        else:
            self.need_learn = True

        try:
            self.megahal = MegaHAL(brainfile=self.brainfile_name)
        except Exception as e:
            print("Database Error: %s"% e)
            # starts with new db
            try:
                os.unlink(self.brainfile_name)
            except:
                pass
            try:
                os.unlink(self.brainfile_name_real)
            except:
                pass

    def log(self,msg):
        # meglio non loggare nulla va
        return
        #try:
            #with open(os.path.join(os.path.split(__file__)[0],"lampone.log"),"a") as logfile:
                #logfile.write("%s\n" % msg)
        #except Exception as e:
            #print(e)

    def log_learn(self,msg):
        # loggo solo le frasi in caso il db si smerdi
        try:
            with open(os.path.join(os.path.split(__file__)[0],"lampone_learn.txt"),"a") as logfile:
                logfile.write("%s\n" % msg)
        except Exception as e:
            print(e)

    def learn(self,message):
        lines = message['text'].splitlines()[1:]
        for l in lines:
            print("learning: %s" % l)
            self.megahal.learn(l)
            self.log_learn(l)
        self.megahal.sync()

    def autolearn(self):
        # learns from previous chats
        if os.path.exists(os.path.join(os.path.split(__file__)[0],"lampone_learn.txt")):
            with open(os.path.join(os.path.split(__file__)[0],"lampone_learn.txt"),"rb") as learnfile:
                for r in learnfile.readlines():
                    l = r.decode('utf8').strip()
                    try:
                        print("learning: %s" % l)
                        self.megahal.learn(l)
                    except:
                        pass
            self.megahal.sync()

    def backupBrain(self):
        # backup brain file in case of crash
        print("Brain Backup! %s" % datetime.now().hour)
        self.megahal.sync()
        self.megahal._MegaHAL__brain.db.close()

        # check for brainfile ext
        if not os.path.exists(self.brainfile_name_real):
            self.brainfile_name_real = self.brainfile_name

        if os.path.exists("%s.%s" % (self.brainfile_name_real,datetime.now().hour)):
            os.unlink("%s.%s" % (self.brainfile_name_real,datetime.now().hour))
        copy2(self.brainfile_name_real, "%s.%s" % (self.brainfile_name_real,datetime.now().hour))

        self.megahal = MegaHAL(brainfile=self.brainfile_name)

    def parsedocument(self,chat_id,message):
        print("Documento ricevuto da %s" % message['from'] )

    def parsemessage(self,chat_id,message):
        print("Messaggio ricevuto da %s" % message['from'] )
        print(message['text'])
        if self.stop:
            # stopping, ignore messages
            return

        if self.lastbackup != datetime.now().hour:
            self.lastbackup = datetime.now().hour
            self.backupBrain()

        if message['text'].startswith('/learn') and message['from']['id'] in self.admins:
            self.learn(message)
            return

        if message['text'].startswith('/update') and message['from']['id'] in self.admins:
            """
            Bot updates from git and then quits,
            to reload automatically use a cron like this:
            */1 * * * * [ `ps aux | grep python3 | grep lampone | grep -v grep | wc -l` -eq 0 ] && /usr/bin/python3 /home/pi/lampone/lampone.py  > /dev/null 2>&1
            """
            self.sendMessage(chat_id,"Updating...")
            res = os.popen("cd %s && git fetch --all && git reset --hard origin" % os.path.join(os.path.split(__file__)[0])).read()
            self.sendMessage(chat_id,res)
            self.sendMessage(chat_id,"Restarting...")
            self.megahal.sync()
            self.megahal._MegaHAL__brain.db.close()
            self.stop = True
            return

        if message['text'].startswith('/f') and message['from']['id'] in self.admins:
            ## fortune auto learning, needs fortune-mod installed
            ## check for params
            count = 1
            if len(message['text'].split()) > 1:
                param = message['text'].split()[1]
                if param.isdecimal():
                    count = int(param)
                    if count > 100:
                        count = 100
            self.sendMessage(chat_id,"Learning %s fortunes"%count)
            for x in range(count):
                txt = os.popen('fortune | grep -v "\-\-\s.*" | grep -v ".*:$" | grep -v ".*http://"').read()
                if txt:
                    self.megahal.learn(txt)
            self.sendMessage(chat_id,"Done")
            return

        if message['text'] == "/start":
            self.sendMessage(chat_id,"Welcome to Lampone Bot")
            return

        if message['text'] == "/groupmode":
            if chat_id > 0:
                self.sendMessage(chat_id,"This is not a group.")
            else:
                self.sendMessage(chat_id,"Select Group Mode:",reply_markup={'keyboard':[['/g1 Respond all messages'],['/g2 Respond some messages'],['/g3 Respond only for Lampone'],]})
            return

        if message['text'].startswith("/g1"):
            if chat_id > 0:
                self.sendMessage(chat_id,"This is not a group.",reply_markup={'hide_keyboard':True})
            else:
                self.groupmode[chat_id] = 1
                self.sendMessage(chat_id,"Groupmode 1 enabled",reply_markup={'hide_keyboard':True})
            return

        if message['text'].startswith("/g2"):
            if chat_id > 0:
                self.sendMessage(chat_id,"This is not a group.",reply_markup={'hide_keyboard':True})
            else:
                self.groupmode[chat_id] = 2
                self.sendMessage(chat_id,"Groupmode 2 enabled",reply_markup={'hide_keyboard':True})
            return

        if message['text'].startswith("/g3"):
            if chat_id > 0:
                self.sendMessage(chat_id,"This is not a group.",reply_markup={'hide_keyboard':True})
            else:
                self.groupmode[chat_id] = 3
                self.sendMessage(chat_id,"Groupmode 3 enabled",reply_markup={'hide_keyboard':True})
            return

        if message['text'] == "/help":
            self.sendMessage(chat_id,"This is a simple AI bot, just talk to him or invite to your group and he will learn and respond\nTry /groupmode for limit group interaction")
            return

        if message['text'] == "/stop":
            self.sendMessage(chat_id,"Command not needed, just close the chat :)")
            return

        if message['text'] == "/s" and message['from']['id'] in self.admins:
            self.megahal.sync()
            self.megahal._MegaHAL__brain.db.close()
            self.megahal = MegaHAL(brainfile=self.brainfile_name)
            self.sendMessage(chat_id,"Sync db")
            return

        if message['text'] == "/autolearn" and message['from']['id'] in self.admins:
            self.autolearn()
            return

        if message['text'] == "/listen" and message['from']['id'] in self.admins:
            self.sendMessage(chat_id,"Listening enabled, stop with /stoplisten")
            if not chat_id in self.listening:
                self.listening.append(chat_id)
            return

        if message['text'] == "/stoplisten" and message['from']['id'] in self.admins:
            self.sendMessage(chat_id,"Listening stopped")
            if chat_id in self.listening:
                self.listening.pop(self.listening.index(chat_id))
            return

        if not message['text'].startswith('/'):
            for ll in self.listening:
                self.sendMessage(ll,"<-- %s" % message['text'])

            #self.log('%s --- MSG FROM:%s --- %s' % (datetime.now(),message['from'],message['text']))

            if len(message['text']) < 3:
                # nothing to learn
                return

            if len(message['text'].split()) > 50:
                # spam received ignore
                return

            if 'http' in message['text']:
                # spam received ignore
                return

            self.log_learn(message['text'])

            rispondi = True
            if chat_id in self.groupmode:
                gm = self.groupmode[chat_id]
                if gm == 2:
                    if not 'lampone' in message['text'].lower():
                        if len(message['text'].split()) > 3:
                            rispondi = True if randrange(0,3) == 0 else False
                        else:
                            rispondi = True if randrange(0,7) == 0 else False
                if gm == 3:
                    if not 'lampone' in message['text'].lower():
                        rispondi = False
            else:
                # default set group mode to 2
                if chat_id < 0:
                    #self.sendMessage(chat_id,'Lampone is here!\ndefault group mode is 2, use /groupmode to switch')
                    self.groupmode[chat_id] = 2
                    rispondi = False

            if rispondi:
                self.action_typing(chat_id)
                reply = self.megahal.get_reply(message['text'])
                #self.log('%s --- MSG TO:%s --- %s' % (datetime.now(),message['from'],reply))
                self.sendMessage(chat_id,reply)

                for ll in self.listening:
                    self.sendMessage(ll,"--> %s" % reply)
            else:
                # learn always
                self.megahal.learn(message['text'])

if __name__ == '__main__':

    # load config
    cf = ConfigParser()
    if os.path.exists(os.path.join(os.path.split(__file__)[0],"lampone.conf")):
        cf.read(os.path.join(os.path.split(__file__)[0],"lampone.conf"))
    else:
        # set defaults
        cf.add_section("telegram")
        cf.set("telegram","token","YOUR TOKEN HERE")
        cf.set("telegram","admins","12345,12345,1232")
        with open(os.path.join(os.path.split(__file__)[0],"lampone.conf"),"w") as cf_file:
            cf.write(cf_file)

    if cf['telegram']['token'] == "YOUR TOKEN HERE":
        print("Token not defined, check config!")
    else:
        l = Lampone(cf['telegram']['token'],admins=cf['telegram']['admins'])
        #l.clearWebHook()
        print(l.get('getMe'))
        if l.need_learn:
            l.autolearn()
        for admin in l.admins:
            # notify admins when online
            l.action_typing(admin)
            l.sendMessage(admin,"Lampone is Online!")
        l.getUpdates()
<|endoftext|>


<commit_before>import match_stats

def schedule_check(matches):
    print '{0} matches total'.format(len(matches))
    match_counter, opponents, collisions = match_stats.match_statistics(matches)
    all_teams = set(match_counter.iterkeys())
    for team, matches in match_counter.iteritems():
        print '{0}: {1} matches, missed opponents: {2}'.format(team, matches, ', '.join(all_teams - opponents[team] - {team}))
        if team in collisions:
            print '\t{0} DO have a match collision'.format(team)
<commit_msg>Make check messages print to stderr<commit_after>import match_stats
import sys

def schedule_check(matches):
    print >>sys.stderr, '{0} matches total'.format(len(matches))
    match_counter, opponents, collisions = match_stats.match_statistics(matches)
    all_teams = set(match_counter.iterkeys())
    for team, matches in match_counter.iteritems():
        print >>sys.stderr, '{0}: {1} matches, missed opponents: {2}'.format(team, matches, ', '.join(all_teams - opponents[team] - {team}))
        if team in collisions:
            print >>sys.stderr, '\t{0} DO have a match collision'.format(team)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
@package
@file brick_split
@author Christopher Mueller
@brief

from coverage_model.test.multi_dim_trials import *
md=MultiDim()
val_arr = np.arange(100).reshape(10,10)
sl = (0,0)
md.put_values_to_bricks(sl, val_arr[sl])
md.reset_bricks()
"""
from ooi.logging import log
from coverage_model import fix_slice

from copy import deepcopy
import itertools
from rtree import index
import numpy as np

class MultiDim(object):

    def __init__(self, total_domain=(10,10), brick_size=5):
        self.total_domain = total_domain
        self.brick_size = tuple(brick_size for x in total_domain)
        self.bricks = {}
        self.rtree = index.Index()

        self.calc_brick_origins()
        self.calc_brick_extents()
        self.build_bricks()
        self.populate_rtree()

    def calc_brick_origins(self):
        bo = list(set(itertools.product(*[range(d)[::self.brick_size[i]] for i,d in enumerate(self.total_domain)])))
        bo.sort()
        self.brick_origins = tuple(bo)

    def calc_brick_extents(self):
        be=[]
        rte=[]
        for ori in self.brick_origins:
            be.append(tuple(zip(ori,map(lambda o,s: o+s-1, ori, self.brick_size))))
            r = ori+tuple(map(lambda o,s: o+s-1, ori, self.brick_size))
            if len(ori) == 1:
                r = tuple([e for ext in zip(r,[0 for x in r]) for e in ext])
            rte.append(r)

        self.brick_extents = tuple(be)
        self.rtree_extents = tuple(rte)

    def build_bricks(self):
        for x in xrange(len(self.brick_origins)):
            self.bricks[x] = np.empty(self.brick_size, dtype='int16')
            self.bricks[x].fill(-1)

    def reset_bricks(self):
        for arr in self.bricks.itervalues():
            arr.fill(-1)

    def populate_rtree(self):
        for i, e in enumerate(self.rtree_extents):
            self.rtree.insert(i, e, obj=self.brick_extents[i])

    def get_bricks_from_slice(self, slice_):
        sl = deepcopy(slice_)
        sl = fix_slice(sl, self.total_domain)

        rank = len(sl)
        if rank == 1:
            rank += 1
            sl += (slice(None),)

        bnds = self.rtree.bounds
        log.debug('slice_ ==> %s', sl)
        log.debug('rtree bounds ==> %s', bnds)

        start=[]
        end=[]
        for x in xrange(rank):
            sx=sl[x]
            if isinstance(sx, slice):
                si=sx.start if sx.start is not None else bnds[x::rank][0]
                start.append(si)
                ei=sx.stop-1 if sx.stop is not None else bnds[x::rank][1]
                end.append(ei)
            elif isinstance(sx, (list, tuple)):
                start.append(min(sx))
                end.append(max(sx))
            elif isinstance(sx, int):
                start.append(sx)
                end.append(sx)

        bricks = list(self.rtree.intersection(tuple(start+end), objects=True))
        bricks = [(b.id, b.object) for b in bricks]
        log.debug('bricks found ==> %s', bricks)
        return bricks

    def put_values_to_bricks(self, slice_, values):
        slice_ = fix_slice(slice_, self.total_domain)
        bricks = self.get_bricks_from_slice(slice_) # this is a list of tuples [(b_id, (bounds...),), ...]

        values = np.asanyarray(values)
        v_shp = values.shape
        log.error('value_shape: %s', v_shp)
        s_shp = self.get_shape_from_slice(slice_)
        log.error('slice_shape: %s', s_shp)
        is_broadcast = False
        if v_shp == ():
            log.info('Broadcast!!')
            is_broadcast = True
            value_slice = ()
        elif v_shp != s_shp:
            # CBM TODO: May be able to leverage np.broadcast() here - raises ValueError if the two arrays can't be broadcast
            # Must account for missing 1's!!
            if v_shp != tuple([i for i in s_shp if i != 1]):
                raise IndexError('Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format(s_shp, v_shp))
        else:
            value_slice = None

        log.error('value_shape: %s', v_shp)

        boo = [0 for x in slice_]
#        v_ori = [0 for x in v_shp]
        for b in bricks:
            # b is (brick_id, (brick_bounds per dim...),)
            bid, bbnds = b
            log.warn('Determining slice for brick: %s', b)
            log.info('bid=%s, bbnds=%s', bid, bbnds)
            brick_slice = []
            brick_mm = []
            for x, sl in enumerate(slice_): # Dimensionality
                log.info('x=%s  sl=%s', x, sl)
                log.warn('bbnds[%s]: %s', x, bbnds[x])
                log.warn('boo[%s]: %s', x, boo[x])
                bsl, nboo, mm = self.calc_brick_slice(sl, bbnds[x], boo[x])
                brick_slice.append(bsl)
                brick_mm.append(mm)
                boo[x] = nboo

            brick_slice = tuple(brick_slice)
            brick_mm = tuple(brick_mm)

            if not is_broadcast:
                value_slice = []
                for x, sl in enumerate(slice_): # Dimensionality
                    vm=v_shp[x] if x < len(v_shp) else 1
                    vs = self.calc_value_slice(sl, bbnds[x], brick_slice=brick_slice[x], brick_sl=brick_mm[x], val_max=vm)
                    value_slice.append(vs)

                value_slice = tuple(value_slice)

            v = values[value_slice]
            bss = self.get_shape_from_slice(brick_slice, self.brick_extents[bid])
            vss = self.get_shape_from_slice(value_slice, v_shp)
            log.warn('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s\n\tvalues %s=\n%s', b, bss, brick_slice, brick_mm, vss, value_slice, v.shape, v)
            self.bricks[bid][brick_slice] = v

    def get_values_from_bricks(self, slice_):
        pass

    def calc_value_slice(self, slice_, brick_ext, brick_slice, brick_sl, val_max):
        log.error('slice_==%s\tbrick_ext==%s\tbrick_slice==%s\tbrick_sl==%s', slice_, brick_ext, brick_slice, brick_sl)

        # Value Slice in Total Domain Notation:
        if isinstance(slice_, int):
            ts = slice_
            es = slice_
        elif isinstance(slice, (list,tuple)):
            ts = slice_[0]
            es = slice_[-1]
        elif isinstance(slice_, slice):
            ts = slice_.start if slice_.start is not None else brick_ext[0]
            es = slice_.stop if slice_.stop is not None else brick_ext[0]
        else:
            ts = 0

        val_sl_tn_min = max(ts, brick_ext[0])
        val_sl_tn_max = brick_sl[1] + brick_ext[1] + 1 -(brick_ext[1] + 1 - brick_ext[0])

        val_sl_min = val_sl_tn_min - ts
        val_sl_max = val_sl_tn_max - ts

        value_slice = slice(val_sl_min, val_sl_max, None)
        return value_slice

#        #  xmin = max(total_sl.xmin, brick_ext.xmin)
#        td_start = max(ts, brick_ext[0])
#        # xmax = sum(brick_sl.xmax, brick_ext.xmax, -brick_ext.xshp)
#        td_stop = brick_sl[1] + brick_ext[1] + 1 - (brick_ext[1] + 1 - brick_ext[0])
#
#        log.warn('td_start==%s\ttd_stop==%s', td_start, td_stop)
#
#        # IF(brick_ext.xmin < brick_sl.xmax, brick_ext.xmin, brick_ext.xmin - brick_sl.xmax)
#        if brick_ext[0] < brick_sl[1]:
#            start = brick_ext[0]
#        else:
#            start = brick_ext[0] - brick_sl[1]
#
#        # sum(brick_ext.xmax, -brick_ext.xmin, -brick_sl.xmin)
#        stop = (brick_ext[1]+1) - brick_ext[0] - brick_sl[0]
#
#        if start >= val_max:
#            offset = start - 0
#            start -= offset
#            stop -= offset
#
#        value_slice = slice(start, stop, None)
#        total_domain_value_slice = slice(td_start, td_stop, None)
#
#        log.warn('\n\ttd_value_slice=%s\n\tvalue_slice=%s', total_domain_value_slice, value_slice)
#
#        if isinstance(slice_, slice) and slice_.start is None:
#            log.error('slice_ starts with None, using total_doman_value_slice')
#            value_slice = total_domain_value_slice
#
#        return value_slice

    def get_shape_from_slice(self, slice_, max_shp=None):
        log.debug('Getting array shape for slice_: %s', slice_)

        if len(self.brick_extents) == 0:
            return 0

        if max_shp is None:
            max_shp = self.total_domain

        shp=[]
        for i, s in enumerate(slice_):
            if isinstance(s, int):
                shp.append(1)
            elif isinstance(s, (list,tuple)):
                shp.append(len(s))
            elif isinstance(s, slice):
                st=min(s.stop, max_shp[i]) if s.stop is not None else max_shp[i]
                shp.append(len(range(*s.indices(st))))

        return tuple(shp)

    def calc_brick_slice(self, slice_, bounds, brick_origin_offset=0):
        log.error('%s  %s', slice_, bounds)
        sl = deepcopy(slice_)
        bo = bounds[0]
        bn = bounds[1] + 1
        bs = bn - bo
        if isinstance(sl, int):
            if bo <= sl < bn:
                return sl-bo, brick_origin_offset, (sl, sl)
            else:
                raise ValueError('Outside brick bounds: %s <= %s < %s', bo, sl, bn)
        elif isinstance(sl, (list,tuple)):
            filt_slice = [x - bo for x in sl if bo <= x < bn]
            if len(filt_slice) > 0:
                return filt_slice, brick_origin_offset, (min(filt_slice), max(filt_slice))
            else:
                raise ValueError('No values within brick bounds: %s <= %s < %s', bo, sl, bn)
        elif isinstance(sl, slice):
            if sl.start is None:
                start = 0
            else:
                if bo <= sl.start < bn:
                    start = sl.start - bo
                elif bo > sl.start:
                    start = 0
                else:
                    raise ValueError('Slice not in brick: %s > %s', sl.start, bn)

            if sl.stop is None:
                stop = bs
            else:
                if bo < sl.stop <= bn:
                    stop = sl.stop - bo
                elif sl.stop > bn:
                    stop = bs
                else: #  bo > sl.stop
                    raise ValueError('Slice not in brick: %s > %s', bo, sl.stop)

            log.debug('start=%s, stop=%s', start, stop)
            log.debug('brick_origin_offset=%s', brick_origin_offset)
            start += brick_origin_offset
            log.debug('start=%s, stop=%s', start, stop)
            nbs = slice(start, stop, sl.step)
            nbsi = range(*nbs.indices(stop))
            nbsl = len(nbsi)
            if nbsl == 0: # No values in this brick!!
                if sl.step is not None:
                    brick_origin_offset -= bs
                return None, brick_origin_offset
            last_index = nbsi[-1]
            log.debug('last_index=%s',last_index)
            log.debug('nbsl=%s',nbsl)

            if sl.step is not None:
                brick_origin_offset = last_index - bs + sl.step
                log.debug('brick_origin_offset = %s', brick_origin_offset)

            bsl_min = nbs.start
            bsl_max = last_index + 1 # ?? Is this right ??

            return nbs, brick_origin_offset, (bsl_min, bsl_max)

    def do_recurse(self, slice_, depth=None):
        if depth is None:
            depth = len(self.total_domain)-1

        if depth != 0:
            self.do_recurse(slice_, depth-1)

        sl=slice_[depth]

        log.warn('depth %s: %s', depth, sl)
        log.error(self.total_domain[depth])

<commit_msg>More improvements to value_slice/brick_slice<commit_after>#!/usr/bin/env python

"""
@package
@file brick_split
@author Christopher Mueller
@brief

from coverage_model.test.multi_dim_trials import *
md=MultiDim()
val_arr = np.arange(100).reshape(10,10)
sl = (slice(None),slice(None))
md.put_values_to_bricks(sl, val_arr[sl])
md.reset_bricks()
"""
from ooi.logging import log
from coverage_model import fix_slice

from copy import deepcopy
import itertools
from rtree import index
import numpy as np

class MultiDim(object):

    def __init__(self, total_domain=(10,10), brick_size=5):
        self.total_domain = total_domain
        self.brick_size = tuple(brick_size for x in total_domain)
        self.bricks = {}
        self.rtree = index.Index()

        self.calc_brick_origins()
        self.calc_brick_extents()
        self.build_bricks()
        self.populate_rtree()

    def calc_brick_origins(self):
        bo = list(set(itertools.product(*[range(d)[::self.brick_size[i]] for i,d in enumerate(self.total_domain)])))
        bo.sort()
        self.brick_origins = tuple(bo)

    def calc_brick_extents(self):
        be=[]
        rte=[]
        for ori in self.brick_origins:
            be.append(tuple(zip(ori,map(lambda o,s: o+s-1, ori, self.brick_size))))
            r = ori+tuple(map(lambda o,s: o+s-1, ori, self.brick_size))
            if len(ori) == 1:
                r = tuple([e for ext in zip(r,[0 for x in r]) for e in ext])
            rte.append(r)

        self.brick_extents = tuple(be)
        self.rtree_extents = tuple(rte)

    def build_bricks(self):
        for x in xrange(len(self.brick_origins)):
            self.bricks[x] = np.empty(self.brick_size, dtype='int16')
            self.bricks[x].fill(-1)

    def reset_bricks(self):
        for arr in self.bricks.itervalues():
            arr.fill(-1)

    def populate_rtree(self):
        for i, e in enumerate(self.rtree_extents):
            self.rtree.insert(i, e, obj=self.brick_extents[i])

    def get_bricks_from_slice(self, slice_):
        sl = deepcopy(slice_)
        sl = fix_slice(sl, self.total_domain)

        rank = len(sl)
        if rank == 1:
            rank += 1
            sl += (slice(None),)

        bnds = self.rtree.bounds
        log.debug('slice_ ==> %s', sl)
        log.debug('rtree bounds ==> %s', bnds)

        start=[]
        end=[]
        for x in xrange(rank):
            sx=sl[x]
            if isinstance(sx, slice):
                si=sx.start if sx.start is not None else bnds[x::rank][0]
                start.append(si)
                ei=sx.stop-1 if sx.stop is not None else bnds[x::rank][1]
                end.append(ei)
            elif isinstance(sx, (list, tuple)):
                start.append(min(sx))
                end.append(max(sx))
            elif isinstance(sx, int):
                start.append(sx)
                end.append(sx)

        bricks = list(self.rtree.intersection(tuple(start+end), objects=True))
        bricks = [(b.id, b.object) for b in bricks]
        log.debug('bricks found ==> %s', bricks)
        return bricks

    def put_values_to_bricks(self, slice_, values):
        slice_ = fix_slice(slice_, self.total_domain)
        bricks = self.get_bricks_from_slice(slice_) # this is a list of tuples [(b_id, (bounds...),), ...]

        values = np.asanyarray(values)
        v_shp = values.shape
        log.error('value_shape: %s', v_shp)
        s_shp = self.get_shape_from_slice(slice_)
        log.error('slice_shape: %s', s_shp)
        is_broadcast = False
        if v_shp == ():
            log.info('Broadcast!!')
            is_broadcast = True
            value_slice = ()
        elif v_shp != s_shp:
            # CBM TODO: May be able to leverage np.broadcast() here - raises ValueError if the two arrays can't be broadcast
            # Must account for missing 1's!!
            if v_shp != tuple([i for i in s_shp if i != 1]):
                raise IndexError('Shape of \'value\' is not compatible with \'slice_\': slice_ shp == {0}\tvalue shp == {1}'.format(s_shp, v_shp))
        else:
            value_slice = None

        log.error('value_shape: %s', v_shp)

        for b in bricks:
            # b is (brick_id, (brick_bounds per dim...),)
            bid, bbnds = b
            log.warn('Determining slice for brick: %s', b)
            log.info('bid=%s, bbnds=%s', bid, bbnds)
            brick_slice = []
            brick_mm = []
            for x, sl in enumerate(slice_): # Dimensionality
                log.info('x=%s  sl=%s', x, sl)
                log.warn('bbnds[%s]: %s', x, bbnds[x])
                bsl, mm = self.calc_brick_slice(sl, bbnds[x])
                brick_slice.append(bsl)
                brick_mm.append(mm)

            brick_slice = tuple(brick_slice)
            brick_mm = tuple(brick_mm)

            if not is_broadcast:
                value_slice = []
                for x, sl in enumerate(slice_): # Dimensionality
                    vm=v_shp[x] if x < len(v_shp) else 1
                    vs = self.calc_value_slice(sl, bbnds[x], brick_slice=brick_slice[x], brick_sl=brick_mm[x], val_shp_max=vm)
                    value_slice.append(vs)

                value_slice = tuple(value_slice)

            v = values[value_slice]
            bss = self.get_shape_from_slice(brick_slice, self.brick_extents[bid])
            vss = self.get_shape_from_slice(value_slice, v_shp)
            log.warn('\nbrick %s:\n\tbrick_slice %s=%s\n\tmin/max=%s\n\tvalue_slice %s=%s\n\tvalues %s=\n%s', b, bss, brick_slice, brick_mm, vss, value_slice, v.shape, v)
            self.bricks[bid][brick_slice] = v

    def get_values_from_bricks(self, slice_):
        pass

    def calc_value_slice(self, slice_, brick_ext, brick_slice, brick_sl, val_shp_max):
        log.error('slice_==%s\tbrick_ext==%s\tbrick_slice==%s\tbrick_sl==%s\tval_shp_max==%s', slice_, brick_ext, brick_slice, brick_sl, val_shp_max)

        brick_ext_min = brick_ext[0]
        brick_ext_max = brick_ext[1] + 1
        brick_shp = brick_ext_max - brick_ext[0]

        # Value Slice in Total Domain Notation:
        if isinstance(slice_, int):
            ts = slice_
        elif isinstance(slice, (list,tuple)):
            ts = slice_[0]
        elif isinstance(slice_, slice):
            ts = slice_.start if slice_.start is not None else 0
            if slice_.step is None or slice_.step == 1:
                pass
            else:
                brick_ext_min = len(xrange(*slice_.indices(brick_ext_min))) + ts
                brick_ext_max = len(xrange(*slice_.indices(brick_ext_max))) + ts
#                if brick_ext_min > 0:
#                    brick_ext_min = xrange(*slice_.indices(brick_ext_min))[0] + ts
#                brick_ext_max = xrange(*slice_.indices(brick_ext_max))[-1]
                log.error('STEP ADJUSTMENT: brick_ext_min=%s\tbrick_ext_max=%s', brick_ext_min, brick_ext_max)
        else:
            ts = 0

        log.error('ts = %s', ts)

        val_sl_tn_min = max(ts, brick_ext_min)
        val_sl_tn_max = brick_sl[1] + brick_ext_max - brick_shp

        log.error('val_sl_tn_min/max = %s/%s', val_sl_tn_min, val_sl_tn_max)

        val_sl_min = val_sl_tn_min - ts
        val_sl_max = val_sl_tn_max - ts

        value_slice = slice(val_sl_min, val_sl_max, None)
        return value_slice

    def get_shape_from_slice(self, slice_, max_shp=None):
        log.debug('Getting array shape for slice_: %s', slice_)

        if len(self.brick_extents) == 0:
            return 0

        if max_shp is None:
            max_shp = self.total_domain

        shp=[]
        for i, s in enumerate(slice_):
            if isinstance(s, int):
                shp.append(1)
            elif isinstance(s, (list,tuple)):
                shp.append(len(s))
            elif isinstance(s, slice):
                st=min(s.stop, max_shp[i]) if s.stop is not None else max_shp[i]
                shp.append(len(range(*s.indices(st))))

        return tuple(shp)

    def calc_brick_slice(self, slice_, bounds):
        log.error('%s  %s', slice_, bounds)
        sl = deepcopy(slice_)
        bo = bounds[0]
        bn = bounds[1] + 1
        bs = bn - bo
        if isinstance(sl, int):
            if bo <= sl < bn:
                return sl-bo, (sl, sl)
            else:
                raise ValueError('Outside brick bounds: %s <= %s < %s', bo, sl, bn)
        elif isinstance(sl, (list,tuple)):
            filt_slice = [x - bo for x in sl if bo <= x < bn]
            if len(filt_slice) > 0:
                return filt_slice, (min(filt_slice), max(filt_slice))
            else:
                raise ValueError('No values within brick bounds: %s <= %s < %s', bo, sl, bn)
        elif isinstance(sl, slice):
            if sl.start is None:
                start = 0
            else:
                if bo <= sl.start < bn:
                    start = sl.start - bo
                elif bo > sl.start:
                    start = 0
                else:
                    raise ValueError('Slice not in brick: %s > %s', sl.start, bn)

            if sl.stop is None:
                stop = bs
            else:
                if bo < sl.stop <= bn:
                    stop = sl.stop - bo
                elif sl.stop > bn:
                    stop = bs
                else: #  bo > sl.stop
                    raise ValueError('Slice not in brick: %s > %s', bo, sl.stop)

            log.warn('start=%s, stop=%s', start, stop)
            if bo != 0 and sl.step is not None and sl.step != 1:
                try:
                    ss = 0 if sl.start is None else sl.start
                    calc_boo = xrange(*slice(ss,bn,sl.step).indices(bo+sl.step))[-1] - bo
                except:
                    calc_boo = 0
                log.warn('calc_boo=%s',calc_boo)
                start += calc_boo
            log.debug('start=%s, stop=%s', start, stop)
            nbs = slice(start, stop, sl.step)
            nbsi = range(*nbs.indices(stop))
            nbsl = len(nbsi)
            if nbsl == 0: # No values in this brick!!
#                if sl.step is not None:
#                    brick_origin_offset -= bs
#                return None, brick_origin_offset
                return None
            log.debug('nbsl=%s',nbsl)

            bsl_min = nbs.start
            bsl_max = nbs.stop

            return nbs, (bsl_min, bsl_max)

    def do_recurse(self, slice_, depth=None):
        if depth is None:
            depth = len(self.total_domain)-1

        if depth != 0:
            self.do_recurse(slice_, depth-1)

        sl=slice_[depth]

        log.warn('depth %s: %s', depth, sl)
        log.error(self.total_domain[depth])

<|endoftext|>


<commit_before>"""Make a Deque object that has access to DoublyLinkedList methods."""
from doubly_linked_list import DoublyLinkedList

class Deque(object):
    """Create a Queue object."""

    def __init__(self):
        """Initialize Queue using DoublyLinkedList methods."""
        self._doubly_linked_list = DoublyLinkedList()

    def append(self, val):
        """Add a node to queue."""
        self._doubly_linked_list.append(val)

    def appendleft(self, val):
        """Add a node to queue."""
        self._doubly_linked_list.push(val)

    def pop(self):
        """Remove first node pushed in."""
        try:
            return self._doubly_linked_list.shift()
        except IndexError:
            raise IndexError('Nothing to pop')

    def popleft(self):
        """Remove first node pushed in."""
        try:
            return self._doubly_linked_list.pop()
        except IndexError:
            raise IndexError('Nothing to pop from front')

    def peek(self):
        """Get value of next node to be dequeued."""
        if self._doubly_linked_list.size() == 0:
            return None
        return self._doubly_linked_list.tail.data

    def peek(self):
        """Get value of next node to be dequeued."""
        if self._doubly_linked_list.size() == 0:
            return None
        return self._doubly_linked_list.head.data

    def size(self):
        """Get the size of the queue."""
        return self._doubly_linked_list.size()

    def __len__(self):
        """Return the length of the queue using Python len function."""
        return self.size()
<commit_msg>fixed docstring<commit_after>"""Make a Deque object that has access to DoublyLinkedList methods."""
from doubly_linked_list import DoublyLinkedList

class Deque(object):
    """Create a Deque object."""

    def __init__(self):
        """Initialize Deque using DoublyLinkedList methods."""
        self._doubly_linked_list = DoublyLinkedList()

    def append(self, val):
        """Add a node to back Deque."""
        self._doubly_linked_list.append(val)

    def appendleft(self, val):
        """Add a node to front Deque."""
        self._doubly_linked_list.push(val)

    def pop(self):
        """Remove node at the back."""
        try:
            return self._doubly_linked_list.shift()
        except IndexError:
            raise IndexError('Nothing to pop')

    def popleft(self):
        """Remove node at the front."""
        try:
            return self._doubly_linked_list.pop()
        except IndexError:
            raise IndexError('Nothing to pop from front')

    def peek(self):
        """Get value of next node to be popped left."""
        if self._doubly_linked_list.size() == 0:
            return None
        return self._doubly_linked_list.tail.data

    def peekleft(self):
        """Get value of next node to be popped."""
        if self._doubly_linked_list.size() == 0:
            return None
        return self._doubly_linked_list.head.data

    def size(self):
        """Get the size of the Deque."""
        return self._doubly_linked_list.size()

    def __len__(self):
        """Return the length of the Deque using Python len function."""
        return self.size()
<|endoftext|>