<commit_before># -*- coding: utf-8 -*-
"""This class maintains the internal dfTimewolf state.

Use it to track errors, abort on global failures, clean up after modules, etc.
"""

import logging
import threading
import traceback
import importlib

from dftimewolf.lib import errors
from dftimewolf.lib import utils
from dftimewolf.lib.modules import manager as modules_manager

# TODO(tomchop): Consider changing this to `dftimewolf.state` if we ever need
# more granularity.
logger = logging.getLogger('dftimewolf')

NEW_ISSUE_URL = 'https://github.com/log2timeline/dftimewolf/issues/new'

class DFTimewolfState(object):
  """The main State class.

  Attributes:
    command_line_options (dict[str, str]): Command line options passed to
        dftimewolf.
    config (dftimewolf.config.Config): Class to be used throughout execution.
    errors (list[tuple[str, bool]]): errors generated by a module. These
        should be cleaned up after each module run using the CleanUp() method.
    global_errors (list[tuple[str, bool]]): the CleanUp() method moves non
        critical errors to this attribute for later reporting.
    input (list[str]): data that the current module will use as input.
    output (list[str]): data that the current module generates.
    recipe: (dict[str, str]): recipe declaring modules to load.
    store (dict[str, object]): arbitrary data for modules.
  """

  def __init__(self, config):
    """Initializes a state."""
    super(DFTimewolfState, self).__init__()
    self.command_line_options = {}
    self._cache = {}
    self._module_pool = {}
    self._state_lock = threading.Lock()
    self._threading_event_per_module = {}
    self.config = config
    self.errors = []
    self.global_errors = []
    self.input = []
    self.output = []
    self.recipe = None
    self.store = {}
    self.streaming_callbacks = {}
    self._abort_execution = False

  def _InvokeModulesInThreads(self, callback):
    """Invokes the callback function on all the modules in separate threads.

    Args:
      callback (function): callback function to invoke on all the modules.
    """
    threads = []
    for module_definition in self.recipe['modules']:
      thread_args = (module_definition, )
      thread = threading.Thread(target=callback, args=thread_args)
      threads.append(thread)
      thread.start()

    for thread in threads:
      thread.join()

    self.CheckErrors(is_global=True)

  def ImportRecipeModules(self, module_locations):
    """Dynamically loads the modules declared in a recipe.

    Args:
      module_location (dict[str, str]): A dfTimewolf module name - Python module
          mapping. e.g.:
            {'GRRArtifactCollector': 'dftimewolf.lib.collectors.grr_hosts'}

    Raises:
      errors.RecipeParseError: if a module requested in a recipe does not
          exist in the mapping.
    """
    for module in self.recipe['modules'] + self.recipe.get('preflights', []):
      name = module['name']
      if name not in module_locations:
        msg = f'Module {name} cannot be found. It may not have been declared.'
        raise errors.RecipeParseError(msg)
      logger.debug('Loading module {0:s} from {1:s}'.format(
          name, module_locations[name]))
      try:
        importlib.import_module(module_locations[name])
      except ModuleNotFoundError as exception:
        msg = f'Cannot find Python module for {name}: {exception}'
        raise errors.RecipeParseError(msg)

  def LoadRecipe(self, recipe, module_locations):
    """Populates the internal module pool with modules declared in a recipe.

    Args:
      recipe (dict[str, str]): recipe declaring modules to load.

    Raises:
      RecipeParseError: if a module in the recipe has not been declared.
    """
    self.recipe = recipe
    module_definitions = recipe.get('modules', [])
    preflight_definitions = recipe.get('preflights', [])
    self.ImportRecipeModules(module_locations)

    for module_definition in module_definitions + preflight_definitions:
      # Combine CLI args with args from the recipe description
      module_name = module_definition['name']
      module_class = modules_manager.ModulesManager.GetModuleByName(module_name)

      runtime_name = module_definition.get('runtime_name')
      if not runtime_name:
        runtime_name = module_name
      self._module_pool[runtime_name] = module_class(self, name=runtime_name)

  def LogExecutionPlan(self):
    """Logs loaded modules and their corresponding arguments to stdout."""
    maxlen = 0

    modules = self.recipe.get('preflights', []) + self.recipe.get('modules', [])

    for module in modules:
      if not module['args']:
        continue
      spacing = len(max(module['args'].keys(), key=len))
      maxlen = maxlen if maxlen > spacing else spacing

    for module in modules:
      runtime_name = module.get('runtime_name')
      if runtime_name:
        logger.debug('{0:s} ({1:s}):'.format(runtime_name, module['name']))
      else:
        logger.debug('{0:s}:'.format(module['name']))

      new_args = utils.ImportArgsFromDict(
          module['args'], self.command_line_options, self.config)

      if not new_args:
        logger.debug('  *No params*')
      for key, value in new_args.items():
        logger.debug('  {0:s}{1:s}'.format(key.ljust(maxlen + 3), repr(value)))

  def AddToCache(self, name, value):
    """Thread-safe method to add data to the state's cache.

    If the cached item is already in the cache it will be
    overwritten with the new value.

    Args:
      name (str): string with the name of the cache variable.
      value (object): the value that will be stored in the cache.
    """
    with self._state_lock:
      self._cache[name] = value

  def GetFromCache(self, name, default_value=None):
    """Thread-safe method to get data from the state's cache.

    Args:
      name (str): string with the name of the cache variable.
      default_value (object): the value that will be returned if
          the item does not exist in the cache. Optional argument
          and defaults to None.

    Returns:
      object: object from the cache that corresponds to the name, or
          the value of "default_value" if the cache does not contain
          the variable.
    """
    with self._state_lock:
      return self._cache.get(name, default_value)

  def StoreContainer(self, container):
    """Thread-safe method to store data in the state's store.

    Args:
      container (AttributeContainer): data to store.
    """
    with self._state_lock:
      self.store.setdefault(container.CONTAINER_TYPE, []).append(container)

  def GetContainers(self, container_class, pop=False):
    """Thread-safe method to retrieve data from the state's store.

    Args:
      container_class (type): AttributeContainer class used to filter data.
      pop (Optional[bool]): Whether to remove the containers from the state when
          they are retrieved.

    Returns:
      list[AttributeContainer]: attribute container objects provided in
          the store that correspond to the container type.
    """
    with self._state_lock:
      containers = self.store.get(container_class.CONTAINER_TYPE, [])
      if pop:
        self.store[container_class.CONTAINER_TYPE] = []
      return containers

  def _SetupModuleThread(self, module_definition):
    """Calls the module's SetUp() function and sets a threading event for it.

    Callback for _InvokeModulesInThreads.

    Args:
      module_definition (dict[str, str]): recipe module definition.
    """
    module_name = module_definition['name']
    runtime_name = module_definition.get('runtime_name', module_name)
    logger.info('Setting up module: {0:s}'.format(runtime_name))
    new_args = utils.ImportArgsFromDict(
        module_definition['args'], self.command_line_options, self.config)
    module = self._module_pool[runtime_name]

    try:
      module.SetUp(**new_args)
    except errors.DFTimewolfError as exception:
      msg = "A critical error occurred in module {0:s}, aborting execution."
      logger.critical(msg.format(module.name))
    except Exception as exception:  # pylint: disable=broad-except
      msg = 'An unknown error occurred in module {0:s}: {1!s}'.format(
          module.name, exception)
      logger.critical(msg)
      # We're catching any exception that is not a DFTimewolfError, so we want
      # to generate an error for further reporting.
      error = errors.DFTimewolfError(
          message=msg, name='state', stacktrace=traceback.format_exc(),
          critical=True, unexpected=True)
      self.AddError(error)

    self._threading_event_per_module[runtime_name] = threading.Event()
    self.CleanUp()

  def SetupModules(self):
    """Performs setup tasks for each module in the module pool.

    Threads declared modules' SetUp() functions. Takes CLI arguments into
    account when replacing recipe parameters for each module.
    """
    # Note that vars() copies the values of argparse.Namespace to a dict.
    self._InvokeModulesInThreads(self._SetupModuleThread)

  def _RunModuleThread(self, module_definition):
    """Runs the module's Process() function.

    Callback for _InvokeModulesInThreads.

    Waits for any blockers to have finished before running Process(), then
    sets an Event flag declaring the module has completed.

    Args:
      module_definition (dict): module definition.
    """
    module_name = module_definition['name']
    runtime_name = module_definition.get('runtime_name', module_name)

    for dependency in module_definition['wants']:
      self._threading_event_per_module[dependency].wait()

    module = self._module_pool[runtime_name]

    # Abort processing if a module has had critical failures before.
    if self._abort_execution:
      logger.critical(
          'Aborting execution of {0:s} due to previous errors'.format(
              module.name))
      self._threading_event_per_module[runtime_name].set()
      self.CleanUp()
      return

    logger.info('Running module: {0:s}'.format(runtime_name))

    try:
      module.Process()
    except errors.DFTimewolfError as exception:
      logger.critical(
          "Critical error in module {0:s}, aborting execution".format(
              module.name))
    except Exception as exception:  # pylint: disable=broad-except
      msg = 'An unknown error occurred in module {0:s}: {1!s}'.format(
          module.name, exception)
      logger.critical(msg)
      # We're catching any exception that is not a DFTimewolfError, so we want
      # to generate an error for further reporting.
      error = errors.DFTimewolfError(
          message=msg, name='state', stacktrace=traceback.format_exc(),
          critical=True, unexpected=True)
      self.AddError(error)

    logger.info('Module {0:s} finished execution'.format(runtime_name))
    self._threading_event_per_module[runtime_name].set()
    self.CleanUp()

  def RunPreflights(self):
    """Runs preflight modules."""
    for preflight_definition in self.recipe.get('preflights', []):
      preflight_name = preflight_definition['name']
      runtime_name = preflight_definition.get('runtime_name', preflight_name)

      args = preflight_definition.get('args', {})

      new_args = utils.ImportArgsFromDict(
          args, self.command_line_options, self.config)
      preflight = self._module_pool[runtime_name]
      try:
        preflight.SetUp(**new_args)
        preflight.Process()
      finally:
        self.CheckErrors(is_global=True)

  def CleanUpPreflights(self):
    """Executes any cleanup actions defined in preflight modules."""
    for preflight_definition in self.recipe.get('preflights', []):
      preflight_name = preflight_definition['name']
      runtime_name = preflight_definition.get('runtime_name', preflight_name)
      preflight = self._module_pool[runtime_name]
      try:
        preflight.CleanUp()
      finally:
        self.CheckErrors(is_global=True)

  def InstantiateModule(self, module_name):
    """Instantiates an arbitrary dfTimewolf module.

    Args:
      module_name (str): The name of the module to instantiate.

    Returns:
      BaseModule: An instance of a dftimewolf Module, which is a subclass of
          BaseModule.
    """
    module_class = modules_manager.ModulesManager.GetModuleByName(module_name)
    return module_class(self)

  def RunModules(self):
    """Performs the actual processing for each module in the module pool."""
    self._InvokeModulesInThreads(self._RunModuleThread)

  def RegisterStreamingCallback(self, target, container_type):
    """Registers a callback for a type of container.

    The function to be registered should a single parameter of type
    interface.AttributeContainer.

    Args:
      target (function): function to be called.
      container_type (type[interface.AttributeContainer]): container type on
          which the callback will be called.
    """
    if container_type not in self.streaming_callbacks:
      self.streaming_callbacks[container_type] = []
    self.streaming_callbacks[container_type].append(target)

  def StreamContainer(self, container):
    """Streams a container to the callbacks that are registered to handle it.

    Args:
      container (interface.AttributeContainer): container instance that will be
          streamed to any registered callbacks.
    """
    for callback in self.streaming_callbacks.get(type(container), []):
      callback(container)

  def AddError(self, error):
    """Adds an error to the state.

    Args:
      error (errors.DFTimewolfError): The dfTimewolf error to add.
    """
    if error.critical:
      self._abort_execution = True
    self.errors.append(error)

  def CleanUp(self):
    """Cleans up after running a module.

    The state's output becomes the input for the next stage. Any errors are
    moved to the global_errors attribute so that they can be reported at a
    later stage.
    """
    # Move any existing errors to global errors
    self.global_errors.extend(self.errors)
    self.errors = []

  def CheckErrors(self, is_global=False):
    """Checks for errors and exits if any of them are critical.

    Args:
      is_global (Optional[bool]): True if the global_errors attribute should
          be checked. False if the error attribute should be checked.
    """
    error_objects = self.global_errors if is_global else self.errors
    critical_errors = False

    if error_objects:
      logger.error('dfTimewolf encountered one or more errors:')

    for index, error in enumerate(error_objects):
      logger.error('{0:d}: error from {1:s}: {2:s}'.format(
          index+1, error.name, error.message))
      if error.stacktrace:
        for line in error.stacktrace.split('\n'):
          logger.error(line)
      if error.critical:
        critical_errors = True

    if any(error.unexpected for error in error_objects):
      logger.critical('One or more unexpected errors occurred.')
      logger.critical(
          'Please consider opening an issue: {0:s}'.format(NEW_ISSUE_URL))

    if critical_errors:
      raise errors.CriticalError('Critical error found. Aborting.')
<commit_msg>Add FormatExecutionPlan (#410)<commit_after># -*- coding: utf-8 -*-
"""This class maintains the internal dfTimewolf state.

Use it to track errors, abort on global failures, clean up after modules, etc.
"""

import logging
import threading
import traceback
import importlib

from dftimewolf.lib import errors
from dftimewolf.lib import utils
from dftimewolf.lib.modules import manager as modules_manager

# TODO(tomchop): Consider changing this to `dftimewolf.state` if we ever need
# more granularity.
logger = logging.getLogger('dftimewolf')

NEW_ISSUE_URL = 'https://github.com/log2timeline/dftimewolf/issues/new'

class DFTimewolfState(object):
  """The main State class.

  Attributes:
    command_line_options (dict[str, str]): Command line options passed to
        dftimewolf.
    config (dftimewolf.config.Config): Class to be used throughout execution.
    errors (list[tuple[str, bool]]): errors generated by a module. These
        should be cleaned up after each module run using the CleanUp() method.
    global_errors (list[tuple[str, bool]]): the CleanUp() method moves non
        critical errors to this attribute for later reporting.
    input (list[str]): data that the current module will use as input.
    output (list[str]): data that the current module generates.
    recipe: (dict[str, str]): recipe declaring modules to load.
    store (dict[str, object]): arbitrary data for modules.
  """

  def __init__(self, config):
    """Initializes a state."""
    super(DFTimewolfState, self).__init__()
    self.command_line_options = {}
    self._cache = {}
    self._module_pool = {}
    self._state_lock = threading.Lock()
    self._threading_event_per_module = {}
    self.config = config
    self.errors = []
    self.global_errors = []
    self.input = []
    self.output = []
    self.recipe = None
    self.store = {}
    self.streaming_callbacks = {}
    self._abort_execution = False

  def _InvokeModulesInThreads(self, callback):
    """Invokes the callback function on all the modules in separate threads.

    Args:
      callback (function): callback function to invoke on all the modules.
    """
    threads = []
    for module_definition in self.recipe['modules']:
      thread_args = (module_definition, )
      thread = threading.Thread(target=callback, args=thread_args)
      threads.append(thread)
      thread.start()

    for thread in threads:
      thread.join()

    self.CheckErrors(is_global=True)

  def ImportRecipeModules(self, module_locations):
    """Dynamically loads the modules declared in a recipe.

    Args:
      module_location (dict[str, str]): A dfTimewolf module name - Python module
          mapping. e.g.:
            {'GRRArtifactCollector': 'dftimewolf.lib.collectors.grr_hosts'}

    Raises:
      errors.RecipeParseError: if a module requested in a recipe does not
          exist in the mapping.
    """
    for module in self.recipe['modules'] + self.recipe.get('preflights', []):
      name = module['name']
      if name not in module_locations:
        msg = f'Module {name} cannot be found. It may not have been declared.'
        raise errors.RecipeParseError(msg)
      logger.debug('Loading module {0:s} from {1:s}'.format(
          name, module_locations[name]))
      try:
        importlib.import_module(module_locations[name])
      except ModuleNotFoundError as exception:
        msg = f'Cannot find Python module for {name}: {exception}'
        raise errors.RecipeParseError(msg)

  def LoadRecipe(self, recipe, module_locations):
    """Populates the internal module pool with modules declared in a recipe.

    Args:
      recipe (dict[str, str]): recipe declaring modules to load.

    Raises:
      RecipeParseError: if a module in the recipe has not been declared.
    """
    self.recipe = recipe
    module_definitions = recipe.get('modules', [])
    preflight_definitions = recipe.get('preflights', [])
    self.ImportRecipeModules(module_locations)

    for module_definition in module_definitions + preflight_definitions:
      # Combine CLI args with args from the recipe description
      module_name = module_definition['name']
      module_class = modules_manager.ModulesManager.GetModuleByName(module_name)

      runtime_name = module_definition.get('runtime_name')
      if not runtime_name:
        runtime_name = module_name
      self._module_pool[runtime_name] = module_class(self, name=runtime_name)

  def FormatExecutionPlan(self):
    """Formats execution plan.

    Returns information about loaded modules and their corresponding arguments
    to stdout.

    Returns:
      str: String representation of loaded modules and their parameters.
    """
    plan = ""
    maxlen = 0

    modules = self.recipe.get('preflights', []) + self.recipe.get('modules', [])

    for module in modules:
      if not module['args']:
        continue
      spacing = len(max(module['args'].keys(), key=len))
      maxlen = maxlen if maxlen > spacing else spacing

    for module in modules:
      runtime_name = module.get('runtime_name')
      if runtime_name:
        plan += '{0:s} ({1:s}):\n'.format(runtime_name, module['name'])
      else:
        plan += '{0:s}:\n'.format(module['name'])

      new_args = utils.ImportArgsFromDict(
          module['args'], self.command_line_options, self.config)

      if not new_args:
        plan += '  *No params*\n'
      for key, value in new_args.items():
        plan += '  {0:s}{1:s}\n'.format(key.ljust(maxlen + 3), repr(value))

    return plan

  def LogExecutionPlan(self):
    """Logs the result of FormatExecutionPlan() using the base logger."""
    for line in self.FormatExecutionPlan().split('\n'):
      logger.debug(line)

  def AddToCache(self, name, value):
    """Thread-safe method to add data to the state's cache.

    If the cached item is already in the cache it will be
    overwritten with the new value.

    Args:
      name (str): string with the name of the cache variable.
      value (object): the value that will be stored in the cache.
    """
    with self._state_lock:
      self._cache[name] = value

  def GetFromCache(self, name, default_value=None):
    """Thread-safe method to get data from the state's cache.

    Args:
      name (str): string with the name of the cache variable.
      default_value (object): the value that will be returned if
          the item does not exist in the cache. Optional argument
          and defaults to None.

    Returns:
      object: object from the cache that corresponds to the name, or
          the value of "default_value" if the cache does not contain
          the variable.
    """
    with self._state_lock:
      return self._cache.get(name, default_value)

  def StoreContainer(self, container):
    """Thread-safe method to store data in the state's store.

    Args:
      container (AttributeContainer): data to store.
    """
    with self._state_lock:
      self.store.setdefault(container.CONTAINER_TYPE, []).append(container)

  def GetContainers(self, container_class, pop=False):
    """Thread-safe method to retrieve data from the state's store.

    Args:
      container_class (type): AttributeContainer class used to filter data.
      pop (Optional[bool]): Whether to remove the containers from the state when
          they are retrieved.

    Returns:
      list[AttributeContainer]: attribute container objects provided in
          the store that correspond to the container type.
    """
    with self._state_lock:
      containers = self.store.get(container_class.CONTAINER_TYPE, [])
      if pop:
        self.store[container_class.CONTAINER_TYPE] = []
      return containers

  def _SetupModuleThread(self, module_definition):
    """Calls the module's SetUp() function and sets a threading event for it.

    Callback for _InvokeModulesInThreads.

    Args:
      module_definition (dict[str, str]): recipe module definition.
    """
    module_name = module_definition['name']
    runtime_name = module_definition.get('runtime_name', module_name)
    logger.info('Setting up module: {0:s}'.format(runtime_name))
    new_args = utils.ImportArgsFromDict(
        module_definition['args'], self.command_line_options, self.config)
    module = self._module_pool[runtime_name]

    try:
      module.SetUp(**new_args)
    except errors.DFTimewolfError as exception:
      msg = "A critical error occurred in module {0:s}, aborting execution."
      logger.critical(msg.format(module.name))
    except Exception as exception:  # pylint: disable=broad-except
      msg = 'An unknown error occurred in module {0:s}: {1!s}'.format(
          module.name, exception)
      logger.critical(msg)
      # We're catching any exception that is not a DFTimewolfError, so we want
      # to generate an error for further reporting.
      error = errors.DFTimewolfError(
          message=msg, name='state', stacktrace=traceback.format_exc(),
          critical=True, unexpected=True)
      self.AddError(error)

    self._threading_event_per_module[runtime_name] = threading.Event()
    self.CleanUp()

  def SetupModules(self):
    """Performs setup tasks for each module in the module pool.

    Threads declared modules' SetUp() functions. Takes CLI arguments into
    account when replacing recipe parameters for each module.
    """
    # Note that vars() copies the values of argparse.Namespace to a dict.
    self._InvokeModulesInThreads(self._SetupModuleThread)

  def _RunModuleThread(self, module_definition):
    """Runs the module's Process() function.

    Callback for _InvokeModulesInThreads.

    Waits for any blockers to have finished before running Process(), then
    sets an Event flag declaring the module has completed.

    Args:
      module_definition (dict): module definition.
    """
    module_name = module_definition['name']
    runtime_name = module_definition.get('runtime_name', module_name)

    for dependency in module_definition['wants']:
      self._threading_event_per_module[dependency].wait()

    module = self._module_pool[runtime_name]

    # Abort processing if a module has had critical failures before.
    if self._abort_execution:
      logger.critical(
          'Aborting execution of {0:s} due to previous errors'.format(
              module.name))
      self._threading_event_per_module[runtime_name].set()
      self.CleanUp()
      return

    logger.info('Running module: {0:s}'.format(runtime_name))

    try:
      module.Process()
    except errors.DFTimewolfError as exception:
      logger.critical(
          "Critical error in module {0:s}, aborting execution".format(
              module.name))
    except Exception as exception:  # pylint: disable=broad-except
      msg = 'An unknown error occurred in module {0:s}: {1!s}'.format(
          module.name, exception)
      logger.critical(msg)
      # We're catching any exception that is not a DFTimewolfError, so we want
      # to generate an error for further reporting.
      error = errors.DFTimewolfError(
          message=msg, name='state', stacktrace=traceback.format_exc(),
          critical=True, unexpected=True)
      self.AddError(error)

    logger.info('Module {0:s} finished execution'.format(runtime_name))
    self._threading_event_per_module[runtime_name].set()
    self.CleanUp()

  def RunPreflights(self):
    """Runs preflight modules."""
    for preflight_definition in self.recipe.get('preflights', []):
      preflight_name = preflight_definition['name']
      runtime_name = preflight_definition.get('runtime_name', preflight_name)

      args = preflight_definition.get('args', {})

      new_args = utils.ImportArgsFromDict(
          args, self.command_line_options, self.config)
      preflight = self._module_pool[runtime_name]
      try:
        preflight.SetUp(**new_args)
        preflight.Process()
      finally:
        self.CheckErrors(is_global=True)

  def CleanUpPreflights(self):
    """Executes any cleanup actions defined in preflight modules."""
    for preflight_definition in self.recipe.get('preflights', []):
      preflight_name = preflight_definition['name']
      runtime_name = preflight_definition.get('runtime_name', preflight_name)
      preflight = self._module_pool[runtime_name]
      try:
        preflight.CleanUp()
      finally:
        self.CheckErrors(is_global=True)

  def InstantiateModule(self, module_name):
    """Instantiates an arbitrary dfTimewolf module.

    Args:
      module_name (str): The name of the module to instantiate.

    Returns:
      BaseModule: An instance of a dftimewolf Module, which is a subclass of
          BaseModule.
    """
    module_class = modules_manager.ModulesManager.GetModuleByName(module_name)
    return module_class(self)

  def RunModules(self):
    """Performs the actual processing for each module in the module pool."""
    self._InvokeModulesInThreads(self._RunModuleThread)

  def RegisterStreamingCallback(self, target, container_type):
    """Registers a callback for a type of container.

    The function to be registered should a single parameter of type
    interface.AttributeContainer.

    Args:
      target (function): function to be called.
      container_type (type[interface.AttributeContainer]): container type on
          which the callback will be called.
    """
    if container_type not in self.streaming_callbacks:
      self.streaming_callbacks[container_type] = []
    self.streaming_callbacks[container_type].append(target)

  def StreamContainer(self, container):
    """Streams a container to the callbacks that are registered to handle it.

    Args:
      container (interface.AttributeContainer): container instance that will be
          streamed to any registered callbacks.
    """
    for callback in self.streaming_callbacks.get(type(container), []):
      callback(container)

  def AddError(self, error):
    """Adds an error to the state.

    Args:
      error (errors.DFTimewolfError): The dfTimewolf error to add.
    """
    if error.critical:
      self._abort_execution = True
    self.errors.append(error)

  def CleanUp(self):
    """Cleans up after running a module.

    The state's output becomes the input for the next stage. Any errors are
    moved to the global_errors attribute so that they can be reported at a
    later stage.
    """
    # Move any existing errors to global errors
    self.global_errors.extend(self.errors)
    self.errors = []

  def CheckErrors(self, is_global=False):
    """Checks for errors and exits if any of them are critical.

    Args:
      is_global (Optional[bool]): True if the global_errors attribute should
          be checked. False if the error attribute should be checked.
    """
    error_objects = self.global_errors if is_global else self.errors
    critical_errors = False

    if error_objects:
      logger.error('dfTimewolf encountered one or more errors:')

    for index, error in enumerate(error_objects):
      logger.error('{0:d}: error from {1:s}: {2:s}'.format(
          index+1, error.name, error.message))
      if error.stacktrace:
        for line in error.stacktrace.split('\n'):
          logger.error(line)
      if error.critical:
        critical_errors = True

    if any(error.unexpected for error in error_objects):
      logger.critical('One or more unexpected errors occurred.')
      logger.critical(
          'Please consider opening an issue: {0:s}'.format(NEW_ISSUE_URL))

    if critical_errors:
      raise errors.CriticalError('Critical error found. Aborting.')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Truncate.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# 26th October 2006
#
# A wrapper for the CCP4 program Truncate, which calculates F's from
# I's and gives a few useful statistics about the data set.
#
# FIXME 26/OCT/06 this needs to be able to take into account the solvent
#                 content of the crystal (at the moment it will be assumed
#                 to be 50%.)
#
# FIXME 16/NOV/06 need to be able to get the estimates B factor from the
#                 Wilson plot and also second moment stuff, perhaps?

import os
import sys

if not os.environ.has_key('XIA2CORE_ROOT'):
    raise RuntimeError, 'XIA2CORE_ROOT not defined'

if not os.path.join(os.environ['XIA2CORE_ROOT'], 'Python') in sys.path:
    sys.path.append(os.path.join(os.environ['XIA2CORE_ROOT'],
                                 'Python'))

from Driver.DriverFactory import DriverFactory
from Decorators.DecoratorFactory import DecoratorFactory
from lib.Guff import transpose_loggraph
from Handlers.Streams import Chatter, Debug

def Truncate(DriverType = None):
    '''A factory for TruncateWrapper classes.'''

    DriverInstance = DriverFactory.Driver(DriverType)
    CCP4DriverInstance = DecoratorFactory.Decorate(DriverInstance, 'ccp4')

    class TruncateWrapper(CCP4DriverInstance.__class__):
        '''A wrapper for Truncate, using the CCP4-ified Driver.'''

        def __init__(self):
            # generic things
            CCP4DriverInstance.__class__.__init__(self)

            self._anomalous = False
            self._nres = 0

            self.set_executable('truncate')

            self._b_factor = 0.0
            self._moments = None

            self._wilson_fit_grad = 0.0
            self._wilson_fit_grad_sd = 0.0
            self._wilson_fit_m = 0.0
            self._wilson_fit_m_sd = 0.0
            self._wilson_fit_range = None

            return

        def set_anomalous(self, anomalous):
            self._anomalous = anomalous
            return

        def truncate(self):
            '''Actually perform the truncation procedure.'''

            self.check_hklin()
            self.check_hklout()

            self.start()

            # write the harvest files in the local directory, not
            # in $HARVESTHOME. Though we have set this for the project
            # so we should be fine to just plough ahead...
            # self.input('usecwd')

            if self._anomalous:
                self.input('anomalous yes')
            else:
                self.input('anomalous no')

            if self._nres:
                self.input('nres %d' % self._nres)

            self.close_wait()

            try:
                self.check_for_errors()
                self.check_ccp4_errors()

            except RuntimeError, e:
                try:
                    os.remove(self.get_hklout())
                except:
                    pass

            # FIXME need to parse the output for interesting things here!

            for line in self.get_all_output():
                if 'Least squares straight line gives' in line:
                    list = line.replace('=', ' ').split()
                    if not '***' in list[6]:
                        self._b_factor = float(list[6])
                    else:
                        Debug.write('no B factor available')

                if 'LSQ Line Gradient' in line:
                    self._wilson_fit_grad = float(line.split()[-1])
                    resol_width = max(self._wilson_fit_range) - \
                                  min(self._wilson_fit_range)
                    if self._wilson_fit_grad > 0 and resol_width > 1.0:
                        raise RuntimeError, \
                              'wilson plot gradient positive: %.2f' % \
                              self._wilson_fit_grad
                    elif self._wilson_fit_grad > 0:
                        Debug.write(
                            'Positive gradient but not much wilson plot')

                if 'Uncertainty in Gradient' in line:
                    self._wilson_fit_grad_sd = float(line.split()[-1])
                if 'X Intercept' in line:
                    self._wilson_fit_m = float(line.split()[-1])
                if 'Uncertainty in Intercept' in line:
                    self._wilson_fit_m_sd = float(line.split()[-1])
                if 'Resolution range' in line:
                    self._wilson_fit_range = map(float, line.split()[-2:])

            results = self.parse_ccp4_loggraph()
            moments = transpose_loggraph(
                results['Acentric Moments of E for k = 1,3,4,6,8'])

            # keys we want in this are "Resln_Range" "1/resol^2" and
            # MomentZ2. The last of these should be around two, but is
            # likely to be a little different to this.
            self._moments = moments

            return

        def get_b_factor(self):
            return self._b_factor

        def get_wilson_fit(self):
            return self._wilson_fit_grad, self._wilson_fit_grad_sd, \
                   self._wilson_fit_m, self._wilson_fit_m_sd

        def get_wilson_fit_range(self):
            return self._wilson_fit_range

        def get_moments(self):
            return self._moments

    return TruncateWrapper()
<commit_msg><commit_after>#!/usr/bin/env python
# Truncate.py
#   Copyright (C) 2006 CCLRC, Graeme Winter
#
#   This code is distributed under the BSD license, a copy of which is
#   included in the root directory of this package.
#
# 26th October 2006
#
# A wrapper for the CCP4 program Truncate, which calculates F's from
# I's and gives a few useful statistics about the data set.
#
# FIXME 26/OCT/06 this needs to be able to take into account the solvent
#                 content of the crystal (at the moment it will be assumed
#                 to be 50%.)
#
# FIXME 16/NOV/06 need to be able to get the estimates B factor from the
#                 Wilson plot and also second moment stuff, perhaps?

import os
import sys

if not os.environ.has_key('XIA2CORE_ROOT'):
    raise RuntimeError, 'XIA2CORE_ROOT not defined'

if not os.path.join(os.environ['XIA2CORE_ROOT'], 'Python') in sys.path:
    sys.path.append(os.path.join(os.environ['XIA2CORE_ROOT'],
                                 'Python'))

from Driver.DriverFactory import DriverFactory
from Decorators.DecoratorFactory import DecoratorFactory
from lib.Guff import transpose_loggraph
from Handlers.Streams import Chatter, Debug

def Truncate(DriverType = None):
    '''A factory for TruncateWrapper classes.'''

    DriverInstance = DriverFactory.Driver(DriverType)
    CCP4DriverInstance = DecoratorFactory.Decorate(DriverInstance, 'ccp4')

    class TruncateWrapper(CCP4DriverInstance.__class__):
        '''A wrapper for Truncate, using the CCP4-ified Driver.'''

        def __init__(self):
            # generic things
            CCP4DriverInstance.__class__.__init__(self)

            self._anomalous = False
            self._nres = 0

            # should we do wilson scaling?
            self._wilson = True

            self.set_executable('truncate')

            self._b_factor = 0.0
            self._moments = None

            self._wilson_fit_grad = 0.0
            self._wilson_fit_grad_sd = 0.0
            self._wilson_fit_m = 0.0
            self._wilson_fit_m_sd = 0.0
            self._wilson_fit_range = None

            return

        def set_anomalous(self, anomalous):
            self._anomalous = anomalous
            return

        def set_wilson(self, wilson):
            '''Set the use of Wilson scaling - if you set this to False
            Wilson scaling will be switched off...'''
            self._wilson = wilson

        def truncate(self):
            '''Actually perform the truncation procedure.'''

            self.check_hklin()
            self.check_hklout()

            self.start()

            # write the harvest files in the local directory, not
            # in $HARVESTHOME. Though we have set this for the project
            # so we should be fine to just plough ahead...
            # self.input('usecwd')

            if self._anomalous:
                self.input('anomalous yes')
            else:
                self.input('anomalous no')

            if self._nres:
                self.input('nres %d' % self._nres)

            if not self._wilson:
                self.input('scale 1')

            self.close_wait()

            try:
                self.check_for_errors()
                self.check_ccp4_errors()

            except RuntimeError, e:
                try:
                    os.remove(self.get_hklout())
                except:
                    pass

            # FIXME need to parse the output for interesting things here!

            for line in self.get_all_output():
                if 'Least squares straight line gives' in line:
                    list = line.replace('=', ' ').split()
                    if not '***' in list[6]:
                        self._b_factor = float(list[6])
                    else:
                        Debug.write('no B factor available')

                if 'LSQ Line Gradient' in line:
                    self._wilson_fit_grad = float(line.split()[-1])
                    resol_width = max(self._wilson_fit_range) - \
                                  min(self._wilson_fit_range)
                    if self._wilson_fit_grad > 0 and resol_width > 1.0:
                        raise RuntimeError, \
                              'wilson plot gradient positive: %.2f' % \
                              self._wilson_fit_grad
                    elif self._wilson_fit_grad > 0:
                        Debug.write(
                            'Positive gradient but not much wilson plot')

                if 'Uncertainty in Gradient' in line:
                    self._wilson_fit_grad_sd = float(line.split()[-1])
                if 'X Intercept' in line:
                    self._wilson_fit_m = float(line.split()[-1])
                if 'Uncertainty in Intercept' in line:
                    self._wilson_fit_m_sd = float(line.split()[-1])
                if 'Resolution range' in line:
                    self._wilson_fit_range = map(float, line.split()[-2:])

            results = self.parse_ccp4_loggraph()
            moments = transpose_loggraph(
                results['Acentric Moments of E for k = 1,3,4,6,8'])

            # keys we want in this are "Resln_Range" "1/resol^2" and
            # MomentZ2. The last of these should be around two, but is
            # likely to be a little different to this.
            self._moments = moments

            return

        def get_b_factor(self):
            return self._b_factor

        def get_wilson_fit(self):
            return self._wilson_fit_grad, self._wilson_fit_grad_sd, \
                   self._wilson_fit_m, self._wilson_fit_m_sd

        def get_wilson_fit_range(self):
            return self._wilson_fit_range

        def get_moments(self):
            return self._moments

    return TruncateWrapper()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import unittest

import tests.common
import core
from core.localisation import _
from core import Rpg
import models.settings
import json
import sqlite3

class rpgTests(tests.common.common):
		def test_unknown_world(self):
			rpgEngine = Rpg.Rpg()
			try:
				rpgEngine.init("some/unexisting/world", "uselessLogin")
			except core.exception.exception as e:
				self.assertEquals(e.message, _('ERROR_UNKNOWN_SELECTED_WORLD'))

		def test_invalid_world(self):
			rpgEngine = Rpg.Rpg()
			try:
				rpgEngine.init("tests/invalidDB", "uselessLogin")
			except sqlite3.OperationalError as e:
				self.assertTrue(True)
<commit_msg>test on the validity of the action added<commit_after># -*- coding: utf-8 -*-
import unittest

import tests.common
import core
from core.localisation import _
from core import Rpg
import models.settings
import json
import sqlite3

class rpgTests(tests.common.common):
		def test_unknown_world(self):
			rpgEngine = Rpg.Rpg()
			try:
				rpgEngine.init("some/unexisting/world", "uselessLogin")
			except core.exception.exception as e:
				self.assertEquals(e.message, _('ERROR_UNKNOWN_SELECTED_WORLD'))

		def test_invalid_world(self):
			rpgEngine = Rpg.Rpg()
			try:
				rpgEngine.init("tests/invalidDB", "uselessLogin")
			except sqlite3.OperationalError as e:
				self.assertTrue(True)

		def test_invalid_action_format(self):
			rpgEngine = Rpg.Rpg()
			rpgEngine.init(self.dbFile, self.login)
			try:
				rpgEngine.setAction("not list action")
			except TypeError as e:
				self.assertEquals(e.message, _('ERROR_INVALID_FORMAT_ACTION'))
<|endoftext|>


<commit_before>import numpy as np
from scipy.spatial import distance

def DistancesMatrix(X,Xt,Nl,Nt,typeD='Euc',T=False,sqr=True,xinterval=None):
##############################################################################################
# Pairwise distances matrix between the points contained in a matrix X (training points) and #
#                           in a matrix Xt (testing points)				     #
#											     #
#          Louis-Francois Arsenault, Columbia University la2518@columbia.edu (2016)	     #
##############################################################################################
#   INPUT:										     #
# 											     #
#   X  	:   Matrix with the training inputs. The form is a numpy array containing	     #
#	    the X's, where one X, X_i is a line vector with p dimensions. Must be in the form#
#	    numpy.array([[x11,x12,...,x1p],[x21,x22,...,x2p],...,[xn1,xn2,...,xnp]])	     #
#           so shape is (n,p)           !!!!!!VERY IMPORTANT!!!!!!			     #
# 					 						     #
#   Xt 	:   Matrix with the test inputs, same form as X				 	     #
#   Nl 	:   Number of learning examples						 	     #
#   Nt 	:   Number of test examples						 	     #
#   typeD : Distance metric. The possible metrics are Euclidean 'Euc' (default),	     #
# 	    Euclidean for continuous functions 'Euc_Cont' and Manhattan 'Man'. Any distance  #
#	    implemented in scipy cdist function can also be passed 	  	     	     #
#   T	:   If calculated for training purpose, T=True and when for testing T=False	     #
#   sq	:   If Euclidean distance, decide if return the square distance sqr=False 	     #
#           (faster) or the distance sqr=True						     #
#   xinterval: tuple containing the interval of integration for Euc_Cont: =(a,b)	     #
#											     #
#   OUTPUT:					 					     #
#											     #
#   D   :   Distances matrix. Can be sqaured distances if sqr=False			     #
##############################################################################################

	if typeD == 'Euc':
                Amat = X**2
                Bmat = Xt**2
                sqvec = np.sum(Amat,1)
                sqvec2 = np.sum(Bmat,1)
                Ainter = np.outer(sqvec,np.ones((1,Nt)))
                Binter = np.outer(sqvec2,np.ones((1,Nl)))
                #Euclidean distance square
                distSQ = Ainter.transpose() + Binter - 2.*np.dot(Xt,X.transpose())

		if T==True:
			#When training, Xt = X and the diagonal should be zero.
			#However, the fast Euclidean calculation we use here gives
			#something very small but not zero. So we just put 0
			np.fill_diagonal(distSQ,0.)
		if sqr==True:    #Return the distance
                	return np.sqrt(distSQ)
		elif sqr==False: #Return the square distance, faster since no sqrt
				 #function to take
			return distSQ

	elif typeD == 'Euc_Cont':
	#Euclidean distance for continuous functions of one variable i.e. d_ij^2= int_a^b dx ( f(x_i)-f(x_j) )^2
	#Using Simpson 1/3
		xinit = xinterval[0]
		xfinal = xinterval[1]
		feat_N = X.shape[1]
		interval=float(feat_N-1)
		hstep  = (xfinal-xinit)/interval

		if interval % 2:
        		raise ValueError('The number of intervals n must be even')

		#Build the coefficients for Simpson 1/3
		indice_Simp=np.arange(0,feat_N)
		coeffs_simp_vec = 3.+(-1.)**(indice_Simp+1) #pattern of alternating 2's and 4'
		coeffs_simp_vec[0]=1.
		coeffs_simp_vec[-1]=1.
		coeffs_simp = np.diag(coeffs_simp_vec)

		Amat = np.dot(X**2,coeffs_simp)
                Bmat = np.dot(Xt**2,coeffs_simp)
                sqvec = np.sum(Amat,1)
                sqvec2 = np.sum(Bmat,1)
                Ainter = np.outer(sqvec,np.ones((1,Nt)))
                Binter = np.outer(sqvec2,np.ones((1,Nl)))
                #Continuous Euclidean distance square
                distSQ = hstep/3*( Ainter.transpose() + Binter - 2.*np.dot(Xt.dot(coeffs_simp),X.transpose()) )

                if T==True:
                        #When training, Xt = X and the diagonal should be zero.
                        #However, the fast Euclidean calculation we use here gives
                        #for some entries something very small but not zero. So we just put 0
                        np.fill_diagonal(distSQ,0.)
                if sqr==True:    #Return the distance
                        return np.sqrt(distSQ)
                elif sqr==False: #Return the square distance, faster since no sqrt
                                 #function to take
                        return distSQ
        elif typeD == 'Man': #Homemade one was too slow! Although scipy one not
			    #that fast. Implementation in C not fast enough at the moment, even with using openmp, but will work on it
		return distance.cdist(Xt,X,'cityblock')
	else:
		#Any distance implemented in cdist
		return distance.cdist(Xt,X,typeD)
	#else:
	#	raise ValueError('The distance metric %r is not included',typeD)
###############################################################################################

#vec = np.ones((Nl,1))
 #               Xtf = np.outer(vec,Xt)
  #              Y = X.copy()
   #             Np=X.shape[1]
    #            for rp in range(0,Np-1):
     #                   Y = np.concatenate((Y,X))

#distances_matrix = np.zeros((Nt,Nl))
                #for count1 in range(0,Nt):
                #       d_row = Xt[count1,:]
                 #       vec = np.ones((Nl,1))
                  #      d_row = np.outer(vec,d_row)
                   #     dist = np.absolute(d_row-X)
                    #    distances_matrix[count1,:] = np.sum(dist,1)
<commit_msg>Update Calc_dist.py<commit_after>import numpy as np
from scipy.spatial import distance

def DistancesMatrix(X,Xt,Nl,Nt,typeD='Euc',T=False,sqr=True,xinterval=None):
##############################################################################################
# Pairwise distances matrix between the points contained in a matrix X (training points) and #
#                           in a matrix Xt (testing points)				     #
#											     #
#          Louis-Francois Arsenault, Columbia University la2518@columbia.edu (2016)	     #
##############################################################################################
#   INPUT:										     #
# 											     #
#   X  	:   Matrix with the training inputs. The form is a numpy array containing	     #
#	    the X's, where one X, X_i is a line vector with p dimensions. Must be in the form#
#	    numpy.array([[x11,x12,...,x1p],[x21,x22,...,x2p],...,[xn1,xn2,...,xnp]])	     #
#           so shape is (n,p)           !!!!!!VERY IMPORTANT!!!!!!			     #
# 					 						     #
#   Xt 	:   Matrix with the test inputs, same form as X				 	     #
#   Nl 	:   Number of learning examples						 	     #
#   Nt 	:   Number of test examples						 	     #
#   typeD : Distance metric. The possible metrics are Euclidean 'Euc' (default),	     #
# 	    Euclidean for continuous functions 'Euc_Cont' and Manhattan 'Man'. Any distance  #
#	    implemented in scipy cdist function can also be passed 	  	     	     #
#   T	:   If calculated for training purpose, T=True and when for testing T=False	     #
#   sq	:   If Euclidean distance, decide if return the square distance sqr=False 	     #
#           (faster) or the distance sqr=True						     #
#   xinterval: tuple containing the interval of integration for Euc_Cont: =(a,b)	     #
#											     #
#   OUTPUT:					 					     #
#											     #
#   D   :   Distances matrix. Can be sqaured distances if sqr=False			     #
##############################################################################################

	if typeD == 'Euc':
                Amat = X**2
                Bmat = Xt**2
                sqvec = np.sum(Amat,1)
                sqvec2 = np.sum(Bmat,1)
                Ainter = np.outer(sqvec,np.ones((1,Nt)))
                Binter = np.outer(sqvec2,np.ones((1,Nl)))
                #Euclidean distance square
                distSQ = Ainter.transpose() + Binter - 2.*np.dot(Xt,X.transpose())

		if T==True:
			#When training, Xt = X and the diagonal should be zero.
			#However, the fast Euclidean calculation we use here gives
			#something very small but not zero. So we just put 0
			np.fill_diagonal(distSQ,0.)
		if sqr==True:    #Return the distance
                	return np.sqrt(distSQ)
		elif sqr==False: #Return the square distance, faster since no sqrt
				 #function to take
			return distSQ

	elif typeD == 'Euc_Cont':
	#Euclidean distance for continuous functions of one variable i.e. d_ij^2= int_a^b dx ( f(x_i)-f(x_j) )^2
	#Using Simpson 1/3
		xinit = xinterval[0]
		xfinal = xinterval[1]
		feat_N = X.shape[1]
		interval=float(feat_N-1)
		hstep  = (xfinal-xinit)/interval

		if interval % 2:
        		raise ValueError('The number of intervals n must be even')

		#Build the coefficients for Simpson 1/3
		indice_Simp=np.arange(0,feat_N)
		coeffs_simp_vec = 3.+(-1.)**(indice_Simp+1) #pattern of alternating 2's and 4'
		coeffs_simp_vec[0]=1.
		coeffs_simp_vec[-1]=1.
		coeffs_simp = np.diag(coeffs_simp_vec)

		Amat = np.dot(X**2,coeffs_simp)
                Bmat = np.dot(Xt**2,coeffs_simp)
                sqvec = np.sum(Amat,1)
                sqvec2 = np.sum(Bmat,1)
                Ainter = np.outer(sqvec,np.ones((1,Nt)))
                Binter = np.outer(sqvec2,np.ones((1,Nl)))
                #Continuous Euclidean distance square
                distSQ = hstep/3*( Ainter.transpose() + Binter - 2.*np.dot(Xt.dot(coeffs_simp),X.transpose()) )

                if T==True:
                        #When training, Xt = X and the diagonal should be zero.
                        #However, the fast Euclidean calculation we use here gives
                        #for some entries something very small but not zero. So we just put 0
                        np.fill_diagonal(distSQ,0.)
                if sqr==True:    #Return the distance
                        return np.sqrt(distSQ)
                elif sqr==False: #Return the square distance, faster since no sqrt
                                 #function to take
                        return distSQ
        elif typeD == 'Man': #Homemade one was too slow! Although scipy one not
			    #that fast. Implementation in C not fast enough at the moment, even with using openmp, but will work on it
		return distance.cdist(Xt,X,'cityblock')
	else:
		#Any distance implemented in cdist
		return distance.cdist(Xt,X,typeD)
	#else:
	#	raise ValueError('The distance metric %r is not included',typeD)
###############################################################################################
<|endoftext|>


<commit_before>#!/usr/bin/python
'''
Usage::

    ./decoder.py [module] [data_file]

Sample::

    ./decoder.py pyroute2.netlink.rtnl.tcmsg.tcmsg ./sample_packet_01.data
    ./decoder.py pyroute2.netlink.nl80211.nl80211cmd ./nl80211.data

Module is a name within rtnl hierarchy. File should be a
binary data in the escaped string format (see samples).
'''
import sys
from pprint import pprint
from importlib import import_module
from pyroute2.common import load_dump

mod = sys.argv[1]
f = open(sys.argv[2], 'r')
s = mod.split('.')
package = '.'.join(s[:-1])
module = s[-1]
m = import_module(package)
met = getattr(m, module)

data = load_dump(f)

offset = 0
inbox = []
while offset < len(data):
    msg = met(data[offset:])
    msg.decode()
    pprint(msg)
    offset += msg['header']['length']
<commit_msg>split NL msgs with a dotted line for readability<commit_after>#!/usr/bin/python
'''
Usage::

    ./decoder.py [module] [data_file]

Sample::

    ./decoder.py pyroute2.netlink.rtnl.tcmsg.tcmsg ./sample_packet_01.data
    ./decoder.py pyroute2.netlink.nl80211.nl80211cmd ./nl80211.data

Module is a name within rtnl hierarchy. File should be a
binary data in the escaped string format (see samples).
'''
import sys
from pprint import pprint
from importlib import import_module
from pyroute2.common import load_dump

mod = sys.argv[1]
f = open(sys.argv[2], 'r')
s = mod.split('.')
package = '.'.join(s[:-1])
module = s[-1]
m = import_module(package)
met = getattr(m, module)

data = load_dump(f)

offset = 0
inbox = []
while offset < len(data):
    msg = met(data[offset:])
    msg.decode()
    pprint(msg)
    print('.'*40)
    offset += msg['header']['length']
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
amazon_api_lookup.py

This module uses the Amazon Product API to lookup barcodes as UPC and
EAN codes, and returns a list of possible product matches in the Amazon
catalog, along with the corresponding Amazon Standard Identification
Number (ASIN).

The actual Amazon Product API credentials necessary for access are
stored in a private, local file (amazon_local_settings.py) via a settings
harness.

"""

import sys
import json

from amazonproduct import API, errors

from amazon_settings import ACCESS_KEY, SECRET_KEY, ASSOCIATE, AMZLOCALE

api = API(locale=AMZLOCALE, access_key_id=ACCESS_KEY, secret_access_key=SECRET_KEY, associate_tag=ASSOCIATE)

def lookup (barcode, ID_TYPES=['UPC','EAN']):
    """Lookup the given barcode and return a list of possible matches"""

    matches = [] # list of {'desc', 'asin', 'type'}

    for idtype in ID_TYPES:
        try:
            result = api.item_lookup(barcode, SearchIndex='All', IdType=idtype)
            for item in result.Items.Item:
                matches.append({'desc': unicode(item.ItemAttributes.Title),
                                'asin': unicode(item.ASIN),
                                'type': idtype})
        except (errors.InvalidClientTokenId, errors.InvalidParameterValue):
            pass

    return matches

if __name__ == "__main__":
    """Create a command-line main() entry point"""

    if len(sys.argv) != 2:
        # Define the usage
        print sys.argv[0], '[barcode]'

    else:
        # lookup the barcode and return
        # a string of the results to stdout,
        # or nothing if there were no matches

        products = lookup(sys.argv[1])
        if len(products) > 0:
            print json.dumps(products)

<commit_msg>Distinguished Amazon API lookup exceptions: bad credentials versus too many requests, versus other errors, versus simply not found<commit_after>#!/usr/bin/env python

"""
amazon_api_lookup.py

This module uses the Amazon Product API to lookup barcodes as UPC and
EAN codes, and returns a list of possible product matches in the Amazon
catalog, along with the corresponding Amazon Standard Identification
Number (ASIN).

The actual Amazon Product API credentials necessary for access are
stored in a private, local file (amazon_local_settings.py) via a settings
harness.

"""

import sys
import json

from amazonproduct import API, errors

from amazon_settings import ACCESS_KEY, SECRET_KEY, ASSOCIATE, AMZLOCALE

api = API(locale=AMZLOCALE, access_key_id=ACCESS_KEY, secret_access_key=SECRET_KEY, associate_tag=ASSOCIATE)

def lookup (barcode, ID_TYPES=['UPC','EAN']):
    """Lookup the given barcode and return a list of possible matches"""

    matches = [] # list of {'desc', 'asin', 'type'}

    for idtype in ID_TYPES:
        try:
            result = api.item_lookup(barcode, SearchIndex='All', IdType=idtype)
            for item in result.Items.Item:
                matches.append({'desc': unicode(item.ItemAttributes.Title),
                                'asin': unicode(item.ASIN),
                                'type': idtype})

        except (errors.InvalidAccount, errors.InvalidClientTokenId, errors.MissingClientTokenId):
            print >>sys.stderr, "Amazon Product API lookup: bad account credentials"

        except errors.TooManyRequests, toomanyerr:
            print >>sys.stderr, "Amazon Product API lookup error:", toomanyerr

        except errors.InternalError, awserr:
            print >>sys.stderr, "Amazon Product API lookup error:", awserr

        except errors.InvalidParameterValue:
            # this simply means the barcode
            # does not exist for the given type,
            # so no need to do anything explicit
            pass

    return matches

if __name__ == "__main__":
    """Create a command-line main() entry point"""

    if len(sys.argv) != 2:
        # Define the usage
        print sys.argv[0], '[barcode]'

    else:
        # lookup the barcode and return
        # a string of the results to stdout,
        # or nothing if there were no matches

        products = lookup(sys.argv[1])
        if len(products) > 0:
            print json.dumps(products)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import json
import shutil
import subprocess
import time

def ReadKddsToBeCalculated(fname):
    """
    self explanatory
    """

    listKdds=[]

    with open(fname,'r') as f:
        for line in f:
            listKdds.append(line.rstrip('\n'))

    return listKdds

def make_cluster(CID, mach_type, nof_machs, ZID):
    """
    Given machine type and # of machines, creates cluster

    Parameters
    ------------

    CID: string
        cluster id

    mach_type: string
        machine type

    nof_machs: integer
        number of machines

    ZID: string
        zone id
    """

    cmd = "gcloud container clusters create {0} --machine-type {1} --zone {3} --num-nodes {2}".format(CID, mach_type, nof_machs, ZID)

    rc = subprocess.call(cmd, shell=True)
    return rc

def auth_cluster(CID, ZID):
    """
    given zone id and cluser id, make authentication
    """

    cmd = "gcloud container clusters get-credentials {0} --zone {1}".format(CID, ZID)
    rc = subprocess.call(cmd, shell=True)
    return rc

def cvt_kdd_to_gname(kdd):
    """
    given Kdd, convert the name into gcloud one
    """
    t = kdd
    t = t.lower()
    t = t. replace('_', '-')

    return t

def read_template(template):
    """
    read and return template file as JSON
    """
    data = None
    with open(template) as data_file:
        data = json.load(data_file)

    return data

def make_pod_from_template(temjson, kdd, docker2run):
    """
    given JSON from template and kdd, make in-memory pod json
    """

    gname = cvt_kdd_to_gname( kdd )

    temjson["metadata"]["name"] = gname

    temjson["spec"]["containers"][0]["name"] = gname

    temjson["spec"]["containers"][0]["image"] = docker2run

    temjson["spec"]["containers"][0]["args"][0] = kdd

    return temjson

def make_json_pod(template, kdd, docker2run):
    """
    from template and Kdd to calc, make appropriate
    """

    temjson = read_template(template)
    outjson = make_pod_from_template(temjson, kdd, docker2run)

    fname = "pod" + "_" + kdd + ".json"
    with open(fname, "w+") as f:
        f.write(json.dumps(outjson, indent=4))

    return fname

def read_config():
    """
    """
    with open("config_cluster.json") as data_file:
        data = json.load(data_file)
    return data

def main(kdds_fname, numberOfGCL):
    """
    This method creates a cluster, and then
    for a given cluster launches pods (one pod per kdd),
    which are read from input file
    """

    cfg = read_config()

    CID   = cfg["CID"]
    ZID   = cfg["ZID"]
    mtype = cfg["machine-type"]

    docker  = cfg["docker"]
    gcr     = cfg["gcr"]
    project = cfg["project"]

    print("From config_cluster.json:")
    print(CID,ZID,mtype,docker,gcr,project)

    print("Reading KDDs list from {0}".format(kdds_fname))

    Kdds = ReadKddsToBeCalculated(kdds_fname)

    print("To compute KDDs: {0}".format(len(Kdds)))

    print("Making cluster with nodes: {0}".format(numberOfGCL))

    rc = make_cluster(CID, mtype, numberOfGCL, ZID)
    if rc != 0:
        print("Cannot make cluster")
        sys.exit(1)

    rc = auth_cluster(CID, ZID)
    if rc != 0:
        print("Cannot make auth")
        sys.exit(1)

    docker2run = os.path.join(gcr, project, docker) # full path to docker

    for kdd in Kdds:
        pod_name = make_json_pod("tempod.json", kdd, docker2run)
        cmd = "kubectl create -f " + pod_name
        rc = 0
        for k in range(0, 12): # several attempts to make a pod
            rc = subprocess.call(cmd, shell=True)
            if rc == 0:
                time.sleep(0.5)
                break

        if rc != 0:
            print("Cannot make kdd {0}".format(kdd))
            sys.exit(1)

if __name__ =='__main__':
    nof_args = len(sys.argv)

    if nof_args == 1:
        print("Use: startCluster list_of_KDDs <optional>number_of_nodes")
        print("Default machine is usually n1-highcpu-2 with 2CPUs, see config_cluster.json")
        print("Default # of nodes is 8")
        sys.exit(1)

    kdds_fname = ""
    if nof_args >= 2:
        kdds_fname = sys.argv[1]

    numberOfGCL = 8 # default number of nodes
    if nof_args > 2:
        numberOfGCL = int(sys.argv[2])

    main(kdds_fname, numberOfGCL)

    sys.exit(0)

<commit_msg>Documentation updated<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import json
import shutil
import subprocess
import time

def ReadKddsToBeCalculated(fname):
    """
    Read list of KDDs from file to be calculated

    Parameters
    ------------

    fname: string
        file name of the KDDs list

    returns: list
        all KDDs from fname
    """

    listKdds=[]

    with open(fname,'r') as f:
        for line in f:
            listKdds.append(line.rstrip('\n'))

    return listKdds

def make_cluster(CID, mach_type, nof_machs, ZID):
    """
    Given machine type and # of machines, creates cluster

    Parameters
    ------------

    CID: string
        cluster id

    mach_type: string
        machine type

    nof_machs: integer
        number of machines

    ZID: string
        zone id

    returns: integer
        return code from gcloud call
    """

    cmd = "gcloud container clusters create {0} --machine-type {1} --zone {3} --num-nodes {2}".format(CID, mach_type, nof_machs, ZID)

    rc = subprocess.call(cmd, shell=True)
    return rc

def auth_cluster(CID, ZID):
    """
    Given zone id and cluser id, make authentication

    Parameters
    ------------

    CID: string
        cluster ID

    ZID: string
        zone ID

    returns: integer
        return code from gcloud call
    """

    cmd = "gcloud container clusters get-credentials {0} --zone {1}".format(CID, ZID)
    rc = subprocess.call(cmd, shell=True)
    return rc

def cvt_kdd_to_gname(kdd):
    """
    Given Kdd, convert the name into gcloud compatible one
    Gcloud shall have small letters and dahs instead of underscore

    Parameters
    ------------

    kdd: string
        KDD name

    returns: string
        name suited for gcloud
    """

    t = kdd
    t = t.lower()
    t = t. replace('_', '-')

    return t

def read_template(template):
    """
    Read and return template file as JSON

    Parameters
    ------------

    kdd: string
        KDD name

    returns: dictionary
        JSON parsed as dictionary
    """
    data = None
    with open(template) as data_file:
        data = json.load(data_file)

    return data

def make_pod_from_template(temjson, kdd, docker2run):
    """
    Given JSON from template and kdd, make in-memory pod json

    Parameters
    ------------

    temjson: dictionary
        In-memory pod JSON

    kdd: string
        KDD to compute name

    docker2run: string
        docker image to run

    returns: dictionary
        modified JSON suitable for computation
    """

    gname = cvt_kdd_to_gname( kdd )

    temjson["metadata"]["name"] = gname

    temjson["spec"]["containers"][0]["name"] = gname

    temjson["spec"]["containers"][0]["image"] = docker2run

    temjson["spec"]["containers"][0]["args"][0] = kdd

    return temjson

def make_json_pod(template, kdd, docker2run):
    """
    From template and Kdd to calc, make appropriate pod JSON

    Parameters
    ------------

    template: string
        file name of the JSON template

    kdd: string
        KDD to compute name

    docker2run: string
        docker image to run

    returns: dictionary
        modified JSON suitable for computation
    """

    temjson = read_template(template)
    outjson = make_pod_from_template(temjson, kdd, docker2run)

    fname = "pod" + "_" + kdd + ".json"
    with open(fname, "w+") as f:
        f.write(json.dumps(outjson, indent=4))

    return fname

def read_config(cfname):
    """
    Read cluster configuration file as JSON

    Parameters
    ------------

    cfname: string
        cluster config name

    returns: dictionary
        JSON parsed as dictionary
    """
    with open(cfname) as data_file:
        data = json.load(data_file)
    return data

def main(kdds_fname, numberOfGCL):
    """
    This method creates a cluster, and then
    for a given cluster launches pods (one pod per kdd),
    which are read from input file

    Parameters
    ------------

    kdds_fname: string
        file name which contains list of KDDs to compute

    numberOfGCL: integer
        number of the nodes in the cluster
    """

    cfg = read_config("config_cluster.json")

    CID   = cfg["CID"]
    ZID   = cfg["ZID"]
    mtype = cfg["machine-type"]

    docker  = cfg["docker"]
    gcr     = cfg["gcr"]
    project = cfg["project"]

    print("From config_cluster.json:")
    print(CID,ZID,mtype,docker,gcr,project)

    print("Reading KDDs list from {0}".format(kdds_fname))

    Kdds = ReadKddsToBeCalculated(kdds_fname)

    print("To compute KDDs: {0}".format(len(Kdds)))

    print("Making cluster with nodes: {0}".format(numberOfGCL))

    rc = make_cluster(CID, mtype, numberOfGCL, ZID)
    if rc != 0:
        print("Cannot make cluster")
        sys.exit(1)

    rc = auth_cluster(CID, ZID)
    if rc != 0:
        print("Cannot make auth")
        sys.exit(1)

    docker2run = os.path.join(gcr, project, docker) # full path to docker

    for kdd in Kdds:
        pod_name = make_json_pod("tempod.json", kdd, docker2run)
        cmd = "kubectl create -f " + pod_name
        rc = 0
        for k in range(0, 12): # several attempts to make a pod
            rc = subprocess.call(cmd, shell=True)
            if rc == 0:
                time.sleep(0.5)
                break

        if rc != 0:
            print("Cannot make kdd {0}".format(kdd))
            sys.exit(1)

if __name__ =='__main__':
    nof_args = len(sys.argv)

    if nof_args == 1:
        print("Use: startCluster list_of_KDDs <optional>number_of_nodes")
        print("Default machine is usually n1-highcpu-2 with 2CPUs, see config_cluster.json")
        print("Default # of nodes is 8")
        sys.exit(1)

    kdds_fname = ""
    if nof_args >= 2:
        kdds_fname = sys.argv[1]

    numberOfGCL = 8 # default number of nodes
    if nof_args > 2:
        numberOfGCL = int(sys.argv[2])

    main(kdds_fname, numberOfGCL)

    sys.exit(0)
<|endoftext|>


<commit_before>import argparse
import httplib2
import os, errno, datetime, threading, logging
import apiclient.discovery
import oauth2client.client
import oauth2client.file
import oauth2client.tools
import isodate
import rumps
from AppKit import NSWorkspace
from Foundation import NSURL

# CLIENT_SECRETS = os.path.join(os.path.dirname(__file__), 'client_secrets.json')
# NOTE: *.py files will be archived in site-packages.zip by py2app. So relative to this __file__ don't work.
import __main__
CLIENT_SECRETS = os.path.join(os.path.dirname(os.path.abspath(__main__.__file__)), "modules/google_calendar/client_secrets.json")
CACERTS = os.path.join(os.path.dirname(os.path.abspath(__main__.__file__)), "modules/google_calendar/cacerts.txt")
# Set up a Flow object to be used for authentication.
# Add one or more of the following scopes. PLEASE ONLY ADD THE SCOPES YOU
# NEED. For more information on using scopes please see
# <https://developers.google.com/+/best-practices>.
FLOW = oauth2client.client.flow_from_clientsecrets(
  CLIENT_SECRETS,
  scope = [
    #'https://www.googleapis.com/auth/calendar',
    'https://www.googleapis.com/auth/calendar.readonly',
  ],
  message = oauth2client.tools.message_if_missing(CLIENT_SECRETS))

def start(app):
  http = httplib2.Http(ca_certs=CACERTS)
  # If the credentials don't exist or are invalid run through the native client
  # flow. The Storage object will ensure that if successful the good
  # credentials will get written back to the file.
  storage_path = os.path.expanduser("~/.quiet/modules/google_calendar/oauth2.credentials")
  storage_dir = os.path.dirname(storage_path)
  try:
    os.makedirs(storage_dir)
  except OSError as e:
    if e.errno == errno.EEXIST and os.path.isdir(storage_dir):
      pass
    else:
      raise
  storage = oauth2client.file.Storage(storage_path)
  credentials = storage.get()
  if credentials is None or credentials.invalid:
      # FIXME: Use argparse with no actual aguments
      parser = argparse.ArgumentParser(
          description=__doc__,
          formatter_class=argparse.RawDescriptionHelpFormatter,
          parents=[oauth2client.tools.argparser])
      flags = parser.parse_args([])
      credentials = oauth2client.tools.run_flow(FLOW, storage, flags, http=http)

  http = credentials.authorize(http)

  # Construct the service object for the interacting with the Calendar API.
  service = apiclient.discovery.build('calendar', 'v3', http=http)

  #_check_calender_and_update(app, service)
  threading.Timer(0, _check_calender_and_update, args=[app, service, []]).start()

#DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S+00:00"

def _check_calender_and_update(app, service, items, retry=0):
  events = []
  now = datetime.datetime.now(tz=isodate.tzinfo.Utc())
  a_day_later = now + datetime.timedelta(days=1)
  try:
    page_token = None
    while True:
      calendar_list = service.calendarList().list(pageToken=page_token, showHidden=True).execute()
      for calendar_list_entry in calendar_list['items']:
        if not 'selected' in calendar_list_entry or calendar_list_entry['selected'] == False:
          continue
        #print calendar_list_entry
        calendar_id = calendar_list_entry['id']
        #print u"%s (%s)" % (calendar_list_entry['summary'], calendar_id)
        list_response = service.events().list(
            calendarId = calendar_id,
            timeMin = isodate.datetime_isoformat(now),
            timeMax = isodate.datetime_isoformat(now + datetime.timedelta(days=7))
        ).execute()
        for event in list_response['items']:
          if 'recurrence' in event:
            recurrence_events = service.events().instances(
              calendarId = calendar_id,
              eventId = event['id'],
              timeMin = isodate.datetime_isoformat(now),
              timeMax = isodate.datetime_isoformat(now + datetime.timedelta(days=7))
            ).execute()
            for recurrence_event in recurrence_events['items']:
              events.append(recurrence_event)
          else:
            events.append(event)
      page_token = calendar_list.get('nextPageToken')
      if not page_token:
        break
  except oauth2client.client.AccessTokenRefreshError:
    print ("The credentials have been revoked or expired, please re-run"
      "the application to re-authorize")
  except:
    logging.exception("Failed to access to Google Calendar.")
    # Retry
    sec = min(10 * 1 << retry, 60 * 5)
    print("Retry after %ds" % sec)
    threading.Timer(sec, _check_calender_and_update, args=[app, service, items, retry + 1]).start()
    return

  # get datetime to be sorted
  for event in events:
    if 'date' in event['start']:
      event['_date'] = isodate.parse_date(event['start']['date'])
      event['_datetime'] = datetime.datetime.combine(event['_date'], datetime.time(0, 0, tzinfo=isodate.tzinfo.Utc()))
    if 'dateTime' in event['start']:
      event['_datetime'] = isodate.parse_datetime(event['start']['dateTime'])
  # Display number of events in 24h as "title"
  in24h = len([e for e in events if e['_datetime'] < a_day_later])
  app.title = in24h
  # Clear items added last time
  for item in items:
    del app.menu[item.title if type(item) == rumps.MenuItem else item]
  # Display events as menu items
  items = []
  items.append("%d events in 24h (Last sync: %s)" % (in24h, datetime.datetime.now().strftime("%H:%M")))

  def open_url(sender):
    NSWorkspace.sharedWorkspace().openURL_(NSURL.URLWithString_(sender._event['htmlLink']))

  for event in sorted(events, key=lambda e: e['_datetime']):
    start = ""
    if '_date' in event:
      date = event['_date']
      start = "%s/%s %s" % (date.month, date.day, date.strftime("(%a)"))
    elif '_datetime' in event:
      dt = event['_datetime']
      start = "%s/%s %s" % (dt.month, dt.day, dt.strftime("(%a) %H:%M"))
    #items.append("%s %s" % (start, event['summary']))
    menu_item = rumps.MenuItem("%s %s" % (start, event['summary']), callback=open_url)
    menu_item._event = event
    items.append(menu_item)
  # Add items
  for item in items:
    app.menu.insert_before('separator_1', item) # FIXME
  #print app.menu

  # Repeat after 5min
  threading.Timer(60 * 5, _check_calender_and_update, args=[app, service, items, 0]).start()

  # tick = int(app.title) + 1
  # print tick
  # app.title = str(tick)
  # threading.Timer(1, timer_func).start()
<commit_msg>Remove somehow duplicated events from menu items<commit_after>import argparse
import httplib2
import os, errno, datetime, threading, logging
import apiclient.discovery
import oauth2client.client
import oauth2client.file
import oauth2client.tools
import isodate
import rumps
from AppKit import NSWorkspace
from Foundation import NSURL

# CLIENT_SECRETS = os.path.join(os.path.dirname(__file__), 'client_secrets.json')
# NOTE: *.py files will be archived in site-packages.zip by py2app. So relative to this __file__ don't work.
import __main__
CLIENT_SECRETS = os.path.join(os.path.dirname(os.path.abspath(__main__.__file__)), "modules/google_calendar/client_secrets.json")
CACERTS = os.path.join(os.path.dirname(os.path.abspath(__main__.__file__)), "modules/google_calendar/cacerts.txt")
# Set up a Flow object to be used for authentication.
# Add one or more of the following scopes. PLEASE ONLY ADD THE SCOPES YOU
# NEED. For more information on using scopes please see
# <https://developers.google.com/+/best-practices>.
FLOW = oauth2client.client.flow_from_clientsecrets(
  CLIENT_SECRETS,
  scope = [
    #'https://www.googleapis.com/auth/calendar',
    'https://www.googleapis.com/auth/calendar.readonly',
  ],
  message = oauth2client.tools.message_if_missing(CLIENT_SECRETS))

def start(app):
  http = httplib2.Http(ca_certs=CACERTS)
  # If the credentials don't exist or are invalid run through the native client
  # flow. The Storage object will ensure that if successful the good
  # credentials will get written back to the file.
  storage_path = os.path.expanduser("~/.quiet/modules/google_calendar/oauth2.credentials")
  storage_dir = os.path.dirname(storage_path)
  try:
    os.makedirs(storage_dir)
  except OSError as e:
    if e.errno == errno.EEXIST and os.path.isdir(storage_dir):
      pass
    else:
      raise
  storage = oauth2client.file.Storage(storage_path)
  credentials = storage.get()
  if credentials is None or credentials.invalid:
      # FIXME: Use argparse with no actual aguments
      parser = argparse.ArgumentParser(
          description=__doc__,
          formatter_class=argparse.RawDescriptionHelpFormatter,
          parents=[oauth2client.tools.argparser])
      flags = parser.parse_args([])
      credentials = oauth2client.tools.run_flow(FLOW, storage, flags, http=http)

  http = credentials.authorize(http)

  # Construct the service object for the interacting with the Calendar API.
  service = apiclient.discovery.build('calendar', 'v3', http=http)

  #_check_calender_and_update(app, service)
  threading.Timer(0, _check_calender_and_update, args=[app, service, []]).start()

#DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%S+00:00"

def _check_calender_and_update(app, service, items, retry=0):
  events = []
  def append(e):
    if not any(x for x in events if x['id'] == e['id']):
      events.append(e)
  now = datetime.datetime.now(tz=isodate.tzinfo.Utc())
  try:
    page_token = None
    while True:
      calendar_list = service.calendarList().list(pageToken=page_token, showHidden=True).execute()
      for calendar_list_entry in calendar_list['items']:
        if not 'selected' in calendar_list_entry or calendar_list_entry['selected'] == False:
          continue
        #print calendar_list_entry
        calendar_id = calendar_list_entry['id']
        #print u"%s (%s)" % (calendar_list_entry['summary'], calendar_id)
        list_response = service.events().list(
            calendarId = calendar_id,
            timeMin = isodate.datetime_isoformat(now),
            timeMax = isodate.datetime_isoformat(now + datetime.timedelta(days=7))
        ).execute()
        for event in list_response['items']:
          if 'recurrence' in event:
            recurrence_events = service.events().instances(
              calendarId = calendar_id,
              eventId = event['id'],
              timeMin = isodate.datetime_isoformat(now),
              timeMax = isodate.datetime_isoformat(now + datetime.timedelta(days=7))
            ).execute()
            for recurrence_event in recurrence_events['items']:
              append(recurrence_event)
          else:
            append(event)
      page_token = calendar_list.get('nextPageToken')
      if not page_token:
        break
  except oauth2client.client.AccessTokenRefreshError:
    print ("The credentials have been revoked or expired, please re-run"
      "the application to re-authorize")
  except:
    logging.exception("Failed to access to Google Calendar.")
    # Retry
    sec = min(10 * 1 << retry, 60 * 5)
    print("Retry after %ds" % sec)
    threading.Timer(sec, _check_calender_and_update, args=[app, service, items, retry + 1]).start()
    return

  # remove somehow duplicated events
  # for x in events:
  #   for y in evens:

  # get datetime to be sorted
  for event in events:
    # print(event['id'])
    if 'date' in event['start']:
      event['_date'] = isodate.parse_date(event['start']['date'])
      event['_datetime'] = datetime.datetime.combine(event['_date'], datetime.time(0, 0, tzinfo=isodate.tzinfo.Utc()))
    if 'dateTime' in event['start']:
      event['_datetime'] = isodate.parse_datetime(event['start']['dateTime'])
  # Display number of events in 24h as "title"
  a_day_later = now + datetime.timedelta(days=1)
  in24h = len([e for e in events if e['_datetime'] < a_day_later])
  app.title = in24h
  # Clear items added last time
  for item in items:
    key = item.title if type(item) == rumps.MenuItem else item
    if key in app.menu:
      del app.menu[key]
  # Display events as menu items
  items = []
  items.append("%d events in 24h (Last sync: %s)" % (in24h, datetime.datetime.now().strftime("%H:%M")))

  def open_url(sender):
    NSWorkspace.sharedWorkspace().openURL_(NSURL.URLWithString_(sender._event['htmlLink']))

  for event in sorted(events, key=lambda e: e['_datetime']):
    start = ""
    if '_date' in event:
      date = event['_date']
      start = "%s/%s %s" % (date.month, date.day, date.strftime("(%a)"))
    elif '_datetime' in event:
      dt = event['_datetime']
      start = "%s/%s %s" % (dt.month, dt.day, dt.strftime("(%a) %H:%M"))
    #items.append("%s %s" % (start, event['summary']))
    menu_item = rumps.MenuItem("%s %s" % (start, event['summary']), callback=open_url)
    menu_item._event = event
    items.append(menu_item)
  # Add items
  for item in items:
    app.menu.insert_before('separator_1', item) # FIXME
  #print app.menu

  # Repeat after 5min
  threading.Timer(60 * 5, _check_calender_and_update, args=[app, service, items, 0]).start()

  # tick = int(app.title) + 1
  # print tick
  # app.title = str(tick)
  # threading.Timer(1, timer_func).start()
<|endoftext|>


<commit_before># Example of dic based on the CherryPy songs tutorial
# See https://cherrypy.readthedocs.org/en/3.3.0/tutorial/REST.html#getting-started

import cherrypy
import dic

songs = {
    '1': {
        'title': 'Lumberjack Song',
        'artist': 'Canadian Guard Choir'
    },

    '2': {
        'title': 'Always Look On the Bright Side of Life',
        'artist': 'Eric Idle'
    },

    '3': {
        'title': 'Spam Spam Spam',
        'artist': 'Monty Python'
    }
}

class Song(object):
    def __init__(self, title, artist):
    pass

class SongDatabase(object):
    def __init__(self, song_factory: dic.Factory(Song)):
        self.song_factory = song_factory
        self.songs =

    def __getitem__(self, item):
        return self.songs[item]

class Songs:
    exposed = True

    def __init__(self, song_database: SongDatabase):
        self.songs = song_database

    def GET(self, id=None):

        if id is None:
            return('Here are all the songs we have: %s' % songs)
        elif id in songs:
            song = songs[id]

            return(
                'Song with the ID %s is called %s, and the artist is %s' % (
                    id, song['title'], song['artist']))
        else:
            return('No song with the ID %s :-(' % id)

    def POST(self, title, artist):

        id = str(max([int(_) for _ in songs.keys()]) + 1)

        songs[id] = {
            'title': title,
            'artist': artist
        }

        return ('Create a new song with the ID: %s' % id)

    def PUT(self, id, title=None, artist=None):
        if id in songs:
            song = songs[id]

            song['title'] = title or song['title']
            song['artist'] = artist or song['artist']

            return(
                'Song with the ID %s is now called %s, '
                'and the artist is now %s' % (
                    id, song['title'], song['artist'])
            )
        else:
            return('No song with the ID %s :-(' % id)

    def DELETE(self, id):
        if id in songs:
            songs.pop(id)

            return('Song with the ID %s has been deleted.' % id)
        else:
            return('No song with the ID %s :-(' % id)

if __name__ == '__main__':
    builder = dic.ContainerBuilder()
    builder.register_class(Songs)
    builder.register_class(SongDatabase)
    builder.register_class(Song)

    container = builder.build()

    cherrypy.tree.mount(
        container.resolve(Songs), '/api/songs',
        {'/':
            {'request.dispatch': cherrypy.dispatch.MethodDispatcher()}
         }
    )

    cherrypy.engine.start()
    cherrypy.engine.block()
<commit_msg>Adds cherrypy songs sample<commit_after># Example of dic based on the CherryPy songs tutorial
# See https://cherrypy.readthedocs.org/en/3.3.0/tutorial/REST.html#getting-started

import cherrypy
import dic

class Song(object):
    def __init__(self, title, artist):
        self.title = title
        self.artist = artist

    def __repr__(self):
        return 'title: %s, artist: %s' % (self.title, self.artist)

class SongDatabase(object):
    def __init__(self, song_factory: dic.rel.Factory(Song)):
        self.song_factory = song_factory
        self.songs = {}

    def add_song(self, ident, title, artist):
        self.songs[ident] = self.song_factory(title=title, artist=artist)

    def __getitem__(self, ident):
        return self.songs[ident]

    def __str__(self):
        return self.songs.__str__()

class Songs:
    exposed = True

    def __init__(self, song_database: SongDatabase):
        self.songs = song_database
        self.songs.add_song('1', 'Lumberjack Song', 'Canadian Guard Choir')
        self.songs.add_song('2', 'Always Look On the Bright Side of Life', 'Eric Idle')
        self.songs.add_song('3', 'Spam Spam Spam', 'Monty Python')

    def GET(self, id=None):

        if id is None:
            return('Here are all the songs we have: %s' % self.songs)
        elif id in self.songs:
            song = self.songs[id]

            return(
                'Song with the ID %s is called %s, and the artist is %s' % (
                    id, song.title, song.artist))
        else:
            return('No song with the ID %s :-(' % id)

if __name__ == '__main__':
    builder = dic.container.ContainerBuilder()
    builder.register_class(Songs)
    builder.register_class(SongDatabase)
    builder.register_class(Song)

    container = builder.build()

    cherrypy.tree.mount(
        container.resolve(Songs), '/api/songs',
        {'/':
            {'request.dispatch': cherrypy.dispatch.MethodDispatcher()}
         }
    )

    cherrypy.engine.start()
    cherrypy.engine.block()
<|endoftext|>


<commit_before>import re
import operator

class GATKTable:
    def __read_data_line(self, fp):
        line = fp.readline()
        # remove whitespace at beginning of line
        line = re.sub(r'^\s+', '', line)
        # transform whitespace into a single field separator
        line = re.sub(r'\s+', '\x00', line)
        fields = re.split(r'\x00', line)

        if len(fields) != self.num_columns:
            raise ParseError(fp, "invalid number of columns")

        return fields

    def __parse_table_header(self, fp):
        # parse table descriptor
        # #:GATKTable:<number of fields>:<number of rows>:<format specifier>:...:<format specifier>:;
        line = fp.readline()
        fields = re.split(r':', line)

        if len(fields) < 6:
            raise ParseError(fp, "not enough fields in table header")

        if fields[0] != "#":
            raise ParseError(fp, "first field in table header is invalid")

        if fields[1] != "GATKTable":
            raise ParseError(fp, "second field in table header is invalid")

        if fields[-1] != ";":
            raise ParseError(fp, "table header terminator not found")

        try:
            self.num_columns = int(fields[2])
        except:
            raise ParseError(fp, "invalid column count in table header")

        try:
            self.num_rows = int(fields[3])
        except:
            raise ParseError(fp, "invalid row count in column header")

        # now parse all the format strings into tuples of the form (<digit specifier>, <format character>)
        for field in fields[4:-1]:
            m = re.match(r'^\%(\.([0-9]+))?([sfd])$', field)
            if m == None:
                raise ParseError(fp, "error parsing field format specifier '" + field + "'")

            fmt = m.groups()[1:]
            self.field_format_specifiers.append(fmt)

        # parse table name and description
        #:GATKTable:Arguments:Recalibration argument collection values used in this run
        line = fp.readline()
        fields = re.split(r':', line)

        if len(fields) != 4:
            raise ParseError(fp, "not enough fields in table description")

        if fields[0] != "#":
            raise ParseError(fp, "first field in table description is invalid")

        if fields[1] != "GATKTable":
            raise ParseError(fp, "second field in table description is invalid")

        self.name = fields[2]
        self.description = fields[3]

        # finally parse the first row of the table, containing the column names
        self.column_names = self.__read_data_line(fp)

    def parse(self, fp):
        # read table header
        self.__parse_table_header(fp)

        for r in xrange(self.num_rows):
            # parse each entry in the table according to the format specifier
            fields = self.__read_data_line(fp)
            record = list()

            for i in xrange(len(self.field_format_specifiers)):
                desc = self.field_format_specifiers[i][1]
                data = fields[i]

                if desc == "s":
                    # read field as string
                    record.append(data)
                elif desc == "d":
                    # read field as integer
                    record.append(int(data))
                elif desc == "f":
                    # read field as floating point
                    record.append(float(data))
                else:
                    raise ParseError(fp, "unknown format field specifier " + desc)

            self.rows.append(record)

        # consume the empty line at the end
        line = fp.readline()
        if line != "":
            raise ParseError(fp, "table delimiter not found")

    # sorts the table using all string and integer fields as keys
    def __sort(self):
        # build list of keys
        keys = list()

        for i in xrange(len(self.field_format_specifiers)):
            desc = self.field_format_specifiers[i][1]
            if desc == "s" or desc == "d":
                keys.append(i)

        self.rows = sorted(self.rows, key = operator.itemgetter(*keys))

    def __init__(self, fp=None):
        # #:GATKTable:2:17:%s:%s:;
        self.num_columns = 0
        self.num_rows = 0
        self.field_format_specifiers = list()

        # #:GATKTable:Arguments:Recalibration argument collection values used in this run
        self.name = ""
        self.description = ""

        # Argument      Value
        self.column_names = list()

        # list of rows, each row is a list of columns
        self.rows = list()

        if fp != None:
            self.parse(fp)
            self.__sort()
<commit_msg>diffreport: Fix imports<commit_after>import re
import operator
from textfile import ParseError

class GATKTable:
    def __read_data_line(self, fp):
        line = fp.readline()
        # remove whitespace at beginning of line
        line = re.sub(r'^\s+', '', line)
        # transform whitespace into a single field separator
        line = re.sub(r'\s+', '\x00', line)
        fields = re.split(r'\x00', line)

        if len(fields) != self.num_columns:
            raise ParseError(fp, "invalid number of columns")

        return fields

    def __parse_table_header(self, fp):
        # parse table descriptor
        # #:GATKTable:<number of fields>:<number of rows>:<format specifier>:...:<format specifier>:;
        line = fp.readline()
        fields = re.split(r':', line)

        if len(fields) < 6:
            raise ParseError(fp, "not enough fields in table header")

        if fields[0] != "#":
            raise ParseError(fp, "first field in table header is invalid")

        if fields[1] != "GATKTable":
            raise ParseError(fp, "second field in table header is invalid")

        if fields[-1] != ";":
            raise ParseError(fp, "table header terminator not found")

        try:
            self.num_columns = int(fields[2])
        except:
            raise ParseError(fp, "invalid column count in table header")

        try:
            self.num_rows = int(fields[3])
        except:
            raise ParseError(fp, "invalid row count in column header")

        # now parse all the format strings into tuples of the form (<digit specifier>, <format character>)
        for field in fields[4:-1]:
            m = re.match(r'^\%(\.([0-9]+))?([sfd])$', field)
            if m == None:
                raise ParseError(fp, "error parsing field format specifier '" + field + "'")

            fmt = m.groups()[1:]
            self.field_format_specifiers.append(fmt)

        # parse table name and description
        #:GATKTable:Arguments:Recalibration argument collection values used in this run
        line = fp.readline()
        fields = re.split(r':', line)

        if len(fields) != 4:
            raise ParseError(fp, "not enough fields in table description")

        if fields[0] != "#":
            raise ParseError(fp, "first field in table description is invalid")

        if fields[1] != "GATKTable":
            raise ParseError(fp, "second field in table description is invalid")

        self.name = fields[2]
        self.description = fields[3]

        # finally parse the first row of the table, containing the column names
        self.column_names = self.__read_data_line(fp)

    def parse(self, fp):
        # read table header
        self.__parse_table_header(fp)

        for r in xrange(self.num_rows):
            # parse each entry in the table according to the format specifier
            fields = self.__read_data_line(fp)
            record = list()

            for i in xrange(len(self.field_format_specifiers)):
                desc = self.field_format_specifiers[i][1]
                data = fields[i]

                if desc == "s":
                    # read field as string
                    record.append(data)
                elif desc == "d":
                    # read field as integer
                    record.append(int(data))
                elif desc == "f":
                    # read field as floating point
                    record.append(float(data))
                else:
                    raise ParseError(fp, "unknown format field specifier " + desc)

            self.rows.append(record)

        # consume the empty line at the end
        line = fp.readline()
        if line != "":
            raise ParseError(fp, "table delimiter not found")

    # sorts the table using all string and integer fields as keys
    def __sort(self):
        # build list of keys
        keys = list()

        for i in xrange(len(self.field_format_specifiers)):
            desc = self.field_format_specifiers[i][1]
            if desc == "s" or desc == "d":
                keys.append(i)

        self.rows = sorted(self.rows, key = operator.itemgetter(*keys))

    def __init__(self, fp=None):
        # #:GATKTable:2:17:%s:%s:;
        self.num_columns = 0
        self.num_rows = 0
        self.field_format_specifiers = list()

        # #:GATKTable:Arguments:Recalibration argument collection values used in this run
        self.name = ""
        self.description = ""

        # Argument      Value
        self.column_names = list()

        # list of rows, each row is a list of columns
        self.rows = list()

        if fp != None:
            self.parse(fp)
            self.__sort()
<|endoftext|>


<commit_before>"""Test clipping."""

# --- import --------------------------------------------------------------------------------------

import random

import WrightTools as wt
from WrightTools import datasets

# --- test ----------------------------------------------------------------------------------------

def test_w1_wa():
    p = datasets.PyCMDS.w1_wa_000
    data = wt.data.from_PyCMDS(p)
    new_max = random.random() * 0.5 * data.array_signal.max() + data.array_signal.min()
    data.array_signal.clip(max=new_max)
    assert data.array_signal.max() <= new_max
    data.close()

# --- run -----------------------------------------------------------------------------------------

if __name__ == "__main__":
    test_w1_wa()
<commit_msg>TST: remove random from clip (#748)<commit_after>"""Test clipping."""

# --- import --------------------------------------------------------------------------------------

import WrightTools as wt
from WrightTools import datasets

# --- test ----------------------------------------------------------------------------------------

def test_w1_wa():
    p = datasets.PyCMDS.w1_wa_000
    data = wt.data.from_PyCMDS(p)
    new_max = 0.5 * data.array_signal.max() + data.array_signal.min()
    data.array_signal.clip(max=new_max)
    assert data.array_signal.max() <= new_max
    data.close()

# --- run -----------------------------------------------------------------------------------------

if __name__ == "__main__":
    test_w1_wa()
<|endoftext|>