<commit_before># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Core conversion logic, serves as main point of access."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import imp

import gast

from tensorflow.contrib.autograph import operators
from tensorflow.contrib.autograph import utils
from tensorflow.contrib.autograph.converters import asserts
from tensorflow.contrib.autograph.converters import break_statements
from tensorflow.contrib.autograph.converters import builtin_functions
from tensorflow.contrib.autograph.converters import call_trees
from tensorflow.contrib.autograph.converters import conditional_expressions
from tensorflow.contrib.autograph.converters import continue_statements
from tensorflow.contrib.autograph.converters import control_flow
from tensorflow.contrib.autograph.converters import decorators
from tensorflow.contrib.autograph.converters import directives
from tensorflow.contrib.autograph.converters import error_handlers
from tensorflow.contrib.autograph.converters import lists
from tensorflow.contrib.autograph.converters import logical_expressions
from tensorflow.contrib.autograph.converters import name_scopes
from tensorflow.contrib.autograph.converters import return_statements
from tensorflow.contrib.autograph.converters import side_effect_guards
from tensorflow.contrib.autograph.converters import slices
from tensorflow.contrib.autograph.core import config
from tensorflow.contrib.autograph.core import converter
from tensorflow.contrib.autograph.core import errors
from tensorflow.contrib.autograph.pyct import ast_util
from tensorflow.contrib.autograph.pyct import inspect_utils
from tensorflow.contrib.autograph.pyct import origin_info
from tensorflow.contrib.autograph.pyct import parser
from tensorflow.contrib.autograph.pyct import qual_names
from tensorflow.contrib.autograph.pyct import transformer
from tensorflow.python.util import tf_inspect

# TODO(mdan): Might we not need any renaming at all?

def is_whitelisted_for_graph(o):
  """Check whether an entity is whitelisted for use in graph mode.

  Examples of whitelisted entities include all members of the tensorflow
  package.

  Args:
    o: A Python entity.
  Returns:
    Boolean
  """
  m = tf_inspect.getmodule(o)
  for prefix, in config.DEFAULT_UNCOMPILED_MODULES:
    if m.__name__.startswith(prefix):
      return True
  return False

def entity_to_graph(o, program_ctx, arg_values, arg_types):
  """Compile a Python entity into equivalent TensorFlow.

  The function will also recursively compile all the entities that `o`
  references, updating `dependency_cache`.

  This function is reentrant, and relies on dependency_cache to avoid
  generating duplicate code.

  Args:
    o: A Python entity.
    program_ctx: A ProgramContext object.
    arg_values: A dict containing value hints for symbols like function
        parameters.
    arg_types: A dict containing type hints for symbols like function
        parameters.

  Returns:
    A tuple (ast, new_name, namespace):
        * ast: An AST representing an entity with interface equivalent to `o`,
            but which when executed it creates TF a graph.
        * new_name: The symbol name under which the new entity can be found.
        * namespace: A dict mapping all symbols visible to the converted entity,
            keyed by their symbol name.

  Raises:
    ValueError: if the entity type is not supported.
  """
  if tf_inspect.isclass(o):
    node, name, ns = class_to_graph(o, program_ctx)
  elif tf_inspect.isfunction(o):
    # TODO(mdan): This is not a reliable mechanism.
    # The most reliable way is to check the source code, the AST will contain
    # a Lambda node instead of a FunctionDef
    if o.__name__ == '<lambda>':
      raise NotImplementedError(
          'lambda functions are not yet supported; declare the function'
          ' using def instead: %s' % o)
    else:
      node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types)
  elif tf_inspect.ismethod(o):
    node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types)
  else:
    raise ValueError(
        'Entity "%s" has unsupported type "%s". Only functions and classes are '
        'supported for now.' % (o, type(o)))

  program_ctx.add_to_cache(o, node)
  if program_ctx.recursive:
    while True:
      candidate = None
      for obj in program_ctx.name_map.keys():
        if obj not in program_ctx.dependency_cache:
          candidate = obj
          break
      if candidate is None:
        break
      if (hasattr(candidate, 'im_class') and
          getattr(candidate, 'im_class') not in program_ctx.partial_types):
        # Class members are converted with their objects, unless they're
        # only converted partially.
        continue
      entity_to_graph(candidate, program_ctx, {}, {})

  return node, name, ns

def class_to_graph(c, program_ctx):
  """Specialization of `entity_to_graph` for classes."""
  converted_members = {}
  method_filter = lambda m: tf_inspect.isfunction(m) or tf_inspect.ismethod(m)
  members = tf_inspect.getmembers(c, predicate=method_filter)
  if not members:
    raise ValueError('Cannot convert %s: it has no member methods.' % c)

  class_namespace = {}
  for _, m in members:
    # Only convert the members that are directly defined by the class.
    if inspect_utils.getdefiningclass(m, c) is not c:
      continue
    node, _, namespace = function_to_graph(
        m,
        program_ctx=program_ctx,
        arg_values={},
        arg_types={'self': (c.__name__, c)},
        owner_type=c)
    if class_namespace is None:
      class_namespace = namespace
    else:
      class_namespace.update(namespace)
    converted_members[m] = node
  namer = program_ctx.new_namer(class_namespace)
  class_name = namer.compiled_class_name(c.__name__, c)

  # TODO(mdan): This needs to be explained more thoroughly.
  # Process any base classes: if the sueprclass if of a whitelisted type, an
  # absolute import line is generated. Otherwise, it is marked for conversion
  # (as a side effect of the call to namer.compiled_class_name() followed by
  # program_ctx.update_name_map(namer)).
  output_nodes = []
  renames = {}
  bases = []
  for base in c.__bases__:
    if isinstance(object, base):
      bases.append('object')
      continue
    if is_whitelisted_for_graph(base):
      alias = namer.new_symbol(base.__name__, ())
      output_nodes.append(
          gast.ImportFrom(
              module=base.__module__,
              names=[gast.alias(name=base.__name__, asname=alias)],
              level=0))
    else:
      # This will trigger a conversion into a class with this name.
      alias = namer.compiled_class_name(base.__name__, base)
    bases.append(alias)
    renames[qual_names.QN(base.__name__)] = qual_names.QN(alias)
  program_ctx.update_name_map(namer)

  # Generate the definition of the converted class.
  output_nodes.append(
      gast.ClassDef(
          class_name,
          bases=bases,
          keywords=[],
          body=list(converted_members.values()),
          decorator_list=[]))
  node = gast.Module(output_nodes)

  # Make a final pass to replace references to the class or its base classes.
  # Most commonly, this occurs when making super().__init__() calls.
  # TODO(mdan): Making direct references to superclass' superclass will fail.
  node = qual_names.resolve(node)
  renames[qual_names.QN(c.__name__)] = qual_names.QN(class_name)
  node = ast_util.rename_symbols(node, renames)

  return node, class_name, class_namespace

def _add_reserved_symbol(namespace, name, entity):
  if name not in namespace:
    namespace[name] = entity
  elif namespace[name] != entity:
    raise ValueError('The name "%s" is reserved and may not be used.' % name)

ag_internal = None

def _add_self_references(namespace, autograph_module):
  """Adds namespace references to the module that exposes the api itself."""
  global ag_internal
  if ag_internal is None:
    # Craft a module that exposes parts of the external API as well as certain
    # internal modules.
    ag_internal = imp.new_module('autograph')
    ag_internal.converted_call = autograph_module.converted_call
    ag_internal.utils = utils
    ag_internal.rewrite_graph_construction_error = (
        errors.rewrite_graph_construction_error)
    # TODO(mdan): Add safeguards against name clashes.
    # We don't want to create a submodule because we want the operators to be
    # accessible as ag__.<operator>
    ag_internal.__dict__.update(operators.__dict__)

  _add_reserved_symbol(namespace, 'ag__', ag_internal)

def function_to_graph(f, program_ctx, arg_values, arg_types, owner_type=None):
  """Specialization of `entity_to_graph` for callable functions."""

  node, source = parser.parse_entity(f)
  node = node.body[0]
  origin_info.resolve(node, source, f)
  namespace = inspect_utils.getnamespace(f)
  _add_self_references(namespace, program_ctx.autograph_module)
  namer = program_ctx.new_namer(namespace)

  entity_info = transformer.EntityInfo(
      source_code=source,
      source_file='<fragment>',
      namespace=namespace,
      arg_values=arg_values,
      arg_types=arg_types,
      owner_type=owner_type)
  context = converter.EntityContext(namer, entity_info, program_ctx)
  node = node_to_graph(node, context)

  # TODO(mdan): This somewhat duplicates the call rename logic in call_treest.py
  new_name, did_rename = namer.compiled_function_name(f.__name__, f, owner_type)
  if not did_rename:
    new_name = f.__name__
    if node.name != f.__name__:
      raise NotImplementedError('Strange corner case. Send us offending code!')

  node.name = new_name
  program_ctx.update_name_map(namer)
  # TODO(mdan): Use this at compilation.

  return node, new_name, namespace

def node_to_graph(node, context):
  """Convert Python code to equivalent TF graph mode code.

  Args:
    node: AST, the code to convert.
    context: converter.EntityContext

  Returns:
    A tuple (node, deps):
        * node: A Python ast node, representing the converted code.
        * deps: A set of strings, the fully qualified names of entity
            dependencies that this node has.
  """
  # TODO(mdan): Insert list_comprehensions somewhere.

  node = converter.standard_analysis(node, context, is_initial=True)
  # Past this point, line numbers are no longer accurate so we ignore the
  # source.
  # TODO(mdan): Is it feasible to reconstruct intermediate source code?
  context.info.source_code = None

  node = converter.apply_(node, context, decorators)
  node = converter.apply_(node, context, directives)
  node = converter.apply_(node, context, break_statements)
  node = converter.apply_(node, context, asserts)
  # Note: sequencing continue canonicalization before for loop one avoids
  # dealing with the extra loop increment operation that the for
  # canonicalization creates.
  node = converter.apply_(node, context, continue_statements)
  context.info.namespace['len'] = len
  node = converter.apply_(node, context, return_statements)
  node = converter.apply_(node, context, lists)
  node = converter.apply_(node, context, slices)
  node = converter.apply_(node, context, builtin_functions)
  node = converter.apply_(node, context, call_trees)
  node = converter.apply_(node, context, control_flow)
  node = converter.apply_(node, context, conditional_expressions)
  node = converter.apply_(node, context, logical_expressions)
  node = converter.apply_(node, context, side_effect_guards)
  node = converter.apply_(node, context, name_scopes)
  node = converter.apply_(node, context, error_handlers)
  return node
<commit_msg>Disable rewriting errors for classes due to source map overwriting.<commit_after># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Core conversion logic, serves as main point of access."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import imp

import gast

from tensorflow.contrib.autograph import operators
from tensorflow.contrib.autograph import utils
from tensorflow.contrib.autograph.converters import asserts
from tensorflow.contrib.autograph.converters import break_statements
from tensorflow.contrib.autograph.converters import builtin_functions
from tensorflow.contrib.autograph.converters import call_trees
from tensorflow.contrib.autograph.converters import conditional_expressions
from tensorflow.contrib.autograph.converters import continue_statements
from tensorflow.contrib.autograph.converters import control_flow
from tensorflow.contrib.autograph.converters import decorators
from tensorflow.contrib.autograph.converters import directives
from tensorflow.contrib.autograph.converters import error_handlers
from tensorflow.contrib.autograph.converters import lists
from tensorflow.contrib.autograph.converters import logical_expressions
from tensorflow.contrib.autograph.converters import name_scopes
from tensorflow.contrib.autograph.converters import return_statements
from tensorflow.contrib.autograph.converters import side_effect_guards
from tensorflow.contrib.autograph.converters import slices
from tensorflow.contrib.autograph.core import config
from tensorflow.contrib.autograph.core import converter
from tensorflow.contrib.autograph.core import errors
from tensorflow.contrib.autograph.pyct import ast_util
from tensorflow.contrib.autograph.pyct import inspect_utils
from tensorflow.contrib.autograph.pyct import origin_info
from tensorflow.contrib.autograph.pyct import parser
from tensorflow.contrib.autograph.pyct import qual_names
from tensorflow.contrib.autograph.pyct import transformer
from tensorflow.python.util import tf_inspect

# TODO(mdan): Might we not need any renaming at all?

def is_whitelisted_for_graph(o):
  """Check whether an entity is whitelisted for use in graph mode.

  Examples of whitelisted entities include all members of the tensorflow
  package.

  Args:
    o: A Python entity.
  Returns:
    Boolean
  """
  m = tf_inspect.getmodule(o)
  for prefix, in config.DEFAULT_UNCOMPILED_MODULES:
    if m.__name__.startswith(prefix):
      return True
  return False

def entity_to_graph(o, program_ctx, arg_values, arg_types):
  """Compile a Python entity into equivalent TensorFlow.

  The function will also recursively compile all the entities that `o`
  references, updating `dependency_cache`.

  This function is reentrant, and relies on dependency_cache to avoid
  generating duplicate code.

  Args:
    o: A Python entity.
    program_ctx: A ProgramContext object.
    arg_values: A dict containing value hints for symbols like function
        parameters.
    arg_types: A dict containing type hints for symbols like function
        parameters.

  Returns:
    A tuple (ast, new_name, namespace):
        * ast: An AST representing an entity with interface equivalent to `o`,
            but which when executed it creates TF a graph.
        * new_name: The symbol name under which the new entity can be found.
        * namespace: A dict mapping all symbols visible to the converted entity,
            keyed by their symbol name.

  Raises:
    ValueError: if the entity type is not supported.
  """
  if tf_inspect.isclass(o):
    node, name, ns = class_to_graph(o, program_ctx)
  elif tf_inspect.isfunction(o):
    # TODO(mdan): This is not a reliable mechanism.
    # The most reliable way is to check the source code, the AST will contain
    # a Lambda node instead of a FunctionDef
    if o.__name__ == '<lambda>':
      raise NotImplementedError(
          'lambda functions are not yet supported; declare the function'
          ' using def instead: %s' % o)
    else:
      node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types)
  elif tf_inspect.ismethod(o):
    node, name, ns = function_to_graph(o, program_ctx, arg_values, arg_types)
  else:
    raise ValueError(
        'Entity "%s" has unsupported type "%s". Only functions and classes are '
        'supported for now.' % (o, type(o)))

  program_ctx.add_to_cache(o, node)
  if program_ctx.recursive:
    while True:
      candidate = None
      for obj in program_ctx.name_map.keys():
        if obj not in program_ctx.dependency_cache:
          candidate = obj
          break
      if candidate is None:
        break
      if (hasattr(candidate, 'im_class') and
          getattr(candidate, 'im_class') not in program_ctx.partial_types):
        # Class members are converted with their objects, unless they're
        # only converted partially.
        continue
      entity_to_graph(candidate, program_ctx, {}, {})

  return node, name, ns

def class_to_graph(c, program_ctx):
  """Specialization of `entity_to_graph` for classes."""
  converted_members = {}
  method_filter = lambda m: tf_inspect.isfunction(m) or tf_inspect.ismethod(m)
  members = tf_inspect.getmembers(c, predicate=method_filter)
  if not members:
    raise ValueError('Cannot convert %s: it has no member methods.' % c)

  class_namespace = {}
  for _, m in members:
    # Only convert the members that are directly defined by the class.
    if inspect_utils.getdefiningclass(m, c) is not c:
      continue
    node, _, namespace = function_to_graph(
        m,
        program_ctx=program_ctx,
        arg_values={},
        arg_types={'self': (c.__name__, c)},
        owner_type=c,
        rewrite_errors=False)
    if class_namespace is None:
      class_namespace = namespace
    else:
      class_namespace.update(namespace)
    converted_members[m] = node
  namer = program_ctx.new_namer(class_namespace)
  class_name = namer.compiled_class_name(c.__name__, c)

  # TODO(mdan): This needs to be explained more thoroughly.
  # Process any base classes: if the sueprclass if of a whitelisted type, an
  # absolute import line is generated. Otherwise, it is marked for conversion
  # (as a side effect of the call to namer.compiled_class_name() followed by
  # program_ctx.update_name_map(namer)).
  output_nodes = []
  renames = {}
  bases = []
  for base in c.__bases__:
    if isinstance(object, base):
      bases.append('object')
      continue
    if is_whitelisted_for_graph(base):
      alias = namer.new_symbol(base.__name__, ())
      output_nodes.append(
          gast.ImportFrom(
              module=base.__module__,
              names=[gast.alias(name=base.__name__, asname=alias)],
              level=0))
    else:
      # This will trigger a conversion into a class with this name.
      alias = namer.compiled_class_name(base.__name__, base)
    bases.append(alias)
    renames[qual_names.QN(base.__name__)] = qual_names.QN(alias)
  program_ctx.update_name_map(namer)

  # Generate the definition of the converted class.
  output_nodes.append(
      gast.ClassDef(
          class_name,
          bases=bases,
          keywords=[],
          body=list(converted_members.values()),
          decorator_list=[]))
  node = gast.Module(output_nodes)

  # Make a final pass to replace references to the class or its base classes.
  # Most commonly, this occurs when making super().__init__() calls.
  # TODO(mdan): Making direct references to superclass' superclass will fail.
  node = qual_names.resolve(node)
  renames[qual_names.QN(c.__name__)] = qual_names.QN(class_name)
  node = ast_util.rename_symbols(node, renames)

  return node, class_name, class_namespace

def _add_reserved_symbol(namespace, name, entity):
  if name not in namespace:
    namespace[name] = entity
  elif namespace[name] != entity:
    raise ValueError('The name "%s" is reserved and may not be used.' % name)

ag_internal = None

def _add_self_references(namespace, autograph_module):
  """Adds namespace references to the module that exposes the api itself."""
  global ag_internal
  if ag_internal is None:
    # Craft a module that exposes parts of the external API as well as certain
    # internal modules.
    ag_internal = imp.new_module('autograph')
    ag_internal.converted_call = autograph_module.converted_call
    ag_internal.utils = utils
    ag_internal.rewrite_graph_construction_error = (
        errors.rewrite_graph_construction_error)
    # TODO(mdan): Add safeguards against name clashes.
    # We don't want to create a submodule because we want the operators to be
    # accessible as ag__.<operator>
    ag_internal.__dict__.update(operators.__dict__)

  _add_reserved_symbol(namespace, 'ag__', ag_internal)

def function_to_graph(f,
                      program_ctx,
                      arg_values,
                      arg_types,
                      owner_type=None,
                      rewrite_errors=True):
  """Specialization of `entity_to_graph` for callable functions."""

  node, source = parser.parse_entity(f)
  node = node.body[0]
  origin_info.resolve(node, source, f)
  namespace = inspect_utils.getnamespace(f)
  _add_self_references(namespace, program_ctx.autograph_module)
  namer = program_ctx.new_namer(namespace)

  entity_info = transformer.EntityInfo(
      source_code=source,
      source_file='<fragment>',
      namespace=namespace,
      arg_values=arg_values,
      arg_types=arg_types,
      owner_type=owner_type)
  context = converter.EntityContext(namer, entity_info, program_ctx)
  node = node_to_graph(node, context, rewrite_errors=rewrite_errors)

  # TODO(mdan): This somewhat duplicates the call rename logic in call_treest.py
  new_name, did_rename = namer.compiled_function_name(f.__name__, f, owner_type)
  if not did_rename:
    new_name = f.__name__
    if node.name != f.__name__:
      raise NotImplementedError('Strange corner case. Send us offending code!')

  node.name = new_name
  program_ctx.update_name_map(namer)
  # TODO(mdan): Use this at compilation.

  return node, new_name, namespace

def node_to_graph(node, context, rewrite_errors=True):
  """Convert Python code to equivalent TF graph mode code.

  Args:
    node: AST, the code to convert.
    context: converter.EntityContext
    rewrite_errors: Boolean, whether or not to rewrite the error traceback.

  Returns:
    A tuple (node, deps):
        * node: A Python ast node, representing the converted code.
        * deps: A set of strings, the fully qualified names of entity
            dependencies that this node has.
  """
  # TODO(mdan): Insert list_comprehensions somewhere.

  node = converter.standard_analysis(node, context, is_initial=True)
  # Past this point, line numbers are no longer accurate so we ignore the
  # source.
  # TODO(mdan): Is it feasible to reconstruct intermediate source code?
  context.info.source_code = None

  node = converter.apply_(node, context, decorators)
  node = converter.apply_(node, context, directives)
  node = converter.apply_(node, context, break_statements)
  node = converter.apply_(node, context, asserts)
  # Note: sequencing continue canonicalization before for loop one avoids
  # dealing with the extra loop increment operation that the for
  # canonicalization creates.
  node = converter.apply_(node, context, continue_statements)
  context.info.namespace['len'] = len
  node = converter.apply_(node, context, return_statements)
  node = converter.apply_(node, context, lists)
  node = converter.apply_(node, context, slices)
  node = converter.apply_(node, context, builtin_functions)
  node = converter.apply_(node, context, call_trees)
  node = converter.apply_(node, context, control_flow)
  node = converter.apply_(node, context, conditional_expressions)
  node = converter.apply_(node, context, logical_expressions)
  node = converter.apply_(node, context, side_effect_guards)
  node = converter.apply_(node, context, name_scopes)
  if rewrite_errors:
    node = converter.apply_(node, context, error_handlers)
  return node
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Displayhook for IPython.

This defines a callable class that IPython uses for `sys.displayhook`.
"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

from __future__ import print_function

import sys

from IPython.core.formatters import _safe_get_formatter_method
from IPython.config.configurable import Configurable
from IPython.utils import io
from IPython.utils.py3compat import builtin_mod
from IPython.utils.traitlets import Instance
from IPython.utils.warn import warn

# TODO: Move the various attributes (cache_size, [others now moved]). Some
# of these are also attributes of InteractiveShell. They should be on ONE object
# only and the other objects should ask that one object for their values.

class DisplayHook(Configurable):
    """The custom IPython displayhook to replace sys.displayhook.

    This class does many things, but the basic idea is that it is a callable
    that gets called anytime user code returns a value.
    """

    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')

    def __init__(self, shell=None, cache_size=1000, **kwargs):
        super(DisplayHook, self).__init__(shell=shell, **kwargs)

        cache_size_min = 3
        if cache_size <= 0:
            self.do_full_cache = 0
            cache_size = 0
        elif cache_size < cache_size_min:
            self.do_full_cache = 0
            cache_size = 0
            warn('caching was disabled (min value for cache size is %s).' %
                 cache_size_min,level=3)
        else:
            self.do_full_cache = 1

        self.cache_size = cache_size

        # we need a reference to the user-level namespace
        self.shell = shell

        self._,self.__,self.___ = '','',''

        # these are deliberately global:
        to_user_ns = {'_':self._,'__':self.__,'___':self.___}
        self.shell.user_ns.update(to_user_ns)

    @property
    def prompt_count(self):
        return self.shell.execution_count

    #-------------------------------------------------------------------------
    # Methods used in __call__. Override these methods to modify the behavior
    # of the displayhook.
    #-------------------------------------------------------------------------

    def check_for_underscore(self):
        """Check if the user has set the '_' variable by hand."""
        # If something injected a '_' variable in __builtin__, delete
        # ipython's automatic one so we don't clobber that.  gettext() in
        # particular uses _, so we need to stay away from it.
        if '_' in builtin_mod.__dict__:
            try:
                del self.shell.user_ns['_']
            except KeyError:
                pass

    def quiet(self):
        """Should we silence the display hook because of ';'?"""
        # do not print output if input ends in ';'
        try:
            cell = self.shell.history_manager.input_hist_parsed[self.prompt_count]
            return cell.rstrip().endswith(';')
        except IndexError:
            # some uses of ipshellembed may fail here
            return False

    def start_displayhook(self):
        """Start the displayhook, initializing resources."""
        pass

    def write_output_prompt(self):
        """Write the output prompt.

        The default implementation simply writes the prompt to
        ``io.stdout``.
        """
        # Use write, not print which adds an extra space.
        io.stdout.write(self.shell.separate_out)
        outprompt = self.shell.prompt_manager.render('out')
        if self.do_full_cache:
            io.stdout.write(outprompt)

    def compute_format_data(self, result):
        """Compute format data of the object to be displayed.

        The format data is a generalization of the :func:`repr` of an object.
        In the default implementation the format data is a :class:`dict` of
        key value pair where the keys are valid MIME types and the values
        are JSON'able data structure containing the raw data for that MIME
        type. It is up to frontends to determine pick a MIME to to use and
        display that data in an appropriate manner.

        This method only computes the format data for the object and should
        NOT actually print or write that to a stream.

        Parameters
        ----------
        result : object
            The Python object passed to the display hook, whose format will be
            computed.

        Returns
        -------
        (format_dict, md_dict) : dict
            format_dict is a :class:`dict` whose keys are valid MIME types and values are
            JSON'able raw data for that MIME type. It is recommended that
            all return values of this should always include the "text/plain"
            MIME type representation of the object.
            md_dict is a :class:`dict` with the same MIME type keys
            of metadata associated with each output.

        """
        return self.shell.display_formatter.format(result)

    def write_format_data(self, format_dict, md_dict=None):
        """Write the format data dict to the frontend.

        This default version of this method simply writes the plain text
        representation of the object to ``io.stdout``. Subclasses should
        override this method to send the entire `format_dict` to the
        frontends.

        Parameters
        ----------
        format_dict : dict
            The format dict for the object passed to `sys.displayhook`.
        md_dict : dict (optional)
            The metadata dict to be associated with the display data.
        """
        if 'text/plain' not in format_dict:
            # nothing to do
            return
        # We want to print because we want to always make sure we have a
        # newline, even if all the prompt separators are ''. This is the
        # standard IPython behavior.
        result_repr = format_dict['text/plain']
        if '\n' in result_repr:
            # So that multi-line strings line up with the left column of
            # the screen, instead of having the output prompt mess up
            # their first line.
            # We use the prompt template instead of the expanded prompt
            # because the expansion may add ANSI escapes that will interfere
            # with our ability to determine whether or not we should add
            # a newline.
            prompt_template = self.shell.prompt_manager.out_template
            if prompt_template and not prompt_template.endswith('\n'):
                # But avoid extraneous empty lines.
                result_repr = '\n' + result_repr

        print(result_repr, file=io.stdout)

    def update_user_ns(self, result):
        """Update user_ns with various things like _, __, _1, etc."""

        # Avoid recursive reference when displaying _oh/Out
        if result is not self.shell.user_ns['_oh']:
            if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
                warn('Output cache limit (currently '+
                      repr(self.cache_size)+' entries) hit.\n'
                     'Flushing cache and resetting history counter...\n'
                     'The only history variables available will be _,__,___ and _1\n'
                     'with the current result.')

                self.flush()
            # Don't overwrite '_' and friends if '_' is in __builtin__ (otherwise
            # we cause buggy behavior for things like gettext).

            if '_' not in builtin_mod.__dict__:
                self.___ = self.__
                self.__ = self._
                self._ = result
                self.shell.push({'_':self._,
                                 '__':self.__,
                                '___':self.___}, interactive=False)

            # hackish access to top-level  namespace to create _1,_2... dynamically
            to_main = {}
            if self.do_full_cache:
                new_result = '_'+repr(self.prompt_count)
                to_main[new_result] = result
                self.shell.push(to_main, interactive=False)
                self.shell.user_ns['_oh'][self.prompt_count] = result

    def log_output(self, format_dict):
        """Log the output."""
        if 'text/plain' not in format_dict:
            # nothing to do
            return
        if self.shell.logger.log_output:
            self.shell.logger.log_write(format_dict['text/plain'], 'output')
        self.shell.history_manager.output_hist_reprs[self.prompt_count] = \
                                                    format_dict['text/plain']

    def finish_displayhook(self):
        """Finish up all displayhook activities."""
        io.stdout.write(self.shell.separate_out2)
        io.stdout.flush()

    def __call__(self, result=None):
        """Printing with history cache management.

        This is invoked everytime the interpreter needs to print, and is
        activated by setting the variable sys.displayhook to it.
        """
        self.check_for_underscore()
        if result is not None and not self.quiet():
            # If _ipython_display_ is defined, use that to display this object.
            display_method = _safe_get_formatter_method(result, '_ipython_display_')
            if display_method is not None:
                try:
                    return display_method()
                except NotImplementedError:
                    pass

            self.start_displayhook()
            self.write_output_prompt()
            format_dict, md_dict = self.compute_format_data(result)
            self.write_format_data(format_dict, md_dict)
            self.update_user_ns(result)
            self.log_output(format_dict)
            self.finish_displayhook()

    def flush(self):
        if not self.do_full_cache:
            raise ValueError("You shouldn't have reached the cache flush "
                             "if full caching is not enabled!")
        # delete auto-generated vars from global namespace

        for n in range(1,self.prompt_count + 1):
            key = '_'+repr(n)
            try:
                del self.shell.user_ns[key]
            except: pass
        # In some embedded circumstances, the user_ns doesn't have the
        # '_oh' key set up.
        oh = self.shell.user_ns.get('_oh', None)
        if oh is not None:
            oh.clear()

        # Release our own references to objects:
        self._, self.__, self.___ = '', '', ''

        if '_' not in builtin_mod.__dict__:
            self.shell.user_ns.update({'_':None,'__':None, '___':None})
        import gc
        # TODO: Is this really needed?
        # IronPython blocks here forever
        if sys.platform != "cli":
            gc.collect()

<commit_msg>cull 20% when display hook is full<commit_after># -*- coding: utf-8 -*-
"""Displayhook for IPython.

This defines a callable class that IPython uses for `sys.displayhook`.
"""

# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.

from __future__ import print_function

import sys

from IPython.core.formatters import _safe_get_formatter_method
from IPython.config.configurable import Configurable
from IPython.utils import io
from IPython.utils.py3compat import builtin_mod
from IPython.utils.traitlets import Instance, Float
from IPython.utils.warn import warn

# TODO: Move the various attributes (cache_size, [others now moved]). Some
# of these are also attributes of InteractiveShell. They should be on ONE object
# only and the other objects should ask that one object for their values.

class DisplayHook(Configurable):
    """The custom IPython displayhook to replace sys.displayhook.

    This class does many things, but the basic idea is that it is a callable
    that gets called anytime user code returns a value.
    """

    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
    cull_fraction = Float(0.2)

    def __init__(self, shell=None, cache_size=1000, **kwargs):
        super(DisplayHook, self).__init__(shell=shell, **kwargs)
        cache_size_min = 3
        if cache_size <= 0:
            self.do_full_cache = 0
            cache_size = 0
        elif cache_size < cache_size_min:
            self.do_full_cache = 0
            cache_size = 0
            warn('caching was disabled (min value for cache size is %s).' %
                 cache_size_min,level=3)
        else:
            self.do_full_cache = 1

        self.cache_size = cache_size

        # we need a reference to the user-level namespace
        self.shell = shell

        self._,self.__,self.___ = '','',''

        # these are deliberately global:
        to_user_ns = {'_':self._,'__':self.__,'___':self.___}
        self.shell.user_ns.update(to_user_ns)

    @property
    def prompt_count(self):
        return self.shell.execution_count

    #-------------------------------------------------------------------------
    # Methods used in __call__. Override these methods to modify the behavior
    # of the displayhook.
    #-------------------------------------------------------------------------

    def check_for_underscore(self):
        """Check if the user has set the '_' variable by hand."""
        # If something injected a '_' variable in __builtin__, delete
        # ipython's automatic one so we don't clobber that.  gettext() in
        # particular uses _, so we need to stay away from it.
        if '_' in builtin_mod.__dict__:
            try:
                del self.shell.user_ns['_']
            except KeyError:
                pass

    def quiet(self):
        """Should we silence the display hook because of ';'?"""
        # do not print output if input ends in ';'
        try:
            cell = self.shell.history_manager.input_hist_parsed[self.prompt_count]
            return cell.rstrip().endswith(';')
        except IndexError:
            # some uses of ipshellembed may fail here
            return False

    def start_displayhook(self):
        """Start the displayhook, initializing resources."""
        pass

    def write_output_prompt(self):
        """Write the output prompt.

        The default implementation simply writes the prompt to
        ``io.stdout``.
        """
        # Use write, not print which adds an extra space.
        io.stdout.write(self.shell.separate_out)
        outprompt = self.shell.prompt_manager.render('out')
        if self.do_full_cache:
            io.stdout.write(outprompt)

    def compute_format_data(self, result):
        """Compute format data of the object to be displayed.

        The format data is a generalization of the :func:`repr` of an object.
        In the default implementation the format data is a :class:`dict` of
        key value pair where the keys are valid MIME types and the values
        are JSON'able data structure containing the raw data for that MIME
        type. It is up to frontends to determine pick a MIME to to use and
        display that data in an appropriate manner.

        This method only computes the format data for the object and should
        NOT actually print or write that to a stream.

        Parameters
        ----------
        result : object
            The Python object passed to the display hook, whose format will be
            computed.

        Returns
        -------
        (format_dict, md_dict) : dict
            format_dict is a :class:`dict` whose keys are valid MIME types and values are
            JSON'able raw data for that MIME type. It is recommended that
            all return values of this should always include the "text/plain"
            MIME type representation of the object.
            md_dict is a :class:`dict` with the same MIME type keys
            of metadata associated with each output.

        """
        return self.shell.display_formatter.format(result)

    def write_format_data(self, format_dict, md_dict=None):
        """Write the format data dict to the frontend.

        This default version of this method simply writes the plain text
        representation of the object to ``io.stdout``. Subclasses should
        override this method to send the entire `format_dict` to the
        frontends.

        Parameters
        ----------
        format_dict : dict
            The format dict for the object passed to `sys.displayhook`.
        md_dict : dict (optional)
            The metadata dict to be associated with the display data.
        """
        if 'text/plain' not in format_dict:
            # nothing to do
            return
        # We want to print because we want to always make sure we have a
        # newline, even if all the prompt separators are ''. This is the
        # standard IPython behavior.
        result_repr = format_dict['text/plain']
        if '\n' in result_repr:
            # So that multi-line strings line up with the left column of
            # the screen, instead of having the output prompt mess up
            # their first line.
            # We use the prompt template instead of the expanded prompt
            # because the expansion may add ANSI escapes that will interfere
            # with our ability to determine whether or not we should add
            # a newline.
            prompt_template = self.shell.prompt_manager.out_template
            if prompt_template and not prompt_template.endswith('\n'):
                # But avoid extraneous empty lines.
                result_repr = '\n' + result_repr

        print(result_repr, file=io.stdout)

    def update_user_ns(self, result):
        """Update user_ns with various things like _, __, _1, etc."""

        # Avoid recursive reference when displaying _oh/Out
        if result is not self.shell.user_ns['_oh']:
            if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
                self.cull_cache()
            # Don't overwrite '_' and friends if '_' is in __builtin__ (otherwise
            # we cause buggy behavior for things like gettext).

            if '_' not in builtin_mod.__dict__:
                self.___ = self.__
                self.__ = self._
                self._ = result
                self.shell.push({'_':self._,
                                 '__':self.__,
                                '___':self.___}, interactive=False)

            # hackish access to top-level  namespace to create _1,_2... dynamically
            to_main = {}
            if self.do_full_cache:
                new_result = '_'+repr(self.prompt_count)
                to_main[new_result] = result
                self.shell.push(to_main, interactive=False)
                self.shell.user_ns['_oh'][self.prompt_count] = result

    def log_output(self, format_dict):
        """Log the output."""
        if 'text/plain' not in format_dict:
            # nothing to do
            return
        if self.shell.logger.log_output:
            self.shell.logger.log_write(format_dict['text/plain'], 'output')
        self.shell.history_manager.output_hist_reprs[self.prompt_count] = \
                                                    format_dict['text/plain']

    def finish_displayhook(self):
        """Finish up all displayhook activities."""
        io.stdout.write(self.shell.separate_out2)
        io.stdout.flush()

    def __call__(self, result=None):
        """Printing with history cache management.

        This is invoked everytime the interpreter needs to print, and is
        activated by setting the variable sys.displayhook to it.
        """
        self.check_for_underscore()
        if result is not None and not self.quiet():
            # If _ipython_display_ is defined, use that to display this object.
            display_method = _safe_get_formatter_method(result, '_ipython_display_')
            if display_method is not None:
                try:
                    return display_method()
                except NotImplementedError:
                    pass

            self.start_displayhook()
            self.write_output_prompt()
            format_dict, md_dict = self.compute_format_data(result)
            self.write_format_data(format_dict, md_dict)
            self.update_user_ns(result)
            self.log_output(format_dict)
            self.finish_displayhook()

    def cull_cache(self):
        """Output cache is full, cull the oldest entries"""
        oh = self.shell.user_ns.get('_oh', {})
        sz = len(oh)
        cull_count = max(int(sz * self.cull_fraction), 2)
        warn('Output cache limit (currently {sz} entries) hit.\n'
             'Flushing oldest {cull_count} entries.'.format(sz=sz, cull_count=cull_count))

        for i, n in enumerate(sorted(oh)):
            if i >= cull_count:
                break
            self.shell.user_ns.pop('_%i' % n, None)
            oh.pop(n, None)

    def flush(self):
        if not self.do_full_cache:
            raise ValueError("You shouldn't have reached the cache flush "
                             "if full caching is not enabled!")
        # delete auto-generated vars from global namespace

        for n in range(1,self.prompt_count + 1):
            key = '_'+repr(n)
            try:
                del self.shell.user_ns[key]
            except: pass
        # In some embedded circumstances, the user_ns doesn't have the
        # '_oh' key set up.
        oh = self.shell.user_ns.get('_oh', None)
        if oh is not None:
            oh.clear()

        # Release our own references to objects:
        self._, self.__, self.___ = '', '', ''

        if '_' not in builtin_mod.__dict__:
            self.shell.user_ns.update({'_':None,'__':None, '___':None})
        import gc
        # TODO: Is this really needed?
        # IronPython blocks here forever
        if sys.platform != "cli":
            gc.collect()

<|endoftext|>


<commit_before>import wx
from ObjectInspector import ObjectInspector
from Region import Region

class RegionInspector( ObjectInspector ):

    @classmethod
    def objectClass(cls):
        return Region

    @classmethod
    def inspectedAttributes(cls):
        return ['parentRegion', 'subRegions', 'ontologyTerm']

    def objectSizer(self, parentWindow):
        if not hasattr(self, '_sizer'):
            self._sizer = wx.FlexGridSizer(1, 2, 8, 8)
            self._sizer.SetFlexibleDirection(wx.HORIZONTAL | wx.VERTICAL)

            regionImage = Region.image()
            if regionImage is None or not regionImage.IsOk():
                self._regionBitmap = wx.EmptyBitmap(16, 16)
            else:
                self._regionBitmap = wx.BitmapFromImage(regionImage.Rescale(16, 16))

            self._sizer.Add(wx.StaticText(parentWindow, wx.ID_ANY, gettext('Ontology term:')))
            ontologySizer = wx.BoxSizer(wx.HORIZONTAL)
            self._ontologyField = wx.StaticText(parentWindow, wx.ID_ANY)
            ontologySizer.Add(self._ontologyField, 1, wx.LEFT, 2)
            self._browseOntologyButton = wx.Button(parentWindow, wx.ID_ANY, gettext('Browse'), style = wx.BU_EXACTFIT)
            self._browseOntologyButton.SetWindowVariant(wx.WINDOW_VARIANT_SMALL)
            self._browseOntologyButton.SetSize(wx.Size(50, self._browseOntologyButton.GetSize().GetHeight()))
            self._browseOntologyButton.SetMinSize(self._browseOntologyButton.GetSize())
            self._window.Bind(wx.EVT_BUTTON, self.onBrowseOntology, self._browseOntologyButton)
            ontologySizer.Add(self._browseOntologyButton, 0, wx.LEFT, 8)
            self._sizer.Add(ontologySizer)

            self._sizer.Add(wx.StaticText(parentWindow, wx.ID_ANY, gettext('Parent region:')))
            parentSizer = wx.BoxSizer(wx.HORIZONTAL)
            parentSizer.Add(wx.StaticBitmap(parentWindow, wx.ID_ANY, self._regionBitmap))
            self._parentNameField = wx.StaticText(parentWindow, wx.ID_ANY)
            parentSizer.Add(self._parentNameField, 1, wx.LEFT, 2)
            self._selectParentButton = wx.Button(parentWindow, wx.ID_ANY, gettext('Select'), style = wx.BU_EXACTFIT)
            self._selectParentButton.SetWindowVariant(wx.WINDOW_VARIANT_SMALL)
            self._selectParentButton.SetSize(wx.Size(50, self._selectParentButton.GetSize().GetHeight()))
            self._selectParentButton.SetMinSize(self._selectParentButton.GetSize())
            self._window.Bind(wx.EVT_BUTTON, self.onSelectParentRegion, self._selectParentButton)
            parentSizer.Add(self._selectParentButton, 0, wx.LEFT, 8)
            self._sizer.Add(parentSizer)

            self._sizer.Add(wx.StaticText(parentWindow, wx.ID_ANY, gettext('Sub-regions:')))
            self._subRegionsField = wx.StaticText(parentWindow, wx.ID_ANY)
            self._sizer.Add(self._subRegionsField, 1, wx.EXPAND | wx.ALL, 5)
            self._subRegionsSizer = wx.FlexGridSizer(0, 3, 2, 5)
            self._subRegionsSizer.SetFlexibleDirection(wx.HORIZONTAL)
            self._sizer.Add(self._subRegionsSizer, 1, wx.EXPAND)
            self._addSubRegionsButton = wx.Button(parentWindow, wx.ID_ANY, gettext('Add sub-regions from ontology'), style = wx.BU_EXACTFIT)
            self._window.Bind(wx.EVT_BUTTON, self.onAddSubRegions, self._addSubRegionsButton)
            self._sizer.Add(self._addSubRegionsButton, 0, wx.ALL, 5)

            self._parentWindow = parentWindow

        return self._sizer

    def populateObjectSizer(self, attribute = None):
        if attribute is None or attribute == 'ontologyTerm':
            if self.objects.haveEqualAttr('ontologyTerm'):
                if self.objects[0].ontologyTerm is None:
                    self._ontologyField.SetLabel(gettext('None'))
                    self._browseOntologyButton.Disable()
                else:
                    self._ontologyField.SetLabel(self.objects[0].ontologyTerm.name)
                    self._browseOntologyButton.Enable()
            else:
                self._ontologyField.SetLabel(gettext('Multiple values'))
                self._browseOntologyButton.Disable()

        if attribute is None or attribute == 'parentRegion':
            if self.objects.haveEqualAttr('parentRegion'):
                if self.objects[0].parentRegion is None:
                    self._parentNameField.SetLabel(gettext('None'))
                    self._selectParentButton.Disable()
                else:
                    self._parentNameField.SetLabel(self.objects[0].parentRegion.name)
                    self._selectParentButton.Enable()
            else:
                self._parentNameField.SetLabel(gettext('Multiple values'))
                self._selectParentButton.Disable()

        self._subRegionsSizer.Clear(True)
        if len(self.objects) > 1:
            self._subRegionsField.Show()
            self._subRegionsField.SetLabel(gettext('Multiple values'))
            self._addSubRegionsButton.Hide()
        else:
            region = self.objects[0]
            if len(region.subRegions) == 0:
                if region.ontologyTerm is not None and len(region.ontologyTerm.parts) > 0:
                    self._subRegionsField.Hide()
                    self._addSubRegionsButton.Show()
                else:
                    self._subRegionsField.Show()
                    self._subRegionsField.SetLabel(gettext('None'))
                    self._addSubRegionsButton.Hide()
            else:
                self._subRegionsField.Hide()
                self._addSubRegionsButton.Hide()
                for subRegion in self.objects[0].subRegions:
                    self._subRegionsSizer.Add(wx.StaticBitmap(self._parentWindow, wx.ID_ANY, self._regionBitmap))
                    self._subRegionsSizer.Add(wx.StaticText(self._parentWindow, wx.ID_ANY, subRegion.name or gettext('Unnamed region')))
                    selectButton = wx.Button(self._parentWindow, wx.ID_ANY, gettext('Select'), style = wx.BU_EXACTFIT)
                    selectButton.SetWindowVariant(wx.WINDOW_VARIANT_SMALL)
                    selectButton.SetSize(wx.Size(50, selectButton.GetSize().GetHeight()))
                    selectButton.SetMinSize(selectButton.GetSize())
                    self._parentWindow.Bind(wx.EVT_BUTTON, self.onSelectSubRegion, selectButton)
                    self._subRegionsSizer.Add(selectButton, 0, 0, 0, subRegion)

        self._window.Layout()

    def onBrowseOntology(self, event):
        term = self.objects[0].ontologyTerm
        term.ontology.browse(term)

    def onSelectParentRegion(self, event):
        self.selectObject(self.objects[0].parentRegion)

    def onAddSubRegions(self, event):
        region = self.objects[0]
        for term in region.ontologyTerm.parts:
            region.network.createRegion(ontologyTerm = term, parentRegion = region, addSubTerms = wx.GetKeyState(wx.WXK_ALT))

    def onSelectSubRegion(self, event):
        sizerItem = self._subRegionsSizer.GetItem(event.GetEventObject())
        subRegion = sizerItem.GetUserData()
        self.selectObject(subRegion)

<commit_msg>Fixed alignment issues in sub-region controls.<commit_after>import wx
from ObjectInspector import ObjectInspector
from Region import Region

class RegionInspector( ObjectInspector ):

    @classmethod
    def objectClass(cls):
        return Region

    @classmethod
    def inspectedAttributes(cls):
        return ['parentRegion', 'subRegions', 'ontologyTerm']

    def objectSizer(self, parentWindow):
        if not hasattr(self, '_sizer'):
            self._sizer = wx.FlexGridSizer(1, 2, 8, 8)
            self._sizer.SetFlexibleDirection(wx.HORIZONTAL | wx.VERTICAL)

            regionImage = Region.image()
            if regionImage is None or not regionImage.IsOk():
                self._regionBitmap = wx.EmptyBitmap(16, 16)
            else:
                self._regionBitmap = wx.BitmapFromImage(regionImage.Rescale(16, 16))

            self._sizer.Add(wx.StaticText(parentWindow, wx.ID_ANY, gettext('Ontology term:')))
            ontologySizer = wx.BoxSizer(wx.HORIZONTAL)
            self._ontologyField = wx.StaticText(parentWindow, wx.ID_ANY)
            ontologySizer.Add(self._ontologyField, 1, wx.LEFT, 2)
            self._browseOntologyButton = wx.Button(parentWindow, wx.ID_ANY, gettext('Browse'), style = wx.BU_EXACTFIT)
            self._browseOntologyButton.SetWindowVariant(wx.WINDOW_VARIANT_SMALL)
            self._browseOntologyButton.SetSize(wx.Size(60, self._browseOntologyButton.GetSize().GetHeight()))
            self._browseOntologyButton.SetMinSize(self._browseOntologyButton.GetSize())
            self._window.Bind(wx.EVT_BUTTON, self.onBrowseOntology, self._browseOntologyButton)
            ontologySizer.Add(self._browseOntologyButton, 0, wx.LEFT, 8)
            self._sizer.Add(ontologySizer)

            self._sizer.Add(wx.StaticText(parentWindow, wx.ID_ANY, gettext('Parent region:')))
            parentSizer = wx.BoxSizer(wx.HORIZONTAL)
            parentSizer.Add(wx.StaticBitmap(parentWindow, wx.ID_ANY, self._regionBitmap))
            self._parentNameField = wx.StaticText(parentWindow, wx.ID_ANY)
            parentSizer.Add(self._parentNameField, 1, wx.LEFT, 2)
            self._selectParentButton = wx.Button(parentWindow, wx.ID_ANY, gettext('Select'), style = wx.BU_EXACTFIT)
            self._selectParentButton.SetWindowVariant(wx.WINDOW_VARIANT_SMALL)
            self._selectParentButton.SetSize(wx.Size(50, self._selectParentButton.GetSize().GetHeight()))
            self._selectParentButton.SetMinSize(self._selectParentButton.GetSize())
            self._window.Bind(wx.EVT_BUTTON, self.onSelectParentRegion, self._selectParentButton)
            parentSizer.Add(self._selectParentButton, 0, wx.LEFT, 8)
            self._sizer.Add(parentSizer)

            self._sizer.Add(wx.StaticText(parentWindow, wx.ID_ANY, gettext('Sub-regions:')))
            self._subRegionsOptions = wx.BoxSizer(wx.VERTICAL)
            self._subRegionsField = wx.StaticText(parentWindow, wx.ID_ANY)
            self._subRegionsOptions.Add(self._subRegionsField, 1, wx.EXPAND)
            self._subRegionsSizer = wx.FlexGridSizer(0, 3, 2, 5)
            self._subRegionsSizer.SetFlexibleDirection(wx.HORIZONTAL)
            self._subRegionsOptions.Add(self._subRegionsSizer, 1, wx.EXPAND)
            self._addSubRegionsButton = wx.Button(parentWindow, wx.ID_ANY, gettext('Add sub-regions from ontology'), style = wx.BU_EXACTFIT)
            self._window.Bind(wx.EVT_BUTTON, self.onAddSubRegions, self._addSubRegionsButton)
            self._subRegionsOptions.Add(self._addSubRegionsButton, 0, wx.EXPAND)
            self._sizer.Add(self._subRegionsOptions)

            self._parentWindow = parentWindow

        return self._sizer

    def populateObjectSizer(self, attribute = None):
        if attribute is None or attribute == 'ontologyTerm':
            if self.objects.haveEqualAttr('ontologyTerm'):
                if self.objects[0].ontologyTerm is None:
                    self._ontologyField.SetLabel(gettext('None'))
                    self._browseOntologyButton.Disable()
                else:
                    self._ontologyField.SetLabel(self.objects[0].ontologyTerm.name)
                    self._browseOntologyButton.Enable()
            else:
                self._ontologyField.SetLabel(gettext('Multiple values'))
                self._browseOntologyButton.Disable()

        if attribute is None or attribute == 'parentRegion':
            if self.objects.haveEqualAttr('parentRegion'):
                if self.objects[0].parentRegion is None:
                    self._parentNameField.SetLabel(gettext('None'))
                    self._selectParentButton.Disable()
                else:
                    self._parentNameField.SetLabel(self.objects[0].parentRegion.name)
                    self._selectParentButton.Enable()
            else:
                self._parentNameField.SetLabel(gettext('Multiple values'))
                self._selectParentButton.Disable()

        self._subRegionsSizer.Clear(True)
        if len(self.objects) > 1:
            self._subRegionsField.Show()
            self._subRegionsField.SetLabel(gettext('Multiple values'))
            self._subRegionsOptions.Hide(self._subRegionsSizer)
            self._addSubRegionsButton.Hide()
        else:
            region = self.objects[0]
            if len(region.subRegions) == 0:
                self._subRegionsOptions.Hide(self._subRegionsSizer)
                if region.ontologyTerm is not None and len(region.ontologyTerm.parts) > 0:
                    self._subRegionsField.Hide()
                    self._addSubRegionsButton.Show()
                else:
                    self._subRegionsField.Show()
                    self._subRegionsField.SetLabel(gettext('None'))
                    self._addSubRegionsButton.Hide()
            else:
                self._subRegionsField.Hide()
                self._addSubRegionsButton.Hide()
                self._subRegionsOptions.Show(self._subRegionsSizer)
                for subRegion in self.objects[0].subRegions:
                    self._subRegionsSizer.Add(wx.StaticBitmap(self._parentWindow, wx.ID_ANY, self._regionBitmap))
                    self._subRegionsSizer.Add(wx.StaticText(self._parentWindow, wx.ID_ANY, subRegion.name or gettext('Unnamed region')))
                    selectButton = wx.Button(self._parentWindow, wx.ID_ANY, gettext('Select'), style = wx.BU_EXACTFIT)
                    selectButton.SetWindowVariant(wx.WINDOW_VARIANT_SMALL)
                    selectButton.SetSize(wx.Size(50, selectButton.GetSize().GetHeight()))
                    selectButton.SetMinSize(selectButton.GetSize())
                    self._parentWindow.Bind(wx.EVT_BUTTON, self.onSelectSubRegion, selectButton)
                    self._subRegionsSizer.Add(selectButton, 0, 0, 0, subRegion)

        self._window.Layout()

    def onBrowseOntology(self, event):
        term = self.objects[0].ontologyTerm
        term.ontology.browse(term)

    def onSelectParentRegion(self, event):
        self.selectObject(self.objects[0].parentRegion)

    def onAddSubRegions(self, event):
        region = self.objects[0]
        for term in region.ontologyTerm.parts:
            region.network.createRegion(ontologyTerm = term, parentRegion = region, addSubTerms = wx.GetKeyState(wx.WXK_ALT))

    def onSelectSubRegion(self, event):
        sizerItem = self._subRegionsSizer.GetItem(event.GetEventObject())
        subRegion = sizerItem.GetUserData()
        self.selectObject(subRegion)

<|endoftext|>


<commit_before># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests of the readline-based CLI."""
import os
import argparse
import tempfile

from tensorflow.python.debug.cli import cli_config
from tensorflow.python.debug.cli import debugger_cli_common
from tensorflow.python.debug.cli import readline_ui
from tensorflow.python.debug.cli import ui_factory
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import file_io
from tensorflow.python.platform import gfile
from tensorflow.python.platform import googletest

class MockReadlineUI(readline_ui.ReadlineUI):
  """Test subclass of ReadlineUI that bypasses terminal manipulations."""

  def __init__(self, on_ui_exit=None, command_sequence=None):
    readline_ui.ReadlineUI.__init__(
        self, on_ui_exit=on_ui_exit,
        config=cli_config.CLIConfig(config_file_path=tempfile.mktemp()))

    self._command_sequence = command_sequence
    self._command_counter = 0

    self.observers = {"screen_outputs": []}

  def _get_user_command(self):
    command = self._command_sequence[self._command_counter]
    self._command_counter += 1
    return command

  def _display_output(self, screen_output):
    self.observers["screen_outputs"].append(screen_output)

class CursesTest(test_util.TensorFlowTestCase):

  def setUp(self):
    self._tmp_dir = tempfile.mkdtemp()
    self._tmp_config_path = os.path.join(self._tmp_dir, ".tfdbg_config")
    self.assertFalse(gfile.Exists(self._tmp_config_path))
    super(CursesTest, self).setUp()

  def tearDown(self):
    file_io.delete_recursively(self._tmp_dir)
    super(CursesTest, self).tearDown()

  def _babble(self, args, screen_info=None):
    ap = argparse.ArgumentParser(
        description="Do babble.", usage=argparse.SUPPRESS)
    ap.add_argument(
        "-n",
        "--num_times",
        dest="num_times",
        type=int,
        default=60,
        help="How many times to babble")

    parsed = ap.parse_args(args)

    lines = ["bar"] * parsed.num_times
    return debugger_cli_common.RichTextLines(lines)

  def testUIFactoryCreatesReadlineUI(self):
    ui = ui_factory.get_ui(
        "readline",
        config=cli_config.CLIConfig(config_file_path=self._tmp_config_path))
    self.assertIsInstance(ui, readline_ui.ReadlineUI)

  def testUIFactoryRaisesExceptionOnInvalidUIType(self):
    with self.assertRaisesRegex(ValueError, "Invalid ui_type: 'foobar'"):
      ui_factory.get_ui(
          "foobar",
          config=cli_config.CLIConfig(config_file_path=self._tmp_config_path))

  def testUIFactoryRaisesExceptionOnInvalidUITypeGivenAvailable(self):
    with self.assertRaisesRegex(ValueError, "Invalid ui_type: 'readline'"):
      ui_factory.get_ui(
          "readline",
          available_ui_types=["curses"],
          config=cli_config.CLIConfig(config_file_path=self._tmp_config_path))

  def testRunUIExitImmediately(self):
    """Make sure that the UI can exit properly after launch."""

    ui = MockReadlineUI(command_sequence=["exit"])
    ui.run_ui()

    # No screen output should have happened.
    self.assertEqual(0, len(ui.observers["screen_outputs"]))

  def testRunUIEmptyCommand(self):
    """Issue an empty command then exit."""

    ui = MockReadlineUI(command_sequence=["", "exit"])
    ui.run_ui()
    self.assertEqual(1, len(ui.observers["screen_outputs"]))

  def testRunUIWithInitCmd(self):
    """Run UI with an initial command specified."""

    ui = MockReadlineUI(command_sequence=["exit"])

    ui.register_command_handler("babble", self._babble, "")
    ui.run_ui(init_command="babble")

    screen_outputs = ui.observers["screen_outputs"]
    self.assertEqual(1, len(screen_outputs))
    self.assertEqual(["bar"] * 60, screen_outputs[0].lines)

  def testRunUIWithValidUsersCommands(self):
    """Run UI with an initial command specified."""

    ui = MockReadlineUI(command_sequence=["babble -n 3", "babble -n 6", "exit"])
    ui.register_command_handler("babble", self._babble, "")
    ui.run_ui()

    screen_outputs = ui.observers["screen_outputs"]
    self.assertEqual(2, len(screen_outputs))
    self.assertEqual(["bar"] * 3, screen_outputs[0].lines)
    self.assertEqual(["bar"] * 6, screen_outputs[1].lines)

  def testRunUIWithInvalidUsersCommands(self):
    """Run UI with an initial command specified."""

    ui = MockReadlineUI(command_sequence=["babble -n 3", "wobble", "exit"])
    ui.register_command_handler("babble", self._babble, "")
    ui.run_ui()

    screen_outputs = ui.observers["screen_outputs"]
    self.assertEqual(2, len(screen_outputs))
    self.assertEqual(["bar"] * 3, screen_outputs[0].lines)
    self.assertEqual(["ERROR: Invalid command prefix \"wobble\""],
                     screen_outputs[1].lines)

  def testRunUIWithOnUIExitCallback(self):
    observer = {"callback_invoked": False}

    def callback_for_test():
      observer["callback_invoked"] = True

    ui = MockReadlineUI(on_ui_exit=callback_for_test, command_sequence=["exit"])

    self.assertFalse(observer["callback_invoked"])
    ui.run_ui()

    self.assertEqual(0, len(ui.observers["screen_outputs"]))
    self.assertTrue(observer["callback_invoked"])

  def testIncompleteRedirectWorks(self):
    output_path = tempfile.mktemp()

    ui = MockReadlineUI(
        command_sequence=["babble -n 2 > %s" % output_path, "exit"])

    ui.register_command_handler("babble", self._babble, "")
    ui.run_ui()

    screen_outputs = ui.observers["screen_outputs"]
    self.assertEqual(1, len(screen_outputs))
    self.assertEqual(["bar"] * 2, screen_outputs[0].lines)

    with gfile.Open(output_path, "r") as f:
      self.assertEqual("bar\nbar\n", f.read())

  def testConfigSetAndShow(self):
    """Run UI with an initial command specified."""

    ui = MockReadlineUI(command_sequence=[
        "config set graph_recursion_depth 5", "config show", "exit"])
    ui.run_ui()
    outputs = ui.observers["screen_outputs"]
    self.assertEqual(
        ["Command-line configuration:",
         "",
         "  graph_recursion_depth: 5"], outputs[1].lines[:3])

if __name__ == "__main__":
  googletest.main()
<commit_msg>Use `tempfile.mkstemp` instead of `tempfile.mktemp`.<commit_after># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests of the readline-based CLI."""
import os
import argparse
import tempfile

from tensorflow.python.debug.cli import cli_config
from tensorflow.python.debug.cli import debugger_cli_common
from tensorflow.python.debug.cli import readline_ui
from tensorflow.python.debug.cli import ui_factory
from tensorflow.python.framework import test_util
from tensorflow.python.lib.io import file_io
from tensorflow.python.platform import gfile
from tensorflow.python.platform import googletest

class MockReadlineUI(readline_ui.ReadlineUI):
  """Test subclass of ReadlineUI that bypasses terminal manipulations."""

  def __init__(self, on_ui_exit=None, command_sequence=None):
    _, config_file_path = tempfile.mkstemp()  # safe to ignore fd
    readline_ui.ReadlineUI.__init__(
        self,
        on_ui_exit=on_ui_exit,
        config=cli_config.CLIConfig(config_file_path=config_file_path))

    self._command_sequence = command_sequence
    self._command_counter = 0

    self.observers = {"screen_outputs": []}

  def _get_user_command(self):
    command = self._command_sequence[self._command_counter]
    self._command_counter += 1
    return command

  def _display_output(self, screen_output):
    self.observers["screen_outputs"].append(screen_output)

class CursesTest(test_util.TensorFlowTestCase):

  def setUp(self):
    self._tmp_dir = tempfile.mkdtemp()
    self._tmp_config_path = os.path.join(self._tmp_dir, ".tfdbg_config")
    self.assertFalse(gfile.Exists(self._tmp_config_path))
    super(CursesTest, self).setUp()

  def tearDown(self):
    file_io.delete_recursively(self._tmp_dir)
    super(CursesTest, self).tearDown()

  def _babble(self, args, screen_info=None):
    ap = argparse.ArgumentParser(
        description="Do babble.", usage=argparse.SUPPRESS)
    ap.add_argument(
        "-n",
        "--num_times",
        dest="num_times",
        type=int,
        default=60,
        help="How many times to babble")

    parsed = ap.parse_args(args)

    lines = ["bar"] * parsed.num_times
    return debugger_cli_common.RichTextLines(lines)

  def testUIFactoryCreatesReadlineUI(self):
    ui = ui_factory.get_ui(
        "readline",
        config=cli_config.CLIConfig(config_file_path=self._tmp_config_path))
    self.assertIsInstance(ui, readline_ui.ReadlineUI)

  def testUIFactoryRaisesExceptionOnInvalidUIType(self):
    with self.assertRaisesRegex(ValueError, "Invalid ui_type: 'foobar'"):
      ui_factory.get_ui(
          "foobar",
          config=cli_config.CLIConfig(config_file_path=self._tmp_config_path))

  def testUIFactoryRaisesExceptionOnInvalidUITypeGivenAvailable(self):
    with self.assertRaisesRegex(ValueError, "Invalid ui_type: 'readline'"):
      ui_factory.get_ui(
          "readline",
          available_ui_types=["curses"],
          config=cli_config.CLIConfig(config_file_path=self._tmp_config_path))

  def testRunUIExitImmediately(self):
    """Make sure that the UI can exit properly after launch."""

    ui = MockReadlineUI(command_sequence=["exit"])
    ui.run_ui()

    # No screen output should have happened.
    self.assertEqual(0, len(ui.observers["screen_outputs"]))

  def testRunUIEmptyCommand(self):
    """Issue an empty command then exit."""

    ui = MockReadlineUI(command_sequence=["", "exit"])
    ui.run_ui()
    self.assertEqual(1, len(ui.observers["screen_outputs"]))

  def testRunUIWithInitCmd(self):
    """Run UI with an initial command specified."""

    ui = MockReadlineUI(command_sequence=["exit"])

    ui.register_command_handler("babble", self._babble, "")
    ui.run_ui(init_command="babble")

    screen_outputs = ui.observers["screen_outputs"]
    self.assertEqual(1, len(screen_outputs))
    self.assertEqual(["bar"] * 60, screen_outputs[0].lines)

  def testRunUIWithValidUsersCommands(self):
    """Run UI with an initial command specified."""

    ui = MockReadlineUI(command_sequence=["babble -n 3", "babble -n 6", "exit"])
    ui.register_command_handler("babble", self._babble, "")
    ui.run_ui()

    screen_outputs = ui.observers["screen_outputs"]
    self.assertEqual(2, len(screen_outputs))
    self.assertEqual(["bar"] * 3, screen_outputs[0].lines)
    self.assertEqual(["bar"] * 6, screen_outputs[1].lines)

  def testRunUIWithInvalidUsersCommands(self):
    """Run UI with an initial command specified."""

    ui = MockReadlineUI(command_sequence=["babble -n 3", "wobble", "exit"])
    ui.register_command_handler("babble", self._babble, "")
    ui.run_ui()

    screen_outputs = ui.observers["screen_outputs"]
    self.assertEqual(2, len(screen_outputs))
    self.assertEqual(["bar"] * 3, screen_outputs[0].lines)
    self.assertEqual(["ERROR: Invalid command prefix \"wobble\""],
                     screen_outputs[1].lines)

  def testRunUIWithOnUIExitCallback(self):
    observer = {"callback_invoked": False}

    def callback_for_test():
      observer["callback_invoked"] = True

    ui = MockReadlineUI(on_ui_exit=callback_for_test, command_sequence=["exit"])

    self.assertFalse(observer["callback_invoked"])
    ui.run_ui()

    self.assertEqual(0, len(ui.observers["screen_outputs"]))
    self.assertTrue(observer["callback_invoked"])

  def testIncompleteRedirectWorks(self):
    _, output_path = tempfile.mkstemp()  # safe to ignore fd

    ui = MockReadlineUI(
        command_sequence=["babble -n 2 > %s" % output_path, "exit"])

    ui.register_command_handler("babble", self._babble, "")
    ui.run_ui()

    screen_outputs = ui.observers["screen_outputs"]
    self.assertEqual(1, len(screen_outputs))
    self.assertEqual(["bar"] * 2, screen_outputs[0].lines)

    with gfile.Open(output_path, "r") as f:
      self.assertEqual("bar\nbar\n", f.read())

  def testConfigSetAndShow(self):
    """Run UI with an initial command specified."""

    ui = MockReadlineUI(command_sequence=[
        "config set graph_recursion_depth 5", "config show", "exit"])
    ui.run_ui()
    outputs = ui.observers["screen_outputs"]
    self.assertEqual(
        ["Command-line configuration:",
         "",
         "  graph_recursion_depth: 5"], outputs[1].lines[:3])

if __name__ == "__main__":
  googletest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#*******************************************************************************
# ALMA - Atacama Large Millimiter Array
# (c) Associated Universities Inc., 2002
# (c) European Southern Observatory, 2002
# Copyright by ESO (in the framework of the ALMA collaboration)
# and Cosylab 2002, All rights reserved
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307  USA
#
# @(#) $Id: acspyTestLoggingConsumer.py,v 1.2 2008/11/06 16:13:02 agrimstrup Exp $
###############################################################################
'''
Tests logging consumer
'''
from Acspy.Nc.LoggingConsumer import LoggingConsumer
from sys  import argv
from time import sleep
###############################################################################
count = 0
magicNumber = int(argv[1])

def myHandler(xml):
    '''
    '''
    global count
    if count < magicNumber:
#        print xml
        count = count + 1
    return

#create the consumer
myConsumer = LoggingConsumer(myHandler)
myConsumer.consumerReady()

while (count!=magicNumber):
    sleep(1)

#shutdown everything cleanly
myConsumer.disconnect()
print "%d of %d messages received" % (count, magicNumber)

<commit_msg>The test produces some log in a attempt to speed up and reduce the dependency from other producers<commit_after>#!/usr/bin/env python
#*******************************************************************************
# ALMA - Atacama Large Millimiter Array
# (c) Associated Universities Inc., 2002
# (c) European Southern Observatory, 2002
# Copyright by ESO (in the framework of the ALMA collaboration)
# and Cosylab 2002, All rights reserved
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307  USA
#
# @(#) $Id: acspyTestLoggingConsumer.py,v 1.2 2008/11/06 16:13:02 agrimstrup Exp $
###############################################################################
'''
Tests logging consumer

The tests waits until a number of logs (passed in the command line) has been
received.
Note that these logs are produced by already running processes like the containers
so it can take and arbitrary time to complete...
'''
from Acspy.Nc.LoggingConsumer import LoggingConsumer
from Acspy.Common.Log         import getLogger
import Acspy.Clients
from sys  import argv
from time import sleep
###############################################################################
count = 0
magicNumber = int(argv[1])

def myHandler(xml):
    '''
    '''
    global count
    if count < magicNumber:
#        print xml
        count = count + 1
    return

#create the consumer
myConsumer = LoggingConsumer(myHandler)
myConsumer.consumerReady()

# A SimpleClient is instantiated to produce logs and
# limit the execution time and the dependency from non identified
# external processes that are supposed to produce logs
logger = getLogger("acspyTestLoggingConsumerLogger")
for i in range(magicNumber+1):
    logger.logInfo('Log #'+str(i))

while (count!=magicNumber):
    sleep(1)

#shutdown everything cleanly
myConsumer.disconnect()
print "%d of %d messages received" % (count, magicNumber)

<|endoftext|>


<commit_before>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""main module."""
import os

import cfscrape
import click
import mechanicalsoup
import requests
import structlog

from iqdb_tagger import models
from iqdb_tagger.__init__ import db_version
from iqdb_tagger.custom_parser import get_tags as get_tags_from_parser
from iqdb_tagger.utils import default_db_path, thumb_folder, user_data_dir

db = '~/images/! tagged'
DEFAULT_SIZE = 150, 150
DEFAULT_PLACE = 'iqdb'
minsim = 75
services = ['1', '2', '3', '4', '5', '6', '10', '11']
forcegray = False
log = structlog.getLogger()
iqdb_url_dict = {
    'iqdb': ('http://iqdb.org', models.ImageMatch.SP_IQDB),
    'danbooru': (
        'http://danbooru.iqdb.org',
        models.ImageMatch.SP_DANBOORU
    ),
}

def get_page_result(image, url, browser=None):
    """Get page result.

    Args:
        image: Image path to be uploaded.
    Returns:
        HTML page from the result.
    """
    # compatibility
    br = browser

    if br is None:
        br = mechanicalsoup.StatefulBrowser(soup_config={'features': 'lxml'})
        br.raise_on_404 = True

    br.open(url)
    html_form = br.select_form('form')
    html_form.input({'file': image})
    br.submit_selected()
    # if ok, will output: <Response [200]>
    return br.get_current_page()

def get_posted_image(
        img_path, resize=False, size=None, output_thumb_folder=None):
    """Get posted image."""
    if output_thumb_folder is None:
        output_thumb_folder = thumb_folder

    img, _ = models.ImageModel.get_or_create_from_path(img_path)
    def_thumb_rel, _ = models.ThumbnailRelationship.get_or_create_from_image(
        image=img, thumb_folder=output_thumb_folder, size=DEFAULT_SIZE)
    resized_thumb_rel = None

    if resize and size:
        resized_thumb_rel, _ = \
            models.ThumbnailRelationship.get_or_create_from_image(
                image=img, thumb_folder=output_thumb_folder, size=size
            )
    elif resize:
        # use thumbnail if no size is given
        resized_thumb_rel = def_thumb_rel
    else:
        log.debug('Unknown config.', resize=resize, size=size)

    return resized_thumb_rel.thumbnail \
        if resized_thumb_rel is not None else img

def init_program(db_path=None):
    """Init program."""
    # create user data dir
    if not os.path.isdir(user_data_dir):
        os.makedirs(user_data_dir, exist_ok=True)
        log.debug('User data dir created.')
    # create thumbnail folder
    if not os.path.isdir(thumb_folder):
        os.makedirs(thumb_folder, exist_ok=True)
        log.debug('Thumbnail folder created.')

    # database
    if db_path is None:
        db_path = default_db_path
    models.init_db(db_path, db_version)

def get_tags(match_result, browser=None, scraper=None):
    """Get tags."""
    # compatibility
    br = browser

    if br is None:
        br = mechanicalsoup.StatefulBrowser(soup_config={'features': 'lxml'})
        br.raise_on_404 = True
    if scraper is None:
        scraper = cfscrape.CloudflareScraper()

    url = match_result.link
    br.open(url, timeout=10)
    page = br.get_current_page()
    tags = get_tags_from_parser(page, url, scraper)
    if tags:
        for tag in tags:
            namespace, tag_name = tag
            tag_model, _ = models.Tag.get_or_create(
                name=tag_name, namespace=namespace)
            models.MatchTagRelationship.get_or_create(
                match=match_result, tag=tag_model)
            yield tag_model
    else:
        log.debug('No tags found.')

def run_program_for_single_img(
        image, resize, size, place, match_filter, write_tags, browser,
        scraper, disable_tag_print=False
):
    """Run program for single image."""
    # compatibility
    br = browser

    post_img = get_posted_image(img_path=image, resize=resize, size=size)
    tag_textfile = image + '.txt'

    result = []
    for img_m_rel_set in post_img.imagematchrelationship_set:
        for item_set in img_m_rel_set.imagematch_set:
            if item_set.search_place_verbose == place:
                result.append(item_set)

    if not result:
        url, im_place = iqdb_url_dict[place]
        page = get_page_result(image=post_img.path, url=url, browser=br)
        # if ok, will output: <Response [200]>
        result = list(models.ImageMatch.get_or_create_from_page(
            page=page, image=post_img, place=im_place))
        result = [x[0] for x in result]

    if match_filter == 'best-match':
        result = [x for x in result if x.status == x.STATUS_BEST_MATCH]

    MatchTagRelationship = models.MatchTagRelationship
    for item in result:
        # type item: models.ImageMatch
        # type match_result: models.Match object
        match_result = item.match.match_result
        url = match_result.link

        print('{}|{}|{}'.format(
            item.similarity, item.status_verbose, url))

        res = MatchTagRelationship.select().where(
            MatchTagRelationship.match == match_result)
        tags = [x.tag for x in res]

        filtered_hosts = ['anime-pictures.net', 'www.theanimegallery.com']
        is_url_in_filtered_hosts = urlparse(match_result.link).netloc in \
            filtered_hosts
        if is_url_in_filtered_hosts:
            log.debug(
                'URL in filtered hosts, no tag fetched', url=match_result.link)
        elif not tags:
            try:
                tags = list(
                    [x for x in get_tags(match_result, br, scraper)])
            except requests.exceptions.ConnectionError as e:
                log.error(str(e), url=url)

        tags_verbose = [x.full_name for x in tags]
        log.debug('{} tag(s) founds'.format(len(tags_verbose)))
        if tags and not disable_tag_print:
            print('\n'.join(tags_verbose))
        else:
            log.debug('No printing tags.')

        if tags and write_tags:
            with open(tag_textfile, 'a') as f:
                f.write('\n'.join(tags_verbose))
                f.write('\n')
            log.debug('tags written')

@click.command()
@click.option(
    '--place', type=click.Choice(['iqdb', 'danbooru']),
    default=DEFAULT_PLACE,
    help='Specify iqdb place, default:{}'.format(DEFAULT_PLACE)
)
@click.option('--resize', is_flag=True, help='Use resized image.')
@click.option('--size', is_flag=True, help='Specify resized image.')
@click.option('--db-path', help='Specify Database path.')
@click.option(
    '--match-filter', type=click.Choice(['default', 'best-match']),
    default='default', help='Filter the result.'
)
@click.option(
    '--write-tags', is_flag=True, help='Write best match\'s tags to text.')
@click.option(
    '--input-mode', type=click.Choice(['default', 'folder']),
    default='default', help='Set input mode.'
)
@click.argument('prog-input')
def main(
    prog_input, resize=False, size=None,
    db_path=None, place=DEFAULT_PLACE, match_filter='default',
    write_tags=False, input_mode='default'
):
    """Get similar image from iqdb."""
    init_program(db_path)
    br = mechanicalsoup.StatefulBrowser(soup_config={'features': 'lxml'})
    br.raise_on_404 = True
    scraper = cfscrape.CloudflareScraper()

    if input_mode == 'folder':
        assert os.path.isdir(prog_input), 'Input is not valid folder'
        files = [os.path.join(prog_input, x) for x in os.listdir(prog_input)]
        if not files:
            print('No files found.')
            return
        err_set = []
        err_found = False
        for idx, ff in enumerate(files):
            log.debug('file', f=ff, idx=idx, total=len(files))
            try:
                run_program_for_single_img(
                    ff, resize, size, place, match_filter, write_tags,
                    browser=br, scraper=scraper, disable_tag_print=True
                )
            except Exception as e:  # pylint:disable=broad-except
                err_set.append((ff, e))
                err_found = True
        if err_found:
            print('Found error(s)')
            list(map(
                lambda x: print('path:{}\nerror:{}\n'.format(x[0], x[1])),
                err_set
            ))
    else:
        image = prog_input
        run_program_for_single_img(
            image, resize, size, place, match_filter, write_tags,
            browser=br, scraper=scraper
        )
<commit_msg>fix: dev: urlparse import error<commit_after>#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""main module."""
import os
from urllib.parse import urlparse

import cfscrape
import click
import mechanicalsoup
import requests
import structlog

from iqdb_tagger import models
from iqdb_tagger.__init__ import db_version
from iqdb_tagger.custom_parser import get_tags as get_tags_from_parser
from iqdb_tagger.utils import default_db_path, thumb_folder, user_data_dir

db = '~/images/! tagged'
DEFAULT_SIZE = 150, 150
DEFAULT_PLACE = 'iqdb'
minsim = 75
services = ['1', '2', '3', '4', '5', '6', '10', '11']
forcegray = False
log = structlog.getLogger()
iqdb_url_dict = {
    'iqdb': ('http://iqdb.org', models.ImageMatch.SP_IQDB),
    'danbooru': (
        'http://danbooru.iqdb.org',
        models.ImageMatch.SP_DANBOORU
    ),
}

def get_page_result(image, url, browser=None):
    """Get page result.

    Args:
        image: Image path to be uploaded.
    Returns:
        HTML page from the result.
    """
    # compatibility
    br = browser

    if br is None:
        br = mechanicalsoup.StatefulBrowser(soup_config={'features': 'lxml'})
        br.raise_on_404 = True

    br.open(url)
    html_form = br.select_form('form')
    html_form.input({'file': image})
    br.submit_selected()
    # if ok, will output: <Response [200]>
    return br.get_current_page()

def get_posted_image(
        img_path, resize=False, size=None, output_thumb_folder=None):
    """Get posted image."""
    if output_thumb_folder is None:
        output_thumb_folder = thumb_folder

    img, _ = models.ImageModel.get_or_create_from_path(img_path)
    def_thumb_rel, _ = models.ThumbnailRelationship.get_or_create_from_image(
        image=img, thumb_folder=output_thumb_folder, size=DEFAULT_SIZE)
    resized_thumb_rel = None

    if resize and size:
        resized_thumb_rel, _ = \
            models.ThumbnailRelationship.get_or_create_from_image(
                image=img, thumb_folder=output_thumb_folder, size=size
            )
    elif resize:
        # use thumbnail if no size is given
        resized_thumb_rel = def_thumb_rel
    else:
        log.debug('Unknown config.', resize=resize, size=size)

    return resized_thumb_rel.thumbnail \
        if resized_thumb_rel is not None else img

def init_program(db_path=None):
    """Init program."""
    # create user data dir
    if not os.path.isdir(user_data_dir):
        os.makedirs(user_data_dir, exist_ok=True)
        log.debug('User data dir created.')
    # create thumbnail folder
    if not os.path.isdir(thumb_folder):
        os.makedirs(thumb_folder, exist_ok=True)
        log.debug('Thumbnail folder created.')

    # database
    if db_path is None:
        db_path = default_db_path
    models.init_db(db_path, db_version)

def get_tags(match_result, browser=None, scraper=None):
    """Get tags."""
    # compatibility
    br = browser

    if br is None:
        br = mechanicalsoup.StatefulBrowser(soup_config={'features': 'lxml'})
        br.raise_on_404 = True
    if scraper is None:
        scraper = cfscrape.CloudflareScraper()

    url = match_result.link
    br.open(url, timeout=10)
    page = br.get_current_page()
    tags = get_tags_from_parser(page, url, scraper)
    if tags:
        for tag in tags:
            namespace, tag_name = tag
            tag_model, _ = models.Tag.get_or_create(
                name=tag_name, namespace=namespace)
            models.MatchTagRelationship.get_or_create(
                match=match_result, tag=tag_model)
            yield tag_model
    else:
        log.debug('No tags found.')

def run_program_for_single_img(
        image, resize, size, place, match_filter, write_tags, browser,
        scraper, disable_tag_print=False
):
    """Run program for single image."""
    # compatibility
    br = browser

    post_img = get_posted_image(img_path=image, resize=resize, size=size)
    tag_textfile = image + '.txt'

    result = []
    for img_m_rel_set in post_img.imagematchrelationship_set:
        for item_set in img_m_rel_set.imagematch_set:
            if item_set.search_place_verbose == place:
                result.append(item_set)

    if not result:
        url, im_place = iqdb_url_dict[place]
        page = get_page_result(image=post_img.path, url=url, browser=br)
        # if ok, will output: <Response [200]>
        result = list(models.ImageMatch.get_or_create_from_page(
            page=page, image=post_img, place=im_place))
        result = [x[0] for x in result]

    if match_filter == 'best-match':
        result = [x for x in result if x.status == x.STATUS_BEST_MATCH]

    MatchTagRelationship = models.MatchTagRelationship
    for item in result:
        # type item: models.ImageMatch
        # type match_result: models.Match object
        match_result = item.match.match_result
        url = match_result.link

        print('{}|{}|{}'.format(
            item.similarity, item.status_verbose, url))

        res = MatchTagRelationship.select().where(
            MatchTagRelationship.match == match_result)
        tags = [x.tag for x in res]

        filtered_hosts = ['anime-pictures.net', 'www.theanimegallery.com']
        is_url_in_filtered_hosts = urlparse(match_result.link).netloc in \
            filtered_hosts
        if is_url_in_filtered_hosts:
            log.debug(
                'URL in filtered hosts, no tag fetched', url=match_result.link)
        elif not tags:
            try:
                tags = list(
                    [x for x in get_tags(match_result, br, scraper)])
            except requests.exceptions.ConnectionError as e:
                log.error(str(e), url=url)

        tags_verbose = [x.full_name for x in tags]
        log.debug('{} tag(s) founds'.format(len(tags_verbose)))
        if tags and not disable_tag_print:
            print('\n'.join(tags_verbose))
        else:
            log.debug('No printing tags.')

        if tags and write_tags:
            with open(tag_textfile, 'a') as f:
                f.write('\n'.join(tags_verbose))
                f.write('\n')
            log.debug('tags written')

@click.command()
@click.option(
    '--place', type=click.Choice(['iqdb', 'danbooru']),
    default=DEFAULT_PLACE,
    help='Specify iqdb place, default:{}'.format(DEFAULT_PLACE)
)
@click.option('--resize', is_flag=True, help='Use resized image.')
@click.option('--size', is_flag=True, help='Specify resized image.')
@click.option('--db-path', help='Specify Database path.')
@click.option(
    '--match-filter', type=click.Choice(['default', 'best-match']),
    default='default', help='Filter the result.'
)
@click.option(
    '--write-tags', is_flag=True, help='Write best match\'s tags to text.')
@click.option(
    '--input-mode', type=click.Choice(['default', 'folder']),
    default='default', help='Set input mode.'
)
@click.argument('prog-input')
def main(
    prog_input, resize=False, size=None,
    db_path=None, place=DEFAULT_PLACE, match_filter='default',
    write_tags=False, input_mode='default'
):
    """Get similar image from iqdb."""
    init_program(db_path)
    br = mechanicalsoup.StatefulBrowser(soup_config={'features': 'lxml'})
    br.raise_on_404 = True
    scraper = cfscrape.CloudflareScraper()

    if input_mode == 'folder':
        assert os.path.isdir(prog_input), 'Input is not valid folder'
        files = [os.path.join(prog_input, x) for x in os.listdir(prog_input)]
        if not files:
            print('No files found.')
            return
        err_set = []
        err_found = False
        for idx, ff in enumerate(files):
            log.debug('file', f=ff, idx=idx, total=len(files))
            try:
                run_program_for_single_img(
                    ff, resize, size, place, match_filter, write_tags,
                    browser=br, scraper=scraper, disable_tag_print=True
                )
            except Exception as e:  # pylint:disable=broad-except
                err_set.append((ff, e))
                err_found = True
        if err_found:
            print('Found error(s)')
            list(map(
                lambda x: print('path:{}\nerror:{}\n'.format(x[0], x[1])),
                err_set
            ))
    else:
        image = prog_input
        run_program_for_single_img(
            image, resize, size, place, match_filter, write_tags,
            browser=br, scraper=scraper
        )
<|endoftext|>