<commit_before># Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Defines the interface TestTypeBase which other test types inherit from.

Also defines the TestArguments "struct" to pass them additional arguments.
"""

import cgi
import difflib
import errno
import logging
import os.path
import subprocess

from layout_package import path_utils

class TestArguments(object):
  """Struct-like wrapper for additional arguments needed by specific tests."""
  # Whether to save new baseline results.
  new_baseline = False

  # Path to the actual PNG file generated by pixel tests
  png_path = None

  # Value of checksum generated by pixel tests.
  hash = None

  # Whether to use wdiff to generate by-word diffs.
  wdiff = False

  # Whether to report the locations of the expected result files used.
  show_sources = False

# Python bug workaround.  See the wdiff code in WriteOutputFiles for an
# explanation.
_wdiff_available = True

class TestTypeBase(object):
  # Filename pieces when writing failures to the test results directory.
  FILENAME_SUFFIX_ACTUAL = "-actual"
  FILENAME_SUFFIX_EXPECTED = "-expected"
  FILENAME_SUFFIX_DIFF = "-diff"
  FILENAME_SUFFIX_WDIFF = "-wdiff.html"
  FILENAME_SUFFIX_COMPARE = "-diff.png"

  def __init__(self, platform, root_output_dir):
    """Initialize a TestTypeBase object.

    Args:
      platform: the platform (e.g., 'chromium-mac-leopard') identifying the
        platform-specific results to be used.
      root_output_dir: The unix style path to the output dir.
    """
    self._root_output_dir = root_output_dir
    self._platform = platform

  def _MakeOutputDirectory(self, filename):
    """Creates the output directory (if needed) for a given test filename."""
    output_filename = os.path.join(self._root_output_dir,
                                   path_utils.RelativeTestFilename(filename))
    path_utils.MaybeMakeDirectory(os.path.split(output_filename)[0])

  def _SaveBaselineData(self, filename, data, modifier):
    """Saves a new baseline file into the platform directory.

    The file will be named simply "<test>-expected<modifier>", suitable for
    use as the expected results in a later run.

    Args:
      filename: path to the test file
      data: result to be saved as the new baseline
      modifier: type of the result file, e.g. ".txt" or ".png"
    """
    relative_dir = os.path.dirname(path_utils.RelativeTestFilename(filename))
    output_dir = os.path.join(path_utils.ChromiumBaselinePath(self._platform),
                              relative_dir)
    output_file = os.path.basename(os.path.splitext(filename)[0] +
                                   self.FILENAME_SUFFIX_EXPECTED + modifier)

    path_utils.MaybeMakeDirectory(output_dir)
    output_path = os.path.join(output_dir, output_file)
    logging.debug('writing new baseline to "%s"' % (output_path))
    open(output_path, "wb").write(data)

  def OutputFilename(self, filename, modifier):
    """Returns a filename inside the output dir that contains modifier.

    For example, if filename is c:/.../fast/dom/foo.html and modifier is
    "-expected.txt", the return value is
    c:/cygwin/tmp/layout-test-results/fast/dom/foo-expected.txt

    Args:
      filename: absolute filename to test file
      modifier: a string to replace the extension of filename with

    Return:
      The absolute windows path to the output filename
    """
    output_filename = os.path.join(self._root_output_dir,
                                   path_utils.RelativeTestFilename(filename))
    return os.path.splitext(output_filename)[0] + modifier

  def RelativeOutputFilename(self, filename, modifier):
    """Returns a relative filename inside the output dir that contains
    modifier.

    For example, if filename is fast\dom\foo.html and modifier is
    "-expected.txt", the return value is fast\dom\foo-expected.txt

    Args:
      filename: relative filename to test file
      modifier: a string to replace the extension of filename with

    Return:
      The relative windows path to the output filename
    """
    return os.path.splitext(filename)[0] + modifier

  def CompareOutput(self, filename, proc, output, test_args, target):
    """Method that compares the output from the test with the expected value.

    This is an abstract method to be implemented by all sub classes.

    Args:
      filename: absolute filename to test file
      proc: a reference to the test_shell process
      output: a string containing the output of the test
      test_args: a TestArguments object holding optional additional arguments
      target: Debug or Release

    Return:
      a list of TestFailure objects, empty if the test passes
    """
    raise NotImplemented

  def WriteOutputFiles(self, filename, test_type, file_type, output, expected,
                       diff=True, wdiff=False):
    """Writes the test output, the expected output and optionally the diff
    between the two to files in the results directory.

    The full output filename of the actual, for example, will be
      <filename><test_type>-actual<file_type>
    For instance,
      my_test-simp-actual.txt

    Args:
      filename: The test filename
      test_type: A string describing the test type, e.g. "simp"
      file_type: A string describing the test output file type, e.g. ".txt"
      output: A string containing the test output
      expected: A string containing the expected test output
      diff: if True, write a file containing the diffs too. This should be
          False for results that are not text
      wdiff: if True, write an HTML file containing word-by-word diffs
    """
    self._MakeOutputDirectory(filename)
    actual_filename = self.OutputFilename(filename,
                      test_type + self.FILENAME_SUFFIX_ACTUAL + file_type)
    expected_filename = self.OutputFilename(filename,
                      test_type + self.FILENAME_SUFFIX_EXPECTED + file_type)
    open(actual_filename, "wb").write(output)
    open(expected_filename, "wb").write(expected)

    if diff:
      diff = difflib.unified_diff(expected.splitlines(True),
                                  output.splitlines(True),
                                  expected_filename,
                                  actual_filename)

      diff_filename = self.OutputFilename(filename,
                           test_type + self.FILENAME_SUFFIX_DIFF + file_type)
      open(diff_filename, "wb").write(''.join(diff))

    if wdiff:
      # Shell out to wdiff to get colored inline diffs.
      executable = path_utils.WDiffPath()
      cmd = [executable,
             '--start-delete=##WDIFF_DEL##', '--end-delete=##WDIFF_END##',
             '--start-insert=##WDIFF_ADD##', '--end-insert=##WDIFF_END##',
             expected_filename, actual_filename]
      filename = self.OutputFilename(filename,
                      test_type + self.FILENAME_SUFFIX_WDIFF)

      global _wdiff_available

      try:
        # Python's Popen has a bug that causes any pipes opened to a process
        # that can't be exceuted to be leaked.  Since this code is specifically
        # designed to tolerate exec failures to gracefully handle cases where
        # wdiff is not installed, the bug results in a massive file descriptor
        # leak.  As a workaround, if an exec failure is ever experienced for
        # wdiff, assume it's not available.  This will leak one file descriptor
        # but that's better than leaking each time wdiff would be run.
        #
        # http://mail.python.org/pipermail/python-list/2008-August/505753.html
        # http://bugs.python.org/issue3210
        #
        # It also has a threading bug, so we don't output wdiff if the Popen
        # raises a ValueError.
        # http://bugs.python.org/issue1236
        if _wdiff_available:
          wdiff = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
          wdiff_failed = False

      except OSError, e:
        if e.errno == errno.ENOENT or e.errno == errno.EACCES:
          _wdiff_available = False
        else:
          raise e
      except ValueError, e:
        wdiff_failed = True

      out = open(filename, 'wb')

      if not _wdiff_available:
        out.write(
            """wdiff not installed.
"""
            """If you're running OS X, you can install via macports.
"""
            """If running Ubuntu linux, you can run "sudo apt-get install"""
            """ wdiff".""")
      elif wdiff_failed:
        out.write('wdiff failed due to running with multiple test_shells in '
                  'parallel.')
      else:
        wdiff = cgi.escape(wdiff)
        wdiff = wdiff.replace('##WDIFF_DEL##', '')
        wdiff = wdiff.replace('##WDIFF_ADD##', '')
        wdiff = wdiff.replace('##WDIFF_END##', '')
        out.write('')
        out.write('
' + wdiff + '
')

      out.close()
<commit_msg>continued attempts to work around python 2.4's popen() flakiness. Trap another error coming back and ignore it.<commit_after># Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Defines the interface TestTypeBase which other test types inherit from.

Also defines the TestArguments "struct" to pass them additional arguments.
"""

import cgi
import difflib
import errno
import logging
import os.path
import subprocess

from layout_package import path_utils

class TestArguments(object):
  """Struct-like wrapper for additional arguments needed by specific tests."""
  # Whether to save new baseline results.
  new_baseline = False

  # Path to the actual PNG file generated by pixel tests
  png_path = None

  # Value of checksum generated by pixel tests.
  hash = None

  # Whether to use wdiff to generate by-word diffs.
  wdiff = False

  # Whether to report the locations of the expected result files used.
  show_sources = False

# Python bug workaround.  See the wdiff code in WriteOutputFiles for an
# explanation.
_wdiff_available = True

class TestTypeBase(object):
  # Filename pieces when writing failures to the test results directory.
  FILENAME_SUFFIX_ACTUAL = "-actual"
  FILENAME_SUFFIX_EXPECTED = "-expected"
  FILENAME_SUFFIX_DIFF = "-diff"
  FILENAME_SUFFIX_WDIFF = "-wdiff.html"
  FILENAME_SUFFIX_COMPARE = "-diff.png"

  def __init__(self, platform, root_output_dir):
    """Initialize a TestTypeBase object.

    Args:
      platform: the platform (e.g., 'chromium-mac-leopard') identifying the
        platform-specific results to be used.
      root_output_dir: The unix style path to the output dir.
    """
    self._root_output_dir = root_output_dir
    self._platform = platform

  def _MakeOutputDirectory(self, filename):
    """Creates the output directory (if needed) for a given test filename."""
    output_filename = os.path.join(self._root_output_dir,
                                   path_utils.RelativeTestFilename(filename))
    path_utils.MaybeMakeDirectory(os.path.split(output_filename)[0])

  def _SaveBaselineData(self, filename, data, modifier):
    """Saves a new baseline file into the platform directory.

    The file will be named simply "<test>-expected<modifier>", suitable for
    use as the expected results in a later run.

    Args:
      filename: path to the test file
      data: result to be saved as the new baseline
      modifier: type of the result file, e.g. ".txt" or ".png"
    """
    relative_dir = os.path.dirname(path_utils.RelativeTestFilename(filename))
    output_dir = os.path.join(path_utils.ChromiumBaselinePath(self._platform),
                              relative_dir)
    output_file = os.path.basename(os.path.splitext(filename)[0] +
                                   self.FILENAME_SUFFIX_EXPECTED + modifier)

    path_utils.MaybeMakeDirectory(output_dir)
    output_path = os.path.join(output_dir, output_file)
    logging.debug('writing new baseline to "%s"' % (output_path))
    open(output_path, "wb").write(data)

  def OutputFilename(self, filename, modifier):
    """Returns a filename inside the output dir that contains modifier.

    For example, if filename is c:/.../fast/dom/foo.html and modifier is
    "-expected.txt", the return value is
    c:/cygwin/tmp/layout-test-results/fast/dom/foo-expected.txt

    Args:
      filename: absolute filename to test file
      modifier: a string to replace the extension of filename with

    Return:
      The absolute windows path to the output filename
    """
    output_filename = os.path.join(self._root_output_dir,
                                   path_utils.RelativeTestFilename(filename))
    return os.path.splitext(output_filename)[0] + modifier

  def RelativeOutputFilename(self, filename, modifier):
    """Returns a relative filename inside the output dir that contains
    modifier.

    For example, if filename is fast\dom\foo.html and modifier is
    "-expected.txt", the return value is fast\dom\foo-expected.txt

    Args:
      filename: relative filename to test file
      modifier: a string to replace the extension of filename with

    Return:
      The relative windows path to the output filename
    """
    return os.path.splitext(filename)[0] + modifier

  def CompareOutput(self, filename, proc, output, test_args, target):
    """Method that compares the output from the test with the expected value.

    This is an abstract method to be implemented by all sub classes.

    Args:
      filename: absolute filename to test file
      proc: a reference to the test_shell process
      output: a string containing the output of the test
      test_args: a TestArguments object holding optional additional arguments
      target: Debug or Release

    Return:
      a list of TestFailure objects, empty if the test passes
    """
    raise NotImplemented

  def WriteOutputFiles(self, filename, test_type, file_type, output, expected,
                       diff=True, wdiff=False):
    """Writes the test output, the expected output and optionally the diff
    between the two to files in the results directory.

    The full output filename of the actual, for example, will be
      <filename><test_type>-actual<file_type>
    For instance,
      my_test-simp-actual.txt

    Args:
      filename: The test filename
      test_type: A string describing the test type, e.g. "simp"
      file_type: A string describing the test output file type, e.g. ".txt"
      output: A string containing the test output
      expected: A string containing the expected test output
      diff: if True, write a file containing the diffs too. This should be
          False for results that are not text
      wdiff: if True, write an HTML file containing word-by-word diffs
    """
    self._MakeOutputDirectory(filename)
    actual_filename = self.OutputFilename(filename,
                      test_type + self.FILENAME_SUFFIX_ACTUAL + file_type)
    expected_filename = self.OutputFilename(filename,
                      test_type + self.FILENAME_SUFFIX_EXPECTED + file_type)
    open(actual_filename, "wb").write(output)
    open(expected_filename, "wb").write(expected)

    if diff:
      diff = difflib.unified_diff(expected.splitlines(True),
                                  output.splitlines(True),
                                  expected_filename,
                                  actual_filename)

      diff_filename = self.OutputFilename(filename,
                           test_type + self.FILENAME_SUFFIX_DIFF + file_type)
      open(diff_filename, "wb").write(''.join(diff))

    if wdiff:
      # Shell out to wdiff to get colored inline diffs.
      executable = path_utils.WDiffPath()
      cmd = [executable,
             '--start-delete=##WDIFF_DEL##', '--end-delete=##WDIFF_END##',
             '--start-insert=##WDIFF_ADD##', '--end-insert=##WDIFF_END##',
             expected_filename, actual_filename]
      filename = self.OutputFilename(filename,
                      test_type + self.FILENAME_SUFFIX_WDIFF)

      global _wdiff_available

      try:
        # Python's Popen has a bug that causes any pipes opened to a process
        # that can't be exceuted to be leaked.  Since this code is specifically
        # designed to tolerate exec failures to gracefully handle cases where
        # wdiff is not installed, the bug results in a massive file descriptor
        # leak.  As a workaround, if an exec failure is ever experienced for
        # wdiff, assume it's not available.  This will leak one file descriptor
        # but that's better than leaking each time wdiff would be run.
        #
        # http://mail.python.org/pipermail/python-list/2008-August/505753.html
        # http://bugs.python.org/issue3210
        #
        # It also has a threading bug, so we don't output wdiff if the Popen
        # raises a ValueError.
        # http://bugs.python.org/issue1236
        if _wdiff_available:
          wdiff = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
          wdiff_failed = False

      except OSError, e:
        if (e.errno == errno.ENOENT or e.errno == errno.EACCES or
            e.errno == errno.ECHILD):
          _wdiff_available = False
        else:
          raise e
      except ValueError, e:
        wdiff_failed = True

      out = open(filename, 'wb')

      if not _wdiff_available:
        out.write(
            """wdiff not installed.
"""
            """If you're running OS X, you can install via macports.
"""
            """If running Ubuntu linux, you can run "sudo apt-get install"""
            """ wdiff".""")
      elif wdiff_failed:
        out.write('wdiff failed due to running with multiple test_shells in '
                  'parallel.')
      else:
        wdiff = cgi.escape(wdiff)
        wdiff = wdiff.replace('##WDIFF_DEL##', '')
        wdiff = wdiff.replace('##WDIFF_ADD##', '')
        wdiff = wdiff.replace('##WDIFF_END##', '')
        out.write('')
        out.write('
' + wdiff + '
')

      out.close()
<|endoftext|>


<commit_before>import os
import numpy as np
import logging
import itertools
from cached_property import cached_property
from sqlalchemy import Column, String, Integer, Text, ForeignKey
from sqlalchemy.orm import relationship, Session
from sqlalchemy import UniqueConstraint

from tmlib.models import Model, DateMixIn
from tmlib.readers import YamlReader
from tmlib.writers import YamlWriter
from tmlib.models.utils import remove_location_upon_delete
from tmlib.models.plate import SUPPORTED_PLATE_FORMATS
from tmlib.models.plate import SUPPORTED_PLATE_AQUISITION_MODES
from tmlib.workflow.registry import get_workflow_description
from tmlib.workflow.metaconfig import SUPPORTED_MICROSCOPE_TYPES
from tmlib.workflow.canonical import CanonicalWorkflowDescription
from tmlib.utils import autocreate_directory_property

logger = logging.getLogger(__name__)

#: Format string for experiment locations.
EXPERIMENT_LOCATION_FORMAT = 'experiment_{id}'

@remove_location_upon_delete
class Experiment(Model, DateMixIn):

    '''An *experiment* is the main organizational unit of `TissueMAPS`.
    It represents a set of images and associated data.

    Images are grouped by *plates* and *cycles*. A *plate* represents a
    container for the imaged biological samples and a *cycle* corresponds to a
    particular time point of image acquisition. In the simplest case,
    an *experiment* is composed of a single *plate* with one *cycle* where
    each sample was imaged once.

    Attributes
    ----------
    name: str
        name of the experiment
    root_directory: str
        absolute path to root directory where experiment directory
        should be created in
    description: str
        description of the experimental setup
    stage: str
        currently processed workflow stage
    step: str
        currently processed workflow step
    microscope_type: str
        microscope that was used to acquire the images
    plate_format: int
        number of wells in the plate, e.g. 384
    plate_acquisition_mode: str
        the way plates were acquired with the microscope
    user_id: int
        ID of the owner
    user: tmlib.models.User
        the user who owns the experiment
    plates: List[tmlib.models.Plate]
        plates that belong to the experiment
    channels: List[tmlib.models.Channel]
        channels that belong to the experiment
    mapobject_types: List[tmlib.models.MapobjectType]
        mapobject types that belong to the experiment
    submissions: List[tmlib.models.Submission]
        submissions that belong to the experiment
    zoom_factor: int
        zoom factor between pyramid levels
    vertical_site_displacement: int, optional
        displacement of neighboring sites within a well along the
        vertical axis
    horizontal_site_displacement: int
        displacement of neighboring sites within a well along the
        horizontal axis
    well_spacer_size: int
        gab between neighboring wells in pixels

    See also
    --------
    :py:class:`tmlib.models.Plate`
    :py:class:`tmlib.models.Cycle`
    '''

    #: str: name of the corresponding database table
    __tablename__ = 'experiments'

    __table_args__ = (UniqueConstraint('name', 'user_id'), )

    # Table columns
    name = Column(String)
    microscope_type = Column(String)
    plate_format = Column(Integer)
    plate_acquisition_mode = Column(String)
    description = Column(Text)
    root_directory = Column(String)
    status = Column(String)
    stage = Column(String)
    step = Column(String)
    zoom_factor = Column(Integer)
    vertical_site_displacement = Column(Integer)
    horizontal_site_displacement = Column(Integer)
    well_spacer_size = Column(Integer)
    user_id = Column(Integer, ForeignKey('users.id'), index=True)

    # Relationships to other tables
    user = relationship('User', backref='experiments')

    def __init__(self, name, microscope_type, plate_format,
                 plate_acquisition_mode, user_id,
                 root_directory='$TMAPS_STORAGE', description='',
                 zoom_factor=2, well_spacer_size=500,
                 vertical_site_displacement=0, horizontal_site_displacement=0):
        '''
        Parameters
        ----------
        name: str
            name of the experiment
        microscope_type: str
            microscope that was used to acquire the images
        plate_format: int
            number of wells in the plate, e.g. 384
        plate_acquisition_mode: str
            the way plates were acquired with the microscope
        user_id: int
            ID of the owner
        root_directory: str, optional
            absolute path to root directory where experiment directory
            should be created in (default: `$TMAPS_STORAGE`)
        description: str, optional
            description of the experimental setup
        zoom_factor: int, optional
            zoom factor between pyramid levels (default: ``2``)
        well_spacer_size: int
            gab between neighboring wells in pixels (default: ``500``)
        vertical_site_displacement: int, optional
            displacement of neighboring sites within a well along the
            vertical axis (default: ``0``)
        horizontal_site_displacement: int, optional
            displacement of neighboring sites within a well along the
            horizontal axis (default: ``0``)

        See also
        --------
        :py:attr:`tmlib.models.experiment.SUPPORTED_MICROSCOPE_TYPES`
        :py:attr:`tmlib.models.plate.SUPPORTED_PLATE_AQUISITION_MODES`
        :py:attr:`tmlib.models.plate.SUPPORTED_PLATE_FORMATS`
        '''
        self.name = name
        self.user_id = user_id
        self.description = description

        if microscope_type not in SUPPORTED_MICROSCOPE_TYPES:
            raise ValueError(
                'Unsupported microscope type! Supported are: "%s"'
                % '", "'.join(SUPPORTED_MICROSCOPE_TYPES)
            )
        self.microscope_type = microscope_type

        if plate_format not in SUPPORTED_PLATE_FORMATS:
            raise ValueError(
                'Unsupported plate format! Supported are: %s'
                % ', '.join(map(str, SUPPORTED_PLATE_FORMATS))
            )
        self.plate_format = plate_format

        if plate_acquisition_mode not in SUPPORTED_PLATE_AQUISITION_MODES:
            raise ValueError(
                'Unsupported acquisition mode! Supported are: "%s"'
                % '", "'.join(SUPPORTED_PLATE_AQUISITION_MODES)
            )
        self.plate_acquisition_mode = plate_acquisition_mode

        self.root_directory = root_directory
        self.status = 'PENDING'  # TODO: get from task table
        self.stage = None
        self.step = None
        self.zoom_factor = zoom_factor
        self.well_spacer_size = well_spacer_size
        # TODO: we may be able to calculate this automatically from OMEXML
        self.vertical_site_displacement = vertical_site_displacement
        self.horizontal_site_displacement = horizontal_site_displacement

    @autocreate_directory_property
    def location(self):
        '''str: location of the experiment,
        e.g. absolute path to a directory on disk
        '''
        if self.id is None:
            raise AttributeError(
                'Experiment "%s" doesn\'t have an entry in the database yet. '
                'Therefore, its location cannot be determined.' % self.name
            )
        return os.path.join(
            os.path.expandvars(self.root_directory),
            EXPERIMENT_LOCATION_FORMAT.format(id=self.id)
        )

    @autocreate_directory_property
    def plates_location(self):
        '''str: location where plates data are stored'''
        return os.path.join(self.location, 'plates')

    @autocreate_directory_property
    def channels_location(self):
        '''str: location where channels data are stored'''
        return os.path.join(self.location, 'channels')

    @autocreate_directory_property
    def mapobject_types_location(self):
        '''str: location where mapobject type data are stored'''
        return os.path.join(self.location, 'mapobject_types')

    @autocreate_directory_property
    def workflow_location(self):
        '''str: location where workflow data are stored'''
        return os.path.join(self.location, 'workflow')

    @property
    def _workflow_descriptor_file(self):
        return os.path.join(
            self.workflow_location, 'workflow_description.yaml'
        )

    @property
    def workflow_description(self):
        '''tmlib.workflow.tmaps.description.WorkflowDescription: description
        of the workflow

        Raises
        ------
        OSError
            when there is no workflow desriptor file available on disk
        TypeError
            when description obtained from file is not a mapping
        KeyError
            when description obtained from file doesn't have key "type"
        '''
        if not os.path.exists(self._workflow_descriptor_file):
            logger.warn('no persistent workflow description found')
            logger.info('default to "canonical" workflow')
            return CanonicalWorkflowDescription()
        with YamlReader(self._workflow_descriptor_file) as f:
            description = f.read()
        if not isinstance(description, dict):
            raise TypeError('Description must be a mapping.')
        if 'type' not in description:
            raise KeyError('Description must have key "type".')
        if 'stages' not in description:
            raise KeyError('Workflow description must have key "stages".')
        WorkflowDescription = get_workflow_description(
            description['type']
        )
        return WorkflowDescription(description['stages'])

    def persist_workflow_description(self, description):
        '''Persists the workflow description.

        Parameters
        ----------
        description: tmlib.workflow.tmaps.description.WorkflowDescription
            description of the workflow
        '''
        with YamlWriter(self._workflow_descriptor_file) as f:
            f.write(description.as_dict())

    @property
    def session_location(self):
        '''str: location where submission data are stored'''
        return os.path.join(self.workflow_location, 'session')

    @cached_property
    def plate_spacer_size(self):
        '''int: gap between neighboring plates in pixels'''
        return self.well_spacer_size * 2

    @cached_property
    def plate_grid(self):
        '''numpy.ndarray[int]: IDs of plates arranged according to
        their relative position of the plate within the experiment overview
        image
        '''
        n = len(self.plates)
        if (n / np.ceil(np.sqrt(n))) % 2 == 0:
            dimensions = (
                int(np.ceil(np.sqrt(n))), int(n / np.ceil(np.sqrt(n)))
            )
        else:
            dimensions = tuple(np.repeat(int(np.ceil(np.sqrt(n))), 2))
        cooridinates = list(
            itertools.product(
                np.arange(dimensions[0]), np.arange(dimensions[1])
            )
        )
        height, width = dimensions
        grid = np.zeros((height, width), dtype=int)
        for i, c in enumerate(cooridinates):
            if i >= n:
                break
            grid[c[0], c[1]] = self.plates[i].id
        return grid

    def belongs_to(self, user):
        '''Determines whether the experiment belongs to a given `user`.

        Parameters
        ----------
        user: tmlib.user.User
            `TissueMAPS` user

        Returns
        -------
        bool
            whether experiment belongs to `user`
        '''
        return self.user_id == user.id

    @property
    def is_ready_for_processing(self):
        '''bool: whether the experiment is ready for processing
        (requires that upload of images is complete)
        '''
        return all([pls.is_ready_for_processing for pls in self.plate_sources])

    def get_mapobject_type(self, name):
        '''
        Return a mapobject type belonging to this experiment by name.

        Parameters
        ----------
        name : str
            the name of the mapobject_type to be returned

        Returns
        -------
        tmlib.models.MapobjectType

        Raises
        ------
        sqlalchemy.orm.exc.MultipleResultsFound
           when multiple mapobject types with this name were found
        sqlalchemy.orm.exc.NoResultFound
           when no mapobject type with this name was found

        '''
        from tmlib.models import MapobjectType
        session = Session.object_session(self)
        return session.query(MapobjectType).\
            filter_by(name=name, experiment_id=self.id).\
            one()

    def as_dict(self):
        '''Returns attributes as key-value pairs.

        Returns
        -------
        dict
        '''
        mapobject_info = []
        for t in self.mapobject_types:
            mapobject_info.append({
                'mapobject_type_name': t.name,
                'features': [{'name': f.name} for f in t.features]
            })

        return {
            'id': self.hash,
            'name': self.name,
            'description': self.description,
            'user': self.user.name,
            'plate_format': self.plate_format,
            'microscope_type': self.microscope_type,
            'plate_acquisition_mode': self.plate_acquisition_mode,
            'status': self.status,
            'stage': self.stage,
            'step': self.step,
            'channels': [ch.as_dict() for ch in self.channels],
            'mapobject_info': mapobject_info,
            'plates': [pl.as_dict() for pl in self.plates]
        }

    def __repr__(self):
        return '<Experiment(id=%r, name=%r)>' % (self.id, self.name)
<commit_msg>Track status of workflow independent of experiments table<commit_after>import os
import numpy as np
import logging
import itertools
from cached_property import cached_property
from sqlalchemy import Column, String, Integer, Text, ForeignKey
from sqlalchemy.orm import relationship, Session
from sqlalchemy import UniqueConstraint

from tmlib.models import Model, DateMixIn
from tmlib.readers import YamlReader
from tmlib.writers import YamlWriter
from tmlib.models.utils import remove_location_upon_delete
from tmlib.models.plate import SUPPORTED_PLATE_FORMATS
from tmlib.models.plate import SUPPORTED_PLATE_AQUISITION_MODES
from tmlib.workflow.registry import get_workflow_description
from tmlib.workflow.metaconfig import SUPPORTED_MICROSCOPE_TYPES
from tmlib.workflow.canonical import CanonicalWorkflowDescription
from tmlib.utils import autocreate_directory_property

logger = logging.getLogger(__name__)

#: Format string for experiment locations.
EXPERIMENT_LOCATION_FORMAT = 'experiment_{id}'

@remove_location_upon_delete
class Experiment(Model, DateMixIn):

    '''An *experiment* is the main organizational unit of `TissueMAPS`.
    It represents a set of images and associated data.

    Images are grouped by *plates* and *cycles*. A *plate* represents a
    container for the imaged biological samples and a *cycle* corresponds to a
    particular time point of image acquisition. In the simplest case,
    an *experiment* is composed of a single *plate* with one *cycle* where
    each sample was imaged once.

    Attributes
    ----------
    name: str
        name of the experiment
    root_directory: str
        absolute path to root directory where experiment directory
        should be created in
    description: str
        description of the experimental setup
    microscope_type: str
        microscope that was used to acquire the images
    plate_format: int
        number of wells in the plate, e.g. 384
    plate_acquisition_mode: str
        the way plates were acquired with the microscope
    user_id: int
        ID of the owner
    user: tmlib.models.User
        the user who owns the experiment
    plates: List[tmlib.models.Plate]
        plates that belong to the experiment
    channels: List[tmlib.models.Channel]
        channels that belong to the experiment
    mapobject_types: List[tmlib.models.MapobjectType]
        mapobject types that belong to the experiment
    submissions: List[tmlib.models.Submission]
        submissions that belong to the experiment
    zoom_factor: int
        zoom factor between pyramid levels
    vertical_site_displacement: int, optional
        displacement of neighboring sites within a well along the
        vertical axis
    horizontal_site_displacement: int
        displacement of neighboring sites within a well along the
        horizontal axis
    well_spacer_size: int
        gab between neighboring wells in pixels

    See also
    --------
    :py:class:`tmlib.models.Plate`
    :py:class:`tmlib.models.Cycle`
    '''

    #: str: name of the corresponding database table
    __tablename__ = 'experiments'

    __table_args__ = (UniqueConstraint('name', 'user_id'), )

    # Table columns
    name = Column(String)
    microscope_type = Column(String)
    plate_format = Column(Integer)
    plate_acquisition_mode = Column(String)
    description = Column(Text)
    root_directory = Column(String)
    zoom_factor = Column(Integer)
    vertical_site_displacement = Column(Integer)
    horizontal_site_displacement = Column(Integer)
    well_spacer_size = Column(Integer)
    user_id = Column(Integer, ForeignKey('users.id'), index=True)

    # Relationships to other tables
    user = relationship('User', backref='experiments')

    def __init__(self, name, microscope_type, plate_format,
                 plate_acquisition_mode, user_id,
                 root_directory='$TMAPS_STORAGE', description='',
                 zoom_factor=2, well_spacer_size=500,
                 vertical_site_displacement=0, horizontal_site_displacement=0):
        '''
        Parameters
        ----------
        name: str
            name of the experiment
        microscope_type: str
            microscope that was used to acquire the images
        plate_format: int
            number of wells in the plate, e.g. 384
        plate_acquisition_mode: str
            the way plates were acquired with the microscope
        user_id: int
            ID of the owner
        root_directory: str, optional
            absolute path to root directory where experiment directory
            should be created in (default: `$TMAPS_STORAGE`)
        description: str, optional
            description of the experimental setup
        zoom_factor: int, optional
            zoom factor between pyramid levels (default: ``2``)
        well_spacer_size: int
            gab between neighboring wells in pixels (default: ``500``)
        vertical_site_displacement: int, optional
            displacement of neighboring sites within a well along the
            vertical axis (default: ``0``)
        horizontal_site_displacement: int, optional
            displacement of neighboring sites within a well along the
            horizontal axis (default: ``0``)

        See also
        --------
        :py:attr:`tmlib.models.experiment.SUPPORTED_MICROSCOPE_TYPES`
        :py:attr:`tmlib.models.plate.SUPPORTED_PLATE_AQUISITION_MODES`
        :py:attr:`tmlib.models.plate.SUPPORTED_PLATE_FORMATS`
        '''
        self.name = name
        self.user_id = user_id
        self.description = description

        if microscope_type not in SUPPORTED_MICROSCOPE_TYPES:
            raise ValueError(
                'Unsupported microscope type! Supported are: "%s"'
                % '", "'.join(SUPPORTED_MICROSCOPE_TYPES)
            )
        self.microscope_type = microscope_type

        if plate_format not in SUPPORTED_PLATE_FORMATS:
            raise ValueError(
                'Unsupported plate format! Supported are: %s'
                % ', '.join(map(str, SUPPORTED_PLATE_FORMATS))
            )
        self.plate_format = plate_format

        if plate_acquisition_mode not in SUPPORTED_PLATE_AQUISITION_MODES:
            raise ValueError(
                'Unsupported acquisition mode! Supported are: "%s"'
                % '", "'.join(SUPPORTED_PLATE_AQUISITION_MODES)
            )
        self.plate_acquisition_mode = plate_acquisition_mode

        self.root_directory = root_directory
        self.zoom_factor = zoom_factor
        self.well_spacer_size = well_spacer_size
        # TODO: we may be able to calculate this automatically from OMEXML
        self.vertical_site_displacement = vertical_site_displacement
        self.horizontal_site_displacement = horizontal_site_displacement

    @autocreate_directory_property
    def location(self):
        '''str: location of the experiment,
        e.g. absolute path to a directory on disk
        '''
        if self.id is None:
            raise AttributeError(
                'Experiment "%s" doesn\'t have an entry in the database yet. '
                'Therefore, its location cannot be determined.' % self.name
            )
        return os.path.join(
            os.path.expandvars(self.root_directory),
            EXPERIMENT_LOCATION_FORMAT.format(id=self.id)
        )

    @autocreate_directory_property
    def plates_location(self):
        '''str: location where plates data are stored'''
        return os.path.join(self.location, 'plates')

    @autocreate_directory_property
    def channels_location(self):
        '''str: location where channels data are stored'''
        return os.path.join(self.location, 'channels')

    @autocreate_directory_property
    def mapobject_types_location(self):
        '''str: location where mapobject type data are stored'''
        return os.path.join(self.location, 'mapobject_types')

    @autocreate_directory_property
    def workflow_location(self):
        '''str: location where workflow data are stored'''
        return os.path.join(self.location, 'workflow')

    @property
    def _workflow_descriptor_file(self):
        return os.path.join(
            self.workflow_location, 'workflow_description.yaml'
        )

    @property
    def workflow_description(self):
        '''tmlib.workflow.tmaps.description.WorkflowDescription: description
        of the workflow

        Raises
        ------
        OSError
            when there is no workflow desriptor file available on disk
        TypeError
            when description obtained from file is not a mapping
        KeyError
            when description obtained from file doesn't have key "type"
        '''
        if not os.path.exists(self._workflow_descriptor_file):
            logger.warn('no persistent workflow description found')
            logger.info('default to "canonical" workflow')
            return CanonicalWorkflowDescription()
        with YamlReader(self._workflow_descriptor_file) as f:
            description = f.read()
        if not isinstance(description, dict):
            raise TypeError('Description must be a mapping.')
        if 'type' not in description:
            raise KeyError('Description must have key "type".')
        if 'stages' not in description:
            raise KeyError('Workflow description must have key "stages".')
        WorkflowDescription = get_workflow_description(
            description['type']
        )
        return WorkflowDescription(description['stages'])

    def persist_workflow_description(self, description):
        '''Persists the workflow description.

        Parameters
        ----------
        description: tmlib.workflow.tmaps.description.WorkflowDescription
            description of the workflow
        '''
        with YamlWriter(self._workflow_descriptor_file) as f:
            f.write(description.as_dict())

    @property
    def session_location(self):
        '''str: location where submission data are stored'''
        return os.path.join(self.workflow_location, 'session')

    @cached_property
    def plate_spacer_size(self):
        '''int: gap between neighboring plates in pixels'''
        return self.well_spacer_size * 2

    @cached_property
    def plate_grid(self):
        '''numpy.ndarray[int]: IDs of plates arranged according to
        their relative position of the plate within the experiment overview
        image
        '''
        n = len(self.plates)
        if (n / np.ceil(np.sqrt(n))) % 2 == 0:
            dimensions = (
                int(np.ceil(np.sqrt(n))), int(n / np.ceil(np.sqrt(n)))
            )
        else:
            dimensions = tuple(np.repeat(int(np.ceil(np.sqrt(n))), 2))
        cooridinates = list(
            itertools.product(
                np.arange(dimensions[0]), np.arange(dimensions[1])
            )
        )
        height, width = dimensions
        grid = np.zeros((height, width), dtype=int)
        for i, c in enumerate(cooridinates):
            if i >= n:
                break
            grid[c[0], c[1]] = self.plates[i].id
        return grid

    def belongs_to(self, user):
        '''Determines whether the experiment belongs to a given `user`.

        Parameters
        ----------
        user: tmlib.user.User
            `TissueMAPS` user

        Returns
        -------
        bool
            whether experiment belongs to `user`
        '''
        return self.user_id == user.id

    @property
    def is_ready_for_processing(self):
        '''bool: whether the experiment is ready for processing
        (requires that upload of images is complete)
        '''
        return all([pls.is_ready_for_processing for pls in self.plate_sources])

    def get_mapobject_type(self, name):
        '''
        Return a mapobject type belonging to this experiment by name.

        Parameters
        ----------
        name : str
            the name of the mapobject_type to be returned

        Returns
        -------
        tmlib.models.MapobjectType

        Raises
        ------
        sqlalchemy.orm.exc.MultipleResultsFound
           when multiple mapobject types with this name were found
        sqlalchemy.orm.exc.NoResultFound
           when no mapobject type with this name was found

        '''
        from tmlib.models import MapobjectType
        session = Session.object_session(self)
        return session.query(MapobjectType).\
            filter_by(name=name, experiment_id=self.id).\
            one()

    def as_dict(self):
        '''Returns attributes as key-value pairs.

        Returns
        -------
        dict
        '''
        mapobject_info = []
        for t in self.mapobject_types:
            mapobject_info.append({
                'mapobject_type_name': t.name,
                'features': [{'name': f.name} for f in t.features]
            })

        return {
            'id': self.hash,
            'name': self.name,
            'description': self.description,
            'user': self.user.name,
            'plate_format': self.plate_format,
            'microscope_type': self.microscope_type,
            'plate_acquisition_mode': self.plate_acquisition_mode,
            'channels': [ch.as_dict() for ch in self.channels],
            'mapobject_info': mapobject_info,
            'plates': [pl.as_dict() for pl in self.plates]
        }

    def __repr__(self):
        return '<Experiment(id=%r, name=%r)>' % (self.id, self.name)
<|endoftext|>


<commit_before>import argparse
import logging

from dvc.command import completion
from dvc.command.base import CmdBase, append_doc_link
from dvc.exceptions import DvcException
from dvc.prompt import ask

logger = logging.getLogger(__name__)

class CmdCheckIgnore(CmdBase):
    def __init__(self, args):
        super().__init__(args)
        self.ignore_filter = self.repo.tree.dvcignore

    def _show_results(self, result):
        if not result.match and not self.args.non_matching:
            return

        if self.args.details:
            patterns = result.patterns
            if not self.args.all:
                patterns = patterns[-1:]

            for pattern in patterns:
                logger.info("{}\t{}".format(pattern, result.file))
        else:
            logger.info(result.file)

    def _check_one_file(self, target):
        result = self.ignore_filter.check_ignore(target)
        self._show_results(result)
        if result.match:
            return 0
        return 1

    def _interactive_mode(self):
        ret = 1
        while True:
            try:
                target = ask("")
            except (KeyboardInterrupt, EOFError):
                break
            if target == "":
                break
            if not self._check_one_file(target):
                ret = 0
        return ret

    def _normal_mode(self):
        ret = 1
        for target in self.args.targets:
            if not self._check_one_file(target):
                ret = 0
        return ret

    def _check_args(self):
        if not self.args.stdin and not self.args.targets:
            raise DvcException("`targets` or `--stdin` needed")

        if self.args.stdin and self.args.targets:
            raise DvcException("cannot have both `targets` and `--stdin`")

        if self.args.non_matching and not self.args.details:
            raise DvcException(
                "`--non-matching` is only valid with `--details`"
            )

        if self.args.all and not self.args.details:
            raise DvcException("`--all` is only valid with `--details`")

        if self.args.quiet and self.args.details:
            raise DvcException("cannot use both `--details` and `--quiet`")

    def run(self):
        self._check_args()
        if self.args.stdin:
            return self._interactive_mode()
        return self._normal_mode()

def add_parser(subparsers, parent_parser):
    ADD_HELP = "Debug DVC ignore/exclude files"

    parser = subparsers.add_parser(
        "check-ignore",
        parents=[parent_parser],
        description=append_doc_link(ADD_HELP, "check-ignore"),
        help=ADD_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument(
        "-d",
        "--details",
        action="store_true",
        default=False,
        help="Show the exclude pattern together with each target path.",
    )
    parser.add_argument(
        "-n",
        "--non-matching",
        action="store_true",
        default=False,
        help="Show the target paths which don’t match any pattern. "
        "Only usable when `--details` is also employed",
    )
    parser.add_argument(
        "--stdin",
        action="store_true",
        default=False,
        help="Read pathnames from the standard input, one per line, "
        "instead of from the command-line.",
    )
    parser.add_argument(
        "-a",
        "--all",
        action="store_true",
        default=False,
        help="Show all of the patterns match the target paths. "
        "Only usable when `--details` is also employed",
    )
    parser.add_argument(
        "targets",
        nargs="*",
        help="Exact or wildcard paths of files or directories to check "
        "ignore patterns.",
    ).complete = completion.FILE
    parser.set_defaults(func=CmdCheckIgnore)
<commit_msg>check-ignore: update help output (#4358)<commit_after>import argparse
import logging

from dvc.command import completion
from dvc.command.base import CmdBase, append_doc_link
from dvc.exceptions import DvcException
from dvc.prompt import ask

logger = logging.getLogger(__name__)

class CmdCheckIgnore(CmdBase):
    def __init__(self, args):
        super().__init__(args)
        self.ignore_filter = self.repo.tree.dvcignore

    def _show_results(self, result):
        if not result.match and not self.args.non_matching:
            return

        if self.args.details:
            patterns = result.patterns
            if not self.args.all:
                patterns = patterns[-1:]

            for pattern in patterns:
                logger.info("{}\t{}".format(pattern, result.file))
        else:
            logger.info(result.file)

    def _check_one_file(self, target):
        result = self.ignore_filter.check_ignore(target)
        self._show_results(result)
        if result.match:
            return 0
        return 1

    def _interactive_mode(self):
        ret = 1
        while True:
            try:
                target = ask("")
            except (KeyboardInterrupt, EOFError):
                break
            if target == "":
                break
            if not self._check_one_file(target):
                ret = 0
        return ret

    def _normal_mode(self):
        ret = 1
        for target in self.args.targets:
            if not self._check_one_file(target):
                ret = 0
        return ret

    def _check_args(self):
        if not self.args.stdin and not self.args.targets:
            raise DvcException("`targets` or `--stdin` needed")

        if self.args.stdin and self.args.targets:
            raise DvcException("cannot have both `targets` and `--stdin`")

        if self.args.non_matching and not self.args.details:
            raise DvcException(
                "`--non-matching` is only valid with `--details`"
            )

        if self.args.all and not self.args.details:
            raise DvcException("`--all` is only valid with `--details`")

        if self.args.quiet and self.args.details:
            raise DvcException("cannot use both `--details` and `--quiet`")

    def run(self):
        self._check_args()
        if self.args.stdin:
            return self._interactive_mode()
        return self._normal_mode()

def add_parser(subparsers, parent_parser):
    ADD_HELP = (
        "Check whether files or directories are excluded due to `.dvcignore`."
    )

    parser = subparsers.add_parser(
        "check-ignore",
        parents=[parent_parser],
        description=append_doc_link(ADD_HELP, "check-ignore"),
        help=ADD_HELP,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument(
        "-d",
        "--details",
        action="store_true",
        default=False,
        help="Show the exclude patterns along with each target path.",
    )
    parser.add_argument(
        "-a",
        "--all",
        action="store_true",
        default=False,
        help="Include the target paths which don’t match any pattern "
        "in the `--details` list.",
    )
    parser.add_argument(
        "-n",
        "--non-matching",
        action="store_true",
        default=False,
        help="Include the target paths which don’t match any pattern "
        "in the `--details` list.",
    )
    parser.add_argument(
        "--stdin",
        action="store_true",
        default=False,
        help="Read paths from standard input instead of providing `targets`.",
    )
    parser.add_argument(
        "targets", nargs="*", help="File or directory paths to check",
    ).complete = completion.FILE
    parser.set_defaults(func=CmdCheckIgnore)
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-
#
# Copyright (C) 2009-2012 by Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import socket
from threading import Lock
from time import sleep, time

from earwigbot.exceptions import BrokenSocketError

__all__ = ["IRCConnection"]

class IRCConnection(object):
    """Interface with an IRC server."""

    def __init__(self, host, port, nick, ident, realname):
        self._host = host
        self._port = port
        self._nick = nick
        self._ident = ident
        self._realname = realname

        self._is_running = False
        self._send_lock = Lock()

        self._last_recv = time()
        self._last_ping = 0

    def __repr__(self):
        """Return the canonical string representation of the IRCConnection."""
        res = "IRCConnection(host={0!r}, port={1!r}, nick={2!r}, ident={3!r}, realname={4!r})"
        return res.format(self.host, self.port, self.nick, self.ident,
                          self.realname)

    def __str__(self):
        """Return a nice string representation of the IRCConnection."""
        res = "<IRCConnection {0}!{1} at {2}:{3}>"
        return res.format(self.nick, self.ident, self.host, self.port)

    def _connect(self):
        """Connect to our IRC server."""
        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            self._sock.connect((self.host, self.port))
        except socket.error:
            self.logger.exception("Couldn't connect to IRC server; retrying")
            sleep(8)
            self._connect()
        self._send("NICK {0}".format(self.nick))
        self._send("USER {0} {1} * :{2}".format(self.ident, self.host, self.realname))

    def _close(self):
        """Completely close our connection with the IRC server."""
        try:
            self._sock.shutdown(socket.SHUT_RDWR)  # Shut down connection first
        except socket.error:
            pass  # Ignore if the socket is already down
        self._sock.close()

    def _get(self, size=4096):
        """Receive (i.e. get) data from the server."""
        data = self._sock.recv(size)
        if not data:
            # Socket isn't giving us any data, so it is dead or broken:
            raise BrokenSocketError()
        return data

    def _send(self, msg, hidelog=False):
        """Send data to the server."""
        with self._send_lock:
            try:
                self._sock.sendall(msg + "\r\n")
            except socket.error:
                self._is_running = False
            else:
                if not hidelog:
                    self.logger.debug(msg)

    def _quit(self, msg=None):
        """Issue a quit message to the server. Doesn't close the connection."""
        if msg:
            self._send("QUIT :{0}".format(msg))
        else:
            self._send("QUIT")

    def _process_defaults(self, line):
        """Default process hooks for lines received on IRC."""
        self._last_recv = time()
        if line[0] == "PING":  # If we are pinged, pong back
            self.pong(line[1][1:])

    @property
    def host(self):
        """The hostname of the IRC server, like ``"irc.freenode.net"``."""
        return self._host

    @property
    def port(self):
        """The port of the IRC server, like ``6667``."""
        return self._port

    @property
    def nick(self):
        """Our nickname on the server, like ``"EarwigBot"``."""
        return self._nick

    @property
    def ident(self):
        """Our ident on the server, like ``"earwig"``.

        See `http://en.wikipedia.org/wiki/Ident`_.
        """
        return self._ident

    @property
    def realname(self):
        """Our realname (gecos field) on the server."""
        return self._realname

    def say(self, target, msg, hidelog=False):
        """Send a private message to a target on the server."""
        msg = "PRIVMSG {0} :{1}".format(target, msg)
        self._send(msg, hidelog)

    def reply(self, data, msg, hidelog=False):
        """Send a private message as a reply to a user on the server."""
        msg = "\x02{0}\x0f: {1}".format(data.nick, msg)
        self.say(data.chan, msg, hidelog)

    def action(self, target, msg, hidelog=False):
        """Send a private message to a target on the server as an action."""
        msg = "\x01ACTION {0}\x01".format(msg)
        self.say(target, msg, hidelog)

    def notice(self, target, msg, hidelog=False):
        """Send a notice to a target on the server."""
        msg = "NOTICE {0} :{1}".format(target, msg)
        self._send(msg, hidelog)

    def join(self, chan, hidelog=False):
        """Join a channel on the server."""
        msg = "JOIN {0}".format(chan)
        self._send(msg, hidelog)

    def part(self, chan, msg=None, hidelog=False):
        """Part from a channel on the server, optionally using an message."""
        if msg:
            self._send("PART {0} :{1}".format(chan, msg), hidelog)
        else:
            self._send("PART {0}".format(chan), hidelog)

    def mode(self, target, level, msg, hidelog=False):
        """Send a mode message to the server."""
        msg = "MODE {0} {1} {2}".format(target, level, msg)
        self._send(msg, hidelog)

    def ping(self, target, hidelog=False):
        """Ping another entity on the server."""
        msg = "PING {0}".format(target)
        self._send(msg, hidelog)

    def pong(self, target, hidelog=False):
        """Pong another entity on the server."""
        msg = "PONG {0}".format(target)
        self._send(msg, hidelog)

    def loop(self):
        """Main loop for the IRC connection."""
        self._is_running = True
        read_buffer = ""
        while 1:
            try:
                read_buffer += self._get()
            except BrokenSocketError:
                self._is_running = False
                break

            lines = read_buffer.split("\n")
            read_buffer = lines.pop()
            for line in lines:
                line = line.strip().split()
                self._process_defaults(line)
                self._process_message(line)
            if self.is_stopped():
                break

        self._close()

    def keep_alive(self):
        """Ensure that we stay connected, stopping if the connection breaks."""
        now = time()
        if now - self._last_recv > 120:
            if self._last_ping < self._last_recv:
                log = "Last message was received over 120 seconds ago. Pinging."
                self.logger.debug(log)
                self.ping(self.host)
                self._last_ping = now
            elif now - self._last_ping > 60:
                self.logger.debug("No ping response in 60 seconds. Stopping.")
                self.stop()

    def stop(self, msg=None):
        """Request the IRC connection to close at earliest convenience."""
        if self._is_running:
            self._quit(msg)
            self._is_running = False

    def is_stopped(self):
        """Return whether the IRC connection has been (or is to be) closed."""
        return not self._is_running
<commit_msg>Fix ident docstring.<commit_after># -*- coding: utf-8  -*-
#
# Copyright (C) 2009-2012 by Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import socket
from threading import Lock
from time import sleep, time

from earwigbot.exceptions import BrokenSocketError

__all__ = ["IRCConnection"]

class IRCConnection(object):
    """Interface with an IRC server."""

    def __init__(self, host, port, nick, ident, realname):
        self._host = host
        self._port = port
        self._nick = nick
        self._ident = ident
        self._realname = realname

        self._is_running = False
        self._send_lock = Lock()

        self._last_recv = time()
        self._last_ping = 0

    def __repr__(self):
        """Return the canonical string representation of the IRCConnection."""
        res = "IRCConnection(host={0!r}, port={1!r}, nick={2!r}, ident={3!r}, realname={4!r})"
        return res.format(self.host, self.port, self.nick, self.ident,
                          self.realname)

    def __str__(self):
        """Return a nice string representation of the IRCConnection."""
        res = "<IRCConnection {0}!{1} at {2}:{3}>"
        return res.format(self.nick, self.ident, self.host, self.port)

    def _connect(self):
        """Connect to our IRC server."""
        self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            self._sock.connect((self.host, self.port))
        except socket.error:
            self.logger.exception("Couldn't connect to IRC server; retrying")
            sleep(8)
            self._connect()
        self._send("NICK {0}".format(self.nick))
        self._send("USER {0} {1} * :{2}".format(self.ident, self.host, self.realname))

    def _close(self):
        """Completely close our connection with the IRC server."""
        try:
            self._sock.shutdown(socket.SHUT_RDWR)  # Shut down connection first
        except socket.error:
            pass  # Ignore if the socket is already down
        self._sock.close()

    def _get(self, size=4096):
        """Receive (i.e. get) data from the server."""
        data = self._sock.recv(size)
        if not data:
            # Socket isn't giving us any data, so it is dead or broken:
            raise BrokenSocketError()
        return data

    def _send(self, msg, hidelog=False):
        """Send data to the server."""
        with self._send_lock:
            try:
                self._sock.sendall(msg + "\r\n")
            except socket.error:
                self._is_running = False
            else:
                if not hidelog:
                    self.logger.debug(msg)

    def _quit(self, msg=None):
        """Issue a quit message to the server. Doesn't close the connection."""
        if msg:
            self._send("QUIT :{0}".format(msg))
        else:
            self._send("QUIT")

    def _process_defaults(self, line):
        """Default process hooks for lines received on IRC."""
        self._last_recv = time()
        if line[0] == "PING":  # If we are pinged, pong back
            self.pong(line[1][1:])

    @property
    def host(self):
        """The hostname of the IRC server, like ``"irc.freenode.net"``."""
        return self._host

    @property
    def port(self):
        """The port of the IRC server, like ``6667``."""
        return self._port

    @property
    def nick(self):
        """Our nickname on the server, like ``"EarwigBot"``."""
        return self._nick

    @property
    def ident(self):
        """Our ident on the server, like ``"earwig"``.

        See http://en.wikipedia.org/wiki/Ident.
        """
        return self._ident

    @property
    def realname(self):
        """Our realname (gecos field) on the server."""
        return self._realname

    def say(self, target, msg, hidelog=False):
        """Send a private message to a target on the server."""
        msg = "PRIVMSG {0} :{1}".format(target, msg)
        self._send(msg, hidelog)

    def reply(self, data, msg, hidelog=False):
        """Send a private message as a reply to a user on the server."""
        msg = "\x02{0}\x0f: {1}".format(data.nick, msg)
        self.say(data.chan, msg, hidelog)

    def action(self, target, msg, hidelog=False):
        """Send a private message to a target on the server as an action."""
        msg = "\x01ACTION {0}\x01".format(msg)
        self.say(target, msg, hidelog)

    def notice(self, target, msg, hidelog=False):
        """Send a notice to a target on the server."""
        msg = "NOTICE {0} :{1}".format(target, msg)
        self._send(msg, hidelog)

    def join(self, chan, hidelog=False):
        """Join a channel on the server."""
        msg = "JOIN {0}".format(chan)
        self._send(msg, hidelog)

    def part(self, chan, msg=None, hidelog=False):
        """Part from a channel on the server, optionally using an message."""
        if msg:
            self._send("PART {0} :{1}".format(chan, msg), hidelog)
        else:
            self._send("PART {0}".format(chan), hidelog)

    def mode(self, target, level, msg, hidelog=False):
        """Send a mode message to the server."""
        msg = "MODE {0} {1} {2}".format(target, level, msg)
        self._send(msg, hidelog)

    def ping(self, target, hidelog=False):
        """Ping another entity on the server."""
        msg = "PING {0}".format(target)
        self._send(msg, hidelog)

    def pong(self, target, hidelog=False):
        """Pong another entity on the server."""
        msg = "PONG {0}".format(target)
        self._send(msg, hidelog)

    def loop(self):
        """Main loop for the IRC connection."""
        self._is_running = True
        read_buffer = ""
        while 1:
            try:
                read_buffer += self._get()
            except BrokenSocketError:
                self._is_running = False
                break

            lines = read_buffer.split("\n")
            read_buffer = lines.pop()
            for line in lines:
                line = line.strip().split()
                self._process_defaults(line)
                self._process_message(line)
            if self.is_stopped():
                break

        self._close()

    def keep_alive(self):
        """Ensure that we stay connected, stopping if the connection breaks."""
        now = time()
        if now - self._last_recv > 120:
            if self._last_ping < self._last_recv:
                log = "Last message was received over 120 seconds ago. Pinging."
                self.logger.debug(log)
                self.ping(self.host)
                self._last_ping = now
            elif now - self._last_ping > 60:
                self.logger.debug("No ping response in 60 seconds. Stopping.")
                self.stop()

    def stop(self, msg=None):
        """Request the IRC connection to close at earliest convenience."""
        if self._is_running:
            self._quit(msg)
            self._is_running = False

    def is_stopped(self):
        """Return whether the IRC connection has been (or is to be) closed."""
        return not self._is_running
<|endoftext|>


<commit_before># -*- coding: utf-8  -*-
#
# Copyright (C) 2009-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from earwigbot import exceptions
from earwigbot import wiki

__all__ = ["Task"]

class Task(object):
    """
    **EarwigBot: Base Bot Task**

    This package provides built-in wiki bot "tasks" EarwigBot runs. Additional
    tasks can be installed as plugins in the bot's working directory.

    This class (import with ``from earwigbot.tasks import Task``) can be
    subclassed to create custom bot tasks.

    To run a task, use :py:meth:`bot.tasks.start(name, **kwargs)
    <earwigbot.managers.TaskManager.start>`. ``**kwargs`` get passed to the
    Task's :meth:`run` method.
    """
    name = None
    number = 0

    def __init__(self, bot):
        """Constructor for new tasks.

        This is called once immediately after the task class is loaded by
        the task manager (in :py:meth:`tasks.load()
        <earwigbot.managers._ResourceManager.load>`). Don't override this
        directly; if you do, remember to place ``super(Task, self).__init()``
        first. Use :py:meth:`setup` for typical task-init/setup needs.
        """
        self.bot = bot
        self.config = bot.config
        self.logger = bot.tasks.logger.getChild(self.name)
        self.setup()

    def __repr__(self):
        """Return the canonical string representation of the Task."""
        res = "Task(name={0!r}, number={1!r}, bot={2!r})"
        return res.format(self.name, self.number, self.bot)

    def __str__(self):
        """Return a nice string representation of the Task."""
        res = "<Task {0} ({1}) of {2}>"
        return res.format(self.name, self.number, self.bot)

    def setup(self):
        """Hook called immediately after the task is loaded.

        Does nothing by default; feel free to override.
        """
        pass

    def run(self, **kwargs):
        """Main entry point to run a given task.

        This is called directly by :py:meth:`tasks.start()
        <earwigbot.managers.TaskManager.start>` and is the main way to make a
        task do stuff. *kwargs* will be any keyword arguments passed to
        :py:meth:`~earwigbot.managers.TaskManager.start`, which are entirely
        optional.
        """
        pass

    def unload(self):
        """Hook called immediately before the task is unloaded.

        Does nothing by default; feel free to override.
        """
        pass

    def make_summary(self, comment):
        """Make an edit summary by filling in variables in a config value.

        :py:attr:`config.wiki["summary"] <earwigbot.config.BotConfig.wiki>` is
        used, where ``$2`` is replaced by the main summary body, given by the
        *comment* argument, and ``$1`` is replaced by the task number.

        If the config value is not found, we'll just return *comment* as-is.
        """
        try:
            summary = self.bot.config.wiki["summary"]
        except KeyError:
            return comment
        return summary.replace("$1", str(self.number)).replace("$2", comment)

    def shutoff_enabled(self, site=None):
        """Return whether on-wiki shutoff is enabled for this task.

        We check a certain page for certain content. This is determined by
        our config file: :py:attr:`config.wiki["shutoff"]["page"]
        <earwigbot.config.BotConfig.wiki>` is used as the title, with any
        embedded ``$1`` replaced by our username and ``$2``  replaced by the
        task number; and :py:attr:`config.wiki["shutoff"]["disabled"]
        <earwigbot.config.BotConfig.wiki>` is used as the content.

        If the page has that exact content or the page does not exist, then
        shutoff is "disabled", meaning the bot is supposed to run normally, and
        we return  ``False``. If the page's content is something other than
        what we expect, shutoff is enabled, and we return ``True``.

        If a site is not provided, we'll try to use :py:attr:`self.site <site>`
        if it's set. Otherwise, we'll use our default site.
        """
        if not site:
            if hasattr(self, "site"):
                site = getattr(self, "site")
            else:
                site = self.bot.wiki.get_site()

        try:
            cfg = self.config.wiki["shutoff"]
        except KeyError:
            return False
        title = cfg.get("page", "User:$1/Shutoff/Task $2")
        username = site.get_user().name
        title = title.replace("$1", username).replace("$2", str(self.number))
        page = site.get_page(title)

        try:
            content = page.get()
        except exceptions.PageNotFoundError:
            return False
        if content == cfg.get("disabled", "run"):
            return False

        self.logger.warn("Emergency task shutoff has been enabled!")
        return True
<commit_msg>Allow specifying task number in config.<commit_after># -*- coding: utf-8  -*-
#
# Copyright (C) 2009-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from earwigbot import exceptions
from earwigbot import wiki

__all__ = ["Task"]

class Task(object):
    """
    **EarwigBot: Base Bot Task**

    This package provides built-in wiki bot "tasks" EarwigBot runs. Additional
    tasks can be installed as plugins in the bot's working directory.

    This class (import with ``from earwigbot.tasks import Task``) can be
    subclassed to create custom bot tasks.

    To run a task, use :py:meth:`bot.tasks.start(name, **kwargs)
    <earwigbot.managers.TaskManager.start>`. ``**kwargs`` get passed to the
    Task's :meth:`run` method.
    """
    name = None
    number = 0

    def __init__(self, bot):
        """Constructor for new tasks.

        This is called once immediately after the task class is loaded by
        the task manager (in :py:meth:`tasks.load()
        <earwigbot.managers._ResourceManager.load>`). Don't override this
        directly; if you do, remember to place ``super(Task, self).__init()``
        first. Use :py:meth:`setup` for typical task-init/setup needs.
        """
        self.bot = bot
        self.config = bot.config
        self.logger = bot.tasks.logger.getChild(self.name)

        number = self.config.tasks.get(self.name, {}).get("number")
        if number is not None:
            self.number = number
        self.setup()

    def __repr__(self):
        """Return the canonical string representation of the Task."""
        res = "Task(name={0!r}, number={1!r}, bot={2!r})"
        return res.format(self.name, self.number, self.bot)

    def __str__(self):
        """Return a nice string representation of the Task."""
        res = "<Task {0} ({1}) of {2}>"
        return res.format(self.name, self.number, self.bot)

    def setup(self):
        """Hook called immediately after the task is loaded.

        Does nothing by default; feel free to override.
        """
        pass

    def run(self, **kwargs):
        """Main entry point to run a given task.

        This is called directly by :py:meth:`tasks.start()
        <earwigbot.managers.TaskManager.start>` and is the main way to make a
        task do stuff. *kwargs* will be any keyword arguments passed to
        :py:meth:`~earwigbot.managers.TaskManager.start`, which are entirely
        optional.
        """
        pass

    def unload(self):
        """Hook called immediately before the task is unloaded.

        Does nothing by default; feel free to override.
        """
        pass

    def make_summary(self, comment):
        """Make an edit summary by filling in variables in a config value.

        :py:attr:`config.wiki["summary"] <earwigbot.config.BotConfig.wiki>` is
        used, where ``$2`` is replaced by the main summary body, given by the
        *comment* argument, and ``$1`` is replaced by the task number.

        If the config value is not found, we'll just return *comment* as-is.
        """
        try:
            summary = self.bot.config.wiki["summary"]
        except KeyError:
            return comment
        return summary.replace("$1", str(self.number)).replace("$2", comment)

    def shutoff_enabled(self, site=None):
        """Return whether on-wiki shutoff is enabled for this task.

        We check a certain page for certain content. This is determined by
        our config file: :py:attr:`config.wiki["shutoff"]["page"]
        <earwigbot.config.BotConfig.wiki>` is used as the title, with any
        embedded ``$1`` replaced by our username and ``$2``  replaced by the
        task number; and :py:attr:`config.wiki["shutoff"]["disabled"]
        <earwigbot.config.BotConfig.wiki>` is used as the content.

        If the page has that exact content or the page does not exist, then
        shutoff is "disabled", meaning the bot is supposed to run normally, and
        we return  ``False``. If the page's content is something other than
        what we expect, shutoff is enabled, and we return ``True``.

        If a site is not provided, we'll try to use :py:attr:`self.site <site>`
        if it's set. Otherwise, we'll use our default site.
        """
        if not site:
            if hasattr(self, "site"):
                site = getattr(self, "site")
            else:
                site = self.bot.wiki.get_site()

        try:
            cfg = self.config.wiki["shutoff"]
        except KeyError:
            return False
        title = cfg.get("page", "User:$1/Shutoff/Task $2")
        username = site.get_user().name
        title = title.replace("$1", username).replace("$2", str(self.number))
        page = site.get_page(title)

        try:
            content = page.get()
        except exceptions.PageNotFoundError:
            return False
        if content == cfg.get("disabled", "run"):
            return False

        self.logger.warn("Emergency task shutoff has been enabled!")
        return True
<|endoftext|>