<commit_before># Debian packaging tools: Relationship parsing and evaluation.
#
# Author: Peter Odding <peter@peterodding.com>
# Last Change: May 18, 2014
# URL: https://github.com/xolox/python-deb-pkg-tools

"""
Relationship parsing and evaluation
===================================

This module provides functions to parse and evaluate Debian package
relationship declarations as defined in `chapter 7`_ of the `Debian policy
manual`_. The most important function is :py:func:`parse_depends()` which
returns a :py:class:`RelationshipSet` object. The
:py:func:`RelationshipSet.matches()` function can be used to evaluate
relationship expressions. The relationship parsing is implemented in pure
Python (no external dependencies) but relationship evaluation uses the external
command ``dpkg --compare-versions`` to ensure compatibility with Debian's
package version comparison algorithm.

To give you an impression of how to use this module:

>>> from deb_pkg_tools.deps import parse_depends
>>> dependencies = parse_depends('python (>= 2.6), python (<< 3) | python (>= 3.4)')
>>> dependencies.matches('python', '2.5')
False
>>> dependencies.matches('python', '3.0')
False
>>> dependencies.matches('python', '2.6')
True
>>> dependencies.matches('python', '3.4')
True
>>> print repr(dependencies)
RelationshipSet(
    AlternativeRelationship(
        VersionedRelationship(name='python', operator='<<', version='3'),
        VersionedRelationship(name='python', operator='>=', version='3.4')),
    VersionedRelationship(name='python', operator='>=', version='2.6'))
>>> print unicode(dependencies)
python (<< 3) | python (>= 3.4), python (>= 2.6)

As you can see the :py:func:`repr()` output of the relationship set shows the
object tree and the :py:func:`unicode()` output (:py:func:`str()` in Python
3.x) is the normalized (sorted) dependency line.

.. warning:: The relationship parsing code does not understand the complete
             syntax defined in the Debian policy manual. More specifically
             architecture restrictions are not supported (because I simply
             don't use them).

.. _Debian policy manual: http://www.debian.org/doc/debian-policy/
.. _chapter 7: http://www.debian.org/doc/debian-policy/ch-relationships.html#s-depsyntax
"""

# Standard library modules.
import logging
import re

# Modules included in our package.
from deb_pkg_tools.utils import (dpkg_compare_versions, OrderedObject,
                                 str_compatible, unicode)

# Initialize a logger.
logger = logging.getLogger(__name__)

def parse_depends(relationships):
    """
    Parse a list of package relationships of the form ``python (>= 2.6), python
    (<< 3)``, i.e. a comma separated list of relationship expressions. Uses
    :py:func:`parse_alternatives()` to parse each comma separated expression.
    Raises :py:exc:`ValueError` when parsing fails. Here's an example:

    >>> from deb_pkg_tools.deps import parse_depends
    >>> dependencies = parse_depends('python (>= 2.6), python (<< 3)')
    >>> print repr(dependencies)
    RelationshipSet(VersionedRelationship(name='python', operator='<<', version='3'),
                    VersionedRelationship(name='python', operator='>=', version='2.6'))
    >>> dependencies.matches('python', '2.5')
    False
    >>> dependencies.matches('python', '2.6')
    True
    >>> dependencies.matches('python', '2.7')
    True
    >>> dependencies.matches('python', '3.0')
    False

    :param relationships: A list of package relationships (a list of strings).
    :returns: A :py:class:`RelationshipSet` object.
    """
    tokens = [r for r in relationships.split(',') if r and not r.isspace()]
    return RelationshipSet(*map(parse_alternatives, tokens))

def parse_alternatives(expression):
    """
    Parse an expression containing one or more alternative relationships of the
    form ``python2.6 | python2.7.``, i.e. a list of relationship expressions
    separated by ``|`` tokens. Uses :py:func:`parse_relationship()` to parse
    each ``|`` separated expression. Raises :py:exc:`ValueError` when parsing
    fails. An example:

    >>> from deb_pkg_tools.deps import parse_alternatives
    >>> parse_alternatives('python2.6')
    Relationship(name='python2.6')
    >>> parse_alternatives('python2.6 | python2.7')
    AlternativeRelationship(Relationship(name='python2.6'),
                            Relationship(name='python2.7'))

    :param expression: A relationship expression (a string).
    :returns: A :py:class:`Relationship` object.
    """
    if '|' in expression:
        return AlternativeRelationship(*map(parse_relationship, expression.split('|')))
    else:
        return parse_relationship(expression)

def parse_relationship(expression):
    """
    Parse a relationship expression containing a package name and (optionally)
    a version relation of the form ``python (>= 2.6)``. Raises
    :py:exc:`ValueError` when parsing fails. An example:

    >>> from deb_pkg_tools.deps import parse_relationship
    >>> parse_relationship('python')
    Relationship(name='python')
    >>> parse_relationship('python (<< 3)')
    VersionedRelationship(name='python', operator='<<', version='3')

    :param expression: A relationship expression (a string).
    :returns: A :py:class:`Relationship` object.
    """
    tokens = [t.strip() for t in re.split('[()]', expression) if t and not t.isspace()]
    if len(tokens) == 1:
        # Just a package name (no version information).
        return Relationship(tokens[0])
    elif len(tokens) != 2:
        # Encountered something unexpected!
        msg = "Corrupt package relationship expression: Splitting name from relationship resulted in more than two tokens! (expression: %r, tokens: %r)"
        raise ValueError(msg % (expression, tokens))
    else:
        # Package name followed by relationship to specific version(s) of package.
        name, relationship = tokens
        tokens = [t.strip() for t in re.split('([0-9A-Za-z_~:.-]+)', relationship) if t and not t.isspace()]
        if len(tokens) != 2:
            # Encountered something unexpected!
            msg = "Corrupt package relationship expression: Splitting operator from version resulted in more than two tokens! (expression: %r, tokens: %r)"
            raise ValueError(msg % (relationship, tokens))
        return VersionedRelationship(name, *tokens)

@str_compatible
class Relationship(OrderedObject):

    """
    A simple package relationship referring only to the name of a package.
    Created by :py:func:`parse_relationship()`.
    """

    def __init__(self, name):
        """
        Initialize a simple relationship.

        :param name: The name of a package (a string).
        """
        self.name = name

    def matches(self, name, version=None):
        """
        Check if the relationship matches a given package and version.

        :param name: The name of a package (a string).
        :param version: The version number of a package (a string, optional).
        :returns: ``True`` if the relationship matches, ``None`` otherwise.
        """
        if self.name == name:
            return True

    def __unicode__(self):
        """
        Serialize a :py:class:`Relationship` object to a Debian package
        relationship expression.
        """
        return self.name

    def __repr__(self):
        """
        Serialize a :py:class:`Relationship` object to a Python expression.
        """
        return "%s(%s)" % (self.__class__.__name__, ', '.join([
            'name=%r' % self.name
        ]))

    def _key(self):
        """
        Get the comparison key of this :py:class:`Relationship` object. Used to
        implement the equality and rich comparison operations.
        """
        return (self.name,)

@str_compatible
class VersionedRelationship(Relationship):

    """
    A conditional package relationship that refers to a package and certain
    versions of that package. Created by :py:func:`parse_relationship()`.
    """

    def __init__(self, name, operator, version):
        """
        Initialize a conditional relationship.

        :param name: The name of a package (a string).
        :param operator: A version comparison operator (a string).
        :param version: The version number of a package (a string).
        """
        self.name = name
        self.operator = operator
        self.version = version

    def matches(self, name, version=None):
        """
        Check if the relationship matches a given package and version. Uses the
        external command ``dpkg --compare-versions`` to ensure compatibility
        with Debian's package version comparison algorithm.

        :param name: The name of a package (a string).
        :param version: The version number of a package (a string, optional).
        :returns: ``True`` if the name and version match, ``False`` if only the
                  name matches, ``None`` otherwise.
        """
        if self.name == name:
            if version:
                return dpkg_compare_versions(version, self.operator, self.version)
            else:
                return False

    def __unicode__(self):
        """
        Serialize a :py:class:`VersionedRelationship` object to a Debian package
        relationship expression.
        """
        return u'%s (%s %s)' % (self.name, self.operator, self.version)

    def __repr__(self):
        """
        Serialize a :py:class:`VersionedRelationship` object to a Python expression.
        """
        return "%s(%s)" % (self.__class__.__name__, ', '.join([
            'name=%r' % self.name,
            'operator=%r' % self.operator,
            'version=%r' % self.version,
        ]))

    def _key(self):
        """
        Get the comparison key of this :py:class:`VersionedRelationship`
        object. Used to implement the equality and rich comparison
        operations.
        """
        return (self.name, self.operator, self.version)

@str_compatible
class AlternativeRelationship(Relationship):

    """
    A package relationship that refers to one of several alternative packages.
    Created by :py:func:`parse_alternatives()`.
    """

    def __init__(self, *relationships):
        """
        Initialize an alternatives relationship.

        :param relationships: One or more :py:class:`Relationship` objects.
        """
        self.relationships = sorted(relationships)

    def matches(self, name, version=None):
        """
        Check if the relationship matches a given package and version.

        :param name: The name of a package (a string).
        :param version: The version number of a package (a string, optional).
        :returns: ``True`` if the name and version of an alternative match,
                  ``False`` if the name of an alternative was matched but the
                  version didn't match, ``None`` otherwise.
        """
        matches = None
        for alternative in self.relationships:
            alternative_matches = alternative.matches(name, version)
            if alternative_matches == True:
                return True
            elif alternative_matches == False:
                # Keep looking for a match but return False if we don't find one.
                matches = False
        return matches

    def __unicode__(self):
        """
        Serialize an :py:class:`AlternativeRelationship` object to a Debian package
        relationship expression.
        """
        return u' | '.join(map(unicode, self.relationships))

    def __repr__(self):
        """
        Serialize an :py:class:`AlternativeRelationship` object to a Python expression.
        """
        return "%s(%s)" % (self.__class__.__name__, ', '.join(repr(r) for r in self.relationships))

    def _key(self):
        """
        Get the comparison key of this :py:class:`AlternativeRelationship` object. Used to
        implement the equality and rich comparison operations.
        """
        return sorted(self.relationships)

class RelationshipSet(OrderedObject):

    """
    A set of package relationships. Created by :py:func:`parse_depends()`.
    """

    def __init__(self, *relationships):
        """
        Initialize a set of relationships.

        :param relationships: One or more :py:class:`Relationship` objects.
        """
        self.relationships = sorted(relationships)

    def matches(self, name, version=None):
        """
        Check if the set of relationships matches a given package and version.

        :param name: The name of a package (a string).
        :param version: The version number of a package (a string, optional).
        :returns: ``True`` if all matched relationships evaluate to true,
                  ``False`` if a relationship is matched and evaluates to false,
                  ``None`` otherwise.
        """
        results = [r.matches(name, version) for r in self.relationships]
        matches = [r for r in results if r is not None]
        if matches:
            return all(matches)

    def __unicode__(self):
        """
        Serialize a :py:class:`RelationshipSet` object to a Debian package
        relationship expression.
        """
        return u', '.join(map(unicode, self.relationships))

    def __repr__(self):
        """
        Serialize a :py:class:`RelationshipSet` object to a Python expression.
        """
        return "%s(%s)" % (self.__class__.__name__, ', '.join(repr(r) for r in self.relationships))

    def _key(self):
        """
        Get the comparison key of this :py:class:`RelationshipSet` object. Used
        to implement the equality and rich comparison operations.
        """
        return self.relationships

    def __iter__(self):
        """
        Iterate over the relationships in a relationship set.
        """
        return iter(self.relationships)

# vim: ts=4 sw=4 et
<commit_msg>Bug fix for Python 3.x compatibility (missing `str_compatible' decorator)<commit_after># Debian packaging tools: Relationship parsing and evaluation.
#
# Author: Peter Odding <peter@peterodding.com>
# Last Change: May 18, 2014
# URL: https://github.com/xolox/python-deb-pkg-tools

"""
Relationship parsing and evaluation
===================================

This module provides functions to parse and evaluate Debian package
relationship declarations as defined in `chapter 7`_ of the `Debian policy
manual`_. The most important function is :py:func:`parse_depends()` which
returns a :py:class:`RelationshipSet` object. The
:py:func:`RelationshipSet.matches()` function can be used to evaluate
relationship expressions. The relationship parsing is implemented in pure
Python (no external dependencies) but relationship evaluation uses the external
command ``dpkg --compare-versions`` to ensure compatibility with Debian's
package version comparison algorithm.

To give you an impression of how to use this module:

>>> from deb_pkg_tools.deps import parse_depends
>>> dependencies = parse_depends('python (>= 2.6), python (<< 3) | python (>= 3.4)')
>>> dependencies.matches('python', '2.5')
False
>>> dependencies.matches('python', '3.0')
False
>>> dependencies.matches('python', '2.6')
True
>>> dependencies.matches('python', '3.4')
True
>>> print repr(dependencies)
RelationshipSet(
    AlternativeRelationship(
        VersionedRelationship(name='python', operator='<<', version='3'),
        VersionedRelationship(name='python', operator='>=', version='3.4')),
    VersionedRelationship(name='python', operator='>=', version='2.6'))
>>> print unicode(dependencies)
python (<< 3) | python (>= 3.4), python (>= 2.6)

As you can see the :py:func:`repr()` output of the relationship set shows the
object tree and the :py:func:`unicode()` output (:py:func:`str()` in Python
3.x) is the normalized (sorted) dependency line.

.. warning:: The relationship parsing code does not understand the complete
             syntax defined in the Debian policy manual. More specifically
             architecture restrictions are not supported (because I simply
             don't use them).

.. _Debian policy manual: http://www.debian.org/doc/debian-policy/
.. _chapter 7: http://www.debian.org/doc/debian-policy/ch-relationships.html#s-depsyntax
"""

# Standard library modules.
import logging
import re

# Modules included in our package.
from deb_pkg_tools.utils import (dpkg_compare_versions, OrderedObject,
                                 str_compatible, unicode)

# Initialize a logger.
logger = logging.getLogger(__name__)

def parse_depends(relationships):
    """
    Parse a list of package relationships of the form ``python (>= 2.6), python
    (<< 3)``, i.e. a comma separated list of relationship expressions. Uses
    :py:func:`parse_alternatives()` to parse each comma separated expression.
    Raises :py:exc:`ValueError` when parsing fails. Here's an example:

    >>> from deb_pkg_tools.deps import parse_depends
    >>> dependencies = parse_depends('python (>= 2.6), python (<< 3)')
    >>> print repr(dependencies)
    RelationshipSet(VersionedRelationship(name='python', operator='<<', version='3'),
                    VersionedRelationship(name='python', operator='>=', version='2.6'))
    >>> dependencies.matches('python', '2.5')
    False
    >>> dependencies.matches('python', '2.6')
    True
    >>> dependencies.matches('python', '2.7')
    True
    >>> dependencies.matches('python', '3.0')
    False

    :param relationships: A list of package relationships (a list of strings).
    :returns: A :py:class:`RelationshipSet` object.
    """
    tokens = [r for r in relationships.split(',') if r and not r.isspace()]
    return RelationshipSet(*map(parse_alternatives, tokens))

def parse_alternatives(expression):
    """
    Parse an expression containing one or more alternative relationships of the
    form ``python2.6 | python2.7.``, i.e. a list of relationship expressions
    separated by ``|`` tokens. Uses :py:func:`parse_relationship()` to parse
    each ``|`` separated expression. Raises :py:exc:`ValueError` when parsing
    fails. An example:

    >>> from deb_pkg_tools.deps import parse_alternatives
    >>> parse_alternatives('python2.6')
    Relationship(name='python2.6')
    >>> parse_alternatives('python2.6 | python2.7')
    AlternativeRelationship(Relationship(name='python2.6'),
                            Relationship(name='python2.7'))

    :param expression: A relationship expression (a string).
    :returns: A :py:class:`Relationship` object.
    """
    if '|' in expression:
        return AlternativeRelationship(*map(parse_relationship, expression.split('|')))
    else:
        return parse_relationship(expression)

def parse_relationship(expression):
    """
    Parse a relationship expression containing a package name and (optionally)
    a version relation of the form ``python (>= 2.6)``. Raises
    :py:exc:`ValueError` when parsing fails. An example:

    >>> from deb_pkg_tools.deps import parse_relationship
    >>> parse_relationship('python')
    Relationship(name='python')
    >>> parse_relationship('python (<< 3)')
    VersionedRelationship(name='python', operator='<<', version='3')

    :param expression: A relationship expression (a string).
    :returns: A :py:class:`Relationship` object.
    """
    tokens = [t.strip() for t in re.split('[()]', expression) if t and not t.isspace()]
    if len(tokens) == 1:
        # Just a package name (no version information).
        return Relationship(tokens[0])
    elif len(tokens) != 2:
        # Encountered something unexpected!
        msg = "Corrupt package relationship expression: Splitting name from relationship resulted in more than two tokens! (expression: %r, tokens: %r)"
        raise ValueError(msg % (expression, tokens))
    else:
        # Package name followed by relationship to specific version(s) of package.
        name, relationship = tokens
        tokens = [t.strip() for t in re.split('([0-9A-Za-z_~:.-]+)', relationship) if t and not t.isspace()]
        if len(tokens) != 2:
            # Encountered something unexpected!
            msg = "Corrupt package relationship expression: Splitting operator from version resulted in more than two tokens! (expression: %r, tokens: %r)"
            raise ValueError(msg % (relationship, tokens))
        return VersionedRelationship(name, *tokens)

@str_compatible
class Relationship(OrderedObject):

    """
    A simple package relationship referring only to the name of a package.
    Created by :py:func:`parse_relationship()`.
    """

    def __init__(self, name):
        """
        Initialize a simple relationship.

        :param name: The name of a package (a string).
        """
        self.name = name

    def matches(self, name, version=None):
        """
        Check if the relationship matches a given package and version.

        :param name: The name of a package (a string).
        :param version: The version number of a package (a string, optional).
        :returns: ``True`` if the relationship matches, ``None`` otherwise.
        """
        if self.name == name:
            return True

    def __unicode__(self):
        """
        Serialize a :py:class:`Relationship` object to a Debian package
        relationship expression.
        """
        return self.name

    def __repr__(self):
        """
        Serialize a :py:class:`Relationship` object to a Python expression.
        """
        return "%s(%s)" % (self.__class__.__name__, ', '.join([
            'name=%r' % self.name
        ]))

    def _key(self):
        """
        Get the comparison key of this :py:class:`Relationship` object. Used to
        implement the equality and rich comparison operations.
        """
        return (self.name,)

@str_compatible
class VersionedRelationship(Relationship):

    """
    A conditional package relationship that refers to a package and certain
    versions of that package. Created by :py:func:`parse_relationship()`.
    """

    def __init__(self, name, operator, version):
        """
        Initialize a conditional relationship.

        :param name: The name of a package (a string).
        :param operator: A version comparison operator (a string).
        :param version: The version number of a package (a string).
        """
        self.name = name
        self.operator = operator
        self.version = version

    def matches(self, name, version=None):
        """
        Check if the relationship matches a given package and version. Uses the
        external command ``dpkg --compare-versions`` to ensure compatibility
        with Debian's package version comparison algorithm.

        :param name: The name of a package (a string).
        :param version: The version number of a package (a string, optional).
        :returns: ``True`` if the name and version match, ``False`` if only the
                  name matches, ``None`` otherwise.
        """
        if self.name == name:
            if version:
                return dpkg_compare_versions(version, self.operator, self.version)
            else:
                return False

    def __unicode__(self):
        """
        Serialize a :py:class:`VersionedRelationship` object to a Debian package
        relationship expression.
        """
        return u'%s (%s %s)' % (self.name, self.operator, self.version)

    def __repr__(self):
        """
        Serialize a :py:class:`VersionedRelationship` object to a Python expression.
        """
        return "%s(%s)" % (self.__class__.__name__, ', '.join([
            'name=%r' % self.name,
            'operator=%r' % self.operator,
            'version=%r' % self.version,
        ]))

    def _key(self):
        """
        Get the comparison key of this :py:class:`VersionedRelationship`
        object. Used to implement the equality and rich comparison
        operations.
        """
        return (self.name, self.operator, self.version)

@str_compatible
class AlternativeRelationship(Relationship):

    """
    A package relationship that refers to one of several alternative packages.
    Created by :py:func:`parse_alternatives()`.
    """

    def __init__(self, *relationships):
        """
        Initialize an alternatives relationship.

        :param relationships: One or more :py:class:`Relationship` objects.
        """
        self.relationships = sorted(relationships)

    def matches(self, name, version=None):
        """
        Check if the relationship matches a given package and version.

        :param name: The name of a package (a string).
        :param version: The version number of a package (a string, optional).
        :returns: ``True`` if the name and version of an alternative match,
                  ``False`` if the name of an alternative was matched but the
                  version didn't match, ``None`` otherwise.
        """
        matches = None
        for alternative in self.relationships:
            alternative_matches = alternative.matches(name, version)
            if alternative_matches == True:
                return True
            elif alternative_matches == False:
                # Keep looking for a match but return False if we don't find one.
                matches = False
        return matches

    def __unicode__(self):
        """
        Serialize an :py:class:`AlternativeRelationship` object to a Debian package
        relationship expression.
        """
        return u' | '.join(map(unicode, self.relationships))

    def __repr__(self):
        """
        Serialize an :py:class:`AlternativeRelationship` object to a Python expression.
        """
        return "%s(%s)" % (self.__class__.__name__, ', '.join(repr(r) for r in self.relationships))

    def _key(self):
        """
        Get the comparison key of this :py:class:`AlternativeRelationship` object. Used to
        implement the equality and rich comparison operations.
        """
        return sorted(self.relationships)

@str_compatible
class RelationshipSet(OrderedObject):

    """
    A set of package relationships. Created by :py:func:`parse_depends()`.
    """

    def __init__(self, *relationships):
        """
        Initialize a set of relationships.

        :param relationships: One or more :py:class:`Relationship` objects.
        """
        self.relationships = sorted(relationships)

    def matches(self, name, version=None):
        """
        Check if the set of relationships matches a given package and version.

        :param name: The name of a package (a string).
        :param version: The version number of a package (a string, optional).
        :returns: ``True`` if all matched relationships evaluate to true,
                  ``False`` if a relationship is matched and evaluates to false,
                  ``None`` otherwise.
        """
        results = [r.matches(name, version) for r in self.relationships]
        matches = [r for r in results if r is not None]
        if matches:
            return all(matches)

    def __unicode__(self):
        """
        Serialize a :py:class:`RelationshipSet` object to a Debian package
        relationship expression.
        """
        return u', '.join(map(unicode, self.relationships))

    def __repr__(self):
        """
        Serialize a :py:class:`RelationshipSet` object to a Python expression.
        """
        return "%s(%s)" % (self.__class__.__name__, ', '.join(repr(r) for r in self.relationships))

    def _key(self):
        """
        Get the comparison key of this :py:class:`RelationshipSet` object. Used
        to implement the equality and rich comparison operations.
        """
        return self.relationships

    def __iter__(self):
        """
        Iterate over the relationships in a relationship set.
        """
        return iter(self.relationships)

# vim: ts=4 sw=4 et
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""IPython Test Suite Runner.

This module provides a main entry point to a user script to test IPython
itself from the command line. There are two ways of running this script:

1. With the syntax `iptest all`.  This runs our entire test suite by
   calling this script (with different arguments) or trial recursively.  This
   causes modules and package to be tested in different processes, using nose
   or trial where appropriate.
2. With the regular nose syntax, like `iptest -vvs IPython`.  In this form
   the script simply calls nose, but with special command line flags and
   plugins loaded.

For now, this script requires that both nose and twisted are installed.  This
will change in the future.
"""

#-----------------------------------------------------------------------------
# Module imports
#-----------------------------------------------------------------------------

import os
import os.path as path
import sys
import subprocess
import time
import warnings

import nose.plugins.builtin
from nose.core import TestProgram

from IPython.testing.plugin.ipdoctest import IPythonDoctest

#-----------------------------------------------------------------------------
# Globals and constants
#-----------------------------------------------------------------------------

# For the IPythonDoctest plugin, we need to exclude certain patterns that cause
# testing problems.  We should strive to minimize the number of skipped
# modules, since this means untested code.  As the testing machinery
# solidifies, this list should eventually become empty.
EXCLUDE = ['IPython/external/',
           'IPython/platutils_win32',
           'IPython/frontend/cocoa',
           'IPython/frontend/process/winprocess.py',
           'IPython_doctest_plugin',
           'IPython/Gnuplot',
           'IPython/Extensions/ipy_',
           'IPython/Extensions/clearcmd',
           'IPython/Extensions/PhysicalQIn',
           'IPython/Extensions/scitedirector',
           'IPython/Extensions/numeric_formats',
           'IPython/testing/attic',
           ]

#-----------------------------------------------------------------------------
# Functions and classes
#-----------------------------------------------------------------------------

def run_iptest():
    """Run the IPython test suite using nose.

    This function is called when this script is **not** called with the form
    `iptest all`.  It simply calls nose with appropriate command line flags
    and accepts all of the standard nose arguments.
    """

    warnings.filterwarnings('ignore',
        'This will be removed soon.  Use IPython.testing.util instead')

    argv = sys.argv + [
                        # Loading ipdoctest causes problems with Twisted.
                        # I am removing this as a temporary fix to get the
                        # test suite back into working shape.  Our nose
                        # plugin needs to be gone through with a fine
                        # toothed comb to find what is causing the problem.
                        '--with-ipdoctest',
                        '--ipdoctest-tests','--ipdoctest-extension=txt',
                        '--detailed-errors',

                        # We add --exe because of setuptools' imbecility (it
                        # blindly does chmod +x on ALL files).  Nose does the
                        # right thing and it tries to avoid executables,
                        # setuptools unfortunately forces our hand here.  This
                        # has been discussed on the distutils list and the
                        # setuptools devs refuse to fix this problem!
                        '--exe',
                        ]

    # Detect if any tests were required by explicitly calling an IPython
    # submodule or giving a specific path
    has_tests = False
    for arg in sys.argv:
        if 'IPython' in arg or arg.endswith('.py') or \
           (':' in arg and  '.py' in arg):
            has_tests = True
            break

    # If nothing was specifically requested, test full IPython
    if not has_tests:
        argv.append('IPython')

    # Construct list of plugins, omitting the existing doctest plugin, which
    # ours replaces (and extends).
    plugins = [IPythonDoctest(EXCLUDE)]
    for p in nose.plugins.builtin.plugins:
        plug = p()
        if plug.name == 'doctest':
            continue

        #print '*** adding plugin:',plug.name  # dbg
        plugins.append(plug)

    TestProgram(argv=argv,plugins=plugins)

class IPTester(object):
    """Call that calls iptest or trial in a subprocess.
    """
    def __init__(self,runner='iptest',params=None):
        """ """
        if runner == 'iptest':
            self.runner = ['iptest','-v']
        else:
            self.runner = ['trial']
        if params is None:
            params = []
        if isinstance(params,str):
            params = [params]
        self.params = params

        # Assemble call
        self.call_args = self.runner+self.params

    def run(self):
        """Run the stored commands"""
        return subprocess.call(self.call_args)

def make_runners():
    """Define the modules and packages that need to be tested.
    """

    # This omits additional top-level modules that should not be doctested.
    # XXX: Shell.py is also ommited because of a bug in the skip_doctest
    # decorator.  See ticket https://bugs.launchpad.net/bugs/366209
    top_mod = \
      ['background_jobs.py', 'ColorANSI.py', 'completer.py', 'ConfigLoader.py',
       'CrashHandler.py', 'Debugger.py', 'deep_reload.py', 'demo.py',
       'DPyGetOpt.py', 'dtutils.py', 'excolors.py', 'FakeModule.py',
       'generics.py', 'genutils.py', 'history.py', 'hooks.py', 'ipapi.py',
       'iplib.py', 'ipmaker.py', 'ipstruct.py', 'irunner.py', 'Itpl.py',
       'Logger.py', 'macro.py', 'Magic.py', 'OInspect.py',
       'OutputTrap.py', 'platutils.py', 'prefilter.py', 'Prompts.py',
       'PyColorize.py', 'Release.py', 'rlineimpl.py', 'shadowns.py',
       'shellglobals.py', 'strdispatch.py', 'twshell.py',
       'ultraTB.py', 'upgrade_dir.py', 'usage.py', 'wildcard.py',
       # See note above for why this is skipped
       # 'Shell.py',
       'winconsole.py']

    if os.name == 'posix':
        top_mod.append('platutils_posix.py')
    elif sys.platform == 'win32':
        top_mod.append('platutils_win32.py')
    else:
        top_mod.append('platutils_dummy.py')

    # These are tested by nose, so skip IPython.kernel
    top_pack = ['config','Extensions','frontend','gui',
                'testing','tests','tools','UserConfig']

    modules  = ['IPython.%s' % m[:-3] for m in top_mod ]
    packages = ['IPython.%s' % m for m in top_pack ]

    # Make runners
    runners = dict(zip(top_pack, [IPTester(params=v) for v in packages]))

    # Test IPython.kernel using trial if twisted is installed
    try:
        import zope.interface
        import twisted
        import foolscap
    except ImportError:
        pass
    else:
        runners['trial'] = IPTester('trial',['IPython'])

    runners['modules'] = IPTester(params=modules)

    return runners

def run_iptestall():
    """Run the entire IPython test suite by calling nose and trial.

    This function constructs :class:`IPTester` instances for all IPython
    modules and package and then runs each of them.  This causes the modules
    and packages of IPython to be tested each in their own subprocess using
    nose or twisted.trial appropriately.
    """
    runners = make_runners()
    # Run all test runners, tracking execution time
    failed = {}
    t_start = time.time()
    for name,runner in runners.iteritems():
        print '*'*77
        print 'IPython test set:',name
        res = runner.run()
        if res:
            failed[name] = res
    t_end = time.time()
    t_tests = t_end - t_start
    nrunners = len(runners)
    nfail = len(failed)
    # summarize results
    print
    print '*'*77
    print 'Ran %s test sets in %.3fs' % (nrunners, t_tests)
    print
    if not failed:
        print 'OK'
    else:
        # If anything went wrong, point out what command to rerun manually to
        # see the actual errors and individual summary
        print 'ERROR - %s out of %s test sets failed.' % (nfail, nrunners)
        for name in failed:
            failed_runner = runners[name]
            print '-'*40
            print 'Runner failed:',name
            print 'You may wish to rerun this one individually, with:'
            print ' '.join(failed_runner.call_args)
            print

def main():
    if sys.argv[1] == 'all':
        run_iptestall()
    else:
        run_iptest()

if __name__ == '__main__':
    main()<commit_msg>Making the doctest exclude paths os independent.<commit_after># -*- coding: utf-8 -*-
"""IPython Test Suite Runner.

This module provides a main entry point to a user script to test IPython
itself from the command line. There are two ways of running this script:

1. With the syntax `iptest all`.  This runs our entire test suite by
   calling this script (with different arguments) or trial recursively.  This
   causes modules and package to be tested in different processes, using nose
   or trial where appropriate.
2. With the regular nose syntax, like `iptest -vvs IPython`.  In this form
   the script simply calls nose, but with special command line flags and
   plugins loaded.

For now, this script requires that both nose and twisted are installed.  This
will change in the future.
"""

#-----------------------------------------------------------------------------
# Module imports
#-----------------------------------------------------------------------------

import os
import os.path as path
import sys
import subprocess
import time
import warnings

import nose.plugins.builtin
from nose.core import TestProgram

from IPython.testing.plugin.ipdoctest import IPythonDoctest

pjoin = path.join

#-----------------------------------------------------------------------------
# Globals and constants
#-----------------------------------------------------------------------------

# For the IPythonDoctest plugin, we need to exclude certain patterns that cause
# testing problems.  We should strive to minimize the number of skipped
# modules, since this means untested code.  As the testing machinery
# solidifies, this list should eventually become empty.
EXCLUDE = [pjoin('IPython', 'external'),
           pjoin('IPython', 'platutils_win32'),
           pjoin('IPython', 'frontend', 'cocoa'),
           pjoin('IPython', 'frontend', 'process', 'winprocess.py'),
           pjoin('IPython_doctest_plugin'),
           pjoin('IPython', 'Gnuplot'),
           pjoin('IPython', 'Extensions', 'ipy_'),
           pjoin('IPython', 'Extensions', 'clearcmd'),
           pjoin('IPython', 'Extensions', 'PhysicalQIn'),
           pjoin('IPython', 'Extensions', 'scitedirector'),
           pjoin('IPython', 'Extensions', 'numeric_formats'),
           pjoin('IPython', 'testing', 'attic'),
           ]

#-----------------------------------------------------------------------------
# Functions and classes
#-----------------------------------------------------------------------------

def run_iptest():
    """Run the IPython test suite using nose.

    This function is called when this script is **not** called with the form
    `iptest all`.  It simply calls nose with appropriate command line flags
    and accepts all of the standard nose arguments.
    """

    warnings.filterwarnings('ignore',
        'This will be removed soon.  Use IPython.testing.util instead')

    argv = sys.argv + [
                        # Loading ipdoctest causes problems with Twisted.
                        # I am removing this as a temporary fix to get the
                        # test suite back into working shape.  Our nose
                        # plugin needs to be gone through with a fine
                        # toothed comb to find what is causing the problem.
                        '--with-ipdoctest',
                        '--ipdoctest-tests','--ipdoctest-extension=txt',
                        '--detailed-errors',

                        # We add --exe because of setuptools' imbecility (it
                        # blindly does chmod +x on ALL files).  Nose does the
                        # right thing and it tries to avoid executables,
                        # setuptools unfortunately forces our hand here.  This
                        # has been discussed on the distutils list and the
                        # setuptools devs refuse to fix this problem!
                        '--exe',
                        ]

    # Detect if any tests were required by explicitly calling an IPython
    # submodule or giving a specific path
    has_tests = False
    for arg in sys.argv:
        if 'IPython' in arg or arg.endswith('.py') or \
           (':' in arg and  '.py' in arg):
            has_tests = True
            break

    # If nothing was specifically requested, test full IPython
    if not has_tests:
        argv.append('IPython')

    # Construct list of plugins, omitting the existing doctest plugin, which
    # ours replaces (and extends).
    plugins = [IPythonDoctest(EXCLUDE)]
    for p in nose.plugins.builtin.plugins:
        plug = p()
        if plug.name == 'doctest':
            continue

        #print '*** adding plugin:',plug.name  # dbg
        plugins.append(plug)

    TestProgram(argv=argv,plugins=plugins)

class IPTester(object):
    """Call that calls iptest or trial in a subprocess.
    """
    def __init__(self,runner='iptest',params=None):
        """ """
        if runner == 'iptest':
            self.runner = ['iptest','-v']
        else:
            self.runner = ['trial']
        if params is None:
            params = []
        if isinstance(params,str):
            params = [params]
        self.params = params

        # Assemble call
        self.call_args = self.runner+self.params

    def run(self):
        """Run the stored commands"""
        return subprocess.call(self.call_args)

def make_runners():
    """Define the modules and packages that need to be tested.
    """

    # This omits additional top-level modules that should not be doctested.
    # XXX: Shell.py is also ommited because of a bug in the skip_doctest
    # decorator.  See ticket https://bugs.launchpad.net/bugs/366209
    top_mod = \
      ['background_jobs.py', 'ColorANSI.py', 'completer.py', 'ConfigLoader.py',
       'CrashHandler.py', 'Debugger.py', 'deep_reload.py', 'demo.py',
       'DPyGetOpt.py', 'dtutils.py', 'excolors.py', 'FakeModule.py',
       'generics.py', 'genutils.py', 'history.py', 'hooks.py', 'ipapi.py',
       'iplib.py', 'ipmaker.py', 'ipstruct.py', 'irunner.py', 'Itpl.py',
       'Logger.py', 'macro.py', 'Magic.py', 'OInspect.py',
       'OutputTrap.py', 'platutils.py', 'prefilter.py', 'Prompts.py',
       'PyColorize.py', 'Release.py', 'rlineimpl.py', 'shadowns.py',
       'shellglobals.py', 'strdispatch.py', 'twshell.py',
       'ultraTB.py', 'upgrade_dir.py', 'usage.py', 'wildcard.py',
       # See note above for why this is skipped
       # 'Shell.py',
       'winconsole.py']

    if os.name == 'posix':
        top_mod.append('platutils_posix.py')
    elif sys.platform == 'win32':
        top_mod.append('platutils_win32.py')
    else:
        top_mod.append('platutils_dummy.py')

    # These are tested by nose, so skip IPython.kernel
    top_pack = ['config','Extensions','frontend','gui',
                'testing','tests','tools','UserConfig']

    modules  = ['IPython.%s' % m[:-3] for m in top_mod ]
    packages = ['IPython.%s' % m for m in top_pack ]

    # Make runners
    runners = dict(zip(top_pack, [IPTester(params=v) for v in packages]))

    # Test IPython.kernel using trial if twisted is installed
    try:
        import zope.interface
        import twisted
        import foolscap
    except ImportError:
        pass
    else:
        runners['trial'] = IPTester('trial',['IPython'])

    runners['modules'] = IPTester(params=modules)

    return runners

def run_iptestall():
    """Run the entire IPython test suite by calling nose and trial.

    This function constructs :class:`IPTester` instances for all IPython
    modules and package and then runs each of them.  This causes the modules
    and packages of IPython to be tested each in their own subprocess using
    nose or twisted.trial appropriately.
    """
    runners = make_runners()
    # Run all test runners, tracking execution time
    failed = {}
    t_start = time.time()
    for name,runner in runners.iteritems():
        print '*'*77
        print 'IPython test set:',name
        res = runner.run()
        if res:
            failed[name] = res
    t_end = time.time()
    t_tests = t_end - t_start
    nrunners = len(runners)
    nfail = len(failed)
    # summarize results
    print
    print '*'*77
    print 'Ran %s test sets in %.3fs' % (nrunners, t_tests)
    print
    if not failed:
        print 'OK'
    else:
        # If anything went wrong, point out what command to rerun manually to
        # see the actual errors and individual summary
        print 'ERROR - %s out of %s test sets failed.' % (nfail, nrunners)
        for name in failed:
            failed_runner = runners[name]
            print '-'*40
            print 'Runner failed:',name
            print 'You may wish to rerun this one individually, with:'
            print ' '.join(failed_runner.call_args)
            print

def main():
    if sys.argv[1] == 'all':
        run_iptestall()
    else:
        run_iptest()

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib import admin

import plugin.models as models
import plugin.registry as pl_registry

def plugin_update(queryset, new_status: bool):
    """
    General function for bulk changing plugins
    """

    apps_changed = False

    # Run through all plugins in the queryset as the save method needs to be overridden
    for plugin in queryset:
        if plugin.active is not new_status:
            plugin.active = new_status
            plugin.save(no_reload=True)
            apps_changed = True

    # Reload plugins if they changed
    if apps_changed:
        pl_registry.registry.reload_plugins()

@admin.action(description='Activate plugin(s)')
def plugin_activate(modeladmin, request, queryset):
    """
    Activate a set of plugins
    """
    plugin_update(queryset, True)

@admin.action(description='Deactivate plugin(s)')
def plugin_deactivate(modeladmin, request, queryset):
    """
    Deactivate a set of plugins
    """

    plugin_update(queryset, False)

class PluginSettingInline(admin.TabularInline):
    """
    Inline admin class for PluginSetting
    """

    model = models.PluginSetting

    read_only_fields = [
        'key',
    ]

    def has_add_permission(self, request, obj):
        return False

class PluginConfigAdmin(admin.ModelAdmin):
    """
    Custom admin with restricted id fields
    """

    readonly_fields = ["key", "name", ]
    list_display = ['name', 'key', '__str__', 'active', ]
    list_filter = ['active']
    actions = [plugin_activate, plugin_deactivate, ]
    inlines = [PluginSettingInline, ]

admin.site.register(models.PluginConfig, PluginConfigAdmin)
<commit_msg>fix reference<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib import admin

import plugin.models as models
import plugin.registry as pl_registry

def plugin_update(queryset, new_status: bool):
    """
    General function for bulk changing plugins
    """

    apps_changed = False

    # Run through all plugins in the queryset as the save method needs to be overridden
    for plugin in queryset:
        if plugin.active is not new_status:
            plugin.active = new_status
            plugin.save(no_reload=True)
            apps_changed = True

    # Reload plugins if they changed
    if apps_changed:
        pl_registry.reload_plugins()

@admin.action(description='Activate plugin(s)')
def plugin_activate(modeladmin, request, queryset):
    """
    Activate a set of plugins
    """
    plugin_update(queryset, True)

@admin.action(description='Deactivate plugin(s)')
def plugin_deactivate(modeladmin, request, queryset):
    """
    Deactivate a set of plugins
    """

    plugin_update(queryset, False)

class PluginSettingInline(admin.TabularInline):
    """
    Inline admin class for PluginSetting
    """

    model = models.PluginSetting

    read_only_fields = [
        'key',
    ]

    def has_add_permission(self, request, obj):
        return False

class PluginConfigAdmin(admin.ModelAdmin):
    """
    Custom admin with restricted id fields
    """

    readonly_fields = ["key", "name", ]
    list_display = ['name', 'key', '__str__', 'active', ]
    list_filter = ['active']
    actions = [plugin_activate, plugin_deactivate, ]
    inlines = [PluginSettingInline, ]

admin.site.register(models.PluginConfig, PluginConfigAdmin)
<|endoftext|>


<commit_before>import asyncio
import io
import json
import backoff

from typing import Literal, List
from itertools import cycle
from urllib.parse import quote

from discord import File
from discord.ext import commands
from aiohttp import ClientResponseError, ClientResponse

from bot import QTBot
from utils.custom_context import CustomContext
from utils.images import stitch_images

class DiffusionError(Exception):
    pass

class Diffusion(commands.Cog):
    INPUT = {
        "version": "a9758cbfbd5f3c2094457d996681af52552901775aa2d6dd0b17fd15df959bef",
        "input": {
            "width": 256,
            "height": 256,
            "num_outputs": 4,
        },
    }
    URL = "https://api.replicate.com/v1/predictions"
    HEADERS = {"Content-Type": "application/json"}

    def __init__(self, bot: QTBot):
        with open("data/apikeys.json") as f:
            self.api_keys = cycle(json.load(f)["stable_diffusion"])

        self.HEADERS["Authorization"] = f"Token {next(self.api_keys)}"
        self.bot = bot

    @backoff.on_exception(backoff.expo, ClientResponseError, max_tries=3, giveup=lambda x: x.status == 402)
    async def req(
        self,
        verb: Literal["GET", "POST"],
        url: str = "",
        params: dict = {},
        headers: dict = {},
        data: dict = None,
    ) -> ClientResponse:
        attempt_count = 0
        while True:
            if attempt_count > 4:
                break
            attempt_count += 1
            resp = await self.bot.aio_session.request(
                verb, f"{self.URL}{url}", params=params, headers={**headers, **self.HEADERS}, json=data
            )
            if resp.status == 402:
                self.HEADERS["Authorization"] = f"Token {next(self.api_keys)}"
                continue

            break

        resp.raise_for_status()

        return resp

    async def start_job(self, prompt: str) -> str:
        payload = {**self.INPUT, "input": {**self.INPUT["input"], "prompt": prompt}}
        resp = await self.req("POST", data=payload)
        resp = await resp.json()
        if resp["error"]:
            raise DiffusionError(resp["error"])

        return resp["id"]

    async def check_progress(self, id: str) -> List[str]:
        total_checks = 0
        while True:
            resp = await self.req("GET", f"/{id}")
            resp = await resp.json()
            if total_checks >= 15:
                raise DiffusionError("Couldn't get a result after 30 seconds. Aborting.")
            if resp["error"]:
                raise DiffusionError(resp["error"])
            if resp["completed_at"]:
                return resp["output"]

            total_checks += 1
            await asyncio.sleep(2)

    @commands.command(aliases=["diffuse", "sd"])
    async def diffusion(self, ctx: CustomContext, *, prompt: str) -> None:
        async with ctx.typing():
            try:
                job_id = await self.start_job(prompt)
            except DiffusionError as e:
                return await ctx.error("API Error", f"{ctx.author.mention} {e}")
            except ClientResponseError as e:
                return await ctx.error(
                    "API Error", f"{ctx.author.mention} Received status code `{e.status}`\n{e.message}"
                )

            try:
                images = await self.check_progress(job_id)
            except DiffusionError as e:
                return await ctx.error("API Error", str(e))
            except ClientResponseError as e:
                return await ctx.error(
                    "API Error", f"{ctx.author.mention} Received status code `{e.status}`\n{e.message}"
                )

            images_b = [io.BytesIO(await (await self.bot.aio_session.get(url)).read()) for url in images]
            file = File(stitch_images(images_b, rows_cols=2), filename=f"{quote(prompt)}.png")

        return await ctx.send(f"{ctx.author.mention}: {prompt}", file=file)

def setup(bot):
    bot.add_cog(Diffusion(bot))
<commit_msg>Reduce image #<commit_after>import asyncio
import io
import json
import backoff

from typing import Literal, List
from itertools import cycle
from urllib.parse import quote

from discord import File
from discord.ext import commands
from aiohttp import ClientResponseError, ClientResponse

from bot import QTBot
from utils.custom_context import CustomContext
from utils.images import stitch_images

class DiffusionError(Exception):
    pass

class Diffusion(commands.Cog):
    INPUT = {
        "version": "a9758cbfbd5f3c2094457d996681af52552901775aa2d6dd0b17fd15df959bef",
        "input": {
            "width": 256,
            "height": 256,
            "num_outputs": 2,
        },
    }
    URL = "https://api.replicate.com/v1/predictions"
    HEADERS = {"Content-Type": "application/json"}

    def __init__(self, bot: QTBot):
        with open("data/apikeys.json") as f:
            self.api_keys = cycle(json.load(f)["stable_diffusion"])

        self.HEADERS["Authorization"] = f"Token {next(self.api_keys)}"
        self.bot = bot

    @backoff.on_exception(backoff.expo, ClientResponseError, max_tries=3, giveup=lambda x: x.status == 402)
    async def req(
        self,
        verb: Literal["GET", "POST"],
        url: str = "",
        params: dict = {},
        headers: dict = {},
        data: dict = None,
    ) -> ClientResponse:
        attempt_count = 0
        while True:
            if attempt_count > 4:
                break
            attempt_count += 1
            resp = await self.bot.aio_session.request(
                verb, f"{self.URL}{url}", params=params, headers={**headers, **self.HEADERS}, json=data
            )
            if resp.status == 402:
                self.HEADERS["Authorization"] = f"Token {next(self.api_keys)}"
                continue

            break

        resp.raise_for_status()

        return resp

    async def start_job(self, prompt: str) -> str:
        payload = {**self.INPUT, "input": {**self.INPUT["input"], "prompt": prompt}}
        resp = await self.req("POST", data=payload)
        resp = await resp.json()
        if resp["error"]:
            raise DiffusionError(resp["error"])

        return resp["id"]

    async def check_progress(self, id: str) -> List[str]:
        total_checks = 0
        while True:
            resp = await self.req("GET", f"/{id}")
            resp = await resp.json()
            if total_checks >= 15:
                raise DiffusionError("Couldn't get a result after 30 seconds. Aborting.")
            if resp["error"]:
                raise DiffusionError(resp["error"])
            if resp["completed_at"]:
                return resp["output"]

            total_checks += 1
            await asyncio.sleep(2)

    @commands.command(aliases=["diffuse", "sd"])
    async def diffusion(self, ctx: CustomContext, *, prompt: str) -> None:
        async with ctx.typing():
            try:
                job_id = await self.start_job(prompt)
            except DiffusionError as e:
                return await ctx.error("API Error", f"{ctx.author.mention} {e}")
            except ClientResponseError as e:
                return await ctx.error(
                    "API Error", f"{ctx.author.mention} Received status code `{e.status}`\n{e.message}"
                )

            try:
                images = await self.check_progress(job_id)
            except DiffusionError as e:
                return await ctx.error("API Error", str(e))
            except ClientResponseError as e:
                return await ctx.error(
                    "API Error", f"{ctx.author.mention} Received status code `{e.status}`\n{e.message}"
                )

            images_b = [io.BytesIO(await (await self.bot.aio_session.get(url)).read()) for url in images]
            file = File(stitch_images(images_b, rows_cols=len(images_b)), filename=f"{quote(prompt)}.png")

        return await ctx.send(f"{ctx.author.mention}: {prompt}", file=file)

def setup(bot):
    bot.add_cog(Diffusion(bot))
<|endoftext|>


<commit_before>#
# sandbox.py :  tools for manipulating a test's working area ("a sandbox")
#
# ====================================================================
#    Licensed to the Apache Software Foundation (ASF) under one
#    or more contributor license agreements.  See the NOTICE file
#    distributed with this work for additional information
#    regarding copyright ownership.  The ASF licenses this file
#    to you under the Apache License, Version 2.0 (the
#    "License"); you may not use this file except in compliance
#    with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing,
#    software distributed under the License is distributed on an
#    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#    KIND, either express or implied.  See the License for the
#    specific language governing permissions and limitations
#    under the License.
# ====================================================================
#

import os
import shutil
import copy

import svntest

class Sandbox:
  """Manages a sandbox (one or more repository/working copy pairs) for
  a test to operate within."""

  dependents = None

  def __init__(self, module, idx):
    self._set_name("%s-%d" % (module, idx))
    # This flag is set to True by build() and returned by is_built()
    self._is_built = False

  def _set_name(self, name, read_only=False):
    """A convenience method for renaming a sandbox, useful when
    working with multiple repositories in the same unit test."""
    if not name is None:
      self.name = name
    self.read_only = read_only
    self.wc_dir = os.path.join(svntest.main.general_wc_dir, self.name)
    if not read_only:
      self.repo_dir = os.path.join(svntest.main.general_repo_dir, self.name)
      self.repo_url = (svntest.main.options.test_area_url + '/'
                       + svntest.main.pathname2url(self.repo_dir))
    else:
      self.repo_dir = svntest.main.pristine_greek_repos_dir
      self.repo_url = svntest.main.pristine_greek_repos_url

    ### TODO: Move this into to the build() method
    # For dav tests we need a single authz file which must be present,
    # so we recreate it each time a sandbox is created with some default
    # contents, making sure that an empty file is never present
    if self.repo_url.startswith("http"):
      # this dir doesn't exist out of the box, so we may have to make it
      if not os.path.exists(svntest.main.work_dir):
        os.makedirs(svntest.main.work_dir)
      self.authz_file = os.path.join(svntest.main.work_dir, "authz")
      tmp_authz_file = os.path.join(svntest.main.work_dir, "authz-" + self.name)
      open(tmp_authz_file, 'w').write("[/]\n* = rw\n")
      shutil.move(tmp_authz_file, self.authz_file)

    # For svnserve tests we have a per-repository authz file, and it
    # doesn't need to be there in order for things to work, so we don't
    # have any default contents.
    elif self.repo_url.startswith("svn"):
      self.authz_file = os.path.join(self.repo_dir, "conf", "authz")

    self.test_paths = [self.wc_dir, self.repo_dir]

  def clone_dependent(self, copy_wc=False):
    """A convenience method for creating a near-duplicate of this
    sandbox, useful when working with multiple repositories in the
    same unit test.  If COPY_WC is true, make an exact copy of this
    sandbox's working copy at the new sandbox's working copy
    directory.  Any necessary cleanup operations are triggered by
    cleanup of the original sandbox."""

    if not self.dependents:
      self.dependents = []
    clone = copy.deepcopy(self)
    self.dependents.append(clone)
    clone._set_name("%s-%d" % (self.name, len(self.dependents)))
    if copy_wc:
      self.add_test_path(clone.wc_dir)
      shutil.copytree(self.wc_dir, clone.wc_dir, symlinks=True)
    return clone

  def build(self, name=None, create_wc=True, read_only=False):
    """Make a 'Greek Tree' repo (or refer to the central one if READ_ONLY),
       and check out a WC from it (unless CREATE_WC is false). Change the
       sandbox's name to NAME. See actions.make_repo_and_wc() for details."""
    self._set_name(name, read_only)
    svntest.actions.make_repo_and_wc(self, create_wc, read_only)
    self._is_built = True

  def add_test_path(self, path, remove=True):
    self.test_paths.append(path)
    if remove:
      svntest.main.safe_rmtree(path)

  def add_repo_path(self, suffix, remove=True):
    """Generate a path, under the general repositories directory, with
       a name that ends in SUFFIX, e.g. suffix="2" -> ".../basic_tests.2".
       If REMOVE is true, remove anything currently on disk at that path.
       Remember that path so that the automatic clean-up mechanism can
       delete it at the end of the test. Generate a repository URL to
       refer to a repository at that path. Do not create a repository.
       Return (REPOS-PATH, REPOS-URL)."""
    path = (os.path.join(svntest.main.general_repo_dir, self.name)
            + '.' + suffix)
    url = svntest.main.options.test_area_url + \
                                        '/' + svntest.main.pathname2url(path)
    self.add_test_path(path, remove)
    return path, url

  def add_wc_path(self, suffix, remove=True):
    """Generate a path, under the general working copies directory, with
       a name that ends in SUFFIX, e.g. suffix="2" -> ".../basic_tests.2".
       If REMOVE is true, remove anything currently on disk at that path.
       Remember that path so that the automatic clean-up mechanism can
       delete it at the end of the test. Do not create a working copy.
       Return the generated WC-PATH."""
    path = self.wc_dir + '.' + suffix
    self.add_test_path(path, remove)
    return path

  tempname_offs = 0 # Counter for get_tempname

  def get_tempname(self, prefix='tmp'):
    """Get a stable name for a temporary file that will be removed after
       running the test"""

    dir = self.add_wc_path('tmp')
    if not os.path.exists(dir):
      os.mkdir(dir)

    self.tempname_offs = self.tempname_offs + 1

    return os.path.join(dir, '%s-%s' % (prefix, self.tempname_offs))

  def cleanup_test_paths(self):
    "Clean up detritus from this sandbox, and any dependents."
    if self.dependents:
      # Recursively cleanup any dependent sandboxes.
      for sbox in self.dependents:
        sbox.cleanup_test_paths()
    # cleanup all test specific working copies and repositories
    for path in self.test_paths:
      if not path is svntest.main.pristine_greek_repos_dir:
        _cleanup_test_path(path)

  def is_built(self):
    "Returns True when build() has been called on this instance."
    return self._is_built

  def ospath(self, relpath, wc_dir=None):
    if wc_dir is None:
      wc_dir = self.wc_dir
    return os.path.join(wc_dir, svntest.wc.to_ospath(relpath))

  def redirected_root_url(self, temporary=False):
    """If TEMPORARY is set, return the URL which should be configured
       to temporarily redirect to the root of this repository;
       otherwise, return the URL which should be configured to
       permanent redirect there.  (Assumes that the sandbox is not
       read-only.)"""
    assert not self.read_only
    assert self.repo_url.startswith("http")
    parts = self.repo_url.rsplit('/', 1)
    return '%s/REDIRECT-%s-%s' % (parts[0],
                                  temporary and 'TEMP' or 'PERM',
                                  parts[1])

  def simple_update(self, target=None):
    assert not self.read_only
    if target is None:
      target = self.wc_dir
    svntest.main.run_svn(False, 'update', target)

  def simple_commit(self, target=None):
    assert not self.read_only
    if target is None:
      target = self.wc_dir
    svntest.main.run_svn(False, 'commit',
                         '-m', svntest.main.make_log_msg(),
                         target)

  def simple_rm(self, *targets):
    assert len(targets) > 0
    if len(targets) == 1 and is_url(targets[0]):
      assert not self.read_only
      targets = ('-m', svntests.main.make_log_msg(), targets[0])
    svntest.main.run_svn(False, 'rm', *targets)

  def simple_mkdir(self, *targets):
    assert len(targets) > 0
    if len(targets) == 1 and is_url(targets[0]):
      assert not self.read_only
      targets = ('-m', svntests.main.make_log_msg(), targets[0])
    svntest.main.run_svn(False, 'mkdir', *targets)

  def simple_add(self, *targets):
    assert len(targets) > 0
    svntest.main.run_svn(False, 'add', *targets)

  def simple_revert(self, *targets):
    assert len(targets) > 0
    svntest.main.run_svn(False, 'revert', *targets)

  def simple_propset(self, name, value, *targets):
    assert len(targets) > 0
    svntest.main.run_svn(False, 'propset', name, value, *targets)

  def simple_propdel(self, name, *targets):
    assert len(targets) > 0
    svntest.main.run_svn(False, 'propdel', name, *targets)

def is_url(target):
  return (target.startswith('^/')
          or target.startswith('file://')
          or target.startswith('http://')
          or target.startswith('https://')
          or target.startswith('svn://')
          or target.startswith('svn+ssh://'))

_deferred_test_paths = []

def cleanup_deferred_test_paths():
  global _deferred_test_paths
  test_paths = _deferred_test_paths
  _deferred_test_paths = []
  for path in test_paths:
    _cleanup_test_path(path, True)

def _cleanup_test_path(path, retrying=False):
  if svntest.main.options.verbose:
    if retrying:
      print("CLEANUP: RETRY: %s" % path)
    else:
      print("CLEANUP: %s" % path)
  try:
    svntest.main.safe_rmtree(path)
  except:
    if svntest.main.options.verbose:
      print("WARNING: cleanup failed, will try again later")
    _deferred_test_paths.append(path)
<commit_msg>* subversion/tests/cmdline/svntest/sandbox.py   (simple_rm, simple_mkdir): Remove unused support for a URL target, as that     would be better in a separate function and gets in the way of     a future change: to make the target arguments relative to the WC dir.<commit_after>#
# sandbox.py :  tools for manipulating a test's working area ("a sandbox")
#
# ====================================================================
#    Licensed to the Apache Software Foundation (ASF) under one
#    or more contributor license agreements.  See the NOTICE file
#    distributed with this work for additional information
#    regarding copyright ownership.  The ASF licenses this file
#    to you under the Apache License, Version 2.0 (the
#    "License"); you may not use this file except in compliance
#    with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing,
#    software distributed under the License is distributed on an
#    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#    KIND, either express or implied.  See the License for the
#    specific language governing permissions and limitations
#    under the License.
# ====================================================================
#

import os
import shutil
import copy

import svntest

class Sandbox:
  """Manages a sandbox (one or more repository/working copy pairs) for
  a test to operate within."""

  dependents = None

  def __init__(self, module, idx):
    self._set_name("%s-%d" % (module, idx))
    # This flag is set to True by build() and returned by is_built()
    self._is_built = False

  def _set_name(self, name, read_only=False):
    """A convenience method for renaming a sandbox, useful when
    working with multiple repositories in the same unit test."""
    if not name is None:
      self.name = name
    self.read_only = read_only
    self.wc_dir = os.path.join(svntest.main.general_wc_dir, self.name)
    if not read_only:
      self.repo_dir = os.path.join(svntest.main.general_repo_dir, self.name)
      self.repo_url = (svntest.main.options.test_area_url + '/'
                       + svntest.main.pathname2url(self.repo_dir))
    else:
      self.repo_dir = svntest.main.pristine_greek_repos_dir
      self.repo_url = svntest.main.pristine_greek_repos_url

    ### TODO: Move this into to the build() method
    # For dav tests we need a single authz file which must be present,
    # so we recreate it each time a sandbox is created with some default
    # contents, making sure that an empty file is never present
    if self.repo_url.startswith("http"):
      # this dir doesn't exist out of the box, so we may have to make it
      if not os.path.exists(svntest.main.work_dir):
        os.makedirs(svntest.main.work_dir)
      self.authz_file = os.path.join(svntest.main.work_dir, "authz")
      tmp_authz_file = os.path.join(svntest.main.work_dir, "authz-" + self.name)
      open(tmp_authz_file, 'w').write("[/]\n* = rw\n")
      shutil.move(tmp_authz_file, self.authz_file)

    # For svnserve tests we have a per-repository authz file, and it
    # doesn't need to be there in order for things to work, so we don't
    # have any default contents.
    elif self.repo_url.startswith("svn"):
      self.authz_file = os.path.join(self.repo_dir, "conf", "authz")

    self.test_paths = [self.wc_dir, self.repo_dir]

  def clone_dependent(self, copy_wc=False):
    """A convenience method for creating a near-duplicate of this
    sandbox, useful when working with multiple repositories in the
    same unit test.  If COPY_WC is true, make an exact copy of this
    sandbox's working copy at the new sandbox's working copy
    directory.  Any necessary cleanup operations are triggered by
    cleanup of the original sandbox."""

    if not self.dependents:
      self.dependents = []
    clone = copy.deepcopy(self)
    self.dependents.append(clone)
    clone._set_name("%s-%d" % (self.name, len(self.dependents)))
    if copy_wc:
      self.add_test_path(clone.wc_dir)
      shutil.copytree(self.wc_dir, clone.wc_dir, symlinks=True)
    return clone

  def build(self, name=None, create_wc=True, read_only=False):
    """Make a 'Greek Tree' repo (or refer to the central one if READ_ONLY),
       and check out a WC from it (unless CREATE_WC is false). Change the
       sandbox's name to NAME. See actions.make_repo_and_wc() for details."""
    self._set_name(name, read_only)
    svntest.actions.make_repo_and_wc(self, create_wc, read_only)
    self._is_built = True

  def add_test_path(self, path, remove=True):
    self.test_paths.append(path)
    if remove:
      svntest.main.safe_rmtree(path)

  def add_repo_path(self, suffix, remove=True):
    """Generate a path, under the general repositories directory, with
       a name that ends in SUFFIX, e.g. suffix="2" -> ".../basic_tests.2".
       If REMOVE is true, remove anything currently on disk at that path.
       Remember that path so that the automatic clean-up mechanism can
       delete it at the end of the test. Generate a repository URL to
       refer to a repository at that path. Do not create a repository.
       Return (REPOS-PATH, REPOS-URL)."""
    path = (os.path.join(svntest.main.general_repo_dir, self.name)
            + '.' + suffix)
    url = svntest.main.options.test_area_url + \
                                        '/' + svntest.main.pathname2url(path)
    self.add_test_path(path, remove)
    return path, url

  def add_wc_path(self, suffix, remove=True):
    """Generate a path, under the general working copies directory, with
       a name that ends in SUFFIX, e.g. suffix="2" -> ".../basic_tests.2".
       If REMOVE is true, remove anything currently on disk at that path.
       Remember that path so that the automatic clean-up mechanism can
       delete it at the end of the test. Do not create a working copy.
       Return the generated WC-PATH."""
    path = self.wc_dir + '.' + suffix
    self.add_test_path(path, remove)
    return path

  tempname_offs = 0 # Counter for get_tempname

  def get_tempname(self, prefix='tmp'):
    """Get a stable name for a temporary file that will be removed after
       running the test"""

    dir = self.add_wc_path('tmp')
    if not os.path.exists(dir):
      os.mkdir(dir)

    self.tempname_offs = self.tempname_offs + 1

    return os.path.join(dir, '%s-%s' % (prefix, self.tempname_offs))

  def cleanup_test_paths(self):
    "Clean up detritus from this sandbox, and any dependents."
    if self.dependents:
      # Recursively cleanup any dependent sandboxes.
      for sbox in self.dependents:
        sbox.cleanup_test_paths()
    # cleanup all test specific working copies and repositories
    for path in self.test_paths:
      if not path is svntest.main.pristine_greek_repos_dir:
        _cleanup_test_path(path)

  def is_built(self):
    "Returns True when build() has been called on this instance."
    return self._is_built

  def ospath(self, relpath, wc_dir=None):
    if wc_dir is None:
      wc_dir = self.wc_dir
    return os.path.join(wc_dir, svntest.wc.to_ospath(relpath))

  def redirected_root_url(self, temporary=False):
    """If TEMPORARY is set, return the URL which should be configured
       to temporarily redirect to the root of this repository;
       otherwise, return the URL which should be configured to
       permanent redirect there.  (Assumes that the sandbox is not
       read-only.)"""
    assert not self.read_only
    assert self.repo_url.startswith("http")
    parts = self.repo_url.rsplit('/', 1)
    return '%s/REDIRECT-%s-%s' % (parts[0],
                                  temporary and 'TEMP' or 'PERM',
                                  parts[1])

  def simple_update(self, target=None):
    assert not self.read_only
    if target is None:
      target = self.wc_dir
    svntest.main.run_svn(False, 'update', target)

  def simple_commit(self, target=None):
    assert not self.read_only
    if target is None:
      target = self.wc_dir
    svntest.main.run_svn(False, 'commit',
                         '-m', svntest.main.make_log_msg(),
                         target)

  def simple_rm(self, *targets):
    assert len(targets) > 0
    svntest.main.run_svn(False, 'rm', *targets)

  def simple_mkdir(self, *targets):
    assert len(targets) > 0
    svntest.main.run_svn(False, 'mkdir', *targets)

  def simple_add(self, *targets):
    assert len(targets) > 0
    svntest.main.run_svn(False, 'add', *targets)

  def simple_revert(self, *targets):
    assert len(targets) > 0
    svntest.main.run_svn(False, 'revert', *targets)

  def simple_propset(self, name, value, *targets):
    assert len(targets) > 0
    svntest.main.run_svn(False, 'propset', name, value, *targets)

  def simple_propdel(self, name, *targets):
    assert len(targets) > 0
    svntest.main.run_svn(False, 'propdel', name, *targets)

def is_url(target):
  return (target.startswith('^/')
          or target.startswith('file://')
          or target.startswith('http://')
          or target.startswith('https://')
          or target.startswith('svn://')
          or target.startswith('svn+ssh://'))

_deferred_test_paths = []

def cleanup_deferred_test_paths():
  global _deferred_test_paths
  test_paths = _deferred_test_paths
  _deferred_test_paths = []
  for path in test_paths:
    _cleanup_test_path(path, True)

def _cleanup_test_path(path, retrying=False):
  if svntest.main.options.verbose:
    if retrying:
      print("CLEANUP: RETRY: %s" % path)
    else:
      print("CLEANUP: %s" % path)
  try:
    svntest.main.safe_rmtree(path)
  except:
    if svntest.main.options.verbose:
      print("WARNING: cleanup failed, will try again later")
    _deferred_test_paths.append(path)
<|endoftext|>


<commit_before>from survey.models import LocationTypeDetails, Location, LocationType, Household, HouseholdMember, \
    HouseholdMemberGroup, Answer, MultiChoiceAnswer, MultiSelectAnswer, NumericalAnswer, QuestionOption, Interview
from survey.utils.views_helper import get_ancestors
from django.core.mail import send_mail, EmailMessage
from django.conf import settings
from datetime import datetime
import csv, StringIO, string

class ResultComposer:
    def __init__(self, user, results_download_service):
        self.results_download_service = results_download_service
        self.user = user

    def send_mail(self):
        attachment_name = '%s.csv' % (self.results_download_service.batch.name if self.results_download_service.batch  \
        else self.results_download_service.survey.name)
        subject = 'Completion report for %s'  % attachment_name
        text = 'Completion report for %s. Date: %s'  % (attachment_name, datetime.now())
        print 'commencing...'
        try:
            mail = EmailMessage(subject, text, settings.DEFAULT_EMAIL_SENDER, [self.user.email, ])
            data = self.results_download_service.generate_interview_reports()
            #data = [[unicode('"%s"' % entry) for entry in entries] for entries in data]
            f = StringIO.StringIO()
            writer = csv.writer(f)
            map(lambda row: writer.writerow(row), data)
            #data = [','.join([unicode('"%s"' % entry) for entry in entries]) for entries in data]
            f.seek(0)
            mail.attach(attachment_name, f.read(), 'text/csv')
            f.close()
            sent = mail.send()
            print 'Emailed!! ', sent
        except Exception, ex:
            print 'error while sending mail: %s', str(ex)

class ResultsDownloadService(object):
    AS_TEXT = 1
    AS_LABEL = 0
    answers = None
    # MEMBER_ATTRS = {
    #     'id' : 'Household ID',
    #      'name' : 'Name', 'Age', 'Date of Birth', 'Gender'
    # }
    def __init__(self, survey=None, batch=None, restrict_to=None, specific_households=None, multi_display=AS_TEXT):
        self.batch = batch
        self.survey, self.questions = self._set_survey_and_questions(survey)
        self.locations = restrict_to or Location.objects.all()
        self.specific_households = specific_households
        self.multi_display = int(multi_display)

    def _set_survey_and_questions(self, survey):
        if self.batch:
            return self.batch.survey, self.batch.survey_questions
        survey_questions = []
        map(lambda batch: survey_questions.extend(list(batch.survey_questions)), survey.batches.all())
        return survey, survey_questions

    def set_report_headers(self):
        header = [loc.name for loc in LocationType.objects.exclude(name__iexact="country")
                  if not loc == LocationType.smallest_unit()]

        other_headers = ['EA', 'Household Number', 'Name', 'Age', 'Date of Birth', 'Gender']
        header.extend(other_headers)
        header.extend(self.question_headers())
        return header

    def question_headers(self):
        header = []
        for question in self.questions:
            header.append(question.identifier)
        return header

    def get_summarised_answers(self):
        data = []
        q_opts = {}
        if self.specific_households is None:
            all_households = Household.objects.filter(listing__survey_houselistings__survey=self.survey)
        else:
            all_households = Household.objects.filter(pk__in=self.specific_households,
                                                      listing__survey_houselistings__survey=self.survey)
        locations = list(set(all_households.values_list('listing__ea__locations', flat=True)))
        for location_id in locations:
            households_in_location = all_households.filter(listing__ea__locations=location_id)
            household_location = Location.objects.get(id=location_id)
            location_ancestors = household_location.get_ancestors().exclude(parent__isnull='country').values_list('name', flat=True)
            answers = []
            for household in households_in_location:
                for member in household.members.all():
                    try:
                        answers = list(location_ancestors)
                        member_gender = 'Male' if member.gender == HouseholdMember.MALE else 'Female'
                        answers.extend([household.listing.ea.name, household.house_number, '%s-%s' % (member.surname, member.first_name), str(member.age),
                                             member.date_of_birth.strftime(settings.DATE_FORMAT),
                                             member_gender])
                        for question in self.questions:
                            reply = member.reply(question)
                            if question.answer_type in [MultiChoiceAnswer.choice_name(), MultiSelectAnswer.choice_name()]\
                                    and self.multi_display == self.AS_LABEL:
                                label = q_opts.get((question.pk, reply), None)
                                if label is None:
                                    try:
                                        label = question.options.get(text__iexact=reply).order
                                    except QuestionOption.DoesNotExist:
                                        label = reply
                                    q_opts[(question.pk, reply)] = label
                                reply = str(label)
                            answers.append(reply.encode('utf8'))
                        data.append(answers)
                    except Exception, ex:
                        print 'Error ', str(ex)
        return data

    def get_interview_answers(self):
        data = []
        q_opts = {}
        interviews = Interview.objects.filter(batch=self.batch).order_by('householdmember__survey_listing',
                                                                         'householdmember__household')
        answer_objs = {}
        for answer_type in Answer.answer_types():
            answer_class = Answer.get_class(answer_type)
            answer_objs[answer_type] = answer_class.objects.filter(interview__batch=self.batch)
        locations_map = {}
        for interview in interviews:
            ea = interview.ea
            location_ancestors = locations_map.get(ea.pk, None)
            if location_ancestors is None:
                location_ancestors = ea.parent_locations().values_list('name', flat=True)
                locations_map[ea.pk] = location_ancestors
            try:
                member = interview.householdmember
                household = member.household
                answers = list(location_ancestors)
                member_gender = 'Male' if member.gender == HouseholdMember.MALE else 'Female'
                answers.extend([ea.name, household.house_number, '%s-%s' % (member.surname, member.first_name), str(member.age),
                                     member.date_of_birth.strftime(settings.DATE_FORMAT),
                                     member_gender])
                for question in self.questions:
                    try:
                        reply = answer_objs[question.answer_type].filter(interview=interview,#interview.get_anwser(question)
                                                                 question=question)[0].to_text()
                    except IndexError:
                        reply = ''
                    reply = unicode(reply)
                    if question.answer_type in [MultiChoiceAnswer.choice_name(), MultiSelectAnswer.choice_name()]\
                            and self.multi_display == self.AS_LABEL:
                        label = q_opts.get((question.pk, reply), None)
                        if label is None:
                            try:
                                label = question.options.get(text__iexact=reply).order
                            except QuestionOption.DoesNotExist:
                                label = reply
                            q_opts[(question.pk, reply)] = label
                        reply = str(label)
                    answers.append(reply.encode('utf8'))
                data.append(answers)
            except Exception, ex:
                print 'Error ', str(ex)
        return data

    def generate_report(self):
        data = [self.set_report_headers(), ]
        data.extend(self.get_summarised_answers())
        return data

    def generate_interview_reports(self):
        data = [self.set_report_headers(), ]
        data.extend(self.get_interview_answers())
        return data

    def _get_ancestors_names(self, household_location, exclude_type='country'):
        location_ancestors = get_ancestors(household_location, include_self=True)
        if exclude_type:
            exclude_location = Location.objects.filter(type__name__iexact=exclude_type.lower())
            for location in exclude_location:
                location_ancestors.remove(location)
        result= [ancestor.name for ancestor in location_ancestors]
        result.reverse()
        return result<commit_msg>fix in reports download<commit_after>from survey.models import LocationTypeDetails, Location, LocationType, Household, HouseholdMember, \
    HouseholdMemberGroup, Answer, MultiChoiceAnswer, MultiSelectAnswer, NumericalAnswer, QuestionOption, Interview
from survey.utils.views_helper import get_ancestors
from django.core.mail import send_mail, EmailMessage
from django.conf import settings
from datetime import datetime
import csv, StringIO, string

class ResultComposer:
    def __init__(self, user, results_download_service):
        self.results_download_service = results_download_service
        self.user = user

    def send_mail(self):
        attachment_name = '%s.csv' % (self.results_download_service.batch.name if self.results_download_service.batch  \
        else self.results_download_service.survey.name)
        subject = 'Completion report for %s'  % attachment_name
        text = 'Completion report for %s. Date: %s'  % (attachment_name, datetime.now())
        print 'commencing...'
        try:
            mail = EmailMessage(subject, text, settings.DEFAULT_EMAIL_SENDER, [self.user.email, ])
            data = self.results_download_service.generate_interview_reports()
            #data = [[unicode('"%s"' % entry) for entry in entries] for entries in data]
            f = StringIO.StringIO()
            writer = csv.writer(f)
            map(lambda row: writer.writerow(row), data)
            #data = [','.join([unicode('"%s"' % entry) for entry in entries]) for entries in data]
            f.seek(0)
            mail.attach(attachment_name, f.read(), 'text/csv')
            f.close()
            sent = mail.send()
            print 'Emailed!! ', sent
        except Exception, ex:
            print 'error while sending mail: %s', str(ex)

class ResultsDownloadService(object):
    AS_TEXT = 1
    AS_LABEL = 0
    answers = None
    # MEMBER_ATTRS = {
    #     'id' : 'Household ID',
    #      'name' : 'Name', 'Age', 'Date of Birth', 'Gender'
    # }
    def __init__(self, survey=None, batch=None, restrict_to=None, specific_households=None, multi_display=AS_TEXT):
        self.batch = batch
        self.survey, self.questions = self._set_survey_and_questions(survey)
        self.locations = restrict_to or Location.objects.all()
        self.specific_households = specific_households
        self.multi_display = int(multi_display)

    def _set_survey_and_questions(self, survey):
        if self.batch:
            return self.batch.survey, self.batch.survey_questions
        survey_questions = []
        map(lambda batch: survey_questions.extend(list(batch.survey_questions)), survey.batches.all())
        return survey, survey_questions

    def set_report_headers(self):
        header = [loc.name for loc in LocationType.objects.exclude(name__iexact="country")
                  if not loc == LocationType.smallest_unit()]

        other_headers = ['EA', 'Household Number', 'Name', 'Age', 'Date of Birth', 'Gender']
        header.extend(other_headers)
        header.extend(self.question_headers())
        return header

    def question_headers(self):
        header = []
        for question in self.questions:
            header.append(question.identifier)
        return header

    def get_summarised_answers(self):
        data = []
        q_opts = {}
        if self.specific_households is None:
            all_households = Household.objects.filter(listing__survey_houselistings__survey=self.survey)
        else:
            all_households = Household.objects.filter(pk__in=self.specific_households,
                                                      listing__survey_houselistings__survey=self.survey)
        locations = list(set(all_households.values_list('listing__ea__locations', flat=True)))
        for location_id in locations:
            households_in_location = all_households.filter(listing__ea__locations=location_id)
            household_location = Location.objects.get(id=location_id)
            location_ancestors = household_location.get_ancestors().exclude(parent__isnull='country').values_list('name', flat=True)
            answers = []
            for household in households_in_location:
                for member in household.members.all():
                    try:
                        answers = list(location_ancestors)
                        member_gender = 'Male' if member.gender == HouseholdMember.MALE else 'Female'
                        answers.extend([household.listing.ea.name, household.house_number, '%s-%s' % (member.surname, member.first_name), str(member.age),
                                             member.date_of_birth.strftime(settings.DATE_FORMAT),
                                             member_gender])
                        for question in self.questions:
                            reply = member.reply(question)
                            if question.answer_type in [MultiChoiceAnswer.choice_name(), MultiSelectAnswer.choice_name()]\
                                    and self.multi_display == self.AS_LABEL:
                                label = q_opts.get((question.pk, reply), None)
                                if label is None:
                                    try:
                                        label = question.options.get(text__iexact=reply).order
                                    except QuestionOption.DoesNotExist:
                                        label = reply
                                    q_opts[(question.pk, reply)] = label
                                reply = str(label)
                            answers.append(reply.encode('utf8'))
                        data.append(answers)
                    except Exception, ex:
                        print 'Error ', str(ex)
        return data

    def get_interview_answers(self):
        data = []
        q_opts = {}
        interviews = Interview.objects.filter(batch=self.batch).order_by('householdmember__survey_listing',
                                                                         'householdmember__household')
        answer_objs = {}
        for answer_type in Answer.answer_types():
            answer_class = Answer.get_class(answer_type)
            answer_objs[answer_type] = dict([((a.interview.pk, a.question.pk), a.to_text())
                                        for a in answer_class.objects.filter(interview__batch=self.batch)])
        locations_map = {}
        for interview in interviews:
            ea = interview.ea
            location_ancestors = locations_map.get(ea.pk, None)
            if location_ancestors is None:
                location_ancestors = ea.parent_locations().values_list('name', flat=True)
                locations_map[ea.pk] = location_ancestors
            try:
                member = interview.householdmember
                household = member.household
                answers = list(location_ancestors)
                member_gender = 'Male' if member.gender == HouseholdMember.MALE else 'Female'
                answers.extend([ea.name, household.house_number, '%s-%s' % (member.surname, member.first_name), str(member.age),
                                     member.date_of_birth.strftime(settings.DATE_FORMAT),
                                     member_gender])
                for question in self.questions:
                    reply = answer_objs[question.answer_type].get((interview.pk, question.pk), '')
                    reply = unicode(reply)
                    if question.answer_type in [MultiChoiceAnswer.choice_name(), MultiSelectAnswer.choice_name()]\
                            and self.multi_display == self.AS_LABEL:
                        label = q_opts.get((question.pk, reply), None)
                        if label is None:
                            try:
                                label = question.options.get(text__iexact=reply).order
                            except QuestionOption.DoesNotExist:
                                label = reply
                            q_opts[(question.pk, reply)] = label
                        reply = str(label)
                    answers.append(reply.encode('utf8'))
                data.append(answers)
            except Exception, ex:
                print 'Error ', str(ex)
        return data

    def generate_report(self):
        data = [self.set_report_headers(), ]
        data.extend(self.get_summarised_answers())
        return data

    def generate_interview_reports(self):
        data = [self.set_report_headers(), ]
        data.extend(self.get_interview_answers())
        return data

    def _get_ancestors_names(self, household_location, exclude_type='country'):
        location_ancestors = get_ancestors(household_location, include_self=True)
        if exclude_type:
            exclude_location = Location.objects.filter(type__name__iexact=exclude_type.lower())
            for location in exclude_location:
                location_ancestors.remove(location)
        result= [ancestor.name for ancestor in location_ancestors]
        result.reverse()
        return result<|endoftext|>