<commit_before># #!/usr/bin/env python
from __future__ import absolute_import

import os
import sys
import re
import textwrap
import site

import scripttest
import virtualenv

from tests.lib.path import Path, curdir, u

DATA_DIR = Path(__file__).folder.folder.join("data").abspath
SRC_DIR = Path(__file__).abspath.folder.folder.folder

pyversion = sys.version[:3]
pyversion_nodot = "%d%d" % (sys.version_info[0], sys.version_info[1])

def path_to_url(path):
    """
    Convert a path to URI. The path will be made absolute and
    will not have quoted path parts.
    (adapted from pip.util)
    """
    path = os.path.normpath(os.path.abspath(path))
    drive, path = os.path.splitdrive(path)
    filepath = path.split(os.path.sep)
    url = '/'.join(filepath)
    if drive:
        return 'file:///' + drive + url
    return 'file://' + url

class TestData(object):
    """
    Represents a bundle of pre-created test data.

    This copies a pristine set of test data into a root location that is
    designed to be test specific. The reason for this is when running the tests
    concurrently errors can be generated because the related tooling uses
    the directory as a work space. This leads to two concurrent processes
    trampling over each other. This class gets around that by copying all
    data into a directory and operating on the copied data.
    """

    def __init__(self, root, source=None):
        self.source = source or DATA_DIR
        self.root = Path(root).abspath

    @classmethod
    def copy(cls, root):
        obj = cls(root)
        obj.reset()
        return obj

    def reset(self):
        self.root.rmtree()
        self.source.copytree(self.root)

    @property
    def packages(self):
        return self.root.join("packages")

    @property
    def packages2(self):
        return self.root.join("packages2")

    @property
    def indexes(self):
        return self.root.join("indexes")

    @property
    def reqfiles(self):
        return self.root.join("reqfiles")

    @property
    def find_links(self):
        return path_to_url(self.packages)

    @property
    def find_links2(self):
        return path_to_url(self.packages2)

    def index_url(self, index="simple"):
        return path_to_url(self.root.join("indexes", index))

class TestFailure(AssertionError):
    """
    An "assertion" failed during testing.
    """
    pass

class TestPipResult(object):

    def __init__(self, impl, verbose=False):
        self._impl = impl

        if verbose:
            print(self.stdout)
            if self.stderr:
                print('======= stderr ========')
                print(self.stderr)
                print('=======================')

    def __getattr__(self, attr):
        return getattr(self._impl, attr)

    if sys.platform == 'win32':

        @property
        def stdout(self):
            return self._impl.stdout.replace('\r\n', '\n')

        @property
        def stderr(self):
            return self._impl.stderr.replace('\r\n', '\n')

        def __str__(self):
            return str(self._impl).replace('\r\n', '\n')
    else:
        # Python doesn't automatically forward __str__ through __getattr__

        def __str__(self):
            return str(self._impl)

    def assert_installed(self, pkg_name, editable=True, with_files=[],
                         without_files=[], without_egg_link=False,
                         use_user_site=False):
        e = self.test_env

        if editable:
            pkg_dir = e.venv/'src'/pkg_name.lower()
        else:
            without_egg_link = True
            pkg_dir = e.site_packages/pkg_name

        if use_user_site:
            egg_link_path = e.user_site / pkg_name + '.egg-link'
        else:
            egg_link_path = e.site_packages / pkg_name + '.egg-link'

        if without_egg_link:
            if egg_link_path in self.files_created:
                raise TestFailure('unexpected egg link file created: '\
                                  '%r\n%s' % (egg_link_path, self))
        else:
            if not egg_link_path in self.files_created:
                raise TestFailure('expected egg link file missing: '\
                                  '%r\n%s' % (egg_link_path, self))

            egg_link_file = self.files_created[egg_link_path]

            if not (# FIXME: I don't understand why there's a trailing . here
                    egg_link_file.bytes.endswith('.')
                and egg_link_file.bytes[:-1].strip().endswith(pkg_dir)):
                raise TestFailure(textwrap.dedent(u('''\
                Incorrect egg_link file %r
                Expected ending: %r
                ------- Actual contents -------
                %s
                -------------------------------''' % (
                        egg_link_file,
                        pkg_dir + u('\n.'),
                        egg_link_file.bytes))))

        if use_user_site:
            pth_file = e.user_site/'easy-install.pth'
        else:
            pth_file = e.site_packages/'easy-install.pth'

        if (pth_file in self.files_updated) == without_egg_link:
            raise TestFailure('%r unexpectedly %supdated by install' % (
                pth_file, (not without_egg_link and 'not ' or '')))

        if (pkg_dir in self.files_created) == (curdir in without_files):
            raise TestFailure(textwrap.dedent('''\
            expected package directory %r %sto be created
            actually created:
            %s
            ''') % (
                pkg_dir,
                (curdir in without_files and 'not ' or ''),
                sorted(self.files_created.keys())))

        for f in with_files:
            if not (pkg_dir/f).normpath in self.files_created:
                raise TestFailure('Package directory %r missing '\
                                  'expected content %r' % (pkg_dir, f))

        for f in without_files:
            if (pkg_dir/f).normpath in self.files_created:
                raise TestFailure('Package directory %r has '\
                                  'unexpected content %f' % (pkg_dir, f))

class PipTestEnvironment(scripttest.TestFileEnvironment):
    """
    A specialized TestFileEnvironment for testing pip
    """

    #
    # Attribute naming convention
    # ---------------------------
    #
    # Instances of this class have many attributes representing paths
    # in the filesystem.  To keep things straight, absolute paths have
    # a name of the form xxxx_path and relative paths have a name that
    # does not end in '_path'.

    exe = sys.platform == 'win32' and '.exe' or ''
    verbose = False

    def __init__(self, base_path, *args, **kwargs):
        # Make our base_path a test.lib.path.Path object
        base_path = Path(base_path)

        # Store paths related to the virtual environment
        _virtualenv = kwargs.pop("virtualenv")
        venv, lib, include, bin = virtualenv.path_locations(_virtualenv)
        self.venv_path = venv
        self.lib_path = lib
        self.include_path = include
        self.bin_path = bin

        if hasattr(sys, "pypy_version_info"):
            self.site_packages_path = self.venv_path.join("site-packages")
        else:
            self.site_packages_path = self.lib_path.join("site-packages")

        self.user_base_path = self.venv_path.join("user")
        self.user_site_path = self.venv_path.join(
            "user",
            site.USER_SITE[len(site.USER_BASE) + 1:],
        )

        # Create a Directory to use as a scratch pad
        self.scratch_path = base_path.join("scratch").mkdir()

        # Set our default working directory
        kwargs.setdefault("cwd", self.scratch_path)

        # Setup our environment
        environ = kwargs.get("environ")
        if environ is None:
            environ = os.environ.copy()

        environ["PIP_LOG_FILE"] = base_path.join("pip-log.txt")
        environ["PATH"] = Path.pathsep.join(
            [self.bin_path] + [environ.get("PATH", [])],
        )
        environ["PYTHONUSERBASE"] = self.user_base_path
        # Writing bytecode can mess up updated file detection
        environ["PYTHONDONTWRITEBYTECODE"] = "1"
        kwargs["environ"] = environ

        # Call the TestFileEnvironment __init__
        super(PipTestEnvironment, self).__init__(base_path, *args, **kwargs)

        # Expand our absolute path directories into relative
        for name in ["base", "venv", "lib", "include", "bin", "site_packages",
                     "user_base", "user_site", "scratch"]:
            real_name = "%s_path" % name
            setattr(self, name, getattr(self, real_name) - self.base_path)

        # Ensure the tmp dir exists, things break horribly if it doesn't
        self.temp_path.mkdir()

        # create easy-install.pth in user_site, so we always have it updated
        #   instead of created
        self.user_site_path.makedirs()
        self.user_site_path.join("easy-install.pth").touch()

    def _ignore_file(self, fn):
        if fn.endswith('__pycache__') or fn.endswith(".pyc"):
            result = True
        else:
            result = super(PipTestEnvironment, self)._ignore_file(fn)
        return result

    def run(self, *args, **kw):
        if self.verbose:
            print('>> running %s %s' % (args, kw))
        cwd = kw.pop('cwd', None)
        run_from = kw.pop('run_from', None)
        assert not cwd or not run_from, "Don't use run_from; it's going away"
        cwd = cwd or run_from or self.cwd
        return TestPipResult(super(PipTestEnvironment, self).run(cwd=cwd, *args, **kw), verbose=self.verbose)

    def pip(self, *args, **kwargs):
        return self.run("pip", *args, **kwargs)

    def pip_install_local(self, *args, **kwargs):
        return self.pip(
            "install", "--no-index",
            "--find-links", path_to_url(os.path.join(DATA_DIR, "packages")),
            *args, **kwargs
        )

# FIXME ScriptTest does something similar, but only within a single
# ProcResult; this generalizes it so states can be compared across
# multiple commands.  Maybe should be rolled into ScriptTest?
def diff_states(start, end, ignore=None):
    """
    Differences two "filesystem states" as represented by dictionaries
    of FoundFile and FoundDir objects.

    Returns a dictionary with following keys:

    ``deleted``
        Dictionary of files/directories found only in the start state.

    ``created``
        Dictionary of files/directories found only in the end state.

    ``updated``
        Dictionary of files whose size has changed (FIXME not entirely
        reliable, but comparing contents is not possible because
        FoundFile.bytes is lazy, and comparing mtime doesn't help if
        we want to know if a file has been returned to its earlier
        state).

    Ignores mtime and other file attributes; only presence/absence and
    size are considered.

    """
    ignore = ignore or []

    def prefix_match(path, prefix):
        if path == prefix:
            return True
        prefix = prefix.rstrip(os.path.sep) + os.path.sep
        return path.startswith(prefix)

    start_keys = set([k for k in start.keys()
                      if not any([prefix_match(k, i) for i in ignore])])
    end_keys = set([k for k in end.keys()
                    if not any([prefix_match(k, i) for i in ignore])])
    deleted = dict([(k, start[k]) for k in start_keys.difference(end_keys)])
    created = dict([(k, end[k]) for k in end_keys.difference(start_keys)])
    updated = {}
    for k in start_keys.intersection(end_keys):
        if (start[k].size != end[k].size):
            updated[k] = end[k]
    return dict(deleted=deleted, created=created, updated=updated)

def assert_all_changes(start_state, end_state, expected_changes):
    """
    Fails if anything changed that isn't listed in the
    expected_changes.

    start_state is either a dict mapping paths to
    scripttest.[FoundFile|FoundDir] objects or a TestPipResult whose
    files_before we'll test.  end_state is either a similar dict or a
    TestPipResult whose files_after we'll test.

    Note: listing a directory means anything below
    that directory can be expected to have changed.
    """
    __tracebackhide__ = True

    start_files = start_state
    end_files = end_state
    if isinstance(start_state, TestPipResult):
        start_files = start_state.files_before
    if isinstance(end_state, TestPipResult):
        end_files = end_state.files_after

    diff = diff_states(start_files, end_files, ignore=expected_changes)
    if list(diff.values()) != [{}, {}, {}]:
        raise TestFailure('Unexpected changes:\n' + '\n'.join(
            [k + ': ' + ', '.join(v.keys()) for k, v in diff.items()]))

    # Don't throw away this potentially useful information
    return diff

def _create_test_package(script):
    script.scratch_path.join("version_pkg").mkdir()
    version_pkg_path = script.scratch_path/'version_pkg'
    version_pkg_path.join("version_pkg.py").write(textwrap.dedent("""
        def main():
            print('0.1')
    """))
    version_pkg_path.join("setup.py").write(textwrap.dedent("""
        from setuptools import setup, find_packages
        setup(
            name='version_pkg',
            version='0.1',
            packages=find_packages(),
            py_modules=['version_pkg'],
            entry_points=dict(console_scripts=['version_pkg=version_pkg:main'])
        )
    """))
    script.run('git', 'init', cwd=version_pkg_path)
    script.run('git', 'add', '.', cwd=version_pkg_path)
    script.run('git', 'commit', '-q',
            '--author', 'Pip <python-virtualenv@googlegroups.com>',
            '-am', 'initial version', cwd=version_pkg_path)
    return version_pkg_path

def _change_test_package_version(script, version_pkg_path):
    version_pkg_path.join("version_pkg.py").write(textwrap.dedent('''\
        def main():
            print("some different version")'''))
    script.run('git', 'clean', '-qfdx',
        cwd=version_pkg_path,
        expect_stderr=True,
    )
    script.run('git', 'commit', '-q',
            '--author', 'Pip <python-virtualenv@googlegroups.com>',
            '-am', 'messed version',
            cwd=version_pkg_path, expect_stderr=True)

def assert_raises_regexp(exception, reg, run, *args, **kwargs):
    """Like assertRaisesRegexp in unittest"""
    try:
        run(*args, **kwargs)
        assert False, "%s should have been thrown" % exception
    except Exception:
        e = sys.exc_info()[1]
        p = re.compile(reg)
        assert p.search(str(e)), str(e)
<commit_msg>Hide assert_raises_regexp in the traceback<commit_after># #!/usr/bin/env python
from __future__ import absolute_import

import os
import sys
import re
import textwrap
import site

import scripttest
import virtualenv

from tests.lib.path import Path, curdir, u

DATA_DIR = Path(__file__).folder.folder.join("data").abspath
SRC_DIR = Path(__file__).abspath.folder.folder.folder

pyversion = sys.version[:3]
pyversion_nodot = "%d%d" % (sys.version_info[0], sys.version_info[1])

def path_to_url(path):
    """
    Convert a path to URI. The path will be made absolute and
    will not have quoted path parts.
    (adapted from pip.util)
    """
    path = os.path.normpath(os.path.abspath(path))
    drive, path = os.path.splitdrive(path)
    filepath = path.split(os.path.sep)
    url = '/'.join(filepath)
    if drive:
        return 'file:///' + drive + url
    return 'file://' + url

class TestData(object):
    """
    Represents a bundle of pre-created test data.

    This copies a pristine set of test data into a root location that is
    designed to be test specific. The reason for this is when running the tests
    concurrently errors can be generated because the related tooling uses
    the directory as a work space. This leads to two concurrent processes
    trampling over each other. This class gets around that by copying all
    data into a directory and operating on the copied data.
    """

    def __init__(self, root, source=None):
        self.source = source or DATA_DIR
        self.root = Path(root).abspath

    @classmethod
    def copy(cls, root):
        obj = cls(root)
        obj.reset()
        return obj

    def reset(self):
        self.root.rmtree()
        self.source.copytree(self.root)

    @property
    def packages(self):
        return self.root.join("packages")

    @property
    def packages2(self):
        return self.root.join("packages2")

    @property
    def indexes(self):
        return self.root.join("indexes")

    @property
    def reqfiles(self):
        return self.root.join("reqfiles")

    @property
    def find_links(self):
        return path_to_url(self.packages)

    @property
    def find_links2(self):
        return path_to_url(self.packages2)

    def index_url(self, index="simple"):
        return path_to_url(self.root.join("indexes", index))

class TestFailure(AssertionError):
    """
    An "assertion" failed during testing.
    """
    pass

class TestPipResult(object):

    def __init__(self, impl, verbose=False):
        self._impl = impl

        if verbose:
            print(self.stdout)
            if self.stderr:
                print('======= stderr ========')
                print(self.stderr)
                print('=======================')

    def __getattr__(self, attr):
        return getattr(self._impl, attr)

    if sys.platform == 'win32':

        @property
        def stdout(self):
            return self._impl.stdout.replace('\r\n', '\n')

        @property
        def stderr(self):
            return self._impl.stderr.replace('\r\n', '\n')

        def __str__(self):
            return str(self._impl).replace('\r\n', '\n')
    else:
        # Python doesn't automatically forward __str__ through __getattr__

        def __str__(self):
            return str(self._impl)

    def assert_installed(self, pkg_name, editable=True, with_files=[],
                         without_files=[], without_egg_link=False,
                         use_user_site=False):
        e = self.test_env

        if editable:
            pkg_dir = e.venv/'src'/pkg_name.lower()
        else:
            without_egg_link = True
            pkg_dir = e.site_packages/pkg_name

        if use_user_site:
            egg_link_path = e.user_site / pkg_name + '.egg-link'
        else:
            egg_link_path = e.site_packages / pkg_name + '.egg-link'

        if without_egg_link:
            if egg_link_path in self.files_created:
                raise TestFailure('unexpected egg link file created: '\
                                  '%r\n%s' % (egg_link_path, self))
        else:
            if not egg_link_path in self.files_created:
                raise TestFailure('expected egg link file missing: '\
                                  '%r\n%s' % (egg_link_path, self))

            egg_link_file = self.files_created[egg_link_path]

            if not (# FIXME: I don't understand why there's a trailing . here
                    egg_link_file.bytes.endswith('.')
                and egg_link_file.bytes[:-1].strip().endswith(pkg_dir)):
                raise TestFailure(textwrap.dedent(u('''\
                Incorrect egg_link file %r
                Expected ending: %r
                ------- Actual contents -------
                %s
                -------------------------------''' % (
                        egg_link_file,
                        pkg_dir + u('\n.'),
                        egg_link_file.bytes))))

        if use_user_site:
            pth_file = e.user_site/'easy-install.pth'
        else:
            pth_file = e.site_packages/'easy-install.pth'

        if (pth_file in self.files_updated) == without_egg_link:
            raise TestFailure('%r unexpectedly %supdated by install' % (
                pth_file, (not without_egg_link and 'not ' or '')))

        if (pkg_dir in self.files_created) == (curdir in without_files):
            raise TestFailure(textwrap.dedent('''\
            expected package directory %r %sto be created
            actually created:
            %s
            ''') % (
                pkg_dir,
                (curdir in without_files and 'not ' or ''),
                sorted(self.files_created.keys())))

        for f in with_files:
            if not (pkg_dir/f).normpath in self.files_created:
                raise TestFailure('Package directory %r missing '\
                                  'expected content %r' % (pkg_dir, f))

        for f in without_files:
            if (pkg_dir/f).normpath in self.files_created:
                raise TestFailure('Package directory %r has '\
                                  'unexpected content %f' % (pkg_dir, f))

class PipTestEnvironment(scripttest.TestFileEnvironment):
    """
    A specialized TestFileEnvironment for testing pip
    """

    #
    # Attribute naming convention
    # ---------------------------
    #
    # Instances of this class have many attributes representing paths
    # in the filesystem.  To keep things straight, absolute paths have
    # a name of the form xxxx_path and relative paths have a name that
    # does not end in '_path'.

    exe = sys.platform == 'win32' and '.exe' or ''
    verbose = False

    def __init__(self, base_path, *args, **kwargs):
        # Make our base_path a test.lib.path.Path object
        base_path = Path(base_path)

        # Store paths related to the virtual environment
        _virtualenv = kwargs.pop("virtualenv")
        venv, lib, include, bin = virtualenv.path_locations(_virtualenv)
        self.venv_path = venv
        self.lib_path = lib
        self.include_path = include
        self.bin_path = bin

        if hasattr(sys, "pypy_version_info"):
            self.site_packages_path = self.venv_path.join("site-packages")
        else:
            self.site_packages_path = self.lib_path.join("site-packages")

        self.user_base_path = self.venv_path.join("user")
        self.user_site_path = self.venv_path.join(
            "user",
            site.USER_SITE[len(site.USER_BASE) + 1:],
        )

        # Create a Directory to use as a scratch pad
        self.scratch_path = base_path.join("scratch").mkdir()

        # Set our default working directory
        kwargs.setdefault("cwd", self.scratch_path)

        # Setup our environment
        environ = kwargs.get("environ")
        if environ is None:
            environ = os.environ.copy()

        environ["PIP_LOG_FILE"] = base_path.join("pip-log.txt")
        environ["PATH"] = Path.pathsep.join(
            [self.bin_path] + [environ.get("PATH", [])],
        )
        environ["PYTHONUSERBASE"] = self.user_base_path
        # Writing bytecode can mess up updated file detection
        environ["PYTHONDONTWRITEBYTECODE"] = "1"
        kwargs["environ"] = environ

        # Call the TestFileEnvironment __init__
        super(PipTestEnvironment, self).__init__(base_path, *args, **kwargs)

        # Expand our absolute path directories into relative
        for name in ["base", "venv", "lib", "include", "bin", "site_packages",
                     "user_base", "user_site", "scratch"]:
            real_name = "%s_path" % name
            setattr(self, name, getattr(self, real_name) - self.base_path)

        # Ensure the tmp dir exists, things break horribly if it doesn't
        self.temp_path.mkdir()

        # create easy-install.pth in user_site, so we always have it updated
        #   instead of created
        self.user_site_path.makedirs()
        self.user_site_path.join("easy-install.pth").touch()

    def _ignore_file(self, fn):
        if fn.endswith('__pycache__') or fn.endswith(".pyc"):
            result = True
        else:
            result = super(PipTestEnvironment, self)._ignore_file(fn)
        return result

    def run(self, *args, **kw):
        if self.verbose:
            print('>> running %s %s' % (args, kw))
        cwd = kw.pop('cwd', None)
        run_from = kw.pop('run_from', None)
        assert not cwd or not run_from, "Don't use run_from; it's going away"
        cwd = cwd or run_from or self.cwd
        return TestPipResult(super(PipTestEnvironment, self).run(cwd=cwd, *args, **kw), verbose=self.verbose)

    def pip(self, *args, **kwargs):
        return self.run("pip", *args, **kwargs)

    def pip_install_local(self, *args, **kwargs):
        return self.pip(
            "install", "--no-index",
            "--find-links", path_to_url(os.path.join(DATA_DIR, "packages")),
            *args, **kwargs
        )

# FIXME ScriptTest does something similar, but only within a single
# ProcResult; this generalizes it so states can be compared across
# multiple commands.  Maybe should be rolled into ScriptTest?
def diff_states(start, end, ignore=None):
    """
    Differences two "filesystem states" as represented by dictionaries
    of FoundFile and FoundDir objects.

    Returns a dictionary with following keys:

    ``deleted``
        Dictionary of files/directories found only in the start state.

    ``created``
        Dictionary of files/directories found only in the end state.

    ``updated``
        Dictionary of files whose size has changed (FIXME not entirely
        reliable, but comparing contents is not possible because
        FoundFile.bytes is lazy, and comparing mtime doesn't help if
        we want to know if a file has been returned to its earlier
        state).

    Ignores mtime and other file attributes; only presence/absence and
    size are considered.

    """
    ignore = ignore or []

    def prefix_match(path, prefix):
        if path == prefix:
            return True
        prefix = prefix.rstrip(os.path.sep) + os.path.sep
        return path.startswith(prefix)

    start_keys = set([k for k in start.keys()
                      if not any([prefix_match(k, i) for i in ignore])])
    end_keys = set([k for k in end.keys()
                    if not any([prefix_match(k, i) for i in ignore])])
    deleted = dict([(k, start[k]) for k in start_keys.difference(end_keys)])
    created = dict([(k, end[k]) for k in end_keys.difference(start_keys)])
    updated = {}
    for k in start_keys.intersection(end_keys):
        if (start[k].size != end[k].size):
            updated[k] = end[k]
    return dict(deleted=deleted, created=created, updated=updated)

def assert_all_changes(start_state, end_state, expected_changes):
    """
    Fails if anything changed that isn't listed in the
    expected_changes.

    start_state is either a dict mapping paths to
    scripttest.[FoundFile|FoundDir] objects or a TestPipResult whose
    files_before we'll test.  end_state is either a similar dict or a
    TestPipResult whose files_after we'll test.

    Note: listing a directory means anything below
    that directory can be expected to have changed.
    """
    __tracebackhide__ = True

    start_files = start_state
    end_files = end_state
    if isinstance(start_state, TestPipResult):
        start_files = start_state.files_before
    if isinstance(end_state, TestPipResult):
        end_files = end_state.files_after

    diff = diff_states(start_files, end_files, ignore=expected_changes)
    if list(diff.values()) != [{}, {}, {}]:
        raise TestFailure('Unexpected changes:\n' + '\n'.join(
            [k + ': ' + ', '.join(v.keys()) for k, v in diff.items()]))

    # Don't throw away this potentially useful information
    return diff

def _create_test_package(script):
    script.scratch_path.join("version_pkg").mkdir()
    version_pkg_path = script.scratch_path/'version_pkg'
    version_pkg_path.join("version_pkg.py").write(textwrap.dedent("""
        def main():
            print('0.1')
    """))
    version_pkg_path.join("setup.py").write(textwrap.dedent("""
        from setuptools import setup, find_packages
        setup(
            name='version_pkg',
            version='0.1',
            packages=find_packages(),
            py_modules=['version_pkg'],
            entry_points=dict(console_scripts=['version_pkg=version_pkg:main'])
        )
    """))
    script.run('git', 'init', cwd=version_pkg_path)
    script.run('git', 'add', '.', cwd=version_pkg_path)
    script.run('git', 'commit', '-q',
            '--author', 'Pip <python-virtualenv@googlegroups.com>',
            '-am', 'initial version', cwd=version_pkg_path)
    return version_pkg_path

def _change_test_package_version(script, version_pkg_path):
    version_pkg_path.join("version_pkg.py").write(textwrap.dedent('''\
        def main():
            print("some different version")'''))
    script.run('git', 'clean', '-qfdx',
        cwd=version_pkg_path,
        expect_stderr=True,
    )
    script.run('git', 'commit', '-q',
            '--author', 'Pip <python-virtualenv@googlegroups.com>',
            '-am', 'messed version',
            cwd=version_pkg_path, expect_stderr=True)

def assert_raises_regexp(exception, reg, run, *args, **kwargs):
    """Like assertRaisesRegexp in unittest"""
    __tracebackhide__ = True

    try:
        run(*args, **kwargs)
        assert False, "%s should have been thrown" % exception
    except Exception:
        e = sys.exc_info()[1]
        p = re.compile(reg)
        assert p.search(str(e)), str(e)
<|endoftext|>


<commit_before>import logging

import psutil

from flask import jsonify

import flask.ext.login

from . import app
from . import MMSupervisor

LOG = logging.getLogger(__name__)

@app.route('/supervisor', methods=['GET'])
@flask.ext.login.login_required
def service_status():
    try:
        supervisorstate = MMSupervisor.supervisor.getState()

    except:
        LOG.exception("Exception connecting to supervisor")
        return jsonify(result={'statename': 'STOPPED'})

    supervisorstate['processes'] = {}
    pinfo = MMSupervisor.supervisor.getAllProcessInfo()
    for p in pinfo:
        process = {
            'statename': p['statename']
        }

        try:
            ps = psutil.Process(pid=p['pid'])
            process['children'] = len(ps.children())

        except:
            LOG.exception("Error retrieving childen of %d" % p['pid'])
            pass

        supervisorstate['processes'][p['name']] = process

    return jsonify(result=supervisorstate)

@app.route('/supervisor/minemeld-core/start', methods=['GET'])
@flask.ext.login.login_required
def start_minemeld_core():
    result = MMSupervisor.supervisor.startProcess('minemeld-core', False)

    return jsonify(result=result)

@app.route('/supervisor/minemeld-core/stop', methods=['GET'])
@flask.ext.login.login_required
def stop_minemeld_core():
    result = MMSupervisor.supervisor.stopProcess('minemeld-core', False)

    return jsonify(result=result)
<commit_msg>added restart<commit_after>import logging

import psutil
import time
import gevent
import xmlrpclib
import supervisor.xmlrpc

from flask import jsonify

import flask.ext.login

from . import app
from . import config
from . import MMSupervisor

LOG = logging.getLogger(__name__)

def _restart_core():
    LOG.info('Restarting minemeld-core')

    supervisorurl = config.get('SUPERVISOR_URL',
                               'unix:///var/run/supervisor.sock')
    sserver = xmlrpclib.ServerProxy(
        'http://127.0.0.1',
        transport=supervisor.xmlrpc.SupervisorTransport(
            None,
            None,
            supervisorurl
        )
    )

    result = sserver.supervisor.stopProcess('minemeld-core', False)
    if not result:
        LOG.error('Stop minemeld-core returned False')
        return
    LOG.info('Stopped minemeld-core for API request')

    now = time.time()
    info = None
    while (time.time()-now) < 60*15*1000:
        info = sserver.supervisor.getProcessInfo('minemeld-core')
        if info['statename'] == 'STOPPED':
            break
        gevent.sleep(5)

    if info is not None and info['statename'] != 'STOPPED':
        LOG.error('Timeout during minemeld-core restart')
        return

    sserver.supervisor.startProcess('minemeld-core', False)
    LOG.info('Started minemeld-core')

@app.route('/supervisor', methods=['GET'])
@flask.ext.login.login_required
def service_status():
    try:
        supervisorstate = MMSupervisor.supervisor.getState()

    except:
        LOG.exception("Exception connecting to supervisor")
        return jsonify(result={'statename': 'STOPPED'})

    supervisorstate['processes'] = {}
    pinfo = MMSupervisor.supervisor.getAllProcessInfo()
    for p in pinfo:
        process = {
            'statename': p['statename'],
            'start': p['start']
        }

        try:
            ps = psutil.Process(pid=p['pid'])
            process['children'] = len(ps.children())

        except:
            LOG.exception("Error retrieving childen of %d" % p['pid'])
            pass

        supervisorstate['processes'][p['name']] = process

    return jsonify(result=supervisorstate)

@app.route('/supervisor/minemeld-core/start', methods=['GET'])
@flask.ext.login.login_required
def start_minemeld_core():
    result = MMSupervisor.supervisor.startProcess('minemeld-core', False)

    return jsonify(result=result)

@app.route('/supervisor/minemeld-core/stop', methods=['GET'])
@flask.ext.login.login_required
def stop_minemeld_core():
    result = MMSupervisor.supervisor.stopProcess('minemeld-core', False)

    return jsonify(result=result)

@app.route('/supervisor/minemeld-core/restart', methods=['GET'])
@flask.ext.login.login_required
def restart_minemeld_core():
    info = MMSupervisor.supervisor.getProcessInfo('minemeld-core')
    if info['statename'] != 'RUNNING':
        return jsonify(error={
            'message': ('minemeld-core not in RUNNING state: %s' %
                        info['statename'])
        }), 400

    gevent.spawn(_restart_core)

    return jsonify(result='OK')
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import re
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import ticker
from astropy.stats import sigma_clip
import astropy.io.ascii as ascii_io
import fitsio
import bass

def _convertfitsreg(regstr):
	regpattern = r'\[(\d+):(\d+),(\d+):(\d+)\]'
	rv =  [ int(d) for d in  re.match(regpattern,regstr).groups() ]
	# FITS region indices are 1-indexed
	rv[0] -= 1
	rv[2] -= 1
	return rv

def colbias(imhdu,method='median',xmargin1=5,xmargin2=2,ymargin=10,**kwargs):
	data = imhdu.read()
	hdr = imhdu.read_header()
	x1,x2,y1,y2 = _convertfitsreg(hdr['DATASEC'])
	im = data[y1:y2,x1:x2].astype(np.float32)
	x1,x2,y1,y2 = _convertfitsreg(hdr['BIASSEC'])
	bias = data[y1:y2,x1:x2].astype(np.float32)
	bias = sigma_clip(bias[ymargin:-ymargin,xmargin1:-xmargin2])
	bias = np.ma.median(bias).filled()[0]
	im -= bias
	return im,bias

def improcess(imhdu,extn=None,biasim=None,pixflatim=None,superskyim=None,
              **kwargs):
	im,bias = colbias(imhdu,**kwargs)
	if biasim is not None:
		im -= biasim[extn]
	if pixflatim is not None:
		im /= pixflatim[extn].data
	if superskyim is not None:
		im /= superskyim[extn]
	return im,bias

def load_flat_im(fn,npix2=100):
	from astropy.io import fits
	# fitsio doesn't like the way these files are written
	#pixflatim = fitsio.FITS(fn)
	pixflatim = fits.open(fn)
	x1,x2,y1,y2 = 1024-npix2,1024+npix2,1008-npix2,1008+npix2
	for extn in range(1,17):
		im = pixflatim[extn].data
		pix = sigma_clip(im[y1:y2,x1:x2],iters=2,sig=2.2)
		im /= pix.mean()
		print 'scaling ext ',extn,' by ',pix.mean()
	return pixflatim

def quickproc(fn,flatname,outfn,**kwargs):
	from astropy.io import fits
	pixflatim = load_flat_im(flatname)
	imhdu = fitsio.FITS(fn)
	hdul = [fits.PrimaryHDU()]
	for extn in range(1,17):
		im,bias = improcess(imhdu[extn],extn,pixflatim=pixflatim,**kwargs)
		hdr = fits.getheader(fn,extn)
		hdul.append(fits.ImageHDU(im,hdr))
	f = fits.HDUList(hdul)
	f.writeto(outfn,clobber=True)

def calc_raw_image_background(imagepath,extNum=None,**kwargs):
	margin = kwargs.get('stat_margin',500)
	stat_dtype = [('ampNum','i4'),('skyMean','f4'),('skyMedian','f4'),
	               ('skyVar','f4'),('biasLevel','f4')]
	fits = fitsio.FITS(imagepath)
	if extNum is None:
		extNums = range(1,17)
	elif type(extNum) in [int,str]:
		extNums = [extNum]
	else:
		extNums = extNum
	rv = np.empty(len(extNums),dtype=stat_dtype)
	for i,extn in enumerate(extNums):
		im,bias = improcess(fits[extn],extn,**kwargs)
		pix = sigma_clip(im[margin:-margin,margin:-margin],iters=3)
		extnum = int(fits[extn].get_extname().replace('IM','')) # "IM4"->4
		rv['ampNum'][i] = int(extnum)
		rv['skyMean'][i] = pix.mean()
		rv['skyMedian'][i] = np.ma.median(pix)
		rv['skyVar'][i] = pix.var()
		rv['biasLevel'][i] = bias
	return rv

def calc_sky_all():
	datadir = os.environ['GSCRATCH']
	master_pixflat = datadir+'/rmreduce/20150315/domeflat_g.fits'
	master_supersky = datadir+'/rmreduce/20150315/superflt_g.fits'
	obsdb = bass.load_obsdb()
	tile_dtype = [('utDate','S8'),('fileName','S12'),('expTime','f4')]
	tileList = []
	statList = []
	pixflatim = load_flat_im(master_pixflat)
	#superskyim = fitsio.FITS(master_supersky)
	for ti,tile in enumerate(obsdb):
		if (ti%10) != 0: continue # takes too long to do all of them
		print 'tile %d/%d (%s)' % (ti+1,len(obsdb),tile['fileName'])
		if tile['filter']=='g':
			try:
				depthstat = calc_depth_tile(tile)
				imagepath = os.path.join(bass.bass_data_dir,
				                         tile['utDate'],
				                         tile['fileName']+'.fits.gz')
				imstat = calc_raw_image_background(imagepath,
				                                   pixflatim=pixflatim)
				tileList.append((tile['utDate'],tile['fileName'],
				                 tile['expTime']))
				statList.append(imstat)
			except:
				print 'skipping ',tile['fileName']
				continue
	tileData = np.array(tileList,dtype=tile_dtype)
	statData = np.concatenate(statList)
	outfn = 'tile_stats.fits'
	fitsio.write(outfn,tileData,clobber=True)
	fitsio.write(outfn,statData)

def calc_depth_tile(tile,**kwargs):
	imagepath = os.path.join(bass.bass_data_dir,
	                         tile['utDate'],tile['fileName']+'.fits.gz')
	imstat = calc_raw_image_background(imagepath,
	                                   extNum=kwargs.get('extNum',4),**kwargs)
	return imstat

def calc_depth_all():
	obsdb = bass.load_obsdb()
	outf = open('imagestat.dat','w')
	for ti,tile in enumerate(obsdb):
		print 'tile %d/%d (%s)' % (ti+1,len(obsdb),tile['fileName'])
		if tile['filter']=='g':
			try:
				depthstat = calc_depth_tile(tile)
			except:
				print 'skipping ',tile['fileName']
				continue
			outf.write('%04d %8s %10s %6.1f ' %
			     (ti,tile['utDate'],tile['fileName'],tile['expTime']))
			outf.write('%8.1f %8.1f %8.1f\n' %
			     (depthstat['bias'],depthstat['medsky'],depthstat['rmssky']))
	outf.close()

def read_image_stats():
	imstat = ascii_io.read('imagestat.dat',
	                       names=['ti','utDate','fileName','expTime',
	                              'biasADU','skyADU','rmsADU'])
	return imstat

def strip_charts():
	imstat = read_image_stats()
	# XXX use per-image zeropoints once available
	zpt = 25.55
	skymag = -2.5*np.log10(imstat['skyADU']/imstat['expTime']*0.455**-2) + zpt
	plt.figure(figsize=(10,4))
	plt.subplots_adjust(0.06,0.13,0.99,0.98)
	ax1 = plt.subplot(111)
	ax1.plot(skymag)
	utds,ii = np.unique(imstat['utDate'],return_index=True)
	for i,utd in zip(ii,utds):
		offset = 1.5 if np.sum(imstat['utDate']==utd) < 30 else 0.2
		ax1.text(i,skymag[i]-offset,imstat['utDate'][i],size=8,
		         rotation='vertical',ha='left',va='bottom')
	ax1.axvspan(ii[utds==20150305],ii[utds==20150313],color='c',alpha=0.5)
	ax1.axhline(22.1,c='r')
	ax1.yaxis.set_minor_locator(ticker.MultipleLocator(0.2))
	ax1.set_ylim(22.5,17)
	ax1.set_xlim(-10,6200)
	ax1.set_xlabel('image number')
	ax1.set_ylabel('g sky brightness [mag/asec2]')

if __name__=='__main__':
	#calc_depth_all()
	calc_sky_all()

<commit_msg>set up for sky calculation of SDSS-RM data<commit_after>#!/usr/bin/env python

import os
import re
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import ticker
from astropy.stats import sigma_clip
import astropy.io.ascii as ascii_io
import fitsio
import bass

def _convertfitsreg(regstr):
	regpattern = r'\[(\d+):(\d+),(\d+):(\d+)\]'
	rv =  [ int(d) for d in  re.match(regpattern,regstr).groups() ]
	# FITS region indices are 1-indexed
	rv[0] -= 1
	rv[2] -= 1
	return rv

def colbias(imhdu,method='median',xmargin1=5,xmargin2=2,ymargin=10,**kwargs):
	data = imhdu.read()
	hdr = imhdu.read_header()
	x1,x2,y1,y2 = _convertfitsreg(hdr['DATASEC'])
	im = data[y1:y2,x1:x2].astype(np.float32)
	x1,x2,y1,y2 = _convertfitsreg(hdr['BIASSEC'])
	bias = data[y1:y2,x1:x2].astype(np.float32)
	bias = sigma_clip(bias[ymargin:-ymargin,xmargin1:-xmargin2])
	bias = np.ma.median(bias).filled()[0]
	im -= bias
	return im,bias

def improcess(imhdu,extn=None,biasim=None,pixflatim=None,superskyim=None,
              **kwargs):
	im,bias = colbias(imhdu,**kwargs)
	if biasim is not None:
		im -= biasim[extn]
	if pixflatim is not None:
		im /= pixflatim[extn].data
	if superskyim is not None:
		im /= superskyim[extn]
	return im,bias

def load_flat_im(fn,npix2=100):
	from astropy.io import fits
	# fitsio doesn't like the way these files are written
	#pixflatim = fitsio.FITS(fn)
	pixflatim = fits.open(fn)
	x1,x2,y1,y2 = 1024-npix2,1024+npix2,1008-npix2,1008+npix2
	for extn in range(1,17):
		im = pixflatim[extn].data
		pix = sigma_clip(im[y1:y2,x1:x2],iters=2,sig=2.2)
		im /= pix.mean()
		print 'scaling ext ',extn,' by ',pix.mean()
	return pixflatim

def quickproc(fn,flatname,outfn,**kwargs):
	from astropy.io import fits
	pixflatim = load_flat_im(flatname)
	imhdu = fitsio.FITS(fn)
	hdul = [fits.PrimaryHDU()]
	for extn in range(1,17):
		im,bias = improcess(imhdu[extn],extn,pixflatim=pixflatim,**kwargs)
		hdr = fits.getheader(fn,extn)
		hdul.append(fits.ImageHDU(im,hdr))
	f = fits.HDUList(hdul)
	f.writeto(outfn,clobber=True)

def calc_raw_image_background(imagepath,extNum=None,**kwargs):
	margin = kwargs.get('stat_margin',500)
	stat_dtype = [('ampNum','i4'),('skyMean','f4'),('skyMedian','f4'),
	               ('skyVar','f4'),('biasLevel','f4')]
	fits = fitsio.FITS(imagepath)
	if extNum is None:
		extNums = range(1,17)
	elif type(extNum) in [int,str]:
		extNums = [extNum]
	else:
		extNums = extNum
	rv = np.empty(len(extNums),dtype=stat_dtype)
	for i,extn in enumerate(extNums):
		im,bias = improcess(fits[extn],extn,**kwargs)
		pix = sigma_clip(im[margin:-margin,margin:-margin],iters=3)
		extnum = int(fits[extn].get_extname().replace('IM','')) # "IM4"->4
		rv['ampNum'][i] = int(extnum)
		rv['skyMean'][i] = pix.mean()
		rv['skyMedian'][i] = np.ma.median(pix)
		rv['skyVar'][i] = pix.var()
		rv['biasLevel'][i] = bias
	return rv

def calc_sky_all(survey='bass'):
	import boklog
	if survey=='bass':
		datadir = os.environ['GSCRATCH']
		master_pixflat = datadir+'/rmreduce/20150315/domeflat_g.fits'
		#master_supersky = datadir+'/rmreduce/20150315/superflt_g.fits'
		logs = boklog.load_Bok_logs('logs/')
		dpfx = ''
	else:
		datadir = os.environ['BOK90PRIMERAWDIR']
		master_pixflat = datadir+'/rmreduce/20140315/domeflat_g.fits'
		logs = boklog.load_Bok_logs()
		dpfx = 'ut'
	tile_dtype = [('utDate','S8'),('fileName','S12'),('expTime','f4')]
	tileList = []
	statList = []
	pixflatim = load_flat_im(master_pixflat)
	#superskyim = fitsio.FITS(master_supersky)
	for utd in utds:
		ii = np.where((logs[utd]['imType']=='object') &
		              (logs[utd]['expTime']>30.0) &
		              (logs[utd]['filter']=='g'))[0]
		# takes too long to do all of them
		stride = 10 if survey=='bass' else 1
		for _i,i in enumerate(ii[::stride]):
			tile = logs[utd][i]
			print '[%s] tile %d/%d (%s)' % (utd,_i+1,len(ii),tile['fileName'])
			try:
				imagepath = os.path.join(datadir,dpfx+tile['utDate'],
				                         tile['fileName']+'.fits.gz')
				imstat = calc_raw_image_background(imagepath,
				                                   pixflatim=pixflatim)
				tileList.append((tile['utDate'],tile['fileName'],
				                 tile['expTime']))
				statList.append(imstat)
			except:
				print 'skipping ',tile['fileName']
				continue
		break
	tileData = np.array(tileList,dtype=tile_dtype)
	statData = np.concatenate(statList)
	outfn = 'tile_stats_%s.fits' % survey
	fitsio.write(outfn,tileData,clobber=True)
	fitsio.write(outfn,statData)

def calc_depth_tile(tile,**kwargs):
	imagepath = os.path.join(bass.bass_data_dir,
	                         tile['utDate'],tile['fileName']+'.fits.gz')
	imstat = calc_raw_image_background(imagepath,
	                                   extNum=kwargs.get('extNum',4),**kwargs)
	return imstat

def calc_depth_all():
	obsdb = bass.load_obsdb()
	outf = open('imagestat.dat','w')
	for ti,tile in enumerate(obsdb):
		print 'tile %d/%d (%s)' % (ti+1,len(obsdb),tile['fileName'])
		if tile['filter']=='g':
			try:
				depthstat = calc_depth_tile(tile)
			except:
				print 'skipping ',tile['fileName']
				continue
			outf.write('%04d %8s %10s %6.1f ' %
			     (ti,tile['utDate'],tile['fileName'],tile['expTime']))
			outf.write('%8.1f %8.1f %8.1f\n' %
			     (depthstat['bias'],depthstat['medsky'],depthstat['rmssky']))
	outf.close()

def read_image_stats():
	imstat = ascii_io.read('imagestat.dat',
	                       names=['ti','utDate','fileName','expTime',
	                              'biasADU','skyADU','rmsADU'])
	return imstat

def strip_charts():
	imstat = read_image_stats()
	# XXX use per-image zeropoints once available
	zpt = 25.55
	skymag = -2.5*np.log10(imstat['skyADU']/imstat['expTime']*0.455**-2) + zpt
	plt.figure(figsize=(10,4))
	plt.subplots_adjust(0.06,0.13,0.99,0.98)
	ax1 = plt.subplot(111)
	ax1.plot(skymag)
	utds,ii = np.unique(imstat['utDate'],return_index=True)
	for i,utd in zip(ii,utds):
		offset = 1.5 if np.sum(imstat['utDate']==utd) < 30 else 0.2
		ax1.text(i,skymag[i]-offset,imstat['utDate'][i],size=8,
		         rotation='vertical',ha='left',va='bottom')
	ax1.axvspan(ii[utds==20150305],ii[utds==20150313],color='c',alpha=0.5)
	ax1.axhline(22.1,c='r')
	ax1.yaxis.set_minor_locator(ticker.MultipleLocator(0.2))
	ax1.set_ylim(22.5,17)
	ax1.set_xlim(-10,6200)
	ax1.set_xlabel('image number')
	ax1.set_ylabel('g sky brightness [mag/asec2]')

if __name__=='__main__':
	#calc_depth_all()
	#calc_sky_all()
	calc_sky_all('sdssrm')

<|endoftext|>


<commit_before>""" Various animation constants.

	Author: Alastair Hughes
"""

# Display constants:
BORDER = 20 # Empty space around the image, in pixels.
BROKEN_COLOUR = (0, 0, 255)
DEFAULT_COLOUR = (255, 255, 255)
EDGE_COLOUR = (0, 0, 0)
EDGE_THICKNESS = 1 # Some integer greater than or equal to one.
MAX_FPS = 24 # Maximum allowed FPS
MIN_FPS = 1 # Minimum allowed FPS
MAX_FRAMES_PER_DAY = 5 # Maximum number of frames per day
MIN_FRAMES_PER_DAY = 1 # Minimum number of frames per day
TEXT_COLOUR = (0, 0, 0) # The colour of any text.
TEXT_AA = False # Whether or not to antialias the text.

# Other:
DATE_FIELD = "Clock.Today" # Field name for dates.
PATCH_NUMBER_FIELD = 'PN' # Field name for patch numbers (in the GIS files).

# Temporary constants:
SCALE_DECIMAL_PLACES = 2 # Decimal places to display on the scale.
SCALE_MARKER_SIZE = 2 # Marker size, in pixels.
SCALE_TEXT_OFFSET = 5 # Offset of text from the scale, in pixels.
SCALE_WIDTH = 60 # Width of the scale, in pixels, including text.
<commit_msg>Added a note for BROKEN_COLOUR<commit_after>""" Various animation constants.

	Author: Alastair Hughes
"""

# Display constants:
BORDER = 20 # Empty space around the image, in pixels.
BROKEN_COLOUR = (0, 0, 255) # Colour for patches missing data.
DEFAULT_COLOUR = (255, 255, 255)
EDGE_COLOUR = (0, 0, 0)
EDGE_THICKNESS = 1 # Some integer greater than or equal to one.
MAX_FPS = 24 # Maximum allowed FPS
MIN_FPS = 1 # Minimum allowed FPS
MAX_FRAMES_PER_DAY = 5 # Maximum number of frames per day
MIN_FRAMES_PER_DAY = 1 # Minimum number of frames per day
TEXT_COLOUR = (0, 0, 0) # The colour of any text.
TEXT_AA = False # Whether or not to antialias the text.

# Other:
DATE_FIELD = "Clock.Today" # Field name for dates.
PATCH_NUMBER_FIELD = 'PN' # Field name for patch numbers (in the GIS files).

# Temporary constants:
SCALE_DECIMAL_PLACES = 2 # Decimal places to display on the scale.
SCALE_MARKER_SIZE = 2 # Marker size, in pixels.
SCALE_TEXT_OFFSET = 5 # Offset of text from the scale, in pixels.
SCALE_WIDTH = 60 # Width of the scale, in pixels, including text.
<|endoftext|>


<commit_before>from xobj import xobj
from collections import deque

from mint.rest.modellib.ordereddict import OrderedDict

_converters = {}

def getConverterFactory(format):
    return _converters[format]

def fromText(format, inputText, outputClass, controller, request):
    factory = getConverterFactory(format)
    converter = factory.getConverter(outputClass)
    context = Context(controller, request)
    return converter.fromText(inputText, outputClass, context)

def toText(format, inputObject, controller, request):
    factory = getConverterFactory(format)
    converter = factory.getConverter(inputObject.__class__)
    context = Context(controller, request)
    return converter.toText(inputObject, context)

class Converter(object):

    def walkModelClass(self, model):
        for fieldName in model._fields:
            field = getattr(model, fieldName)
            if hasattr(field, 'displayName') and field.displayName:
                fieldName = field.displayName
            yield fieldName, field, field.getModel()

    def walkModelInstance(self, modelInstance, context):
        for fieldName in modelInstance._fields:
            field = getattr(modelInstance.__class__, fieldName)
            value = getattr(modelInstance, fieldName)
            if hasattr(field, 'displayName') and field.displayName:
                fieldName = field.displayName

            childModel = field.getModelInstance(value, modelInstance, context)
            if childModel is None:
                value = field.valueToString(value, modelInstance, context)
            yield fieldName, field, value, childModel

    def walkModelClassAndObject(self, modelClass, object, accessMethod=getattr):
        for fieldName in modelClass._fields:
            field = getattr(modelClass, fieldName)
            if hasattr(field, 'displayName') and field.displayName:
                attrName = field.displayName
            else:
                attrName = fieldName
            value = accessMethod(object, attrName, None)
            childModel = field.getModel()
            yield fieldName, field, value, childModel

class XobjConverterFactory(Converter):
    _xobjClassCache = {None : str}

    def getConverter(self, outputClass):
        if outputClass in self._xobjClassCache:
            # shortcut for the common case
            return XobjConverter(self._xobjClassCache[outputClass])
        attrs = {}
        entry = (outputClass, None, None, None, attrs)

        # toProcess is FIFO - deque is better.
        # toCreate is FILO - so use standard list.
        toProcess = deque([entry])
        toCreate =  [ entry ]

        while toProcess:
            # esentially a DFS the order in which to build xobj classes.
            # last to be found is the first to be converted into xobj.
            # likely we could avoid creating as long a toCreate list
            # if we didn't
            modelClass, field, attrName, parentDict, attrs = toProcess.popleft()
            for entry in self.walkModelClass(modelClass):
                name, field, childModel  = entry
                if childModel:
                    entry = (childModel, field, name, attrs, {})
                    if childModel not in self._xobjClassCache:
                        toProcess.append(entry)
                        # holder so we don't create twice while processing
                        # this same model.
                        self._xobjClassCache[childModel] = None
                    toCreate.append(entry)
                else:
                    toCreate.append((None, field, name, attrs, None))

        while toCreate:
            modelClass, field, attrName, parentDict, attrs = toCreate.pop()
            xobjClass = self._xobjClassCache.get(modelClass, None)
            if not xobjClass:
                xobjClass = self.newXobjClass(modelClass, attrs)
                self._xobjClassCache[modelClass] = xobjClass
            if parentDict is not None:
                if field.isList():
                    parentDict[attrName] = [xobjClass]
                else:
                    parentDict[attrName] = xobjClass
            else:
                return XobjConverter(xobjClass)

    def newXobjClass(self, modelClass, attrs):
        attrs['__module__'] =  __name__
        attrs['_xobj'] = xobj.XObjMetadata(
            elements = modelClass._elements,
            attributes = OrderedDict.fromkeys(modelClass._attributes))
        className = modelClass._meta.name
        return type.__new__(type, className, xobj.XObj.__mro__, attrs)

class Context(object):
    def __init__(self, controller, request):
        self.controller = controller
        self.request = request

class XobjConverter(Converter):
    def __init__(self, xobjClass):
        self.xobjClass = xobjClass

    def getXobjObject(self, modelInstance, context):
        returnObject = self.xobjClass()
        origModelInstance = modelInstance
        toProcess = deque([ (returnObject, modelInstance) ])

        while toProcess:
            xobject, modelInstance = toProcess.popleft()
            xobjClass = xobject.__class__
            for entry in self.walkModelInstance(modelInstance, context):
                name, field, value, childModel = entry
                if field.isText:
                    xobject._xobj.text = value
                elif childModel is None:
                    setattr(xobject, name, value)
                elif field.isList():
                    childXobjClass = getattr(xobject.__class__, name)[0]
                    lst = []
                    setattr(xobject, name, lst)
                    for subModel in value:
                        childXobj = childXobjClass()
                        lst.append(childXobj)
                        toProcess.append((childXobj, subModel))
                else:
                    childXobj = getattr(xobject.__class__, name)()
                    setattr(xobject, name, childXobj)
                    toProcess.append((childXobj, childModel))
        return returnObject

    def fromXobjObject(self, xobjObject, modelClass, context):
        origModelClass = modelClass
        entry = (modelClass, xobjObject, {}, None, None)

        # toProcess is FIFO - deque is better.
        toProcess = [(entry, False)]

        while toProcess:
            entry, finished = toProcess.pop()
            modelClass, xobject, attrs, parentDict, attrName = entry
            if not finished:
                toProcess.append((entry, True))
                for fieldData in self.walkModelClassAndObject(modelClass,
                                                              xobject):
                    name, field, value, childModel  = fieldData
                    if childModel:
                        entry = (childModel, value, {}, attrs, name)
                        toProcess.append((entry, False))
                    elif field.isList():
                        lst = []
                        entry = (None, lst, childAttrs, attrs, name)
                        toProcess.append((entry, True))
                        for lstValue in value:
                            # doesn't work for non-Model lists.
                            subEntry = (childModel, lstValue, childAttrs, lst,
                                        None)
                            toProcess.append((subEntry, False))
                    else:
                        attrs[name] = field.valueFromString(value)
            else:
                if modelClass:
                    item = modelClass(**attrs)
                else:
                    item = field.valueFromString(xobject)
                if parentDict is not None:
                    parentDict[attrName] = item
                elif modelClass is origModelClass:
                    return item

    def toText(self, modelInstance, context):
        xobjObject = self.getXobjObject(modelInstance, context)
        return xobj.toxml(xobjObject, xobjObject.__class__.__name__)

    def fromText(self, text, modelClass, context):
        className = self.xobjClass.__name__
        obj = xobj.parse(text, typeMap={className : self.xobjClass})
        xobjObject = getattr(obj, className)
        return self.fromXobjObject(xobjObject, modelClass, context)

_converters['xml'] = XobjConverterFactory()
<commit_msg>Properly parse list objects<commit_after>from xobj import xobj
from collections import deque

from mint.rest.modellib.ordereddict import OrderedDict

_converters = {}

def getConverterFactory(format):
    return _converters[format]

def fromText(format, inputText, outputClass, controller, request):
    factory = getConverterFactory(format)
    converter = factory.getConverter(outputClass)
    context = Context(controller, request)
    return converter.fromText(inputText, outputClass, context)

def toText(format, inputObject, controller, request):
    factory = getConverterFactory(format)
    converter = factory.getConverter(inputObject.__class__)
    context = Context(controller, request)
    return converter.toText(inputObject, context)

class Converter(object):

    def walkModelClass(self, model):
        for fieldName in model._fields:
            field = getattr(model, fieldName)
            if hasattr(field, 'displayName') and field.displayName:
                fieldName = field.displayName
            yield fieldName, field, field.getModel()

    def walkModelInstance(self, modelInstance, context):
        for fieldName in modelInstance._fields:
            field = getattr(modelInstance.__class__, fieldName)
            value = getattr(modelInstance, fieldName)
            if hasattr(field, 'displayName') and field.displayName:
                fieldName = field.displayName

            childModel = field.getModelInstance(value, modelInstance, context)
            if childModel is None:
                value = field.valueToString(value, modelInstance, context)
            yield fieldName, field, value, childModel

    def walkModelClassAndObject(self, modelClass, object, accessMethod=getattr):
        for fieldName in modelClass._fields:
            field = getattr(modelClass, fieldName)
            if hasattr(field, 'displayName') and field.displayName:
                attrName = field.displayName
            else:
                attrName = fieldName
            value = accessMethod(object, attrName, None)
            childModel = field.getModel()
            yield fieldName, field, value, childModel

class XobjConverterFactory(Converter):
    _xobjClassCache = {None : str}

    def getConverter(self, outputClass):
        if outputClass in self._xobjClassCache:
            # shortcut for the common case
            return XobjConverter(self._xobjClassCache[outputClass])
        attrs = {}
        entry = (outputClass, None, None, None, attrs)

        # toProcess is FIFO - deque is better.
        # toCreate is FILO - so use standard list.
        toProcess = deque([entry])
        toCreate =  [ entry ]

        while toProcess:
            # esentially a DFS the order in which to build xobj classes.
            # last to be found is the first to be converted into xobj.
            # likely we could avoid creating as long a toCreate list
            # if we didn't
            modelClass, field, attrName, parentDict, attrs = toProcess.popleft()
            for entry in self.walkModelClass(modelClass):
                name, field, childModel  = entry
                if childModel:
                    entry = (childModel, field, name, attrs, {})
                    if childModel not in self._xobjClassCache:
                        toProcess.append(entry)
                        # holder so we don't create twice while processing
                        # this same model.
                        self._xobjClassCache[childModel] = None
                    toCreate.append(entry)
                else:
                    toCreate.append((None, field, name, attrs, None))

        while toCreate:
            modelClass, field, attrName, parentDict, attrs = toCreate.pop()
            xobjClass = self._xobjClassCache.get(modelClass, None)
            if not xobjClass:
                xobjClass = self.newXobjClass(modelClass, attrs)
                self._xobjClassCache[modelClass] = xobjClass
            if parentDict is not None:
                if field.isList():
                    parentDict[attrName] = [xobjClass]
                else:
                    parentDict[attrName] = xobjClass
            else:
                return XobjConverter(xobjClass)

    def newXobjClass(self, modelClass, attrs):
        attrs['__module__'] =  __name__
        attrs['_xobj'] = xobj.XObjMetadata(
            elements = modelClass._elements,
            attributes = OrderedDict.fromkeys(modelClass._attributes))
        className = modelClass._meta.name
        return type.__new__(type, className, xobj.XObj.__mro__, attrs)

class Context(object):
    def __init__(self, controller, request):
        self.controller = controller
        self.request = request

class XobjConverter(Converter):
    def __init__(self, xobjClass):
        self.xobjClass = xobjClass

    def getXobjObject(self, modelInstance, context):
        returnObject = self.xobjClass()
        origModelInstance = modelInstance
        toProcess = deque([ (returnObject, modelInstance) ])

        while toProcess:
            xobject, modelInstance = toProcess.popleft()
            xobjClass = xobject.__class__
            for entry in self.walkModelInstance(modelInstance, context):
                name, field, value, childModel = entry
                if field.isText:
                    xobject._xobj.text = value
                elif childModel is None:
                    setattr(xobject, name, value)
                elif field.isList():
                    childXobjClass = getattr(xobject.__class__, name)[0]
                    lst = []
                    setattr(xobject, name, lst)
                    for subModel in value:
                        childXobj = childXobjClass()
                        lst.append(childXobj)
                        toProcess.append((childXobj, subModel))
                else:
                    childXobj = getattr(xobject.__class__, name)()
                    setattr(xobject, name, childXobj)
                    toProcess.append((childXobj, childModel))
        return returnObject

    def fromXobjObject(self, xobjObject, modelClass, context):
        entry = (modelClass, xobjObject, {}, None, None)

        # toProcess is FIFO - deque is better.
        toProcess = [(entry, False)]

        while toProcess:
            entry, finished = toProcess.pop()
            modelClass, xobject, attrs, parentDict, attrName = entry
            if not finished:
                toProcess.append((entry, True))
                for fieldData in self.walkModelClassAndObject(modelClass,
                                                              xobject):
                    name, field, value, childModel = fieldData
                    if field.isList():
                        # Lists get automatically processed by their children,
                        # by appending to the valie of parentDict (which is
                        # now a list :-/ )
                        # There is no need to finalize a list
                        lst = attrs[name] = []
                        for lstValue in value:
                            # doesn't work for non-Model lists.
                            # List sub-entries do not get their own name, it
                            # is part of the child model
                            subEntry = (childModel, lstValue, {}, lst,
                                        None)
                            toProcess.append((subEntry, False))
                    elif childModel:
                        entry = (childModel, value, {}, attrs, name)
                        toProcess.append((entry, False))
                    else:
                        attrs[name] = field.valueFromString(value)
            else:
                if modelClass:
                    item = modelClass(**attrs)
                else:
                    item = field.valueFromString(xobject)
                if parentDict is not None:
                    if attrName is None:
                        # This is a sub-node of a list
                        parentDict.append(item)
                    else:
                        parentDict[attrName] = item
                elif not toProcess:
                    # queue is empty, return the results
                    return item

    def toText(self, modelInstance, context):
        xobjObject = self.getXobjObject(modelInstance, context)
        return xobj.toxml(xobjObject, xobjObject.__class__.__name__)

    def fromText(self, text, modelClass, context):
        className = self.xobjClass.__name__
        obj = xobj.parse(text, typeMap={className : self.xobjClass})
        xobjObject = getattr(obj, className)
        return self.fromXobjObject(xobjObject, modelClass, context)

_converters['xml'] = XobjConverterFactory()
<|endoftext|>


<commit_before>#
# server.py - Max Usachev <maxusachev@gmail.com>
#             Ed Bartosh <bartosh@gmail.com>
#             Peter Bienstman <Peter.Bienstman@UGent.be>

import os
import cgi
import select
import tarfile
import tempfile
import cStringIO
from wsgiref.simple_server import WSGIServer, WSGIRequestHandler

from utils import tar_file_size
from data_format import DataFormat

# Avoid delays caused by Nagle's algorithm.
# http://www.cmlenz.net/archives/2008/03/python-httplib-performance-problems

import socket
realsocket = socket.socket
def socketwrap(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
    sockobj = realsocket(family, type, proto)
    sockobj.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
    return sockobj
socket.socket = socketwrap

# Work around http://bugs.python.org/issue6085.

def not_insane_address_string(self):
    host, port = self.client_address[:2]
    return "%s (no getfqdn)" % host

WSGIRequestHandler.address_string = not_insane_address_string

# Don't pollute our testsuite output.

def dont_log(*kwargs):
    pass

WSGIRequestHandler.log_message = dont_log

class Server(WSGIServer):

    program_name = "unknown-SRS-app"
    program_version = "unknown"
    capabilities = None  # TODO: list possibilies.

    stop_after_sync = False # Setting this True is useful for the testsuite.

    def __init__(self, host, port, ui):
        WSGIServer.__init__(self, (host, port), WSGIRequestHandler)
        self.set_app(self.wsgi_app)
        self.ui = ui
        self.data_format = DataFormat()
        self.stopped = False
        self.logged_in = False
        self.id = "TODO"
        self.client_info = {}

    def wsgi_app(self, environ, start_response):
        status, mime, method, args = self.get_method(environ)
        response_headers = [("Content-type", mime)]
        start_response(status, response_headers)
        if method:
            return getattr(self, method)(environ, **args)
        else:
            return status

    def serve_forever(self):
        self.ui.status_bar_message("Waiting for client connection...")
        timeout = 1
        while not self.stopped:
            if select.select([self.socket], [], [], timeout)[0]:
                self.handle_request()
        self.socket.close()

    def get_method(self, environ):
        # Convert e.g. GET /foo/bar into get_foo_bar.
        method = (environ["REQUEST_METHOD"] + \
                "_".join(environ["PATH_INFO"].split("/"))).lower()
        if method != "put_login" and not self.logged_in:
            return "403 Forbidden", "text/plain", None, None
        if hasattr(self, method) and callable(getattr(self, method)):
            args = cgi.parse_qs(environ["QUERY_STRING"])
            args = dict([(key, val[0]) for key, val in args.iteritems()])
            if len(args) == 0:
                return "200 OK", "xml/text", method, args
            else:
                return "400 Bad Request", "text/plain", None, None
        else:
            return "404 Not Found", "text/plain", None, None

    def stop(self):
        self.stopped = True

    # The following functions are to be overridden by the actual server code,
    # to implement e.g. authorisation and storage.

    def authorise(self, username, password):

        """Returns true if password correct for username."""

        raise NotImplementedError

    def open_database(self, database_name):

        """Sets self.database to a database object for the database named
        'database_name'.

        """

        raise NotImplementedError

    # The following are methods that are supported by the server through GET
    # and PUT calls. 'get_foo_bar' gets executed after a 'GET /foo/bar'
    # request. Similarly, 'put_foo_bar' gets executed after a 'PUT /foo/bar'
    # request.

    def put_login(self, environ):
        self.ui.status_bar_message("Client logging in...")
        client_info_repr = environ["wsgi.input"].readline()
        self.client_info = \
            self.data_format.parse_partner_info(client_info_repr)
        if not self.authorise(self.client_info["username"],
            self.client_info["password"]):
            self.logged_in = False
            return "403 Forbidden"
        self.logged_in = True
        self.open_database(self.client_info["database_name"])
        self.database.backup()
        self.database.create_partnership_if_needed_for(self.client_info["id"])
        server_info = {"id": self.id, "program_name": self.program_name,
            "program_version": self.program_version,
            "capabilities": self.capabilities}
        return self.data_format.repr_partner_info(server_info)

    def put_client_log_entries(self, environ):
        self.ui.status_bar_message("Receiving client log entries...")
        # Since chunked requests are not supported by the WSGI 1.x standard,
        # the client does not set Content-Length in order to be able to
        # stream the log entries. Therefore, it is our responsability that we
        # consume the entire stream, nothing more and nothing less. For that,
        # we use the file format footer as a sentinel.
        # For simplicity, we also keep the entire stream in memory, as the
        # server is not expected to be resource limited.
        sentinel = self.data_format.log_entries_footer
        socket = environ["wsgi.input"]
        number_of_entries = int(socket.readline())
        progress_dialog = self.ui.get_progress_dialog()
        progress_dialog.set_range(0, number_of_entries)
        lines = []
        line = socket.readline()
        lines.append(line)
        while not line.endswith(sentinel):
            line = socket.readline()
            lines.append(line)
        # In order to do conflict resolution easily, one of the sync partners
        # has to have both logs in memory. We do this at the server side, as
        # the client could be resource-limited mobile device.
        self.client_log = []
        count = 0
        data_stream = cStringIO.StringIO("".join(lines))
        for log_entry in self.data_format.parse_log_entries(data_stream):
            self.client_log.append(log_entry)
            count += 1
            progress_dialog.set_value(count)
        self.ui.status_bar_message("Waiting for client to finish...")
        return "OK"

    def get_server_log_entries(self, environ):
        self.ui.status_bar_message("Sending log entries to client...")
        number_of_entries = self.database.number_of_log_entries_to_sync_for(\
            self.client_info["id"])
        progress_dialog = self.ui.get_progress_dialog()
        progress_dialog.set_range(0, number_of_entries)
        progress_dialog.set_text("Sending log entries to client...")
        buffer = str(number_of_entries) + "\n"
        buffer += self.data_format.log_entries_header
        BUFFER_SIZE = 8192
        count = 0
        for log_entry in self.database.log_entries_to_sync_for(\
            self.client_info["id"]):
            count += 1
            progress_dialog.set_value(count)
            buffer += self.data_format.repr_log_entry(log_entry)
            if len(buffer) > BUFFER_SIZE:
                yield buffer.encode("utf-8")
                buffer = ""
        buffer += self.data_format.log_entries_footer
        yield buffer.encode("utf-8")

    def put_client_media_files(self, environ):
        self.ui.status_bar_message("Receiving client media files...")
        try:
            socket = environ["wsgi.input"]
            size = int(socket.readline())
            tar_pipe = tarfile.open(mode="r|", fileobj=socket)
            # Work around http://bugs.python.org/issue7693.
            tar_pipe.extractall(self.database.mediadir().encode("utf-8"))
        except:
            return "CANCEL"
        return "OK"

    def get_server_media_files(self, environ):
        self.ui.status_bar_message("Sending media files to client...")
        try:
            # Determine files to send across.
            self.database.check_for_updated_media_files()
            filenames = list(self.database.media_filenames_to_sync_for(\
                self.client_info["id"]))
            if len(filenames) == 0:
                yield "0\n"
                return
            # Create a temporary tar file with the files.
            tmp_file = tempfile.NamedTemporaryFile(delete=False)
            tmp_file_name = tmp_file.name
            saved_path = os.getcwdu()
            os.chdir(self.database.mediadir())
            BUFFER_SIZE = 8192
            tar_pipe = tarfile.open(mode="w|", fileobj=tmp_file,
                bufsize=BUFFER_SIZE, format=tarfile.PAX_FORMAT)
            for filename in filenames:
                tar_pipe.add(filename)
            tar_pipe.close()
            # Determine tar file size.
            tmp_file = file(tmp_file_name, "rb")
            file_size = os.path.getsize(tmp_file_name)
            progress_dialog = self.ui.get_progress_dialog()
            progress_dialog.set_range(0, file_size)
            # Send tar file across.
            progress_dialog.set_text("Sending media files to client...")
            buffer = str(file_size) + "\n" + tmp_file.read(BUFFER_SIZE)
            count = BUFFER_SIZE
            while buffer:
                progress_dialog.set_value(count)
                yield buffer
                buffer = tmp_file.read(BUFFER_SIZE)
                count += BUFFER_SIZE
            progress_dialog.set_value(file_size)
            os.remove(tmp_file_name)
            os.chdir(saved_path)
        except:
            yield "CANCEL"

    def get_sync_finish(self, environ):
        self.ui.status_bar_message("Waiting for client to finish...")
        for log_entry in self.client_log:
            self.database.apply_log_entry(log_entry)
        self.database.update_last_sync_log_entry_for(self.client_info["id"])
        self.logged_in = False
        if self.stop_after_sync:
            self.stop()
        return "OK"
<commit_msg>Start working on applying the client log entries as soon as possible.<commit_after>#
# server.py - Max Usachev <maxusachev@gmail.com>
#             Ed Bartosh <bartosh@gmail.com>
#             Peter Bienstman <Peter.Bienstman@UGent.be>

import os
import cgi
import select
import tarfile
import tempfile
import cStringIO
from wsgiref.simple_server import WSGIServer, WSGIRequestHandler

from utils import tar_file_size
from data_format import DataFormat

# Avoid delays caused by Nagle's algorithm.
# http://www.cmlenz.net/archives/2008/03/python-httplib-performance-problems

import socket
realsocket = socket.socket
def socketwrap(family=socket.AF_INET, type=socket.SOCK_STREAM, proto=0):
    sockobj = realsocket(family, type, proto)
    sockobj.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
    return sockobj
socket.socket = socketwrap

# Work around http://bugs.python.org/issue6085.

def not_insane_address_string(self):
    host, port = self.client_address[:2]
    return "%s (no getfqdn)" % host

WSGIRequestHandler.address_string = not_insane_address_string

# Don't pollute our testsuite output.

def dont_log(*kwargs):
    pass

WSGIRequestHandler.log_message = dont_log

class Server(WSGIServer):

    program_name = "unknown-SRS-app"
    program_version = "unknown"
    capabilities = None  # TODO: list possibilies.

    stop_after_sync = False # Setting this True is useful for the testsuite.

    def __init__(self, host, port, ui):
        WSGIServer.__init__(self, (host, port), WSGIRequestHandler)
        self.set_app(self.wsgi_app)
        self.ui = ui
        self.data_format = DataFormat()
        self.stopped = False
        self.logged_in = False
        self.id = "TODO"
        self.client_info = {}

    def wsgi_app(self, environ, start_response):
        status, mime, method, args = self.get_method(environ)
        response_headers = [("Content-type", mime)]
        start_response(status, response_headers)
        if method:
            return getattr(self, method)(environ, **args)
        else:
            return status

    def serve_forever(self):
        self.ui.status_bar_message("Waiting for client connection...")
        timeout = 1
        while not self.stopped:
            if select.select([self.socket], [], [], timeout)[0]:
                self.handle_request()
        self.socket.close()

    def get_method(self, environ):
        # Convert e.g. GET /foo/bar into get_foo_bar.
        method = (environ["REQUEST_METHOD"] + \
                "_".join(environ["PATH_INFO"].split("/"))).lower()
        if method != "put_login" and not self.logged_in:
            return "403 Forbidden", "text/plain", None, None
        if hasattr(self, method) and callable(getattr(self, method)):
            args = cgi.parse_qs(environ["QUERY_STRING"])
            args = dict([(key, val[0]) for key, val in args.iteritems()])
            if len(args) == 0:
                return "200 OK", "xml/text", method, args
            else:
                return "400 Bad Request", "text/plain", None, None
        else:
            return "404 Not Found", "text/plain", None, None

    def stop(self):
        self.stopped = True

    # The following functions are to be overridden by the actual server code,
    # to implement e.g. authorisation and storage.

    def authorise(self, username, password):

        """Returns true if password correct for username."""

        raise NotImplementedError

    def open_database(self, database_name):

        """Sets self.database to a database object for the database named
        'database_name'.

        """

        raise NotImplementedError

    # The following are methods that are supported by the server through GET
    # and PUT calls. 'get_foo_bar' gets executed after a 'GET /foo/bar'
    # request. Similarly, 'put_foo_bar' gets executed after a 'PUT /foo/bar'
    # request.

    def put_login(self, environ):
        self.ui.status_bar_message("Client logging in...")
        client_info_repr = environ["wsgi.input"].readline()
        self.client_info = \
            self.data_format.parse_partner_info(client_info_repr)
        if not self.authorise(self.client_info["username"],
            self.client_info["password"]):
            self.logged_in = False
            return "403 Forbidden"
        self.logged_in = True
        self.open_database(self.client_info["database_name"])
        self.database.backup()
        self.database.create_partnership_if_needed_for(self.client_info["id"])
        server_info = {"id": self.id, "program_name": self.program_name,
            "program_version": self.program_version,
            "capabilities": self.capabilities}
        return self.data_format.repr_partner_info(server_info)

    def put_client_log_entries(self, environ):
        self.ui.status_bar_message("Receiving client log entries...")
        # Since chunked requests are not supported by the WSGI 1.x standard,
        # the client does not set Content-Length in order to be able to
        # stream the log entries. Therefore, it is our responsability that we
        # consume the entire stream, nothing more and nothing less. For that,
        # we use the file format footer as a sentinel.
        # For simplicity, we also keep the entire stream in memory, as the
        # server is not expected to be resource limited.
        sentinel = self.data_format.log_entries_footer
        socket = environ["wsgi.input"]
        number_of_entries = int(socket.readline())
        progress_dialog = self.ui.get_progress_dialog()
        progress_dialog.set_range(0, number_of_entries)
        lines = []
        line = socket.readline()
        lines.append(line)
        while not line.endswith(sentinel):
            line = socket.readline()
            lines.append(line)
        # In order to do conflict resolution easily, one of the sync partners
        # has to have both logs in memory. We do this at the server side, as
        # the client could be resource-limited mobile device.
        self.client_log = []
        count = 0
        data_stream = cStringIO.StringIO("".join(lines))
        for log_entry in self.data_format.parse_log_entries(data_stream):
            self.client_log.append(log_entry)
            count += 1
            progress_dialog.set_value(count)
        self.ui.status_bar_message("Waiting for client to finish...")
        return "OK"

    def get_server_log_entries(self, environ):
        self.ui.status_bar_message("Sending log entries to client...")
        number_of_entries = self.database.number_of_log_entries_to_sync_for(\
            self.client_info["id"])
        progress_dialog = self.ui.get_progress_dialog()
        progress_dialog.set_range(0, number_of_entries)
        progress_dialog.set_text("Sending log entries to client...")
        buffer = str(number_of_entries) + "\n"
        buffer += self.data_format.log_entries_header
        BUFFER_SIZE = 8192
        count = 0
        for log_entry in self.database.log_entries_to_sync_for(\
            self.client_info["id"]):
            count += 1
            progress_dialog.set_value(count)
            buffer += self.data_format.repr_log_entry(log_entry)
            if len(buffer) > BUFFER_SIZE:
                yield buffer.encode("utf-8")
                buffer = ""
        buffer += self.data_format.log_entries_footer
        yield buffer.encode("utf-8")
        # Now that all the data is underway to the client, we can start
        # applying the client log entries.
        for log_entry in self.client_log:
            self.database.apply_log_entry(log_entry)
        self.database.update_last_sync_log_entry_for(self.client_info["id"])

    def put_client_media_files(self, environ):
        self.ui.status_bar_message("Receiving client media files...")
        try:
            socket = environ["wsgi.input"]
            size = int(socket.readline())
            tar_pipe = tarfile.open(mode="r|", fileobj=socket)
            # Work around http://bugs.python.org/issue7693.
            tar_pipe.extractall(self.database.mediadir().encode("utf-8"))
        except:
            return "CANCEL"
        return "OK"

    def get_server_media_files(self, environ):
        self.ui.status_bar_message("Sending media files to client...")
        try:
            # Determine files to send across.
            self.database.check_for_updated_media_files()
            filenames = list(self.database.media_filenames_to_sync_for(\
                self.client_info["id"]))
            if len(filenames) == 0:
                yield "0\n"
                return
            # Create a temporary tar file with the files.
            tmp_file = tempfile.NamedTemporaryFile(delete=False)
            tmp_file_name = tmp_file.name
            saved_path = os.getcwdu()
            os.chdir(self.database.mediadir())
            BUFFER_SIZE = 8192
            tar_pipe = tarfile.open(mode="w|", fileobj=tmp_file,
                bufsize=BUFFER_SIZE, format=tarfile.PAX_FORMAT)
            for filename in filenames:
                tar_pipe.add(filename)
            tar_pipe.close()
            # Determine tar file size.
            tmp_file = file(tmp_file_name, "rb")
            file_size = os.path.getsize(tmp_file_name)
            progress_dialog = self.ui.get_progress_dialog()
            progress_dialog.set_range(0, file_size)
            # Send tar file across.
            progress_dialog.set_text("Sending media files to client...")
            buffer = str(file_size) + "\n" + tmp_file.read(BUFFER_SIZE)
            count = BUFFER_SIZE
            while buffer:
                progress_dialog.set_value(count)
                yield buffer
                buffer = tmp_file.read(BUFFER_SIZE)
                count += BUFFER_SIZE
            progress_dialog.set_value(file_size)
            os.remove(tmp_file_name)
            os.chdir(saved_path)
        except:
            yield "CANCEL"

    def get_sync_finish(self, environ):
        self.ui.status_bar_message("Waiting for client to finish...")
        self.logged_in = False
        if self.stop_after_sync:
            self.stop()
        return "OK"
<|endoftext|>


<commit_before>from PyQt5 import QtWidgets
import re
import json
import subprocess

class Monitor(object):
    LAYOUT_TILED   = 1
    LAYOUT_MONOCLE = 2

    def __init__(self):
        self.name       = None
        self.focused    = False
        self.workspaces = []
        self.layout     = None
        self.x          = 0
        self.y          = 0
        self.width      = 0
        self.height     = 0

class Workspace(object):
    def __init__(self):
        self.name    = None
        self.rect    = None
        self.focused = False
        self.free    = True
        self.urgent  = False

class WorkspacesProvider(object):
    delay = 0
    window_title_provider = None

    @staticmethod
    def get_monitors():
        proc = subprocess.Popen(['bspc', 'query', '-M'], stdout=subprocess.PIPE)
        monitor_names = [l for l in proc.stdout.read().decode('utf8').splitlines() if l]
        monitors = []
        for monitor_name in monitor_names:
            proc = subprocess.Popen(['bspc', 'query', '-T', '-m', monitor_name], stdout=subprocess.PIPE)
            monitor_spec = json.loads(proc.stdout.read().decode('utf8'))
            monitor = Monitor()
            monitor.original_id = len(monitors)
            monitor.name = monitor_name
            monitor.width = int(monitor_spec['rectangle']['width'])
            monitor.height = int(monitor_spec['rectangle']['height'])
            monitor.x = int(monitor_spec['rectangle']['x'])
            monitor.y = int(monitor_spec['rectangle']['y'])
            for desktop_spec in monitor_spec['desktops']:
                workspace = Workspace()
                workspace.name = desktop_spec['name']
                workspace.focused = workspace.name == monitor_spec['focusedDesktopName']
                workspace.free = desktop_spec['root'] is not None
                monitor.workspaces.append(workspace)
            monitors.append(monitor)
        monitors.sort(key=lambda m: m.x)
        for i, m in enumerate(monitors):
            m.display_id = i
        return monitors

    def __init__(self, main_window):
        self.main_window = main_window

        self.bspc_process = subprocess.Popen(
            ['bspc', 'subscribe'], stdout=subprocess.PIPE)
        self.monitors = self.get_monitors()
        self.refresh_workspaces()

        self.widgets = {}
        for i, monitor in enumerate(self.monitors):
            monitor_widget = main_window[i].left_widget
            monitor_widget.ws_widgets = {}
            monitor_widget.wheelEvent = lambda event, monitor=monitor: self.wheel(event, monitor)
            for j, ws in enumerate(monitor.workspaces):
                ws_widget = QtWidgets.QPushButton(ws.name)
                ws_widget.setProperty('class', 'workspace')
                ws_widget.mouseReleaseEvent = lambda event, ws=ws: self.click(event, ws)
                monitor_widget.ws_widgets[j] = ws_widget
                monitor_widget.layout().addWidget(ws_widget)
            self.widgets[i] = monitor_widget
        self.render()

    def wheel(self, event, monitor):
        subprocess.call(['bspc', 'monitor', '-f', monitor.name])
        subprocess.call(['bspc', 'desktop', '-f', ['prev', 'next'][event.angleDelta().y() > 0]])

    def click(self, event, ws):
        subprocess.call(['bspc', 'desktop', '-f', ws.name])

    def refresh_workspaces(self):
        line = self.bspc_process.stdout.readline().decode('utf8').strip()
        line = re.sub('^W', '', line)
        if line == '':
            return

        for item in line.split(':'):
            key, value = item[0], item[1:]

            if key in 'mM':
                chosen_monitor = [m for m in self.monitors if m.name == value]
                current_monitor = chosen_monitor[0]
                current_monitor.focused = key.isupper()

            elif key in 'oOfFuU':
                workspace = [w for w in current_monitor.workspaces if w.name == value][0]
                workspace.focused = key.isupper()
                workspace.free = key in 'fF'
                workspace.urgent = key in 'uU'

            elif key in 'lL':
                if value in 'mM':
                    current_monitor.layout = Monitor.LAYOUT_MONOCLE
                elif value in 'tT':
                    current_monitor.layout = Monitor.LAYOUT_TILED

    def refresh(self):
        self.refresh_workspaces()

    def render(self):
        for i, monitor in enumerate(self.monitors):
            for j, ws in enumerate(monitor.workspaces):
                self.widgets[i].ws_widgets[j].setProperty('ws_focused', '%s' % ws.focused)
                self.widgets[i].ws_widgets[j].setProperty('ws_urgent', '%s' % ws.urgent)
                self.widgets[i].ws_widgets[j].setProperty('ws_free', '%s' % ws.free)
                if ws.focused and ws.free and WorkspacesProvider.window_title_provider is not None:
                    WorkspacesProvider.window_title_provider.labels[monitor.display_id].setText('')
                    WorkspacesProvider.window_title_provider.window_names[monitor.display_id] = ''
        self.main_window.reloadStyleSheet()
<commit_msg>mod-bspwm: simplified subscription<commit_after>from PyQt5 import QtWidgets
import re
import json
import subprocess

def run(process):
    proc = subprocess.Popen(process, stdout=subprocess.PIPE)
    return proc.stdout.read().decode('utf8')

class Monitor(object):
    def __init__(self):
        self.name         = None
        self.workspaces   = []
        self.x            = 0
        self.y            = 0
        self.width        = 0
        self.height       = 0

class Workspace(object):
    def __init__(self):
        self.name    = None
        self.rect    = None
        self.focused = False
        self.free    = True
        self.urgent  = False

class WorkspacesProvider(object):
    delay = 0
    monitor_names = None

    @staticmethod
    def get_monitors():
        if WorkspacesProvider.monitor_names is None:
            WorkspacesProvider.monitor_names = (
                [l for l in run(['bspc', 'query', '-M']).splitlines() if l])
        monitors = []
        for monitor_name in WorkspacesProvider.monitor_names:
            monitor_spec = json.loads(run(['bspc', 'query', '-T', '-m', monitor_name]))
            monitor = Monitor()
            monitor.original_id = len(monitors)
            monitor.name = monitor_name
            monitor.width = int(monitor_spec['rectangle']['width'])
            monitor.height = int(monitor_spec['rectangle']['height'])
            monitor.x = int(monitor_spec['rectangle']['x'])
            monitor.y = int(monitor_spec['rectangle']['y'])
            for desktop_spec in monitor_spec['desktops']:
                workspace = Workspace()
                workspace.name = desktop_spec['name']
                workspace.focused = workspace.name == monitor_spec['focusedDesktopName']
                workspace.free = desktop_spec['root'] is None
                workspace.urgent = False
                children = [desktop_spec['root']]
                while children:
                    child = children.pop()
                    if child is None:
                        continue
                    children.append(child['firstChild'])
                    children.append(child['secondChild'])
                    if child['client']:
                        if bool(child['client']['urgent']):
                            workspace.urgent = True
                monitor.workspaces.append(workspace)
            monitors.append(monitor)
        monitors.sort(key=lambda m: m.x)
        for i, m in enumerate(monitors):
            m.display_id = i
        return monitors

    def __init__(self, main_window):
        self.main_window = main_window

        self.bspc_process = subprocess.Popen(
            ['bspc', 'subscribe'], stdout=subprocess.PIPE)
        self.monitors = self.get_monitors()
        self.refresh_workspaces()

        self.widgets = {}
        for i, monitor in enumerate(self.monitors):
            monitor_widget = main_window[i].left_widget
            monitor_widget.ws_widgets = {}
            monitor_widget.wheelEvent = lambda event, monitor=monitor: self.wheel(event, monitor)
            for j, ws in enumerate(monitor.workspaces):
                ws_widget = QtWidgets.QPushButton(ws.name)
                ws_widget.setProperty('class', 'workspace')
                ws_widget.mouseReleaseEvent = lambda event, ws=ws: self.click(event, ws)
                monitor_widget.ws_widgets[j] = ws_widget
                monitor_widget.layout().addWidget(ws_widget)
            self.widgets[i] = monitor_widget
        self.render()

    def wheel(self, event, monitor):
        subprocess.call(['bspc', 'monitor', '-f', monitor.name])
        subprocess.call(['bspc', 'desktop', '-f', ['prev', 'next'][event.angleDelta().y() > 0]])

    def click(self, event, ws):
        subprocess.call(['bspc', 'desktop', '-f', ws.name])

    def refresh_workspaces(self):
        line = self.bspc_process.stdout.readline().decode('utf8').strip()
        self.monitors = self.get_monitors()

    def refresh(self):
        self.refresh_workspaces()

    def render(self):
        for i, monitor in enumerate(self.monitors):
            for j, ws in enumerate(monitor.workspaces):
                self.widgets[i].ws_widgets[j].setProperty('ws_focused', '%s' % ws.focused)
                self.widgets[i].ws_widgets[j].setProperty('ws_urgent', '%s' % ws.urgent)
                self.widgets[i].ws_widgets[j].setProperty('ws_free', '%s' % ws.free)
                if ws.focused and ws.free and WorkspacesProvider.window_title_provider is not None:
                    WorkspacesProvider.window_title_provider.labels[monitor.display_id].setText('')
                    WorkspacesProvider.window_title_provider.window_names[monitor.display_id] = ''
        self.main_window.reloadStyleSheet()
<|endoftext|>