<commit_before>import re;

# this function switches the MainWindow of creator to the specified view
def switchViewTo(view):
    # make sure that no tooltip is shown, so move the mouse away and wait until all disappear
    mouseMove(waitForObject(':Qt Creator_Core::Internal::MainWindow'), -20, -20)
    waitFor("not QToolTip.isVisible()", 15000)
    if view < ViewConstants.WELCOME or view > ViewConstants.LAST_AVAILABLE:
        return
    tabBar = waitForObject("{type='Core::Internal::FancyTabBar' unnamed='1' visible='1' "
                           "window=':Qt Creator_Core::Internal::MainWindow'}")
    mouseMove(tabBar, 20, 20 + 52 * view)
    if waitFor("QToolTip.isVisible()", 10000):
        text = str(QToolTip.text())
    else:
        test.warning("Waiting for ToolTip timed out.")
        text = ""
    pattern = ViewConstants.getToolTipForViewTab(view)
    if re.match(pattern, unicode(text), re.UNICODE):
       test.passes("ToolTip verified")
    else:
        test.warning("ToolTip does not match", "Expected pattern: %s\nGot: %s" % (pattern, text))
    mouseClick(waitForObject("{type='Core::Internal::FancyTabBar' unnamed='1' visible='1' "
                             "window=':Qt Creator_Core::Internal::MainWindow'}"), 20, 20 + 52 * view, 0, Qt.LeftButton)

# this function is used to make sure that simple building prerequisites are met
# param targetCount specifies how many build targets had been selected (it's important that this one is correct)
# param currentTarget specifies which target should be selected for the next build (zero based index)
# param setReleaseBuild defines whether the current target(s) will be set to a Release or a Debug build
# param disableShadowBuild defines whether to disable shadow build or leave it unchanged (no matter what is defined)
# param setForAll defines whether to set Release or Debug and ShadowBuild option for all targets or only for the currentTarget
def prepareBuildSettings(targetCount, currentTarget, setReleaseBuild=True, disableShadowBuild=True, setForAll=True):
    switchViewTo(ViewConstants.PROJECTS)
    success = True
    for current in range(targetCount):
        if setForAll or current == currentTarget:
            switchToBuildOrRunSettingsFor(targetCount, current, ProjectSettings.BUILD)
            qtCombo = waitForObject(":scrollArea.qtVersionComboBox_QComboBox")
            chooseThis = None
            wait = False
            try:
                if qtCombo.currentText != defaultQtVersion:
                    selectFromCombo(qtCombo, defaultQtVersion)
                if setReleaseBuild:
                    chooseThis = "%s Release" % defaultQtVersion
                else:
                    chooseThis = "%s Debug" % defaultQtVersion
                editBuildCfg = waitForObject("{container={type='QScrollArea' name='scrollArea'} "
                                             "leftWidget={container={type='QScrollArea' name='scrollArea'} "
                                             "text='Edit build configuration:' type='QLabel'}"
                                             "unnamed='1' type='QComboBox' visible='1'}", 20000)
                if editBuildCfg.currentText != chooseThis:
                    wait = True
                    clickItem(editBuildCfg, chooseThis.replace(".", "\\."), 5, 5, 0, Qt.LeftButton)
                else:
                    wait = False
            except:
                if current == currentTarget:
                    success = False
            if wait and chooseThis != None:
                waitFor("editBuildCfg.currentText==chooseThis")
            ensureChecked("{name='shadowBuildCheckBox' type='QCheckBox' visible='1'}", not disableShadowBuild)
    # get back to the current target
    if currentTarget < 0 or currentTarget >= targetCount:
        test.warning("Parameter currentTarget is out of range - will be ignored this time!")
    else:
        switchToBuildOrRunSettingsFor(targetCount, currentTarget, ProjectSettings.BUILD)
    switchViewTo(ViewConstants.EDIT)
    return success

# this function switches to the build or the run settings (inside the Projects view)
# if you haven't already switched to the Projects view this will fail and return False
# param currentTarget specifies the target for which to switch into the specified settings (zero based index)
# param targetCount specifies the number of targets currently defined (must be correct!)
# param projectSettings specifies where to switch to (must be one of ProjectSettings.BUILD or ProjectSettings.RUN)
def switchToBuildOrRunSettingsFor(targetCount, currentTarget, projectSettings, isQtQuickUI=False):
    try:
        targetSel = waitForObject("{type='ProjectExplorer::Internal::TargetSelector' unnamed='1' "
                                  "visible='1' window=':Qt Creator_Core::Internal::MainWindow'}", 5000)
    except LookupError:
        if isQtQuickUI:
            if projectSettings == ProjectSettings.RUN:
                mouseClick(waitForObject(":*Qt Creator.DoubleTabWidget_ProjectExplorer::Internal::DoubleTabWidget"), 70, 44, 0, Qt.LeftButton)
                return True
            else:
                test.fatal("Don't know what you're trying to switch to")
                return False
        # there's only one target defined so use the DoubleTabWidget instead
        if projectSettings == ProjectSettings.RUN:
            mouseClick(waitForObject(":*Qt Creator.DoubleTabWidget_ProjectExplorer::Internal::DoubleTabWidget"), 170, 44, 0, Qt.LeftButton)
        elif projectSettings == ProjectSettings.BUILD:
            mouseClick(waitForObject(":*Qt Creator.DoubleTabWidget_ProjectExplorer::Internal::DoubleTabWidget"), 70, 44, 0, Qt.LeftButton)
        else:
            test.fatal("Don't know what you're trying to switch to")
            return False
        return True
    ADD_BUTTON_WIDTH = 27 # bad... (taken from source)
    selectorWidth = (targetSel.width - 3 - 2 * (ADD_BUTTON_WIDTH + 1)) / targetCount - 1
    yToClick = targetSel.height * 3 / 5 + 5
    if projectSettings == ProjectSettings.RUN:
        xToClick = ADD_BUTTON_WIDTH + (selectorWidth + 1) * currentTarget - 2 + selectorWidth / 2 + 5
    elif projectSettings == ProjectSettings.BUILD:
        xToClick = ADD_BUTTON_WIDTH + (selectorWidth + 1) * currentTarget - 2 + selectorWidth / 2 - 5
    else:
        test.fatal("Don't know what you're trying to switch to")
        return False
    mouseClick(targetSel, xToClick, yToClick, 0, Qt.LeftButton)
    return True

# this function switches "Run in terminal" on or off in a project's run settings
# param targetCount specifies the number of targets currently defined (must be correct!)
# param currentTarget specifies the target for which to switch into the specified settings (zero based index)
# param runInTerminal specifies if "Run in terminal should be turned on (True) or off (False)
def setRunInTerminal(targetCount, currentTarget, runInTerminal=True):
    switchViewTo(ViewConstants.PROJECTS)
    switchToBuildOrRunSettingsFor(targetCount, currentTarget, ProjectSettings.RUN)
    ensureChecked("{container=':Qt Creator.scrollArea_QScrollArea' text='Run in terminal'\
                    type='QCheckBox' unnamed='1' visible='1'}", runInTerminal)
    switchViewTo(ViewConstants.EDIT)
<commit_msg>Squish: Fix setRunInTerminal for 1-target-projects<commit_after>import re;

# this function switches the MainWindow of creator to the specified view
def switchViewTo(view):
    # make sure that no tooltip is shown, so move the mouse away and wait until all disappear
    mouseMove(waitForObject(':Qt Creator_Core::Internal::MainWindow'), -20, -20)
    waitFor("not QToolTip.isVisible()", 15000)
    if view < ViewConstants.WELCOME or view > ViewConstants.LAST_AVAILABLE:
        return
    tabBar = waitForObject("{type='Core::Internal::FancyTabBar' unnamed='1' visible='1' "
                           "window=':Qt Creator_Core::Internal::MainWindow'}")
    mouseMove(tabBar, 20, 20 + 52 * view)
    if waitFor("QToolTip.isVisible()", 10000):
        text = str(QToolTip.text())
    else:
        test.warning("Waiting for ToolTip timed out.")
        text = ""
    pattern = ViewConstants.getToolTipForViewTab(view)
    if re.match(pattern, unicode(text), re.UNICODE):
       test.passes("ToolTip verified")
    else:
        test.warning("ToolTip does not match", "Expected pattern: %s\nGot: %s" % (pattern, text))
    mouseClick(waitForObject("{type='Core::Internal::FancyTabBar' unnamed='1' visible='1' "
                             "window=':Qt Creator_Core::Internal::MainWindow'}"), 20, 20 + 52 * view, 0, Qt.LeftButton)

# this function is used to make sure that simple building prerequisites are met
# param targetCount specifies how many build targets had been selected (it's important that this one is correct)
# param currentTarget specifies which target should be selected for the next build (zero based index)
# param setReleaseBuild defines whether the current target(s) will be set to a Release or a Debug build
# param disableShadowBuild defines whether to disable shadow build or leave it unchanged (no matter what is defined)
# param setForAll defines whether to set Release or Debug and ShadowBuild option for all targets or only for the currentTarget
def prepareBuildSettings(targetCount, currentTarget, setReleaseBuild=True, disableShadowBuild=True, setForAll=True):
    switchViewTo(ViewConstants.PROJECTS)
    success = True
    for current in range(targetCount):
        if setForAll or current == currentTarget:
            switchToBuildOrRunSettingsFor(targetCount, current, ProjectSettings.BUILD)
            qtCombo = waitForObject(":scrollArea.qtVersionComboBox_QComboBox")
            chooseThis = None
            wait = False
            try:
                if qtCombo.currentText != defaultQtVersion:
                    selectFromCombo(qtCombo, defaultQtVersion)
                if setReleaseBuild:
                    chooseThis = "%s Release" % defaultQtVersion
                else:
                    chooseThis = "%s Debug" % defaultQtVersion
                editBuildCfg = waitForObject("{container={type='QScrollArea' name='scrollArea'} "
                                             "leftWidget={container={type='QScrollArea' name='scrollArea'} "
                                             "text='Edit build configuration:' type='QLabel'}"
                                             "unnamed='1' type='QComboBox' visible='1'}", 20000)
                if editBuildCfg.currentText != chooseThis:
                    wait = True
                    clickItem(editBuildCfg, chooseThis.replace(".", "\\."), 5, 5, 0, Qt.LeftButton)
                else:
                    wait = False
            except:
                if current == currentTarget:
                    success = False
            if wait and chooseThis != None:
                waitFor("editBuildCfg.currentText==chooseThis")
            ensureChecked("{name='shadowBuildCheckBox' type='QCheckBox' visible='1'}", not disableShadowBuild)
    # get back to the current target
    if currentTarget < 0 or currentTarget >= targetCount:
        test.warning("Parameter currentTarget is out of range - will be ignored this time!")
    else:
        switchToBuildOrRunSettingsFor(targetCount, currentTarget, ProjectSettings.BUILD)
    switchViewTo(ViewConstants.EDIT)
    return success

# this function switches to the build or the run settings (inside the Projects view)
# if you haven't already switched to the Projects view this will fail and return False
# param currentTarget specifies the target for which to switch into the specified settings (zero based index)
# param targetCount specifies the number of targets currently defined (must be correct!)
# param projectSettings specifies where to switch to (must be one of ProjectSettings.BUILD or ProjectSettings.RUN)
def switchToBuildOrRunSettingsFor(targetCount, currentTarget, projectSettings, isQtQuickUI=False):
    try:
        targetSel = waitForObject("{type='ProjectExplorer::Internal::TargetSelector' unnamed='1' "
                                  "visible='1' window=':Qt Creator_Core::Internal::MainWindow'}", 5000)
    except LookupError:
        if isQtQuickUI:
            if projectSettings == ProjectSettings.RUN:
                mouseClick(waitForObject(":*Qt Creator.DoubleTabWidget_ProjectExplorer::Internal::DoubleTabWidget"), 70, 44, 0, Qt.LeftButton)
                return True
            else:
                test.fatal("Don't know what you're trying to switch to")
                return False
        # there's only one target defined so use the DoubleTabWidget instead
        if projectSettings == ProjectSettings.RUN:
            mouseClick(waitForObject(":*Qt Creator.DoubleTabWidget_ProjectExplorer::Internal::DoubleTabWidget"), 170, 44, 0, Qt.LeftButton)
        elif projectSettings == ProjectSettings.BUILD:
            mouseClick(waitForObject(":*Qt Creator.DoubleTabWidget_ProjectExplorer::Internal::DoubleTabWidget"), 70, 44, 0, Qt.LeftButton)
        else:
            test.fatal("Don't know what you're trying to switch to")
            return False
        return True
    ADD_BUTTON_WIDTH = 27 # bad... (taken from source)
    selectorWidth = (targetSel.width - 3 - 2 * (ADD_BUTTON_WIDTH + 1)) / targetCount - 1
    yToClick = targetSel.height * 3 / 5 + 5
    if projectSettings == ProjectSettings.RUN:
        xToClick = ADD_BUTTON_WIDTH + (selectorWidth + 1) * currentTarget - 2 + selectorWidth / 2 + 5
    elif projectSettings == ProjectSettings.BUILD:
        xToClick = ADD_BUTTON_WIDTH + (selectorWidth + 1) * currentTarget - 2 + selectorWidth / 2 - 5
    else:
        test.fatal("Don't know what you're trying to switch to")
        return False
    mouseClick(targetSel, xToClick, yToClick, 0, Qt.LeftButton)
    return True

# this function switches "Run in terminal" on or off in a project's run settings
# param targetCount specifies the number of targets currently defined (must be correct!)
# param currentTarget specifies the target for which to switch into the specified settings (zero based index)
# param runInTerminal specifies if "Run in terminal should be turned on (True) or off (False)
def setRunInTerminal(targetCount, currentTarget, runInTerminal=True):
    switchViewTo(ViewConstants.PROJECTS)
    switchToBuildOrRunSettingsFor(targetCount, currentTarget, ProjectSettings.RUN)
    ensureChecked("{window=':Qt Creator_Core::Internal::MainWindow' text='Run in terminal'\
                    type='QCheckBox' unnamed='1' visible='1'}", runInTerminal)
    switchViewTo(ViewConstants.EDIT)
<|endoftext|>


<commit_before># coding=utf-8
"""
A wrapper extending GitPython's repo.git.

This wrapper class provides support for stdout messages in Git Exceptions
and (nearly) realtime stdout output. In addition, some methods of the
original repo.git are shadowed by custom methods providing functionality
needed for `git up`.
"""

from __future__ import print_function

__all__ = ['GitWrapper', 'GitError']

###############################################################################
# IMPORTS
###############################################################################

# Python libs
import sys
import re
import subprocess
import platform
from contextlib import contextmanager

# 3rd party libs
from termcolor import colored  # Assume, colorama is already initialized
from git import GitCommandError, CheckoutError as OrigCheckoutError, Git

# PyGitUp libs
from PyGitUp.utils import find

###############################################################################
# GitWrapper
###############################################################################

class GitWrapper(object):
    """
    A wrapper for repo.git providing better stdout handling + better exeptions.

    It is preferred to repo.git because it doesn't print to stdout
    in real time. In addition, this wrapper provides better error
    handling (it provides stdout messages inside the exception, too).
    """

    def __init__(self, repo):
        if repo:
            #: :type: git.Repo
            self.repo = repo
            #: :type: git.Git
            self.git = self.repo.git
        else:
            #: :type: git.Git
            self.git = Git()

    def __del__(self):
        # Is the following true?

        # GitPython runs persistent git processes in  the working directory.
        # Therefore, when we use 'git up' in something like a test environment,
        # this might cause troubles because of the open file handlers (like
        # trying to remove the directory right after the test has finished).
        # 'clear_cache' kills the processes...

        if platform.system() == 'Windows':
            pass
            # ... or rather "should kill", because but somehow it recently
            # started to not kill cat_file_header out of the blue (I even
            # tried running old code, but the once working code failed).
            # Thus, we kill it  manually here.
            if self.git.cat_file_header is not None:
                subprocess.call(("TASKKILL /F /T /PID {0} 2>nul 1>nul".format(
                    str(self.git.cat_file_header.proc.pid)
                )), shell=True)
            if self.git.cat_file_all is not None:
                subprocess.call(("TASKKILL /F /T /PID {0} 2>nul 1>nul".format(
                    str(self.git.cat_file_all.proc.pid)
                )), shell=True)

        self.git.clear_cache()

    def run(self, name, *args, **kwargs):
        """ Run a git command specified by name and args/kwargs. """

        tostdout = kwargs.pop('tostdout', False)
        stdout = ''

        # Execute command
        cmd = getattr(self.git, name)(as_process=True, *args, **kwargs)

        # Capture output
        while True:
            output = cmd.stdout.read(1)

            # Print to stdout
            if tostdout:
                sys.stdout.write(output)
                sys.stdout.flush()

            stdout += output

            if output == "":
                break

        # Wait for the process to quit
        try:
            cmd.wait()
        except GitCommandError as error:
            # Add more meta-information to errors
            message = "'{0}' returned exit status {1}".format(
                ' '.join(str(c) for c in error.command),
                error.status
            )

            raise GitError(message, stderr=error.stderr, stdout=stdout)

        return stdout.strip()

    def __getattr__(self, name):
        return lambda *args, **kwargs: self.run(name, *args, **kwargs)

    ###########################################################################
    # Overwrite some methods and add new ones
    ###########################################################################

    @contextmanager
    def stash(self):
        """
        A stashing contextmanager.
        It  stashes all changes inside and unstashed when done.
        """
        stashed = False

        if self.repo.is_dirty():
            print(colored(
                'stashing {0} changes'.format(self.change_count),
                'magenta'
            ))
            self.git.stash()
            stashed = True

        yield

        if stashed:
            print(colored('unstashing', 'magenta'))
            try:
                self.run('stash', 'pop')
            except GitError as e:
                raise UnstashError(stderr=e.stderr, stdout=e.stdout)

    def checkout(self, branch_name):
        """ Checkout a branch by name. """
        try:
            find(
                self.repo.branches, lambda b: b.name == branch_name
            ).checkout()
        except OrigCheckoutError as e:
            raise CheckoutError(branch_name, details=e)

    def rebase(self, target_branch):
        """ Rebase to target branch. """
        current_branch = self.repo.active_branch

        arguments = (
            ([self.config('git-up.rebase.arguments')] or []) +
            [target_branch.name]
        )
        try:
            self.run('rebase', *arguments)
        except GitError as e:
            raise RebaseError(current_branch.name, target_branch.name,
                              **e.__dict__)

    def config(self, key):
        """ Return `git config key` output or None. """
        try:
            return self.git.config(key)
        except GitCommandError:
            return None

    @property
    def change_count(self):
        """ The number of changes in the working directory. """
        return len(
            self.git.status(porcelain=True, untracked_files='no').split(
                '\n')
        )

    @property
    def version(self):
        """
        Return git's version as a list of numbers.

        The original repo.git.version_info has problems with tome types of
        git version strings.
        """
        return re.search(r'\d+(\.\d+)+', self.git.version()).group(0)

    def is_version_min(self, required_version):
        """ Does git's version match the requirements? """
        return self.version.split('.') >= required_version.split('.')

###############################################################################
# GitError + subclasses
###############################################################################

class GitError(GitCommandError):
    """
    Extension of the GitCommandError class.

    New:
    - stdout
    - details: a 'nested' exception with more details
    """

    def __init__(self, message=None, stderr=None, stdout=None, details=None):
        super(GitError, self).__init__(None, None, stderr)
        self.stdout = stdout
        self.details = details
        self.message = message

    def __str__(self):
        return self.message

class UnstashError(GitError):
    """
    Error while unstashing
    """
    def __init__(self, **kwargs):
        kwargs.pop('message', None)
        GitError.__init__(self, 'Unstash failed!', **kwargs)

class CheckoutError(GitError):
    """
    Error during checkout
    """
    def __init__(self, branch_name, **kwargs):
        kwargs.pop('message', None)
        GitError.__init__(self, 'Failed to checkout ' + branch_name,
                          **kwargs)

class RebaseError(GitError):
    """
    Error during rebase command
    """
    def __init__(self, current_branch, target_branch, **kwargs):
        # Remove kwargs we won't pass to GitError
        kwargs.pop('message', None)
        kwargs.pop('command', None)
        kwargs.pop('status', None)

        message = "Failed to rebase {1} onto {0}".format(
            current_branch, target_branch
        )
        GitError.__init__(self, message, **kwargs)
<commit_msg>Use correct plural for 'stashing {0} changes'<commit_after># coding=utf-8
"""
A wrapper extending GitPython's repo.git.

This wrapper class provides support for stdout messages in Git Exceptions
and (nearly) realtime stdout output. In addition, some methods of the
original repo.git are shadowed by custom methods providing functionality
needed for `git up`.
"""

from __future__ import print_function

__all__ = ['GitWrapper', 'GitError']

###############################################################################
# IMPORTS
###############################################################################

# Python libs
import sys
import re
import subprocess
import platform
from contextlib import contextmanager

# 3rd party libs
from termcolor import colored  # Assume, colorama is already initialized
from git import GitCommandError, CheckoutError as OrigCheckoutError, Git

# PyGitUp libs
from PyGitUp.utils import find

###############################################################################
# GitWrapper
###############################################################################

class GitWrapper(object):
    """
    A wrapper for repo.git providing better stdout handling + better exeptions.

    It is preferred to repo.git because it doesn't print to stdout
    in real time. In addition, this wrapper provides better error
    handling (it provides stdout messages inside the exception, too).
    """

    def __init__(self, repo):
        if repo:
            #: :type: git.Repo
            self.repo = repo
            #: :type: git.Git
            self.git = self.repo.git
        else:
            #: :type: git.Git
            self.git = Git()

    def __del__(self):
        # Is the following true?

        # GitPython runs persistent git processes in  the working directory.
        # Therefore, when we use 'git up' in something like a test environment,
        # this might cause troubles because of the open file handlers (like
        # trying to remove the directory right after the test has finished).
        # 'clear_cache' kills the processes...

        if platform.system() == 'Windows':
            pass
            # ... or rather "should kill", because but somehow it recently
            # started to not kill cat_file_header out of the blue (I even
            # tried running old code, but the once working code failed).
            # Thus, we kill it  manually here.
            if self.git.cat_file_header is not None:
                subprocess.call(("TASKKILL /F /T /PID {0} 2>nul 1>nul".format(
                    str(self.git.cat_file_header.proc.pid)
                )), shell=True)
            if self.git.cat_file_all is not None:
                subprocess.call(("TASKKILL /F /T /PID {0} 2>nul 1>nul".format(
                    str(self.git.cat_file_all.proc.pid)
                )), shell=True)

        self.git.clear_cache()

    def run(self, name, *args, **kwargs):
        """ Run a git command specified by name and args/kwargs. """

        tostdout = kwargs.pop('tostdout', False)
        stdout = ''

        # Execute command
        cmd = getattr(self.git, name)(as_process=True, *args, **kwargs)

        # Capture output
        while True:
            output = cmd.stdout.read(1)

            # Print to stdout
            if tostdout:
                sys.stdout.write(output)
                sys.stdout.flush()

            stdout += output

            if output == "":
                break

        # Wait for the process to quit
        try:
            cmd.wait()
        except GitCommandError as error:
            # Add more meta-information to errors
            message = "'{0}' returned exit status {1}".format(
                ' '.join(str(c) for c in error.command),
                error.status
            )

            raise GitError(message, stderr=error.stderr, stdout=stdout)

        return stdout.strip()

    def __getattr__(self, name):
        return lambda *args, **kwargs: self.run(name, *args, **kwargs)

    ###########################################################################
    # Overwrite some methods and add new ones
    ###########################################################################

    @contextmanager
    def stash(self):
        """
        A stashing contextmanager.
        It  stashes all changes inside and unstashed when done.
        """
        stashed = False

        if self.repo.is_dirty():
            if self.change_count > 1:
                text = 'stashing {0} changes'
            else:
                text = 'stashing {0} change'
            print(colored(
                text.format(self.change_count),
                'magenta'
            ))
            self.git.stash()
            stashed = True

        yield

        if stashed:
            print(colored('unstashing', 'magenta'))
            try:
                self.run('stash', 'pop')
            except GitError as e:
                raise UnstashError(stderr=e.stderr, stdout=e.stdout)

    def checkout(self, branch_name):
        """ Checkout a branch by name. """
        try:
            find(
                self.repo.branches, lambda b: b.name == branch_name
            ).checkout()
        except OrigCheckoutError as e:
            raise CheckoutError(branch_name, details=e)

    def rebase(self, target_branch):
        """ Rebase to target branch. """
        current_branch = self.repo.active_branch

        arguments = (
            ([self.config('git-up.rebase.arguments')] or []) +
            [target_branch.name]
        )
        try:
            self.run('rebase', *arguments)
        except GitError as e:
            raise RebaseError(current_branch.name, target_branch.name,
                              **e.__dict__)

    def config(self, key):
        """ Return `git config key` output or None. """
        try:
            return self.git.config(key)
        except GitCommandError:
            return None

    @property
    def change_count(self):
        """ The number of changes in the working directory. """
        return len(
            self.git.status(porcelain=True, untracked_files='no').split(
                '\n')
        )

    @property
    def version(self):
        """
        Return git's version as a list of numbers.

        The original repo.git.version_info has problems with tome types of
        git version strings.
        """
        return re.search(r'\d+(\.\d+)+', self.git.version()).group(0)

    def is_version_min(self, required_version):
        """ Does git's version match the requirements? """
        return self.version.split('.') >= required_version.split('.')

###############################################################################
# GitError + subclasses
###############################################################################

class GitError(GitCommandError):
    """
    Extension of the GitCommandError class.

    New:
    - stdout
    - details: a 'nested' exception with more details
    """

    def __init__(self, message=None, stderr=None, stdout=None, details=None):
        super(GitError, self).__init__(None, None, stderr)
        self.stdout = stdout
        self.details = details
        self.message = message

    def __str__(self):
        return self.message

class UnstashError(GitError):
    """
    Error while unstashing
    """
    def __init__(self, **kwargs):
        kwargs.pop('message', None)
        GitError.__init__(self, 'Unstash failed!', **kwargs)

class CheckoutError(GitError):
    """
    Error during checkout
    """
    def __init__(self, branch_name, **kwargs):
        kwargs.pop('message', None)
        GitError.__init__(self, 'Failed to checkout ' + branch_name,
                          **kwargs)

class RebaseError(GitError):
    """
    Error during rebase command
    """
    def __init__(self, current_branch, target_branch, **kwargs):
        # Remove kwargs we won't pass to GitError
        kwargs.pop('message', None)
        kwargs.pop('command', None)
        kwargs.pop('status', None)

        message = "Failed to rebase {1} onto {0}".format(
            current_branch, target_branch
        )
        GitError.__init__(self, message, **kwargs)
<|endoftext|>


<commit_before>"""
Celery tasks.

.. moduleauthor:: Martijn Vermaat <martijn@vermaat.name>

.. Licensed under the MIT license, see the LICENSE file.
"""

from __future__ import division

from collections import defaultdict
from contextlib import contextmanager
import hashlib
import os
import time
import uuid

from celery import current_task, current_app, Task
from celery.utils.log import get_task_logger
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import NoResultFound
from vcf.parser import _Info as VcfInfo, field_counts as vcf_field_counts
import vcf

from . import db, celery
from .models import Annotation, Coverage, DataSource, DataUnavailable, Observation, Sample, Region, Variant, Variation
from .region_binning import all_bins
from .utils import digest, normalize_variant, normalize_chromosome, normalize_region, ReferenceMismatch

logger = get_task_logger(__name__)

class ReadError(Exception):
    """
    Exception thrown on failed data reading.
    """
    pass

class TaskError(Exception):
    """
    Exception thrown on failed task execution.
    """
    def __init__(self, code, message):
        self.code = code
        self.message = message
        super(TaskError, self).__init__(code, message)

class CleanTask(Task):
    """
    Ordinary Celery task, but with a way of registering cleanup routines that
    are executed after the task failed (on the worker node).
    """
    abstract = True

    # We maintain a list of cleanups per task id since the worker node only
    # instantiates the task class once (not for every run). Just storing one
    # list of cleanups in this instance would mean they are shared between
    # different task runs.
    _cleanups = defaultdict(list)

    def register_cleanup(self, task_id, cleanup):
        self._cleanups[task_id].append(cleanup)

    def on_failure(self, exc, task_id, args, kwargs, einfo):
        for cleanup in reversed(self._cleanups[task_id]):
            cleanup()
        del self._cleanups[task_id]

def annotate_variants(original_variants, annotated_variants, original_filetype='vcf', annotated_filetype='vcf', ignore_sample_ids=None, original_records=1):
    """
    .. todo:: Merge back population study annotation (see implementation in
        the old-population-study branch).

    .. todo:: Calculate frequencies instead of counts, and take chromosome
        into account (config.CHROMOSOMES).
    """
    ignore_sample_ids = ignore_sample_ids or []

    if original_filetype != 'vcf':
        raise ReadError('Original data must be in VCF format')

    if annotated_filetype != 'vcf':
        raise ReadError('Annotated data must be in VCF format')

    reader = vcf.Reader(original_variants)

    reader.infos['OBS'] = VcfInfo('OBS', vcf_field_counts['A'], 'Integer',
        'Samples with variant (out of %i)' % Sample.query.count())
    reader.infos['COV'] = VcfInfo('COV', vcf_field_counts['A'], 'Integer',
        'Samples with coverage (out of %i)' % Sample.query.count())
    writer = vcf.Writer(annotated_variants, reader, lineterminator='\n')

    old_percentage = -1
    for i, record in enumerate(reader):
        # Task progress is updated in whole percentages, so for a maximum of
        # 100 times per task.
        percentage = min(int(i / original_records * 100), 99)
        if percentage > old_percentage:
            current_task.update_state(state='PROGRESS', meta={'percentage': percentage})
            old_percentage = percentage

        observations = []
        coverage = []
        for index, allele in enumerate(record.ALT):
            try:
                chromosome, position, reference, observed = normalize_variant(record.CHROM, record.POS, record.REF, str(allele))
            except ReferenceMismatch as e:
                raise ReadError(str(e))

            end_position = position + max(1, len(reference)) - 1
            bins = all_bins(position, end_position)

            try:
                variant = Variant.query.filter_by(chromosome=chromosome, position=position, reference=reference, observed=observed).one()
                observations.append(variant.observations.join(Variation).filter(~Variation.sample_id.in_(ignore_sample_ids)).count())
            except NoResultFound:
                observations.append(0)
            coverage.append(Region.query.join(Coverage).filter(Region.chromosome == chromosome,
                                                               Region.begin <= position,
                                                               Region.end >= end_position,
                                                               Region.bin.in_(bins),
                                                               ~Coverage.sample_id.in_(ignore_sample_ids)).count())

        record.add_info('OBS', observations)
        record.add_info('COV', coverage)
        writer.write_record(record)

def read_observations(observations, filetype='vcf'):
    # Todo: Merge back population study importing (see implementation in the
    #     old-population-study branch).
    if filetype != 'vcf':
        raise ReadError('Data must be in VCF format')

    reader = vcf.Reader(observations)

    for record in reader:
        if 'SV' in record.INFO:
            # For now we ignore these, reference is likely to be larger than
            # the maximum of 200 by the database schema.
            # Example use of this type are large deletions in 1000 Genomes.
            continue

        for index, allele in enumerate(record.ALT):
            try:
                chromosome, position, reference, observed = normalize_variant(
                    record.CHROM, record.POS, record.REF, str(allele))
            except ReferenceMismatch as e:
                logger.info('Reference mismatch: %s', str(e))
                if current_app.conf['REFERENCE_MISMATCH_ABORT']:
                    raise ReadError(str(e))
                continue

            # Variant support is defined by the number of samples in which a
            # variant allele was called, ignoring homo-/heterozygocity.
            # Todo: This check can break if index > 9.
            support = sum(1 for sample in record.samples if str(index + 1) in sample['GT'])

            #if 'SF' in record.INFO and False:
            #    # Todo: Per alt allele?
            #    support = len(record.INFO['SF'])
            #elif 'AC' in record.INFO and False:
            #    # Todo: Per alt allele?
            #    support = record.INFO['AC'][0]
            #else:
            #    support = 1

            yield chromosome, position, reference, observed, support

def read_regions(regions, filetype='bed'):
    # Todo: Use pybedtools to parse BED file?
    if filetype != 'bed':
        raise ReadError('Data must be in BED format')

    for line in regions:
        fields = line.split()
        if len(fields) < 1 or fields[0] == 'track':
            continue
        try:
            chromosome, begin, end = normalize_region(
                fields[0], int(fields[1]), int(fields[2]))
        except (IndexError, ValueError):
            raise ReadError('Invalid line in BED file: "%s"' % line)
        except ReferenceMismatch as e:
            logger.info('Reference mismatch: %s', str(e))
            if current_app.conf['REFERENCE_MISMATCH_ABORT']:
                raise ReadError(str(e))
            continue
        yield chromosome, begin + 1, end

@celery.task(base=CleanTask)
def import_variation(variation_id):
    """
    Import variation as observations.
    """
    logger.info('Started task: import_variation(%d)', variation_id)

    variation = Variation.query.get(variation_id)
    if variation is None:
        raise TaskError('variation_not_found', 'Variation not found')

    if variation.imported:
        raise TaskError('variation_imported', 'Variation already imported')

    if variation.import_task_uuid:
        # Todo: Check somehow if the importing task is still running.
        # http://stackoverflow.com/questions/9824172/find-out-whether-celery-task-exists
        raise TaskError('variation_importing', 'Variation is being imported')

    # Todo: This has a possible race condition, but I'm not bothered to fix it
    #     at the moment. Reading and setting import_task_uuid should be an
    #     atomic action.
    variation.import_task_uuid = current_task.request.id
    db.session.commit()

    data_source = variation.data_source

    # Calculate data digest if it is not yet known.
    # Todo: Can we somehow factor this out into a separate (singleton) task,
    #     on which we wait?
    if not data_source.checksum:
        with data_source.data() as data:
            data_source.checksum, data_source.records = digest(data)
        db.session.commit()

    # Check if checksum is not in imported data sources.
    if DataSource.query.filter_by(checksum=data_source.checksum).join(Variation).filter_by(imported=True).count() > 0:
        raise TaskError('duplicate_data_source', 'Identical data source already imported')

    # Note: Since we are dealing with huge numbers of entries here, we commit
    # after each INSERT and manually rollback. Using builtin session rollback
    # would fill up all our memory.
    def delete_observations():
        variation.observations.delete()
        db.session.commit()
    current_task.register_cleanup(current_task.request.id, delete_observations)

    try:
        data = data_source.data()
    except DataUnavailable as e:
        raise TaskError(e.code, e.message)

    with data as observations:
        try:
            old_percentage = -1
            for i, (chromosome, position, reference, observed, support) in enumerate(read_observations(observations, filetype=data_source.filetype)):
                # Task progress is updated in whole percentages, so for a
                # maximum of 100 times per task.
                percentage = min(int(i / data_source.records * 100), 99)
                if percentage > old_percentage:
                    current_task.update_state(state='PROGRESS', meta={'percentage': percentage})
                    old_percentage = percentage
                    time.sleep(1)
                # SQLAlchemy doesn't seem to have anything like INSERT IGNORE
                # or INSERT ... ON DUPLICATE KEY UPDATE, so we have to work
                # our way around the situation.
                try:
                    # Todo: Check for errors (binning on begin/end may fail,
                    #     sequences might be too long).
                    variant = Variant(chromosome, position, reference, observed)
                    db.session.add(variant)
                    db.session.commit()
                except IntegrityError:
                    db.session.rollback()
                    try:
                        variant = Variant.query.filter_by(chromosome=chromosome, position=position, reference=reference, observed=observed).one()
                    except NoResultFound:
                        # Should never happen.
                        raise TaskError('database_inconsistency', 'Unrecoverable inconsistency of the database observed')
                observation = Observation(variant, variation, support=support)
                db.session.add(observation)
                db.session.commit()
        except ReadError as e:
            raise TaskError('invalid_observations', str(e))

    current_task.update_state(state='PROGRESS', meta={'percentage': 100})
    variation.imported = True
    db.session.commit()

    logger.info('Finished task: import_variation(%d)', variation_id)

@celery.task(base=CleanTask)
def import_coverage(coverage_id):
    """
    Import coverage as regions.
    """
    logger.info('Started task: import_coverage(%d)', coverage_id)

    coverage = Coverage.query.get(coverage_id)
    if coverage is None:
        raise TaskError('coverage_not_found', 'Coverage not found')

    if coverage.imported:
        raise TaskError('coverage_imported', 'Coverage already imported')

    if coverage.import_task_uuid is not None:
        # Todo: Check somehow if the importing task is still running.
        # http://stackoverflow.com/questions/9824172/find-out-whether-celery-task-exists
        raise TaskError('coverage_importing', 'Coverage is being imported')

    # Todo: This has a possible race condition, but I'm not bothered to fix it
    #     at the moment. Reading and setting import_task_uuid should be an
    #     atomic action.
    coverage.import_task_uuid = current_task.request.id
    db.session.commit()

    data_source = coverage.data_source

    # Calculate data digest if it is not yet known.
    if not data_source.checksum:
        with data_source.data() as data:
            data_source.checksum, data_source.records = digest(data)
        db.session.commit()

    # Check if checksum is not in imported data sources.
    if DataSource.query.filter_by(checksum=data_source.checksum).join(Coverage).filter_by(imported=True).count() > 0:
        raise TaskError('duplicate_data_source', 'Identical data source already imported')

    try:
        data = data_source.data()
    except DataUnavailable as e:
        raise TaskError(e.code, e.message)

    # Note: Since we are dealing with huge numbers of entries here, we commit
    # after each INSERT and manually rollback. Using builtin session rollback
    # would fill up all our memory.
    def delete_regions():
        coverage.regions.delete()
        db.session.commit()
    current_task.register_cleanup(current_task.request.id, delete_regions)

    with data as regions:
        try:
            old_percentage = -1
            for i, (chromosome, begin, end) in enumerate(read_regions(regions, filetype=data_source.filetype)):
                # Task progress is updated in whole percentages, so for a
                # maximum of 100 times per task.
                percentage = min(int(i / data_source.records * 100), 99)
                if percentage > old_percentage:
                    current_task.update_state(state='PROGRESS', meta={'percentage': percentage})
                    old_percentage = percentage
                db.session.add(Region(coverage, chromosome, begin, end))
                db.session.commit()
        except ReadError as e:
            raise TaskError('invalid_regions', str(e))

    current_task.update_state(state='PROGRESS', meta={'percentage': 100})
    coverage.imported = True
    db.session.commit()

    logger.info('Finished task: import_coverage(%d)', coverage_id)

@celery.task(base=CleanTask)
def write_annotation(annotation_id, ignore_sample_ids=None):
    """
    Annotate variants with frequencies from the database.
    """
    logger.info('Started task: write_annotation(%d)', annotation_id)

    ignore_sample_ids = ignore_sample_ids or []

    annotation = Annotation.query.get(annotation_id)
    if annotation is None:
        raise TaskError('annotation_not_found', 'Annotation not found')

    if annotation.written:
        raise TaskError('annotation_written', 'Annotation already written')

    if annotation.write_task_uuid is not None:
        # Todo: Check somehow if the writing task is still running, it might
        #     also be a failed task.
        # http://stackoverflow.com/questions/9824172/find-out-whether-celery-task-exists
        raise TaskError('annotation_writing', 'Annotation is being written')

    # Todo: This has a possible race condition, but I'm not bothered to fix it
    #     at the moment. Reading and setting write_task_uuid should be an
    #     atomic action.
    annotation.write_task_uuid = current_task.request.id
    db.session.commit()

    original_data_source = annotation.original_data_source
    annotated_data_source = annotation.annotated_data_source

    # Calculate data digest if it is not yet known.
    if not original_data_source.checksum:
        with original_data_source.data() as data:
            original_data_source.checksum, original_data_source.records = digest(data)
        db.session.commit()

    try:
        original_data = original_data_source.data()
        annotated_data = annotated_data_source.data_writer()
    except DataUnavailable as e:
        raise TaskError(e.code, e.message)

    with original_data as original_variants, annotated_data as annotated_variants:
        try:
            annotate_variants(original_variants, annotated_variants,
                              original_filetype=original_data_source.filetype,
                              annotated_filetype=annotated_data_source.filetype,
                              ignore_sample_ids=ignore_sample_ids,
                              original_records=original_data_source.records)
        except ReadError as e:
            raise TaskError('invalid_observations', str(e))

    current_task.update_state(state='PROGRESS', meta={'percentage': 100})
    annotation.written = True
    db.session.commit()

    logger.info('Finished task: write_annotation(%d)', annotation_id)

@celery.task
def ping():
    """
    Ping-pong task usefull for testing purposes.
    """
    logger.info('Started task: ping')
    logger.info('Finished task: ping')
    return 'pong'
<commit_msg>Handle VCF without genotypes<commit_after>"""
Celery tasks.

.. moduleauthor:: Martijn Vermaat <martijn@vermaat.name>

.. Licensed under the MIT license, see the LICENSE file.
"""

from __future__ import division

from collections import defaultdict
from contextlib import contextmanager
import hashlib
import os
import time
import uuid

from celery import current_task, current_app, Task
from celery.utils.log import get_task_logger
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import NoResultFound
from vcf.parser import _Info as VcfInfo, field_counts as vcf_field_counts
import vcf

from . import db, celery
from .models import Annotation, Coverage, DataSource, DataUnavailable, Observation, Sample, Region, Variant, Variation
from .region_binning import all_bins
from .utils import digest, normalize_variant, normalize_chromosome, normalize_region, ReferenceMismatch

logger = get_task_logger(__name__)

class ReadError(Exception):
    """
    Exception thrown on failed data reading.
    """
    pass

class TaskError(Exception):
    """
    Exception thrown on failed task execution.
    """
    def __init__(self, code, message):
        self.code = code
        self.message = message
        super(TaskError, self).__init__(code, message)

class CleanTask(Task):
    """
    Ordinary Celery task, but with a way of registering cleanup routines that
    are executed after the task failed (on the worker node).
    """
    abstract = True

    # We maintain a list of cleanups per task id since the worker node only
    # instantiates the task class once (not for every run). Just storing one
    # list of cleanups in this instance would mean they are shared between
    # different task runs.
    _cleanups = defaultdict(list)

    def register_cleanup(self, task_id, cleanup):
        self._cleanups[task_id].append(cleanup)

    def on_failure(self, exc, task_id, args, kwargs, einfo):
        for cleanup in reversed(self._cleanups[task_id]):
            cleanup()
        del self._cleanups[task_id]

def annotate_variants(original_variants, annotated_variants, original_filetype='vcf', annotated_filetype='vcf', ignore_sample_ids=None, original_records=1):
    """
    .. todo:: Merge back population study annotation (see implementation in
        the old-population-study branch).

    .. todo:: Calculate frequencies instead of counts, and take chromosome
        into account (config.CHROMOSOMES).
    """
    ignore_sample_ids = ignore_sample_ids or []

    if original_filetype != 'vcf':
        raise ReadError('Original data must be in VCF format')

    if annotated_filetype != 'vcf':
        raise ReadError('Annotated data must be in VCF format')

    reader = vcf.Reader(original_variants)

    reader.infos['OBS'] = VcfInfo('OBS', vcf_field_counts['A'], 'Integer',
        'Samples with variant (out of %i)' % Sample.query.count())
    reader.infos['COV'] = VcfInfo('COV', vcf_field_counts['A'], 'Integer',
        'Samples with coverage (out of %i)' % Sample.query.count())
    writer = vcf.Writer(annotated_variants, reader, lineterminator='\n')

    old_percentage = -1
    for i, record in enumerate(reader):
        # Task progress is updated in whole percentages, so for a maximum of
        # 100 times per task.
        percentage = min(int(i / original_records * 100), 99)
        if percentage > old_percentage:
            current_task.update_state(state='PROGRESS', meta={'percentage': percentage})
            old_percentage = percentage

        observations = []
        coverage = []
        for index, allele in enumerate(record.ALT):
            try:
                chromosome, position, reference, observed = normalize_variant(record.CHROM, record.POS, record.REF, str(allele))
            except ReferenceMismatch as e:
                raise ReadError(str(e))

            end_position = position + max(1, len(reference)) - 1
            bins = all_bins(position, end_position)

            try:
                variant = Variant.query.filter_by(chromosome=chromosome, position=position, reference=reference, observed=observed).one()
                observations.append(variant.observations.join(Variation).filter(~Variation.sample_id.in_(ignore_sample_ids)).count())
            except NoResultFound:
                observations.append(0)
            coverage.append(Region.query.join(Coverage).filter(Region.chromosome == chromosome,
                                                               Region.begin <= position,
                                                               Region.end >= end_position,
                                                               Region.bin.in_(bins),
                                                               ~Coverage.sample_id.in_(ignore_sample_ids)).count())

        record.add_info('OBS', observations)
        record.add_info('COV', coverage)
        writer.write_record(record)

def read_observations(observations, filetype='vcf'):
    # Todo: Merge back population study importing (see implementation in the
    #     old-population-study branch).
    if filetype != 'vcf':
        raise ReadError('Data must be in VCF format')

    reader = vcf.Reader(observations)

    for record in reader:
        if 'SV' in record.INFO:
            # For now we ignore these, reference is likely to be larger than
            # the maximum of 200 by the database schema.
            # Example use of this type are large deletions in 1000 Genomes.
            continue

        for index, allele in enumerate(record.ALT):
            try:
                chromosome, position, reference, observed = normalize_variant(
                    record.CHROM, record.POS, record.REF, str(allele))
            except ReferenceMismatch as e:
                logger.info('Reference mismatch: %s', str(e))
                if current_app.conf['REFERENCE_MISMATCH_ABORT']:
                    raise ReadError(str(e))
                continue

            # Variant support is defined by the number of samples in which a
            # variant allele was called, ignoring homo-/heterozygocity.
            # Todo: This check can break if index > 9.
            try:
                support = sum(1 for sample in record.samples if str(index + 1) in sample['GT'])
            except AttributeError:
                support = 1

            #if 'SF' in record.INFO and False:
            #    # Todo: Per alt allele?
            #    support = len(record.INFO['SF'])
            #elif 'AC' in record.INFO and False:
            #    # Todo: Per alt allele?
            #    support = record.INFO['AC'][0]
            #else:
            #    support = 1

            yield chromosome, position, reference, observed, support

def read_regions(regions, filetype='bed'):
    # Todo: Use pybedtools to parse BED file?
    if filetype != 'bed':
        raise ReadError('Data must be in BED format')

    for line in regions:
        fields = line.split()
        if len(fields) < 1 or fields[0] == 'track':
            continue
        try:
            chromosome, begin, end = normalize_region(
                fields[0], int(fields[1]), int(fields[2]))
        except (IndexError, ValueError):
            raise ReadError('Invalid line in BED file: "%s"' % line)
        except ReferenceMismatch as e:
            logger.info('Reference mismatch: %s', str(e))
            if current_app.conf['REFERENCE_MISMATCH_ABORT']:
                raise ReadError(str(e))
            continue
        yield chromosome, begin + 1, end

@celery.task(base=CleanTask)
def import_variation(variation_id):
    """
    Import variation as observations.
    """
    logger.info('Started task: import_variation(%d)', variation_id)

    variation = Variation.query.get(variation_id)
    if variation is None:
        raise TaskError('variation_not_found', 'Variation not found')

    if variation.imported:
        raise TaskError('variation_imported', 'Variation already imported')

    if variation.import_task_uuid:
        # Todo: Check somehow if the importing task is still running.
        # http://stackoverflow.com/questions/9824172/find-out-whether-celery-task-exists
        raise TaskError('variation_importing', 'Variation is being imported')

    # Todo: This has a possible race condition, but I'm not bothered to fix it
    #     at the moment. Reading and setting import_task_uuid should be an
    #     atomic action.
    variation.import_task_uuid = current_task.request.id
    db.session.commit()

    data_source = variation.data_source

    # Calculate data digest if it is not yet known.
    # Todo: Can we somehow factor this out into a separate (singleton) task,
    #     on which we wait?
    if not data_source.checksum:
        with data_source.data() as data:
            data_source.checksum, data_source.records = digest(data)
        db.session.commit()

    # Check if checksum is not in imported data sources.
    if DataSource.query.filter_by(checksum=data_source.checksum).join(Variation).filter_by(imported=True).count() > 0:
        raise TaskError('duplicate_data_source', 'Identical data source already imported')

    # Note: Since we are dealing with huge numbers of entries here, we commit
    # after each INSERT and manually rollback. Using builtin session rollback
    # would fill up all our memory.
    def delete_observations():
        variation.observations.delete()
        db.session.commit()
    current_task.register_cleanup(current_task.request.id, delete_observations)

    try:
        data = data_source.data()
    except DataUnavailable as e:
        raise TaskError(e.code, e.message)

    with data as observations:
        try:
            old_percentage = -1
            for i, (chromosome, position, reference, observed, support) in enumerate(read_observations(observations, filetype=data_source.filetype)):
                # Task progress is updated in whole percentages, so for a
                # maximum of 100 times per task.
                percentage = min(int(i / data_source.records * 100), 99)
                if percentage > old_percentage:
                    current_task.update_state(state='PROGRESS', meta={'percentage': percentage})
                    old_percentage = percentage
                    time.sleep(1)
                # SQLAlchemy doesn't seem to have anything like INSERT IGNORE
                # or INSERT ... ON DUPLICATE KEY UPDATE, so we have to work
                # our way around the situation.
                try:
                    # Todo: Check for errors (binning on begin/end may fail,
                    #     sequences might be too long).
                    variant = Variant(chromosome, position, reference, observed)
                    db.session.add(variant)
                    db.session.commit()
                except IntegrityError:
                    db.session.rollback()
                    try:
                        variant = Variant.query.filter_by(chromosome=chromosome, position=position, reference=reference, observed=observed).one()
                    except NoResultFound:
                        # Should never happen.
                        raise TaskError('database_inconsistency', 'Unrecoverable inconsistency of the database observed')
                observation = Observation(variant, variation, support=support)
                db.session.add(observation)
                db.session.commit()
        except ReadError as e:
            raise TaskError('invalid_observations', str(e))

    current_task.update_state(state='PROGRESS', meta={'percentage': 100})
    variation.imported = True
    db.session.commit()

    logger.info('Finished task: import_variation(%d)', variation_id)

@celery.task(base=CleanTask)
def import_coverage(coverage_id):
    """
    Import coverage as regions.
    """
    logger.info('Started task: import_coverage(%d)', coverage_id)

    coverage = Coverage.query.get(coverage_id)
    if coverage is None:
        raise TaskError('coverage_not_found', 'Coverage not found')

    if coverage.imported:
        raise TaskError('coverage_imported', 'Coverage already imported')

    if coverage.import_task_uuid is not None:
        # Todo: Check somehow if the importing task is still running.
        # http://stackoverflow.com/questions/9824172/find-out-whether-celery-task-exists
        raise TaskError('coverage_importing', 'Coverage is being imported')

    # Todo: This has a possible race condition, but I'm not bothered to fix it
    #     at the moment. Reading and setting import_task_uuid should be an
    #     atomic action.
    coverage.import_task_uuid = current_task.request.id
    db.session.commit()

    data_source = coverage.data_source

    # Calculate data digest if it is not yet known.
    if not data_source.checksum:
        with data_source.data() as data:
            data_source.checksum, data_source.records = digest(data)
        db.session.commit()

    # Check if checksum is not in imported data sources.
    if DataSource.query.filter_by(checksum=data_source.checksum).join(Coverage).filter_by(imported=True).count() > 0:
        raise TaskError('duplicate_data_source', 'Identical data source already imported')

    try:
        data = data_source.data()
    except DataUnavailable as e:
        raise TaskError(e.code, e.message)

    # Note: Since we are dealing with huge numbers of entries here, we commit
    # after each INSERT and manually rollback. Using builtin session rollback
    # would fill up all our memory.
    def delete_regions():
        coverage.regions.delete()
        db.session.commit()
    current_task.register_cleanup(current_task.request.id, delete_regions)

    with data as regions:
        try:
            old_percentage = -1
            for i, (chromosome, begin, end) in enumerate(read_regions(regions, filetype=data_source.filetype)):
                # Task progress is updated in whole percentages, so for a
                # maximum of 100 times per task.
                percentage = min(int(i / data_source.records * 100), 99)
                if percentage > old_percentage:
                    current_task.update_state(state='PROGRESS', meta={'percentage': percentage})
                    old_percentage = percentage
                db.session.add(Region(coverage, chromosome, begin, end))
                db.session.commit()
        except ReadError as e:
            raise TaskError('invalid_regions', str(e))

    current_task.update_state(state='PROGRESS', meta={'percentage': 100})
    coverage.imported = True
    db.session.commit()

    logger.info('Finished task: import_coverage(%d)', coverage_id)

@celery.task(base=CleanTask)
def write_annotation(annotation_id, ignore_sample_ids=None):
    """
    Annotate variants with frequencies from the database.
    """
    logger.info('Started task: write_annotation(%d)', annotation_id)

    ignore_sample_ids = ignore_sample_ids or []

    annotation = Annotation.query.get(annotation_id)
    if annotation is None:
        raise TaskError('annotation_not_found', 'Annotation not found')

    if annotation.written:
        raise TaskError('annotation_written', 'Annotation already written')

    if annotation.write_task_uuid is not None:
        # Todo: Check somehow if the writing task is still running, it might
        #     also be a failed task.
        # http://stackoverflow.com/questions/9824172/find-out-whether-celery-task-exists
        raise TaskError('annotation_writing', 'Annotation is being written')

    # Todo: This has a possible race condition, but I'm not bothered to fix it
    #     at the moment. Reading and setting write_task_uuid should be an
    #     atomic action.
    annotation.write_task_uuid = current_task.request.id
    db.session.commit()

    original_data_source = annotation.original_data_source
    annotated_data_source = annotation.annotated_data_source

    # Calculate data digest if it is not yet known.
    if not original_data_source.checksum:
        with original_data_source.data() as data:
            original_data_source.checksum, original_data_source.records = digest(data)
        db.session.commit()

    try:
        original_data = original_data_source.data()
        annotated_data = annotated_data_source.data_writer()
    except DataUnavailable as e:
        raise TaskError(e.code, e.message)

    with original_data as original_variants, annotated_data as annotated_variants:
        try:
            annotate_variants(original_variants, annotated_variants,
                              original_filetype=original_data_source.filetype,
                              annotated_filetype=annotated_data_source.filetype,
                              ignore_sample_ids=ignore_sample_ids,
                              original_records=original_data_source.records)
        except ReadError as e:
            raise TaskError('invalid_observations', str(e))

    current_task.update_state(state='PROGRESS', meta={'percentage': 100})
    annotation.written = True
    db.session.commit()

    logger.info('Finished task: write_annotation(%d)', annotation_id)

@celery.task
def ping():
    """
    Ping-pong task usefull for testing purposes.
    """
    logger.info('Started task: ping')
    logger.info('Finished task: ping')
    return 'pong'
<|endoftext|>


<commit_before>import json
from decimal import Decimal
from datetime import datetime, date, time
from service import legacy_transmission_queue

FakeReturnSearchRowFound = [datetime(2016, 1, 26, 13, 0, 30, 5449),
                            '1234',
                            'Test User',
                            datetime(2015, 12, 1, 14, 34, 14, 362556),
                            "D",
                            "drvSummaryView",
                            Decimal('2'),
                            "374f501f4567",
                            "GR12345",
                            "Search"]

FakeReturnNoSearchRowFound = None

FakeSearchTransmissionJSON = {"SEARCH_DATETIME": "2016-01-26 13:00:30.005449",
                              "LRO_TRANS_REF": "1234",
                              "USER_ID": "Test User",
                              "VIEWED_DATETIME": "2015-12-01 14:34:14.362556",
                              "SEARCH_TYPE": "D",
                              "PURCHASE_TYPE": "drvSummaryView",
                              "AMOUNT": 2,
                              "CART_ID": "374f501f4567",
                              "TITLE_NUMBER": "GR12345",
                              "EVENT_ID": "Search"
                              }

FakeEmptySearchTransmissionJSON = {}

class TestCreateSearchMessage:

    def test_message_is_created_when_db_row_is_returned(self):
        created_message = legacy_transmission_queue.create_user_search_message(FakeReturnSearchRowFound)
        created_message = json.loads(created_message)
        assert created_message['TITLE_NUMBER'] == FakeSearchTransmissionJSON['TITLE_NUMBER']

    def test_message_is_not_created_when_no_row_returned(self):
        created_message = legacy_transmission_queue.create_user_search_message(FakeReturnNoSearchRowFound)
        assert created_message == {}

    def test_message_is_published_when_created_message_is_not_empty(self):
        sent_message = legacy_transmission_queue.send_legacy_transmission(FakeSearchTransmissionJSON)
        assert sent_message is True

    def test_message_is_not_published_when_created_message_is_empty(self):
        sent_message = legacy_transmission_queue.send_legacy_transmission(FakeEmptySearchTransmissionJSON)
        assert sent_message is False
<commit_msg>Make two tests integration tests<commit_after>import json
from decimal import Decimal
from datetime import datetime, date, time
from service import legacy_transmission_queue

FakeReturnSearchRowFound = [datetime(2016, 1, 26, 13, 0, 30, 5449),
                            '1234',
                            'Test User',
                            datetime(2015, 12, 1, 14, 34, 14, 362556),
                            "D",
                            "drvSummaryView",
                            Decimal('2'),
                            "374f501f4567",
                            "GR12345",
                            "Search"]

FakeReturnNoSearchRowFound = None

FakeSearchTransmissionJSON = {"SEARCH_DATETIME": "2016-01-26 13:00:30.005449",
                              "LRO_TRANS_REF": "1234",
                              "USER_ID": "Test User",
                              "VIEWED_DATETIME": "2015-12-01 14:34:14.362556",
                              "SEARCH_TYPE": "D",
                              "PURCHASE_TYPE": "drvSummaryView",
                              "AMOUNT": 2,
                              "CART_ID": "374f501f4567",
                              "TITLE_NUMBER": "GR12345",
                              "EVENT_ID": "Search"
                              }

FakeEmptySearchTransmissionJSON = {}

class TestCreateSearchMessage:

    def test_message_is_created_when_db_row_is_returned(self):
        created_message = legacy_transmission_queue.create_user_search_message(FakeReturnSearchRowFound)
        created_message = json.loads(created_message)
        assert created_message['TITLE_NUMBER'] == FakeSearchTransmissionJSON['TITLE_NUMBER']

    def test_message_is_not_created_when_no_row_returned(self):
        created_message = legacy_transmission_queue.create_user_search_message(FakeReturnNoSearchRowFound)
        assert created_message == {}

    def integration_test_message_is_published_when_created_message_is_not_empty(self):
        sent_message = legacy_transmission_queue.send_legacy_transmission(FakeSearchTransmissionJSON)
        assert sent_message is True

    def integration_test_message_is_not_published_when_created_message_is_empty(self):
        sent_message = legacy_transmission_queue.send_legacy_transmission(FakeEmptySearchTransmissionJSON)
        assert sent_message is False
<|endoftext|>


<commit_before>from django.forms import ModelForm, Textarea, ChoiceField
from django.urls import reverse_lazy

from .models import Meeting, Folder

MD_INPUT = {
    'class': 'markdown-input',
    'data-endpoint': reverse_lazy('utilities:preview_safe')
}

def sorted_folders():
    return sorted([(x.pk, str(x)) for x in Folder.objects.all()], key=lambda x: x[1])

class MeetingForm(ModelForm):
    folder = ChoiceField(choices=sorted_folders)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    class Meta:
        model = Meeting
        fields = ['name', 'folder', 'title', 'body', 'date']
        widgets = {
            'body': Textarea(attrs=MD_INPUT),
        }

    def clean_folder(self):
        return Folder.objects.get(pk=self.cleaned_data['folder'])
<commit_msg>Revert "Allow admin preview in minutes as well"<commit_after>from django.forms import ModelForm, Textarea, ChoiceField
from django.utils.functional import lazy

from .models import Meeting, Folder

MD_INPUT = {
    'class': 'markdown-input'
}

def sorted_folders():
    return sorted([(x.pk, str(x)) for x in Folder.objects.all()], key=lambda x: x[1])

class MeetingForm(ModelForm):
    folder = ChoiceField(choices=sorted_folders)

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    class Meta:
        model = Meeting
        fields = ['name', 'folder', 'title', 'body', 'date']
        widgets = {
            'body': Textarea(attrs=MD_INPUT),
        }

    def clean_folder(self):
        return Folder.objects.get(pk=self.cleaned_data['folder'])
<|endoftext|>