<commit_before># Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Dictionary of configuration types for cbuildbot.

Each dictionary entry is in turn a dictionary of config_param->value.

config_param's:
board -- The board of the image to build.

master -- This bot pushes changes to the overlays.
important -- Master bot uses important bots to determine overall status.
             i.e. if master bot succeeds and other important slaves succeed
             then the master will uprev packages.  This should align
             with info vs. closer except for the master.and options.tests
hostname -- Needed for 'important' slaves.  The hostname of the bot.  Should
            match hostname in slaves.cfg in buildbot checkout.

uprev -- Uprevs the local ebuilds to build new changes since last stable.
         build.  If master then also pushes these changes on success.
rev_overlays -- Select what overlays to look at for revving. This can be
                'public', 'private' or 'both'.
push_overlays -- Select what overlays to push at. This should be a subset of
                 rev_overlays for the particular builder.  Must be None if
                 not a master.  There should only be one master bot pushing
                 changes to each overlay per branch.

unittests -- Runs unittests for packages.
vm_tests -- Runs the smoke suite and au test harness in a qemu-based VM
            using KVM.

usepkg -- Use binary packages to bootstrap, when possible. (emerge --usepkg)
chroot_replace -- wipe and replace chroot, but not source.

archive_build -- Do we run archive_build.sh
test_mod -- Create a test mod image for archival.
factory_install_mod -- Create a factory install image for archival.
factory_test_mod -- Create a factory test image for archival.
"""

# TODO(dgarrett) Make test_mod, factory_install_mod, factory_test_mod options
# go away when these options work for arm.

default = {
  # 'board' No default value

  'master' : False,
  'important' : False,
  # 'hostname' No default value

  'uprev' : False,
  'rev_overlays': 'public',
  'push_overlays': None,

  'unittests' : True,
  'vm_tests' : True,

  'usepkg' : True,
  'chroot_replace' : False,

  'archive_build' : False,
  'test_mod' : True,
  'factory_install_mod' : True,
  'factory_test_mod' : True,
}

arm = {
  # VM/tests are broken on arm.
  'unittests' : False,
  'vm_tests' : False,

  # These images don't work for arm.
  'factory_install_mod' : False,
  'factory_test_mod' : False,
}

full = {
  # Full builds are test build to show that we can build from scratch,
  # so use settings to build from scratch, and archive the results.
  'usepkg' : False,
  'chroot_replace' : True,

  'archive_build' : True
}

config = {}

config['x86-generic-pre-flight-queue'] = default.copy()
config['x86-generic-pre-flight-queue'].update({
  'board' : 'x86-generic',
  'master' : True,
  'hostname' : 'chromeosbuild2',

  'uprev' : True,
  'rev_overlays': 'public',
  'push_overlays': 'public',
})

config['x86-generic-chrome-pre-flight-queue'] = default.copy()
config['x86-generic-chrome-pre-flight-queue'].update({
  'board' : 'x86-generic',
  'master' : True,

  'uprev' : True,
  'rev_overlays': 'public',
  'push_overlays': 'public',
})

config['x86-mario-pre-flight-queue'] = default.copy()
config['x86-mario-pre-flight-queue'].update({
  'board' : 'x86-mario',
  'master' : True,

  'uprev' : True,
  'rev_overlays': 'both',
  'push_overlays': 'private',
})

config['x86-mario-pre-flight-branch'] = default.copy()
config['x86-mario-pre-flight-branch'].update({
  'board' : 'x86-mario',
  'master' : True,

  'uprev' : True,
  'rev_overlays': 'both',
  'push_overlays': 'both',
})

config['x86-agz-bin'] = default.copy()
config['x86-agz-bin'].update({
  'board' : 'x86-agz',

  'uprev' : True,
  'rev_overlays': 'both',
  'push_overlays': None,
})

config['x86-dogfood-bin'] = default.copy()
config['x86-dogfood-bin'].update({
  'board' : 'x86-dogfood',

  'uprev' : True,
  'rev_overlays': 'both',
  'push_overlays': None,
})

config['x86-pineview-bin'] = default.copy()
config['x86-pineview-bin'].update({
  'board' : 'x86-pineview',

  'uprev' : True,
  'rev_overlays': 'public',
  'push_overlays': None,
})

config['arm-tegra2-bin'] = default.copy()
config['arm-tegra2-bin'].update(arm)
config['arm-tegra2-bin'].update({
  'board' : 'tegra2_dev-board',

  'uprev' : True,
  'rev_overlays': 'public',
  'push_overlays': None,
})

config['arm-generic-bin'] = default.copy()
config['arm-generic-bin'].update(arm)
config['arm-generic-bin'].update({
  'board' : 'arm-generic',

  'uprev' : True,
  'rev_overlays': 'public',
  'push_overlays': None,
})

config['arm-generic-full'] = default.copy()
config['arm-generic-full'].update(arm)
config['arm-generic-full'].update(full)
config['arm-generic-full'].update({
  'board' : 'arm-generic',
})

config['arm-tegra2-full'] = default.copy()
config['arm-tegra2-full'].update(arm)
config['arm-tegra2-full'].update(full)
config['arm-tegra2-full'].update({
  'board' : 'tegra2_dev-board',
})

config['arm-tegra2-seaboard-full'] = default.copy()
config['arm-tegra2-seaboard-full'].update(arm)
config['arm-tegra2-seaboard-full'].update(full)
config['arm-tegra2-seaboard-full'].update({
  'board' : 'tegra2_seaboard',
})

config['x86-generic-full'] = default.copy()
config['x86-generic-full'].update(full)
config['x86-generic-full'].update({
  'board' : 'x86-generic',
})

config['x86-pineview-full'] = default.copy()
config['x86-pineview-full'].update(full)
config['x86-pineview-full'].update({
  'board' : 'x86-pineview',
})

# TODO(dgarrett) delete when buildbot updated to use new names
config['x86_agz_bin'] = config['x86-agz-bin']
config['x86_dogfood_bin'] = config['x86-dogfood-bin']
config['x86_pineview_bin'] = config['x86-pineview-bin']
config['arm_tegra2_bin'] = config['arm-tegra2-bin']
config['arm_generic_bin'] = config['arm-generic-bin']
<commit_msg>Change uprev to False for Chrome pfq.<commit_after># Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Dictionary of configuration types for cbuildbot.

Each dictionary entry is in turn a dictionary of config_param->value.

config_param's:
board -- The board of the image to build.

master -- This bot pushes changes to the overlays.
important -- Master bot uses important bots to determine overall status.
             i.e. if master bot succeeds and other important slaves succeed
             then the master will uprev packages.  This should align
             with info vs. closer except for the master.and options.tests
hostname -- Needed for 'important' slaves.  The hostname of the bot.  Should
            match hostname in slaves.cfg in buildbot checkout.

uprev -- Uprevs the local ebuilds to build new changes since last stable.
         build.  If master then also pushes these changes on success.
rev_overlays -- Select what overlays to look at for revving. This can be
                'public', 'private' or 'both'.
push_overlays -- Select what overlays to push at. This should be a subset of
                 rev_overlays for the particular builder.  Must be None if
                 not a master.  There should only be one master bot pushing
                 changes to each overlay per branch.

unittests -- Runs unittests for packages.
vm_tests -- Runs the smoke suite and au test harness in a qemu-based VM
            using KVM.

usepkg -- Use binary packages to bootstrap, when possible. (emerge --usepkg)
chroot_replace -- wipe and replace chroot, but not source.

archive_build -- Do we run archive_build.sh
test_mod -- Create a test mod image for archival.
factory_install_mod -- Create a factory install image for archival.
factory_test_mod -- Create a factory test image for archival.
"""

# TODO(dgarrett) Make test_mod, factory_install_mod, factory_test_mod options
# go away when these options work for arm.

default = {
  # 'board' No default value

  'master' : False,
  'important' : False,
  # 'hostname' No default value

  'uprev' : False,
  'rev_overlays': 'public',
  'push_overlays': None,

  'unittests' : True,
  'vm_tests' : True,

  'usepkg' : True,
  'chroot_replace' : False,

  'archive_build' : False,
  'test_mod' : True,
  'factory_install_mod' : True,
  'factory_test_mod' : True,
}

arm = {
  # VM/tests are broken on arm.
  'unittests' : False,
  'vm_tests' : False,

  # These images don't work for arm.
  'factory_install_mod' : False,
  'factory_test_mod' : False,
}

full = {
  # Full builds are test build to show that we can build from scratch,
  # so use settings to build from scratch, and archive the results.
  'usepkg' : False,
  'chroot_replace' : True,

  'archive_build' : True
}

config = {}

config['x86-generic-pre-flight-queue'] = default.copy()
config['x86-generic-pre-flight-queue'].update({
  'board' : 'x86-generic',
  'master' : True,
  'hostname' : 'chromeosbuild2',

  'uprev' : True,
  'rev_overlays': 'public',
  'push_overlays': 'public',
})

config['x86-generic-chrome-pre-flight-queue'] = default.copy()
config['x86-generic-chrome-pre-flight-queue'].update({
  'board' : 'x86-generic',
  'master' : True,

  'uprev' : False,
  'rev_overlays': 'public',
  'push_overlays': 'public',
})

config['x86-mario-pre-flight-queue'] = default.copy()
config['x86-mario-pre-flight-queue'].update({
  'board' : 'x86-mario',
  'master' : True,

  'uprev' : True,
  'rev_overlays': 'both',
  'push_overlays': 'private',
})

config['x86-mario-pre-flight-branch'] = default.copy()
config['x86-mario-pre-flight-branch'].update({
  'board' : 'x86-mario',
  'master' : True,

  'uprev' : True,
  'rev_overlays': 'both',
  'push_overlays': 'both',
})

config['x86-agz-bin'] = default.copy()
config['x86-agz-bin'].update({
  'board' : 'x86-agz',

  'uprev' : True,
  'rev_overlays': 'both',
  'push_overlays': None,
})

config['x86-dogfood-bin'] = default.copy()
config['x86-dogfood-bin'].update({
  'board' : 'x86-dogfood',

  'uprev' : True,
  'rev_overlays': 'both',
  'push_overlays': None,
})

config['x86-pineview-bin'] = default.copy()
config['x86-pineview-bin'].update({
  'board' : 'x86-pineview',

  'uprev' : True,
  'rev_overlays': 'public',
  'push_overlays': None,
})

config['arm-tegra2-bin'] = default.copy()
config['arm-tegra2-bin'].update(arm)
config['arm-tegra2-bin'].update({
  'board' : 'tegra2_dev-board',

  'uprev' : True,
  'rev_overlays': 'public',
  'push_overlays': None,
})

config['arm-generic-bin'] = default.copy()
config['arm-generic-bin'].update(arm)
config['arm-generic-bin'].update({
  'board' : 'arm-generic',

  'uprev' : True,
  'rev_overlays': 'public',
  'push_overlays': None,
})

config['arm-generic-full'] = default.copy()
config['arm-generic-full'].update(arm)
config['arm-generic-full'].update(full)
config['arm-generic-full'].update({
  'board' : 'arm-generic',
})

config['arm-tegra2-full'] = default.copy()
config['arm-tegra2-full'].update(arm)
config['arm-tegra2-full'].update(full)
config['arm-tegra2-full'].update({
  'board' : 'tegra2_dev-board',
})

config['arm-tegra2-seaboard-full'] = default.copy()
config['arm-tegra2-seaboard-full'].update(arm)
config['arm-tegra2-seaboard-full'].update(full)
config['arm-tegra2-seaboard-full'].update({
  'board' : 'tegra2_seaboard',
})

config['x86-generic-full'] = default.copy()
config['x86-generic-full'].update(full)
config['x86-generic-full'].update({
  'board' : 'x86-generic',
})

config['x86-pineview-full'] = default.copy()
config['x86-pineview-full'].update(full)
config['x86-pineview-full'].update({
  'board' : 'x86-pineview',
})

# TODO(dgarrett) delete when buildbot updated to use new names
config['x86_agz_bin'] = config['x86-agz-bin']
config['x86_dogfood_bin'] = config['x86-dogfood-bin']
config['x86_pineview_bin'] = config['x86-pineview-bin']
config['arm_tegra2_bin'] = config['arm-tegra2-bin']
config['arm_generic_bin'] = config['arm-generic-bin']
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
##
## This file is part of CDS Indico.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Indico; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
`fossilize` allows us to "serialize" complex python objects into dictionaries
and lists. Such operation is very useful for generating JSON data structures
from business objects. It works as a wrapper around `zope.interface`.

Some of the features are:
 * Different "fossil" types for the same source class;
 * Built-in inheritance support;
"""

import inspect
import re
import zope.interface
from types import NoneType, ClassType, TypeType

def fossilizes(*classList):
    """
    Simple wrapper around 'implements'
    """
    zope.interface.declarations._implements("fossilizes",
                                            classList,
                                            zope.interface.classImplements)

def addFossil(klazz, fossils):
    """
    Declares fossils for a class

    :param klazz: a class object
    :type klass: class object
    :param fossils: a fossil class (or a list of fossil classes)
    """
    if not type(fossils) is list:
        fossils = [fossils]

    for fossil in fossils:
        zope.interface.classImplements(klazz, fossil)

class NonFossilizableException(Exception):
    """
    Object is not fossilizable (doesn't implement Fossilizable)
    """

class WrongFossilTypeException(Exception):
    """
    Fossil type doesn't apply to target object
    """

class InvalidFossilException(Exception):
    """
    The fossil name doesn't follow the convention I(\w+)Fossil
    or has an invalid method name and did not declare a .name tag for it
    """

class IFossil(zope.interface.Interface):
    """
    Fossil base interface. All fossil classes should derive from this one.
    """

class Fossilizable(object):
    """
    Base class for all the objects that can be fossilized
    """

    __fossilNameRE = re.compile('^I(\w+)Fossil$')
    __methodNameRE = re.compile('^get(\w+)|(has\w+)|(is\w+)$')
    __methodNameCache = {}
    __fossilNameCache = {}

    @classmethod
    def __extractName(cls, name):
        """
        'De-camelcase' the name
        """

        if name in cls.__methodNameCache:
            return cls.__methodNameCache[name]
        else:
            nmatch = cls.__methodNameRE.match(name)

            if not nmatch:
                raise InvalidFossilException("method name '%s' is not valid! "
                                             "has to start by 'get', 'has', 'is' "
                                             "or use 'name' tag" % name)
            else:
                group = nmatch.group(1) or nmatch.group(2) or nmatch.group(3)
                extractedName = group[0:1].lower() + group[1:]
                cls.__methodNameCache[name] = extractedName
                return extractedName

    @classmethod
    def __extractFossilName(cls, name):
        """
        Extracts the fossil name from a I(.*)Fossil
        class name.
        IMyObjectBasicFossil -> myObjectBasic
        """

        if name in cls.__fossilNameCache:
            fossilName = cls.__fossilNameCache[name]
        else:
            fossilNameMatch = Fossilizable.__fossilNameRE.match(name)
            if fossilNameMatch is None:
                raise InvalidFossilException("Invalid fossil name: %s."
                                             " A fossil name should follow the"
                                             " pattern: I(\w+)Fossil." % name)
            else:
                fossilName = fossilNameMatch.group(1)[0].lower() + fossilNameMatch.group(1)[1:]
                cls.__fossilNameCache[name] = fossilName
        return fossilName

    def __obtainInterface(self, interfaceArg):
        """
        Obtains the appropriate interface for this object.

        :param interfaceArg: the target fossile type
        :type interfaceArg: IFossil, NoneType, or dict

            * If IFossil, we will use it.
            * If None, we will take the default fossil (the first one of this class's "fossilizes" list)
            * If a dict, we will use the object's class, class name, or full class name as key.

        Also verifies that the interface obtained through these 3 methods is effectively provided by the object.
        """
        if interfaceArg is None:
            #we try to take the 1st interface declared with fossilizes
            implementedInterfaces = list(i for i in zope.interface.implementedBy(self.__class__) if i.extends(IFossil))
            if not implementedInterfaces:
                raise NonFossilizableException("Object %s of class %s cannot be fossilized,"
                                               "no fossils were declared for it" %
                                               (str(self),
                                                self.__class__.__name__))
            else:
                interface = implementedInterfaces[0]

        elif type(interfaceArg) is dict:

            className = self.__class__.__module__ + '.' + \
                        self.__class__.__name__

            # interfaceArg is a dictionary of class:Fossil pairs
            if className in interfaceArg:
                interface = interfaceArg[className]
            else:
                raise NonFossilizableException("Object %s of class %s cannot be fossilized; "
                                               "its class was not a key in the provided fossils dictionary" %
                                               (str(self),
                                                self.__class__.__name__))
        else:
            interface = interfaceArg

        if not interface.providedBy(self):
            raise WrongFossilTypeException("Interface '%s' not provided"
                                           " by '%s'" %
                                           (interface.__name__,
                                            self.__class__.__name__))

        return interface

    @classmethod
    def _fossilizeIterable(cls, target, interface, **kwargs):
        """
        Fossilizes an object, be it a 'direct' fossilizable
        object, or an iterable (dict, list, set);
        """

        if isinstance(target, Fossilizable):
            return target.fossilize(interface, **kwargs)
        else:
            ttype = type(target)
            if ttype in [int, str, float, NoneType]:
                return target
            elif ttype in [list, set, tuple]:
                container = [] #we turn sets and tuples into lists since JSON does not have sets / tuples
                for elem in target:
                    container.append(fossilize(elem, interface, **kwargs))
                return container
            elif ttype is dict:
                container = {}
                for key, value in target.iteritems():
                    container[key] = fossilize(value, interface, **kwargs)
                return container
            else:
                raise NonFossilizableException()

            return fossilize(target, interface)

    def fossilize(self, interfaceArg = None, **kwargs):
        """
        Fossilizes the object, using the fossil provided by `interface`.

        :param interfaceArg: the target fossile type
        :type interfaceArg: IFossil, NoneType, or dict
        """

        interface = self.__obtainInterface(interfaceArg)

        name = interface.getName()
        fossilName = self.__extractFossilName(name)

        result = {}

        for method in interface:

            tags = interface[method].getTaggedValueTags()
            #Please use 'produce' as little as possible; there is almost always a more elegant and modular solution!
            if 'produce' in tags:
                methodResult = interface[method].getTaggedValue('produce')(self)
            else:
                methodResult = getattr(self, method)()

            # Result conversion
            if 'result' in tags:
                targetInterface = interface[method].getTaggedValue('result')
                #targetInterface = globals()[targetInterfaceName]
                methodResult = Fossilizable._fossilizeIterable(
                    methodResult, targetInterface, **kwargs)

            # Conversion function
            if 'convert' in tags:
                convertFunction = interface[method].getTaggedValue('convert')
                converterArgNames = inspect.getargspec(convertFunction)[0]
                converterArgs = dict((name, kwargs[name])
                                     for name in converterArgNames
                                     if name in kwargs)
                methodResult = convertFunction(methodResult, **converterArgs)

            # Re-name the attribute produced by the method
            if 'name' in tags:
                attrName = interface[method].getTaggedValue('name')
            else:
                attrName = self.__extractName(method)

            result[attrName] = methodResult

        if "_type" in result or "_fossil" in result:
            raise InvalidFossilException('"_type" or "_fossil"'
                                         ' cannot be a fossil attribute  name')
        else:
            result["_type"] = self.__class__.__name__
            if fossilName: #we check that it's not an empty string
                result["_fossil"] = fossilName
            else:
                result["_fossil"] = ""

        return result

def fossilize(target, interfaceArg = None, **kwargs):
    """
    Method that allows the "fossilization" process to
    be called on data structures (lists, dictionaries
    and sets) as well as normal `Fossilizable` objects.

    :param target: target object to be fossilized
    :type target: Fossilizable
    :param interfaceArg: target fossil type
    :type interfaceArg: IFossil, NoneType, or dict
    """
    return Fossilizable._fossilizeIterable(target, interfaceArg, **kwargs)
<commit_msg>[FIX] Fix for CausedError exception problem<commit_after># -*- coding: utf-8 -*-
##
##
## This file is part of CDS Indico.
## Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007 CERN.
##
## CDS Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## CDS Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with CDS Indico; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
`fossilize` allows us to "serialize" complex python objects into dictionaries
and lists. Such operation is very useful for generating JSON data structures
from business objects. It works as a wrapper around `zope.interface`.

Some of the features are:
 * Different "fossil" types for the same source class;
 * Built-in inheritance support;
"""

import inspect
import re
import zope.interface
from types import NoneType, ClassType, TypeType

def fossilizes(*classList):
    """
    Simple wrapper around 'implements'
    """
    zope.interface.declarations._implements("fossilizes",
                                            classList,
                                            zope.interface.classImplements)

def addFossil(klazz, fossils):
    """
    Declares fossils for a class

    :param klazz: a class object
    :type klass: class object
    :param fossils: a fossil class (or a list of fossil classes)
    """
    if not type(fossils) is list:
        fossils = [fossils]

    for fossil in fossils:
        zope.interface.classImplements(klazz, fossil)

class NonFossilizableException(Exception):
    """
    Object is not fossilizable (doesn't implement Fossilizable)
    """

class WrongFossilTypeException(Exception):
    """
    Fossil type doesn't apply to target object
    """

class InvalidFossilException(Exception):
    """
    The fossil name doesn't follow the convention I(\w+)Fossil
    or has an invalid method name and did not declare a .name tag for it
    """

class IFossil(zope.interface.Interface):
    """
    Fossil base interface. All fossil classes should derive from this one.
    """

class Fossilizable:
    """
    Base class for all the objects that can be fossilized
    """

    __fossilNameRE = re.compile('^I(\w+)Fossil$')
    __methodNameRE = re.compile('^get(\w+)|(has\w+)|(is\w+)$')
    __methodNameCache = {}
    __fossilNameCache = {}

    @classmethod
    def __extractName(cls, name):
        """
        'De-camelcase' the name
        """

        if name in cls.__methodNameCache:
            return cls.__methodNameCache[name]
        else:
            nmatch = cls.__methodNameRE.match(name)

            if not nmatch:
                raise InvalidFossilException("method name '%s' is not valid! "
                                             "has to start by 'get', 'has', 'is' "
                                             "or use 'name' tag" % name)
            else:
                group = nmatch.group(1) or nmatch.group(2) or nmatch.group(3)
                extractedName = group[0:1].lower() + group[1:]
                cls.__methodNameCache[name] = extractedName
                return extractedName

    @classmethod
    def __extractFossilName(cls, name):
        """
        Extracts the fossil name from a I(.*)Fossil
        class name.
        IMyObjectBasicFossil -> myObjectBasic
        """

        if name in cls.__fossilNameCache:
            fossilName = cls.__fossilNameCache[name]
        else:
            fossilNameMatch = Fossilizable.__fossilNameRE.match(name)
            if fossilNameMatch is None:
                raise InvalidFossilException("Invalid fossil name: %s."
                                             " A fossil name should follow the"
                                             " pattern: I(\w+)Fossil." % name)
            else:
                fossilName = fossilNameMatch.group(1)[0].lower() + fossilNameMatch.group(1)[1:]
                cls.__fossilNameCache[name] = fossilName
        return fossilName

    def __obtainInterface(self, interfaceArg):
        """
        Obtains the appropriate interface for this object.

        :param interfaceArg: the target fossile type
        :type interfaceArg: IFossil, NoneType, or dict

            * If IFossil, we will use it.
            * If None, we will take the default fossil (the first one of this class's "fossilizes" list)
            * If a dict, we will use the object's class, class name, or full class name as key.

        Also verifies that the interface obtained through these 3 methods is effectively provided by the object.
        """
        if interfaceArg is None:
            #we try to take the 1st interface declared with fossilizes
            implementedInterfaces = list(i for i in zope.interface.implementedBy(self.__class__) if i.extends(IFossil))
            if not implementedInterfaces:
                raise NonFossilizableException("Object %s of class %s cannot be fossilized,"
                                               "no fossils were declared for it" %
                                               (str(self),
                                                self.__class__.__name__))
            else:
                interface = implementedInterfaces[0]

        elif type(interfaceArg) is dict:

            className = self.__class__.__module__ + '.' + \
                        self.__class__.__name__

            # interfaceArg is a dictionary of class:Fossil pairs
            if className in interfaceArg:
                interface = interfaceArg[className]
            else:
                raise NonFossilizableException("Object %s of class %s cannot be fossilized; "
                                               "its class was not a key in the provided fossils dictionary" %
                                               (str(self),
                                                self.__class__.__name__))
        else:
            interface = interfaceArg

        if not interface.providedBy(self):
            raise WrongFossilTypeException("Interface '%s' not provided"
                                           " by '%s'" %
                                           (interface.__name__,
                                            self.__class__.__name__))

        return interface

    @classmethod
    def _fossilizeIterable(cls, target, interface, **kwargs):
        """
        Fossilizes an object, be it a 'direct' fossilizable
        object, or an iterable (dict, list, set);
        """

        if isinstance(target, Fossilizable):
            return target.fossilize(interface, **kwargs)
        else:
            ttype = type(target)
            if ttype in [int, str, float, NoneType]:
                return target
            elif ttype in [list, set, tuple]:
                container = [] #we turn sets and tuples into lists since JSON does not have sets / tuples
                for elem in target:
                    container.append(fossilize(elem, interface, **kwargs))
                return container
            elif ttype is dict:
                container = {}
                for key, value in target.iteritems():
                    container[key] = fossilize(value, interface, **kwargs)
                return container
            else:
                raise NonFossilizableException()

            return fossilize(target, interface)

    def fossilize(self, interfaceArg = None, **kwargs):
        """
        Fossilizes the object, using the fossil provided by `interface`.

        :param interfaceArg: the target fossile type
        :type interfaceArg: IFossil, NoneType, or dict
        """

        interface = self.__obtainInterface(interfaceArg)

        name = interface.getName()
        fossilName = self.__extractFossilName(name)

        result = {}

        for method in interface:

            tags = interface[method].getTaggedValueTags()
            #Please use 'produce' as little as possible; there is almost always a more elegant and modular solution!
            if 'produce' in tags:
                methodResult = interface[method].getTaggedValue('produce')(self)
            else:
                methodResult = getattr(self, method)()

            # Result conversion
            if 'result' in tags:
                targetInterface = interface[method].getTaggedValue('result')
                #targetInterface = globals()[targetInterfaceName]
                methodResult = Fossilizable._fossilizeIterable(
                    methodResult, targetInterface, **kwargs)

            # Conversion function
            if 'convert' in tags:
                convertFunction = interface[method].getTaggedValue('convert')
                converterArgNames = inspect.getargspec(convertFunction)[0]
                converterArgs = dict((name, kwargs[name])
                                     for name in converterArgNames
                                     if name in kwargs)
                methodResult = convertFunction(methodResult, **converterArgs)

            # Re-name the attribute produced by the method
            if 'name' in tags:
                attrName = interface[method].getTaggedValue('name')
            else:
                attrName = self.__extractName(method)

            result[attrName] = methodResult

        if "_type" in result or "_fossil" in result:
            raise InvalidFossilException('"_type" or "_fossil"'
                                         ' cannot be a fossil attribute  name')
        else:
            result["_type"] = self.__class__.__name__
            if fossilName: #we check that it's not an empty string
                result["_fossil"] = fossilName
            else:
                result["_fossil"] = ""

        return result

def fossilize(target, interfaceArg = None, **kwargs):
    """
    Method that allows the "fossilization" process to
    be called on data structures (lists, dictionaries
    and sets) as well as normal `Fossilizable` objects.

    :param target: target object to be fossilized
    :type target: Fossilizable
    :param interfaceArg: target fossil type
    :type interfaceArg: IFossil, NoneType, or dict
    """
    return Fossilizable._fossilizeIterable(target, interfaceArg, **kwargs)
<|endoftext|>


<commit_before>from contextlib import contextmanager
from zipfile import ZipFile, ZIP_DEFLATED
import csv
import datetime
import os
import pytz
import shutil
import tempfile

from ichnaea.backup.s3 import S3Backend, compute_hash
from ichnaea.models import (
    MEASURE_TYPE_CODE,
    MEASURE_TYPE_CODE_INVERSE,
    MEASURE_TYPE_META,
    MeasureBlock,
)
from ichnaea.tasks import DatabaseTask
from ichnaea.worker import celery
from sqlalchemy import func

@contextmanager
def selfdestruct_tempdir(s3_key):
    """
    We need two temp directories to do this properly.

    The base_path is a temp directory that holds all the content that
    will go into our zip file. This is effectively a working
    directory that gets immediately deleted once the zip file is
    ready.

    The zip_path is the filename of the zip file that will get
    uploaded into S3. It is the responsibility of the caller of
    selfdestruct_tempdir to remove the zip_path and parent directory.
    """
    short_name = os.path.split(s3_key)[-1]

    base_path = tempfile.mkdtemp()
    s3_path = os.path.join(tempfile.mkdtemp(), short_name)
    try:
        zip_path = os.path.join(base_path, s3_path)
        yield base_path, zip_path
    finally:
        try:
            z = ZipFile(zip_path, "w", ZIP_DEFLATED)
            try:
                for root, dirs, files in os.walk(base_path):
                    for fn in files:
                        absfn = os.path.join(root, fn)
                        zip_fn = absfn[len(base_path) + len(os.sep):]
                        z.write(absfn, zip_fn)
            finally:
                z.close()
        finally:
            shutil.rmtree(base_path)

@celery.task(base=DatabaseTask, bind=True)
def write_cellmeasure_s3_backups(self,
                                 limit=100,
                                 batch=10000,
                                 cleanup_zip=True):
    measure_type = MEASURE_TYPE_CODE['cell']
    return write_measure_s3_backups(self,
                                    measure_type,
                                    limit=limit,
                                    batch=batch,
                                    cleanup_zip=cleanup_zip)

@celery.task(base=DatabaseTask, bind=True)
def write_wifimeasure_s3_backups(self,
                                 limit=100,
                                 batch=10000,
                                 cleanup_zip=True):
    measure_type = MEASURE_TYPE_CODE['wifi']
    return write_measure_s3_backups(self,
                                    measure_type,
                                    limit=limit,
                                    batch=batch,
                                    cleanup_zip=cleanup_zip)

def write_measure_s3_backups(self,
                             measure_type,
                             limit=100,
                             batch=10000,
                             cleanup_zip=True):
    """
    Iterate over each of the measure block records that aren't
    backed up yet and back them up.

    Assume that this is running in a single task.
    """
    with self.db_session() as session:

        query = session.query(MeasureBlock).filter(
            MeasureBlock.measure_type == measure_type).filter(
            MeasureBlock.s3_key.is_(None)).order_by(
            MeasureBlock.end_id).limit(limit)

        for block in query:
            write_block_to_s3.delay(block.id,
                                    batch,
                                    cleanup_zip)

@celery.task(base=DatabaseTask, bind=True)
def write_block_to_s3(self, block_id, batch=10000, cleanup_zip=True):

    with self.db_session() as session:
        block = session.query(MeasureBlock).filter(
            MeasureBlock.id == block_id).first()

        measure_type = block.measure_type
        measure_cls = MEASURE_TYPE_META[measure_type]['class']
        csv_name = MEASURE_TYPE_META[measure_type]['csv_name']
        zip_prefix = MEASURE_TYPE_CODE_INVERSE[measure_type]

        start_id = block.start_id
        end_id = block.end_id

        rset = session.execute("select version_num from alembic_version")
        alembic_rev = rset.first()[0]

        s3_backend = S3Backend(
            self.app.s3_settings['backup_bucket'],
            self.app.s3_settings['backup_prefix'],
            self.heka_client)

        utcnow = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
        s3_key = '%s/%s_%d_%d.zip' % (utcnow.strftime("%Y%m"),
                                      zip_prefix,
                                      start_id,
                                      end_id)

        with selfdestruct_tempdir(s3_key) as (tmp_path, zip_path):
            with open(os.path.join(tmp_path,
                                   'alembic_revision.txt'), 'w') as f:
                f.write('%s\n' % alembic_rev)

            # avoid ORM session overhead
            table = measure_cls.__table__

            cm_fname = os.path.join(tmp_path, csv_name)
            with open(cm_fname, 'w') as f:
                csv_out = csv.writer(f, dialect='excel')
                columns = table.c.keys()
                csv_out.writerow(columns)
                for this_start in range(start_id,
                                        end_id,
                                        batch):
                    this_end = min(this_start + batch, end_id)

                    query = table.select().where(
                        table.c.id >= this_start).where(
                        table.c.id < this_end)

                    rproxy = session.execute(query)
                    csv_out.writerows(rproxy)

        archive_sha = compute_hash(zip_path)

        try:
            if not s3_backend.backup_archive(s3_key, zip_path):
                return
            self.heka_client.incr('s3.backup.%s' % measure_type,
                                  (end_id - start_id))
        finally:
            if cleanup_zip:
                if os.path.exists(zip_path):
                    zip_dir, zip_file = os.path.split(zip_path)
                    if os.path.exists(zip_dir):
                        shutil.rmtree(zip_dir)
            else:
                self.heka_client.debug("s3.backup:%s" % zip_path)

        # only set archive_sha / s3_key if upload was successful
        block.archive_sha = archive_sha
        block.s3_key = s3_key
        session.commit()

def schedule_measure_archival(self, measure_type, limit=100, batch=1000000):
    blocks = []
    measure_cls = MEASURE_TYPE_META[measure_type]['class']
    with self.db_session() as session:
        query = session.query(MeasureBlock.end_id).filter(
            MeasureBlock.measure_type == measure_type).order_by(
            MeasureBlock.end_id.desc())
        record = query.first()
        if record is not None:
            min_id = record[0]
        else:
            query = session.query(measure_cls.id).order_by(
                measure_cls.id.asc())
            record = query.first()
            if record is not None:
                min_id = record[0]
            else:
                # no data in the table
                return blocks

        query = session.query(measure_cls.id).order_by(
            measure_cls.id.desc())
        record = query.first()
        if record is None:
            # no data in the table
            return blocks

        max_id = record[0]
        if max_id - min_id < batch - 1:
            # Not enough to fill a block
            return blocks

        # We're using half-open ranges, so we need to bump the max_id
        max_id += 1

        this_max_id = min_id + batch

        i = 0
        while i < limit and (this_max_id - min_id) == batch:
            cm_blk = MeasureBlock(start_id=min_id,
                                  end_id=this_max_id,
                                  measure_type=measure_type)
            blocks.append((cm_blk.start_id, cm_blk.end_id))
            session.add(cm_blk)

            min_id = this_max_id
            this_max_id = min(batch + this_max_id, max_id)
            i += 1
        session.commit()
    return blocks

@celery.task(base=DatabaseTask, bind=True)
def schedule_cellmeasure_archival(self, limit=100, batch=1000000):
    return schedule_measure_archival(
        self, MEASURE_TYPE_CODE['cell'], limit=limit, batch=batch)

@celery.task(base=DatabaseTask, bind=True)
def schedule_wifimeasure_archival(self, limit=100, batch=1000000):
    return schedule_measure_archival(
        self, MEASURE_TYPE_CODE['wifi'], limit=limit, batch=batch)

def delete_measure_records(self,
                           measure_type,
                           limit=100,
                           days_old=7):
    utcnow = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)

    with self.db_session() as session:
        query = session.query(MeasureBlock).filter(
            MeasureBlock.measure_type == measure_type).filter(
            MeasureBlock.s3_key.isnot(None)).filter(
            MeasureBlock.archive_sha.isnot(None)).filter(
            MeasureBlock.archive_date.is_(None)).order_by(
            MeasureBlock.end_id.asc()).limit(limit)
        for block in query.all():
            # Note that 'created' is indexed for both CellMeasure
            # and WifiMeasure
            measure_cls = MEASURE_TYPE_META[measure_type]['class']
            tbl = measure_cls.__table__
            qry = session.query(func.max(tbl.c.created)).filter(
                tbl.c.id < block.end_id)
            max_created = qry.first()[0].replace(tzinfo=pytz.UTC)
            if (utcnow - max_created).days < days_old:
                # Skip this block from deletion, it's not old
                # enough
                continue

            dispatch_delete.delay(block.id)

@celery.task(base=DatabaseTask, bind=True)
def dispatch_delete(self, block_id):
    s3_backend = S3Backend(self.app.s3_settings['backup_bucket'],
                           self.app.s3_settings['backup_prefix'],
                           self.heka_client)
    utcnow = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
    with self.db_session() as session:
        block = session.query(MeasureBlock).filter(
            MeasureBlock.id == block_id).first()
        measure_type = block.measure_type
        measure_cls = MEASURE_TYPE_META[measure_type]['class']
        if s3_backend.check_archive(block.archive_sha, block.s3_key):
            q = session.query(measure_cls).filter(
                measure_cls.id >= block.start_id,
                measure_cls.id <= block.end_id)
            q.delete()
            block.archive_date = utcnow
            session.commit()

@celery.task(base=DatabaseTask, bind=True)
def delete_cellmeasure_records(self, limit=100, days_old=7):
    return delete_measure_records(
        self,
        MEASURE_TYPE_CODE['cell'],
        limit=limit,
        days_old=days_old)

@celery.task(base=DatabaseTask, bind=True)
def delete_wifimeasure_records(self, limit=100, days_old=7):
    return delete_measure_records(
        self,
        MEASURE_TYPE_CODE['wifi'],
        limit=limit,
        days_old=days_old)
<commit_msg>Use nicer metric name for backup counters.<commit_after>from contextlib import contextmanager
from zipfile import ZipFile, ZIP_DEFLATED
import csv
import datetime
import os
import pytz
import shutil
import tempfile

from ichnaea.backup.s3 import S3Backend, compute_hash
from ichnaea.models import (
    MEASURE_TYPE_CODE,
    MEASURE_TYPE_CODE_INVERSE,
    MEASURE_TYPE_META,
    MeasureBlock,
)
from ichnaea.tasks import DatabaseTask
from ichnaea.worker import celery
from sqlalchemy import func

@contextmanager
def selfdestruct_tempdir(s3_key):
    """
    We need two temp directories to do this properly.

    The base_path is a temp directory that holds all the content that
    will go into our zip file. This is effectively a working
    directory that gets immediately deleted once the zip file is
    ready.

    The zip_path is the filename of the zip file that will get
    uploaded into S3. It is the responsibility of the caller of
    selfdestruct_tempdir to remove the zip_path and parent directory.
    """
    short_name = os.path.split(s3_key)[-1]

    base_path = tempfile.mkdtemp()
    s3_path = os.path.join(tempfile.mkdtemp(), short_name)
    try:
        zip_path = os.path.join(base_path, s3_path)
        yield base_path, zip_path
    finally:
        try:
            z = ZipFile(zip_path, "w", ZIP_DEFLATED)
            try:
                for root, dirs, files in os.walk(base_path):
                    for fn in files:
                        absfn = os.path.join(root, fn)
                        zip_fn = absfn[len(base_path) + len(os.sep):]
                        z.write(absfn, zip_fn)
            finally:
                z.close()
        finally:
            shutil.rmtree(base_path)

@celery.task(base=DatabaseTask, bind=True)
def write_cellmeasure_s3_backups(self,
                                 limit=100,
                                 batch=10000,
                                 cleanup_zip=True):
    measure_type = MEASURE_TYPE_CODE['cell']
    return write_measure_s3_backups(self,
                                    measure_type,
                                    limit=limit,
                                    batch=batch,
                                    cleanup_zip=cleanup_zip)

@celery.task(base=DatabaseTask, bind=True)
def write_wifimeasure_s3_backups(self,
                                 limit=100,
                                 batch=10000,
                                 cleanup_zip=True):
    measure_type = MEASURE_TYPE_CODE['wifi']
    return write_measure_s3_backups(self,
                                    measure_type,
                                    limit=limit,
                                    batch=batch,
                                    cleanup_zip=cleanup_zip)

def write_measure_s3_backups(self,
                             measure_type,
                             limit=100,
                             batch=10000,
                             cleanup_zip=True):
    """
    Iterate over each of the measure block records that aren't
    backed up yet and back them up.

    Assume that this is running in a single task.
    """
    with self.db_session() as session:

        query = session.query(MeasureBlock).filter(
            MeasureBlock.measure_type == measure_type).filter(
            MeasureBlock.s3_key.is_(None)).order_by(
            MeasureBlock.end_id).limit(limit)

        for block in query:
            write_block_to_s3.delay(block.id,
                                    batch,
                                    cleanup_zip)

@celery.task(base=DatabaseTask, bind=True)
def write_block_to_s3(self, block_id, batch=10000, cleanup_zip=True):

    with self.db_session() as session:
        block = session.query(MeasureBlock).filter(
            MeasureBlock.id == block_id).first()

        measure_type = block.measure_type
        measure_cls = MEASURE_TYPE_META[measure_type]['class']
        csv_name = MEASURE_TYPE_META[measure_type]['csv_name']
        name = MEASURE_TYPE_CODE_INVERSE[measure_type]

        start_id = block.start_id
        end_id = block.end_id

        rset = session.execute("select version_num from alembic_version")
        alembic_rev = rset.first()[0]

        s3_backend = S3Backend(
            self.app.s3_settings['backup_bucket'],
            self.app.s3_settings['backup_prefix'],
            self.heka_client)

        utcnow = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
        s3_key = '%s/%s_%d_%d.zip' % (utcnow.strftime("%Y%m"),
                                      name,
                                      start_id,
                                      end_id)

        with selfdestruct_tempdir(s3_key) as (tmp_path, zip_path):
            with open(os.path.join(tmp_path,
                                   'alembic_revision.txt'), 'w') as f:
                f.write('%s\n' % alembic_rev)

            # avoid ORM session overhead
            table = measure_cls.__table__

            cm_fname = os.path.join(tmp_path, csv_name)
            with open(cm_fname, 'w') as f:
                csv_out = csv.writer(f, dialect='excel')
                columns = table.c.keys()
                csv_out.writerow(columns)
                for this_start in range(start_id,
                                        end_id,
                                        batch):
                    this_end = min(this_start + batch, end_id)

                    query = table.select().where(
                        table.c.id >= this_start).where(
                        table.c.id < this_end)

                    rproxy = session.execute(query)
                    csv_out.writerows(rproxy)

        archive_sha = compute_hash(zip_path)

        try:
            if not s3_backend.backup_archive(s3_key, zip_path):
                return
            self.heka_client.incr('s3.backup.%s' % name,
                                  (end_id - start_id))
        finally:
            if cleanup_zip:
                if os.path.exists(zip_path):
                    zip_dir, zip_file = os.path.split(zip_path)
                    if os.path.exists(zip_dir):
                        shutil.rmtree(zip_dir)
            else:
                self.heka_client.debug("s3.backup:%s" % zip_path)

        # only set archive_sha / s3_key if upload was successful
        block.archive_sha = archive_sha
        block.s3_key = s3_key
        session.commit()

def schedule_measure_archival(self, measure_type, limit=100, batch=1000000):
    blocks = []
    measure_cls = MEASURE_TYPE_META[measure_type]['class']
    with self.db_session() as session:
        query = session.query(MeasureBlock.end_id).filter(
            MeasureBlock.measure_type == measure_type).order_by(
            MeasureBlock.end_id.desc())
        record = query.first()
        if record is not None:
            min_id = record[0]
        else:
            query = session.query(measure_cls.id).order_by(
                measure_cls.id.asc())
            record = query.first()
            if record is not None:
                min_id = record[0]
            else:
                # no data in the table
                return blocks

        query = session.query(measure_cls.id).order_by(
            measure_cls.id.desc())
        record = query.first()
        if record is None:
            # no data in the table
            return blocks

        max_id = record[0]
        if max_id - min_id < batch - 1:
            # Not enough to fill a block
            return blocks

        # We're using half-open ranges, so we need to bump the max_id
        max_id += 1

        this_max_id = min_id + batch

        i = 0
        while i < limit and (this_max_id - min_id) == batch:
            cm_blk = MeasureBlock(start_id=min_id,
                                  end_id=this_max_id,
                                  measure_type=measure_type)
            blocks.append((cm_blk.start_id, cm_blk.end_id))
            session.add(cm_blk)

            min_id = this_max_id
            this_max_id = min(batch + this_max_id, max_id)
            i += 1
        session.commit()
    return blocks

@celery.task(base=DatabaseTask, bind=True)
def schedule_cellmeasure_archival(self, limit=100, batch=1000000):
    return schedule_measure_archival(
        self, MEASURE_TYPE_CODE['cell'], limit=limit, batch=batch)

@celery.task(base=DatabaseTask, bind=True)
def schedule_wifimeasure_archival(self, limit=100, batch=1000000):
    return schedule_measure_archival(
        self, MEASURE_TYPE_CODE['wifi'], limit=limit, batch=batch)

def delete_measure_records(self,
                           measure_type,
                           limit=100,
                           days_old=7):
    utcnow = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)

    with self.db_session() as session:
        query = session.query(MeasureBlock).filter(
            MeasureBlock.measure_type == measure_type).filter(
            MeasureBlock.s3_key.isnot(None)).filter(
            MeasureBlock.archive_sha.isnot(None)).filter(
            MeasureBlock.archive_date.is_(None)).order_by(
            MeasureBlock.end_id.asc()).limit(limit)
        for block in query.all():
            # Note that 'created' is indexed for both CellMeasure
            # and WifiMeasure
            measure_cls = MEASURE_TYPE_META[measure_type]['class']
            tbl = measure_cls.__table__
            qry = session.query(func.max(tbl.c.created)).filter(
                tbl.c.id < block.end_id)
            max_created = qry.first()[0].replace(tzinfo=pytz.UTC)
            if (utcnow - max_created).days < days_old:
                # Skip this block from deletion, it's not old
                # enough
                continue

            dispatch_delete.delay(block.id)

@celery.task(base=DatabaseTask, bind=True)
def dispatch_delete(self, block_id):
    s3_backend = S3Backend(self.app.s3_settings['backup_bucket'],
                           self.app.s3_settings['backup_prefix'],
                           self.heka_client)
    utcnow = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC)
    with self.db_session() as session:
        block = session.query(MeasureBlock).filter(
            MeasureBlock.id == block_id).first()
        measure_type = block.measure_type
        measure_cls = MEASURE_TYPE_META[measure_type]['class']
        if s3_backend.check_archive(block.archive_sha, block.s3_key):
            q = session.query(measure_cls).filter(
                measure_cls.id >= block.start_id,
                measure_cls.id <= block.end_id)
            q.delete()
            block.archive_date = utcnow
            session.commit()

@celery.task(base=DatabaseTask, bind=True)
def delete_cellmeasure_records(self, limit=100, days_old=7):
    return delete_measure_records(
        self,
        MEASURE_TYPE_CODE['cell'],
        limit=limit,
        days_old=days_old)

@celery.task(base=DatabaseTask, bind=True)
def delete_wifimeasure_records(self, limit=100, days_old=7):
    return delete_measure_records(
        self,
        MEASURE_TYPE_CODE['wifi'],
        limit=limit,
        days_old=days_old)
<|endoftext|>


<commit_before>"""
Check whether there's good concordance between training and test data
in various ways.
"""

from __future__ import print_function
from collections import defaultdict
import sys
import re
import logging

"""
First pass:
- Collect MAPQ distributions for tandem & input
- Collect ZT:Z distributions for tandem & input

Second pass:
- For tandem and input:
  + For each read, check for incorrectness.  If incorrect:
    - Record vector of percentiles for MAPQ and ZT:Zs

Finally:
- For tandem and input:
  + Sort the percentile vector by MAPQ percentile and record

Example usage:
pypy eval_concordance.py $HOME/input.sam $HOME/predictions.csv dists.txt incor.mapq incor.mapq_orig

TODO:
- Are we doing paired-end correctly for the various name formats?
"""

def pass1_fh(fh_sam, fh_pred):
    """ Histograms all the alignments """
    mapq_dist, mapq_orig_dist = defaultdict(int), defaultdict(int)
    ztz_dist = defaultdict(lambda: defaultdict(int))
    line = 0
    for ln in fh_sam:
        line += 1
        if ln[0] == '@':
            continue
        toks = ln.rstrip().split('\t')
        flags = int(toks[1])
        if (flags & 4) != 0 or (flags & 2048) != 0:
            continue
        pred = fh_pred.readline()
        assert len(pred) > 0
        predline, pred = pred.split(',')
        assert int(predline) == line, (predline, line)
        mapq_dist[int(round(float(pred)))] += 1
        mapq_orig_dist[int(toks[4])] += 1
        for tok in toks[12:]:
            if tok.startswith('ZT:Z:'):
                for i, ztok in enumerate(tok[5:].split(',')):
                    if ztok == 'NA':
                        pass
                    else:
                        ztz_dist[i][int(ztok)] += 1  # ZT:Z
                break
    return mapq_dist, mapq_orig_dist, ztz_dist

def pass1_fn(fn_sam, fn_pred):
    """ Histograms all the input and tandem alignments """
    with open(fn_sam) as fh_sam:
        with open(fn_pred) as fh_pred:
            return pass1_fh(fh_sam, fh_pred)

"""
Example: 10_26049747_26049846_0:0:0_0:0:0_100_100_0_3999999
offset is 0-based?
"""
_wgsimex_re = re.compile('(.+)_([^_]+)_([^_]+)_([^:]+):([^:]+):([^_]+)_([^:]+):([^:]+):([^_]+)_([^_]+)_([^_]+)_([^_]+)_([^/]+).*')
#                           1   2       3       4       5       6       7       8       9       10      11      12      13

def name_is_extended_wgsim(nm):
    return _wgsimex_re.match(nm) is not None

def pos_from_extended_wgsim(name, mate2=False):
    res = _wgsimex_re.match(name)
    refid, fragst1, fragen1 = res.group(1), int(res.group(2))-1, int(res.group(3))-1
    len1, len2 = int(res.group(10)), int(res.group(11))
    flip = res.group(12) == '1'
    ln = len2 if mate2 else len1
    if flip == mate2:
        return refid, fragst1, True
    else:
        return refid, fragen1 - (ln-1), False

"""
Example: qsim!:GL000229.1:+:6005:100:u
pretty sure offset is 0-based
"""
_qsim_re = re.compile('qsim!:([^:]+):([+-]):([^:]+):.*')

def name_is_qsim(nm):
    return _qsim_re.match(nm) is not None

def pos_from_qsim(name):
    res = _qsim_re.match(name)
    return res.group(1), int(res.group(3)), res.group(2) == '+'

"""
Example: !h!chr9!118085975!+!50!0
offset is 0-based
"""
_hint_re = re.compile('!h!([^!]+)!([0-9]+)!([+-])!([0-9]+)!([0-9]+)')

def name_is_hint(name):
    return _hint_re.match(name) is not None

def pos_from_hint(name):
    res = _hint_re.match(name)
    return res.group(1), int(res.group(2)), res.group(3) == '+'

"""
Example:
hg38_50nt_mason1_unp.fastq.000999999 contig=chr9 haplotype=1 length=50 orig_begin=118085976 orig_end=118086026 snps=0 indels=0 haplotype_infix=ATGACTCTTGAAGCTGGGCGCAGTGGCTCATGCCTGTAATCCTAGCACTT edit_string=MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM strand=forward
"""
_mason_re = re.compile('[^ ]+ contig=([^ ]+) .* orig_begin=([^ ]+) .* strand=([fr]).*')

def name_is_mason1(name):
    return _mason_re.match(name) is not None

def pos_from_mason1(name):
    res = _hint_re.match(name)
    return res.group(1), int(res.group(2)), res.group(3) == 'f'

def same_pos(pos1, pos2, wiggle=30):
    """ Returns true when the two positions are basically the same """
    refid1, pos1, strand1 = pos1
    refid2, pos2, strand2 = pos2
    if refid1 != refid2 or strand1 != strand2:
        return False
    return abs(pos1 - pos2) < wiggle

def is_correct(toks, wiggle=30):
    """ Checks whether alignment, tokenized in toks, is correct """
    flags = int(toks[1])
    aligned_pos = (toks[2], int(toks[3])-1, (flags & 16) == 0)
    paired = (flags & 1) != 0
    mate2 = paired and (flags & 128) != 0
    if name_is_extended_wgsim(toks[0]):
        true_pos = pos_from_extended_wgsim(toks[0], mate2)
    elif name_is_qsim(toks[0]):
        true_pos = pos_from_qsim(toks[0])
    elif name_is_mason1(toks[0]):
        true_pos = pos_from_mason1(toks[0])
    elif name_is_hint(toks[0]):
        true_pos = pos_from_hint(toks[0])
    else:
        raise RuntimeError('Name was not formatted as expected: "%s"' % toks[0])
    return same_pos(true_pos, aligned_pos, wiggle=wiggle)

def percentileize(dist):
    last_v = 0
    cum = {}
    for k, v in sorted(dist.items()):
        assert k not in cum
        cum[k] = last_v
        last_v += v
    pct_dict = {}
    for k in dist.keys():
        pct_dict[k] = (cum[k], float(cum[k])/last_v, dist[k])
    return pct_dict

def pass2_fh(fh_sam, fh_preds, mapq_dist, mapq_orig_dist, ztz_dist, ofh, ofh_orig):
    logging.info('  Calculating percentiles')
    mapq_pctile_dict = percentileize(mapq_dist)
    mapq_orig_pctile_dict = percentileize(mapq_orig_dist)
    ztz_pctile_dict = {x: percentileize(v) for x, v in ztz_dist.items()}
    incors, incors_orig = [], []
    line = 0
    for ln in fh_sam:
        line += 1
        if ln[0] == '@':
            continue
        toks = ln.rstrip().split('\t')
        flags = int(toks[1])
        if (flags & 4) != 0 or (flags & 2048) != 0:
            continue
        pred = fh_preds.readline()
        assert len(pred) > 0
        predline, pred = pred.split(',')
        assert int(predline) == line
        if not is_correct(toks):
            mapq = int(round(float(pred)))
            mapq_orig = int(toks[4])  # TODO: make mapq_orig dist
            assert mapq in mapq_pctile_dict
            mapq_pctile = mapq_pctile_dict[mapq]
            mapq_orig_pctile = mapq_orig_pctile_dict[mapq_orig]
            ls = [mapq_pctile[1]]
            ls_orig = [mapq_orig_pctile[1]]
            for tok in toks[12:]:
                if tok.startswith('ZT:Z:'):
                    for i, ztok in enumerate(tok[5:].split(',')):
                        if ztok == 'NA':
                            ls.append('NA')
                            ls_orig.append('NA')
                        else:
                            iztok = int(ztok)
                            assert iztok in ztz_pctile_dict[i]
                            ztz_pctile = ztz_pctile_dict[i][iztok]
                            ls.append(ztz_pctile[1])
                            ls_orig.append(ztz_pctile[1])
                    break
            incors.append(ls)
            incors_orig.append(ls_orig)
    logging.info('  Sorting and printing')
    for ls in sorted(incors, reverse=True):
        ofh.write(','.join(map(str, ls)) + '\n')
    for ls in sorted(incors_orig, reverse=True):
        ofh_orig.write(','.join(map(str, ls)) + '\n')

def pass2_fn(sam_fn, predictions_fn, mapq_dist, mapq_orig_dist, ztz_dist, ofn, ofn_orig):
    with open(ofn, 'w') as ofh:
        with open(ofn_orig, 'w') as ofh_orig:
            with open(sam_fn) as fh_sam:
                with open(predictions_fn) as fh_preds:
                    pass2_fh(fh_sam, fh_preds, mapq_dist, mapq_orig_dist, ztz_dist, ofh, ofh_orig)

def write_dists_fh(mapq_dist, mapq_orig_dist, ztz_dist, ofh):
    for k, v in sorted(mapq_dist.items()):
        ofh.write('%d:%d ' % (int(k), v))
    ofh.write('\n')
    for k, v in sorted(mapq_orig_dist.items()):
        ofh.write('%d:%d ' % (k, v))
    ofh.write('\n')
    for k_outer, v_outer in sorted(ztz_dist.items()):
        ofh.write('ztz%d: ' % k_outer)
        for k, v in sorted(v_outer.items()):
            ofh.write('%d:%d ' % (k, v))
        ofh.write('\n')

def write_dists_fn(mapq_dist, mapq_orig_dist, ztz_dist, ofn):
    with open(ofn, 'w') as ofh:
        write_dists_fh(mapq_dist, mapq_orig_dist, ztz_dist, ofh)

def go():
    format_str = '%(asctime)s:%(levelname)s:%(message)s'
    level = logging.INFO
    logging.basicConfig(format=format_str, datefmt='%m/%d/%y-%H:%M:%S', level=level)

    logging.info('Pass 1')
    mapq_dist, mapq_orig_dist, ztz_dist = pass1_fn(sys.argv[1], sys.argv[2])
    logging.info('Writing distributions')
    write_dists_fn(mapq_dist, mapq_orig_dist, ztz_dist, sys.argv[3])
    logging.info('Pass 2')
    pass2_fn(sys.argv[1], sys.argv[2], mapq_dist, mapq_orig_dist, ztz_dist, sys.argv[4], sys.argv[5])
    logging.info('Done')

if __name__ == '__main__':
    go()
<commit_msg>refining script for evaluating ztz and mapq dists<commit_after>"""
Turn ZT:Zs and MAPQs into percentiles for further analysis.
"""

from __future__ import print_function
from collections import defaultdict
import sys
import re
import logging

"""
First pass:
- Collect MAPQ distributions for tandem & input
- Collect ZT:Z distributions for tandem & input

Second pass:
- For tandem and input:
  + For each read, check for incorrectness.  If incorrect:
    - Record vector of percentiles for MAPQ and ZT:Zs

Finally:
- For tandem and input:
  + Sort the percentile vector by MAPQ percentile and record

Example usage:
pypy eval_concordance.py $HOME/input.sam $HOME/predictions.csv dists.txt incor.mapq incor.mapq_orig incor.both

TODO:
- Are we doing paired-end correctly for the various name formats?

# Load data and give appropriate column names
setwd('~/git/qsim-experiments/bin')
mq <- read.table('incor.mapq', sep=',', header=F, quote="", comment.char="")
mo <- read.table('incor.mapq_orig', sep=',', header=F, quote="", comment.char="")
mb <- read.table('incor.both', sep=',', header=F, quote="", comment.char="")
ztn <- c('ztz0', 'ztz1', 'ztz2', 'ztz3', 'ztz4', 'ztz5', 'ztz6', 'ztz7', 'ztz8', 'ztz9')
cn <- c('mapq', ztn)
colnames(mq) <- cn
colnames(mo) <- cn
colnames(mb) <- c('mapq', 'mapq_orig', ztn)

# Plot some basics
plot(jitter(mq$ztz1), jitter(mq$mapq))
points(jitter(mo$ztz1), jitter(mo$mapq), col='blue')

# Plot mapq difference as a function of diff percentile
library(ggplot2)
library(ggExtra)
p <- ggplot(mb, aes(x=ztz1, y=mapq - mapq_orig)) + geom_point() + theme_classic()
ggExtra::ggMarginal(p)
"""

def pass1_fh(fh_sam, fh_pred):
    """ Histograms all the alignments """
    mapq_dist, mapq_orig_dist = defaultdict(int), defaultdict(int)
    ztz_dist = defaultdict(lambda: defaultdict(int))
    line = 0
    for ln in fh_sam:
        line += 1
        if ln[0] == '@':
            continue
        toks = ln.rstrip().split('\t')
        flags = int(toks[1])
        if (flags & 4) != 0 or (flags & 2048) != 0:
            continue
        pred = fh_pred.readline()
        assert len(pred) > 0
        predline, pred = pred.split(',')
        assert int(predline) == line, (predline, line)
        mapq_dist[int(round(float(pred)))] += 1
        mapq_orig_dist[int(toks[4])] += 1
        for tok in toks[12:]:
            if tok.startswith('ZT:Z:'):
                for i, ztok in enumerate(tok[5:].split(',')):
                    if ztok == 'NA':
                        pass
                    else:
                        ztz_dist[i][float(ztok)] += 1  # ZT:Z
                break
    return mapq_dist, mapq_orig_dist, ztz_dist

def pass1_fn(fn_sam, fn_pred):
    """ Histograms all the input and tandem alignments """
    with open(fn_sam) as fh_sam:
        with open(fn_pred) as fh_pred:
            return pass1_fh(fh_sam, fh_pred)

"""
Example: 10_26049747_26049846_0:0:0_0:0:0_100_100_0_3999999
offset is 0-based?
"""
_wgsimex_re = re.compile('(.+)_([^_]+)_([^_]+)_([^:]+):([^:]+):([^_]+)_([^:]+):([^:]+):([^_]+)_([^_]+)_([^_]+)_([^_]+)_([^/]+).*')
#                           1   2       3       4       5       6       7       8       9       10      11      12      13

def name_is_extended_wgsim(nm):
    return _wgsimex_re.match(nm) is not None

def pos_from_extended_wgsim(name, mate2=False):
    res = _wgsimex_re.match(name)
    refid, fragst1, fragen1 = res.group(1), int(res.group(2))-1, int(res.group(3))-1
    len1, len2 = int(res.group(10)), int(res.group(11))
    flip = res.group(12) == '1'
    ln = len2 if mate2 else len1
    if flip == mate2:
        return refid, fragst1, True
    else:
        return refid, fragen1 - (ln-1), False

"""
Example: qsim!:GL000229.1:+:6005:100:u
pretty sure offset is 0-based
"""
_qsim_re = re.compile('qsim!:([^:]+):([+-]):([^:]+):.*')

def name_is_qsim(nm):
    return _qsim_re.match(nm) is not None

def pos_from_qsim(name):
    res = _qsim_re.match(name)
    return res.group(1), int(res.group(3)), res.group(2) == '+'

"""
Example: !h!chr9!118085975!+!50!0
offset is 0-based
"""
_hint_re = re.compile('!h!([^!]+)!([0-9]+)!([+-])!([0-9]+)!([0-9]+)')

def name_is_hint(name):
    return _hint_re.match(name) is not None

def pos_from_hint(name):
    res = _hint_re.match(name)
    return res.group(1), int(res.group(2)), res.group(3) == '+'

"""
Example:
hg38_50nt_mason1_unp.fastq.000999999 contig=chr9 haplotype=1 length=50 orig_begin=118085976 orig_end=118086026 snps=0 indels=0 haplotype_infix=ATGACTCTTGAAGCTGGGCGCAGTGGCTCATGCCTGTAATCCTAGCACTT edit_string=MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM strand=forward
"""
_mason_re = re.compile('[^ ]+ contig=([^ ]+) .* orig_begin=([^ ]+) .* strand=([fr]).*')

def name_is_mason1(name):
    return _mason_re.match(name) is not None

def pos_from_mason1(name):
    res = _hint_re.match(name)
    return res.group(1), int(res.group(2)), res.group(3) == 'f'

def same_pos(pos1, pos2, wiggle=30):
    """ Returns true when the two positions are basically the same """
    refid1, pos1, strand1 = pos1
    refid2, pos2, strand2 = pos2
    if refid1 != refid2 or strand1 != strand2:
        return False
    return abs(pos1 - pos2) < wiggle

def is_correct(toks, wiggle=30):
    """ Checks whether alignment, tokenized in toks, is correct """
    flags = int(toks[1])
    aligned_pos = (toks[2], int(toks[3])-1, (flags & 16) == 0)
    paired = (flags & 1) != 0
    mate2 = paired and (flags & 128) != 0
    if name_is_extended_wgsim(toks[0]):
        true_pos = pos_from_extended_wgsim(toks[0], mate2)
    elif name_is_qsim(toks[0]):
        true_pos = pos_from_qsim(toks[0])
    elif name_is_mason1(toks[0]):
        true_pos = pos_from_mason1(toks[0])
    elif name_is_hint(toks[0]):
        true_pos = pos_from_hint(toks[0])
    else:
        raise RuntimeError('Name was not formatted as expected: "%s"' % toks[0])
    return same_pos(true_pos, aligned_pos, wiggle=wiggle)

def percentileize(dist):
    last_v = 0
    cum = {}
    for k, v in sorted(dist.items()):
        assert k not in cum
        cum[k] = last_v + v / 2.0
        last_v += v
    pct_dict = {}
    for k in dist.keys():
        pct_dict[k] = (cum[k], float(cum[k])/last_v, dist[k])
    return pct_dict

def pass2_fh(fh_sam, fh_preds, mapq_dist, mapq_orig_dist, ztz_dist, ofh, ofh_orig, ofh_both):
    logging.info('  Calculating percentiles')
    mapq_pctile_dict = percentileize(mapq_dist)
    mapq_orig_pctile_dict = percentileize(mapq_orig_dist)
    ztz_pctile_dict = {x: percentileize(v) for x, v in ztz_dist.items()}
    incor_mapq, incor_mapq_orig, incor_both = [], [], []
    line = 0
    for ln in fh_sam:
        line += 1
        if ln[0] == '@':
            continue
        toks = ln.rstrip().split('\t')
        flags = int(toks[1])
        if (flags & 4) != 0 or (flags & 2048) != 0:
            continue
        pred = fh_preds.readline()
        assert len(pred) > 0
        predline, pred = pred.split(',')
        assert int(predline) == line
        if not is_correct(toks):
            mapq = int(round(float(pred)))
            mapq_orig = int(toks[4])  # TODO: make mapq_orig dist
            assert mapq in mapq_pctile_dict
            mapq_pctile = mapq_pctile_dict[mapq]
            mapq_orig_pctile = mapq_orig_pctile_dict[mapq_orig]
            ls = [mapq_pctile[1]]
            ls_orig = [mapq_orig_pctile[1]]
            ls_both = [mapq_pctile[1], mapq_orig_pctile[1]]
            for tok in toks[12:]:
                if tok.startswith('ZT:Z:'):
                    for i, ztok in enumerate(tok[5:].split(',')):
                        if ztok == 'NA':
                            ls.append('NA')
                            ls_orig.append('NA')
                            ls_both.append('NA')
                        else:
                            iztok = float(ztok)
                            assert iztok in ztz_pctile_dict[i]
                            ztz_pctile = ztz_pctile_dict[i][iztok]
                            ls.append(ztz_pctile[1])
                            ls_orig.append(ztz_pctile[1])
                            ls_both.append(ztz_pctile[1])
                    break
            incor_mapq.append(ls)
            incor_mapq_orig.append(ls_orig)
            incor_both.append(ls_both)
    logging.info('  Sorting and printing')
    for ls in sorted(incor_mapq, reverse=True):
        ofh.write(','.join(map(str, ls)) + '\n')
    for ls in sorted(incor_mapq_orig, reverse=True):
        ofh_orig.write(','.join(map(str, ls)) + '\n')
    for ls in sorted(incor_both, reverse=True):
        ofh_both.write(','.join(map(str, ls)) + '\n')

def pass2_fn(sam_fn, predictions_fn, mapq_dist, mapq_orig_dist, ztz_dist, ofn, ofn_orig, ofn_both):
    with open(ofn, 'w') as ofh:
        with open(ofn_orig, 'w') as ofh_orig:
            with open(ofn_both, 'w') as ofh_both:
                with open(sam_fn) as fh_sam:
                    with open(predictions_fn) as fh_preds:
                        pass2_fh(fh_sam, fh_preds, mapq_dist, mapq_orig_dist, ztz_dist, ofh, ofh_orig, ofh_both)

def write_dists_fh(mapq_dist, mapq_orig_dist, ztz_dist, ofh):
    for k, v in sorted(mapq_dist.items()):
        ofh.write('%d:%d ' % (int(k), v))
    ofh.write('\n')
    for k, v in sorted(mapq_orig_dist.items()):
        ofh.write('%d:%d ' % (k, v))
    ofh.write('\n')
    for k_outer, v_outer in sorted(ztz_dist.items()):
        ofh.write('ztz%d: ' % k_outer)
        for k, v in sorted(v_outer.items()):
            ofh.write('%d:%d ' % (k, v))
        ofh.write('\n')

def write_dists_fn(mapq_dist, mapq_orig_dist, ztz_dist, ofn):
    with open(ofn, 'w') as ofh:
        write_dists_fh(mapq_dist, mapq_orig_dist, ztz_dist, ofh)

def go():
    format_str = '%(asctime)s:%(levelname)s:%(message)s'
    level = logging.INFO
    logging.basicConfig(format=format_str, datefmt='%m/%d/%y-%H:%M:%S', level=level)

    logging.info('Pass 1')
    mapq_dist, mapq_orig_dist, ztz_dist = pass1_fn(sys.argv[1], sys.argv[2])
    logging.info('Writing distributions')
    write_dists_fn(mapq_dist, mapq_orig_dist, ztz_dist, sys.argv[3])
    logging.info('Pass 2')
    pass2_fn(sys.argv[1], sys.argv[2], mapq_dist, mapq_orig_dist, ztz_dist, sys.argv[4], sys.argv[5], sys.argv[6])
    logging.info('Done')

if __name__ == '__main__':
    go()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import subprocess

def current_sha():
    return (
        subprocess.check_output(["git", "rev-parse", "HEAD"])
        .strip()
        .decode("utf-8", errors="replace")
    )

def diff(start, end):
    return subprocess.check_output(
        ["git", "diff", "-M", "{}..{}".format(start, end)]
    ).decode("utf-8", errors="replace")

def parent_sha(sha):
    return (
        subprocess.check_output(["git", "rev-list", "--parents", "-n", "1", sha])
        .strip()
        .split()[1]
        .decode("utf-8", errors="replace")
    )

def current_branch():
    return (
        subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"])
        .strip()
        .decode("utf-8", errors="replace")
    )

def url():
    return (
        subprocess.check_output(["git", "config", "--get", "remote.origin.url"])
        .strip()
        .decode("utf-8", errors="replace")
    )

def fetch(git_url):
    return (
        subprocess.check_output(["git", "fetch", git_url])
        .strip()
        .decode("utf-8", errors="replace")
    )

def add(filename):
    return (
        subprocess.check_output(["git", "add", filename])
        .strip()
        .decode("utf-8", errors="replace")
    )

def commit(message):
    return (
        subprocess.check_output(["git", "commit", "-m", message])
        .strip()
        .decode("utf-8", errors="replace")
    )

def push(branch):
    return (
        subprocess.check_output(["git", "push", "origin", "{}".format(branch)])
        .strip()
        .decode("utf-8", errors="replace")
    )

def files_changed(files):
    files_with_changes = []
    for filename in files:
        if (
            subprocess.check_output(["git", "diff", "--name-only", filename])
            .strip()
            .decode("utf-8", errors="replace")
        ):
            files_with_changes.append(filename)
    return files_with_changes

def set_remote(remote):
    return (
        subprocess.check_output(["git", "config", "remote.origin.url", remote])
        .strip()
        .decode("utf-8", errors="replace")
    )

def command(*args):
    git_command = ["git"]
    git_command.extend(args)
    return (
        subprocess.check_output(git_command).strip().decode("utf-8", errors="replace")
    )
<commit_msg>Update git.py<commit_after># -*- coding: utf-8 -*-

import os
import subprocess

def current_sha():
    return (
        subprocess.check_output(["git", "rev-parse", "HEAD"], env=os.environ)
        .strip()
        .decode("utf-8", errors="replace")
    )

def diff(start, end):
    return subprocess.check_output(
        ["git", "diff", "-M", "{}..{}".format(start, end)], env=os.environ
    ).decode("utf-8", errors="replace")

def parent_sha(sha):
    return (
        subprocess.check_output(["git", "rev-list", "--parents", "-n", "1", sha], env=os.environ)
        .strip()
        .split()[1]
        .decode("utf-8", errors="replace")
    )

def current_branch():
    return (
        subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], env=os.environ)
        .strip()
        .decode("utf-8", errors="replace")
    )

def url():
    return (
        subprocess.check_output(["git", "config", "--get", "remote.origin.url"], env=os.environ)
        .strip()
        .decode("utf-8", errors="replace")
    )

def fetch(git_url):
    return (
        subprocess.check_output(["git", "fetch", git_url], env=os.environ)
        .strip()
        .decode("utf-8", errors="replace")
    )

def add(filename):
    return (
        subprocess.check_output(["git", "add", filename], env=os.environ)
        .strip()
        .decode("utf-8", errors="replace")
    )

def commit(message):
    return (
        subprocess.check_output(["git", "commit", "-m", message], env=os.environ)
        .strip()
        .decode("utf-8", errors="replace")
    )

def push(branch):
    return (
        subprocess.check_output(["git", "push", "origin", "{}".format(branch)], env=os.environ)
        .strip()
        .decode("utf-8", errors="replace")
    )

def files_changed(files):
    files_with_changes = []
    for filename in files:
        if (
            subprocess.check_output(["git", "diff", "--name-only", filename], env=os.environ)
            .strip()
            .decode("utf-8", errors="replace")
        ):
            files_with_changes.append(filename)
    return files_with_changes

def set_remote(remote):
    return (
        subprocess.check_output(["git", "config", "remote.origin.url", remote], env=os.environ)
        .strip()
        .decode("utf-8", errors="replace")
    )

def command(*args):
    git_command = ["git"]
    git_command.extend(args)
    return (
        subprocess.check_output(git_command, env=os.environ).strip().decode("utf-8", errors="replace")
    )
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import json
import logging
import requests
from sympy.physics import units
from indra.databases import chebi_client, uniprot_client
from indra.statements import Inhibition, Agent, Evidence
from collections import defaultdict

logger = logging.getLogger('chembl')

def get_inhibition(drug, target):
    chebi_id = drug.db_refs.get('CHEBI')
    mesh_id = drug.db_refs.get('MESH')
    if chebi_id:
        drug_chembl_id = chebi_client.get_chembl_id(chebi_id)
    elif mesh_id:
        drug_chembl_id = get_chembl_id(mesh_id)
    else:
        logger.error('Drug missing ChEBI or MESH grounding.')
        return None

    target_upid = target.db_refs.get('UP')
    if not target_upid:
        logger.error('Target missing UniProt grounding.')
        return None
    target_chembl_id = get_target_chemblid(target_upid)

    logger.info('Drug: %s, Target: %s' % (drug_chembl_id, target_chembl_id))
    query_dict = {'query': 'activity',
                  'params': {'molecule_chembl_id': drug_chembl_id,
                             'target_chembl_id': target_chembl_id,
                             'limit': 10000}}
    res = send_query(query_dict)
    evidence = []
    for assay in res['activities']:
        ev = get_evidence(assay)
        if not ev:
            continue
        evidence.append(ev)
    st = Inhibition(drug, target, evidence=evidence)
    return st

def get_all_protein_activities(drug):
    """Query ChEMBL for kinetics data given drug as Agent get back statements

    Parameters
    ----------
    drug : Agent
        Agent representing drug with MESH or CHEBI grounding

    Returns
    -------
    stmts : list of INDRA statements
        INDRA statements generated by querying ChEMBL for all kinetics data of
        a drug interacting with protein targets
    """
    chebi_id = drug.db_refs.get('CHEBI')
    mesh_id = drug.db_refs.get('MESH')
    if chebi_id:
        drug_chembl_id = chebi_client.get_chembl_id(chebi_id)
    elif mesh_id:
        drug_chembl_id = get_chembl_id(mesh_id)
    else:
        logger.error('Drug missing ChEBI or MESH grounding.')
        return None
    logger.info('Drug: %s' % (drug_chembl_id))
    query_dict = {'query': 'activity',
                  'params': {'molecule_chembl_id': drug_chembl_id,
                             'limit': 10000}
                  }
    res = send_query(query_dict)
    activities = res['activities']
    targ_act_dict = activities_by_target(activities)
    target_chembl_ids = [x for x in targ_act_dict]
    protein_targets = get_protein_targets_only(target_chembl_ids)
    filtered_targ_act_dict = {t: targ_act_dict[t]
                              for t in [x for x in protein_targets]}
    stmts = []
    for target_chembl_id in filtered_targ_act_dict:
        target_activity_ids = filtered_targ_act_dict[target_chembl_id]
        target_activites = [x for x in activities
                            if x['activity_id'] in target_activity_ids]
        target_upids = []
        targ_comp = protein_targets[target_chembl_id]['target_components']
        for t_c in targ_comp:
            target_upids.append(t_c['accession'])
        evidence = []
        for assay in target_activites:
            ev = get_evidence(assay)
            if not ev:
                continue
            evidence.append(ev)
        for target_upid in target_upids:
            agent_name = uniprot_client.get_gene_name(target_upid)
            target_agent = Agent(agent_name, db_refs={'UP': target_upid})
            st = Inhibition(drug, target_agent, evidence=evidence)
            stmts.append(st)
    return stmts

def send_query(query_dict):
    """Query ChEMBL API

    Parameters
    ----------
    query_dict : dict
        'query' : string of the endpoint to query
        'params' : dict of params for the query
    Returns
    -------
    js : dict
        dict parsed from json that is unique to the submitted query

    Example
    -------
    >>> query_dict = {'query': 'target',
    >>>               'params': {'target_chembl_id': 'CHEMBL5145',
    >>>               'limit': 1}}
    >>> send_query(query_dict)
    """
    query = query_dict['query']
    params = query_dict['params']
    url = 'https://www.ebi.ac.uk/chembl/api/data/' + query + '.json'
    r = requests.get(url, params=params)
    r.raise_for_status()
    js = r.json()
    return js

def query_target(target_chembl_id):
    """Query ChEMBL API target by id

    Parameters
    ----------
    target_chembl_id : str
    Returns
    -------
    target : dict
        dict parsed from json that is unique for the target
    """
    query_dict = {'query': 'target',
                  'params': {'target_chembl_id': target_chembl_id,
                             'limit': 1}}
    res = send_query(query_dict)
    assert(res['page_meta']['total_count'] == 1)
    target = res['targets'][0]
    return target

def activities_by_target(activities):
    """Get back lists of activities in a dict keyed by ChEMBL target id
    Parameters
    ----------
    activities : dict
        response from a query returning activities for a drug
    Returns
    -------
    targ_act_dict : dict
        dictionary keyed to ChEMBL target ids with lists of activity ids
    """
    targ_act_dict = defaultdict(lambda: [])
    for activity in activities:
        target_chembl_id = activity['target_chembl_id']
        activity_id = activity['activity_id']
        targ_act_dict[target_chembl_id].append(activity_id)
    for target_chembl_id in targ_act_dict:
        targ_act_dict[target_chembl_id] = \
            list(set(targ_act_dict[target_chembl_id]))
    return targ_act_dict

def get_protein_targets_only(target_chembl_ids):
    """Given list of ChEMBL target ids, return dict of only SINGLE PROTEIN targ
    Parameters
    ----------
    target_chembl_ids : list
        list of chembl_ids as strings
    Returns
    -------
    protein_targets : dict
        dictionary keyed to ChEMBL target ids with lists of activity ids
    """
    protein_targets = {}
    for target_chembl_id in target_chembl_ids:
        target = query_target(target_chembl_id)
        if 'SINGLE PROTEIN' in target['target_type']:
            protein_targets[target_chembl_id] = target
    return protein_targets

def get_evidence(assay):
    kin = get_kinetics(assay)
    source_id = assay.get('assay_chembl_id')
    if not kin:
        return None
    annotations = {'kinetics': kin}
    chembl_doc_id = str(assay.get('document_chembl_id'))
    pmid = get_pmid(chembl_doc_id)
    ev = Evidence(source_api='chembl', pmid=pmid, source_id=source_id,
                  annotations=annotations)
    return ev

def get_kinetics(assay):
    try:
        val = float(assay.get('standard_value'))
    except TypeError:
        logger.warning('Invalid assay value: %s' % assay.get('standard_value'))
        return None
    unit = assay.get('standard_units')
    if unit == 'nM':
        unit_sym = 1e-9 * units.mol / units.liter
    elif unit == 'uM':
        unit_sym = 1e-6 * units.mol / units.liter
    else:
        logger.warning('Unhandled unit: %s' % unit)
        return None
    param_type = assay.get('standard_type')
    if param_type not in ['IC50', 'Kd']:
        logger.warning('Unhandled parameter type: %s' % param_type)
        return None
    kin = {param_type: val * unit_sym}
    return kin

def get_pmid(doc_id):
    url_pmid = 'https://www.ebi.ac.uk/chembl/api/data/document.json'
    params = {'document_chembl_id': doc_id}
    res = requests.get(url_pmid, params=params)
    js = res.json()
    pmid = str(js['documents'][0]['pubmed_id'])
    return pmid

def get_target_chemblid(target_upid):
    url = 'https://www.ebi.ac.uk/chembl/api/data/target.json'
    params = {'target_components__accession': target_upid}
    r = requests.get(url, params=params)
    r.raise_for_status()
    js = r.json()
    target_chemblid = js['targets'][0]['target_chembl_id']
    return target_chemblid

def get_mesh_id(nlm_mesh):
    url_nlm2mesh = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi'
    params = {'db': 'mesh', 'term': nlm_mesh, 'retmode': 'JSON'}
    r = requests.get(url_nlm2mesh, params=params)
    res = r.json()
    mesh_id = res['esearchresult']['idlist'][0]
    return mesh_id

def get_pcid(mesh_id):
    url_mesh2pcid = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi'
    params = {'dbfrom': 'mesh', 'id': mesh_id,
              'db': 'pccompound', 'retmode': 'JSON'}
    r = requests.get(url_mesh2pcid, params=params)
    res = r.json()
    pcid = res['linksets'][0]['linksetdbs'][0]['links'][0]
    return pcid

def get_chembl_id(nlm_mesh):
    mesh_id = get_mesh_id(nlm_mesh)
    pcid = get_pcid(mesh_id)
    url_mesh2pcid = 'https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/' + \
                    'cid/%s/synonyms/JSON' % pcid
    r = requests.get(url_mesh2pcid)
    res = r.json()
    synonyms = res['InformationList']['Information'][0]['Synonym']
    chembl_id = [syn for syn in synonyms
                 if 'CHEMBL' in syn and 'SCHEMBL' not in syn][0]
    return chembl_id
<commit_msg>Only make Inhibition statements for which there is usable evidence<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import json
import logging
import requests
from sympy.physics import units
from indra.databases import chebi_client, uniprot_client
from indra.statements import Inhibition, Agent, Evidence
from collections import defaultdict

logger = logging.getLogger('chembl')

def get_inhibition(drug, target):
    chebi_id = drug.db_refs.get('CHEBI')
    mesh_id = drug.db_refs.get('MESH')
    if chebi_id:
        drug_chembl_id = chebi_client.get_chembl_id(chebi_id)
    elif mesh_id:
        drug_chembl_id = get_chembl_id(mesh_id)
    else:
        logger.error('Drug missing ChEBI or MESH grounding.')
        return None

    target_upid = target.db_refs.get('UP')
    if not target_upid:
        logger.error('Target missing UniProt grounding.')
        return None
    target_chembl_id = get_target_chemblid(target_upid)

    logger.info('Drug: %s, Target: %s' % (drug_chembl_id, target_chembl_id))
    query_dict = {'query': 'activity',
                  'params': {'molecule_chembl_id': drug_chembl_id,
                             'target_chembl_id': target_chembl_id,
                             'limit': 10000}}
    res = send_query(query_dict)
    evidence = []
    for assay in res['activities']:
        ev = get_evidence(assay)
        if not ev:
            continue
        evidence.append(ev)
    st = Inhibition(drug, target, evidence=evidence)
    return st

def get_all_protein_activities(drug):
    """Query ChEMBL for kinetics data given drug as Agent get back statements

    Parameters
    ----------
    drug : Agent
        Agent representing drug with MESH or CHEBI grounding

    Returns
    -------
    stmts : list of INDRA statements
        INDRA statements generated by querying ChEMBL for all kinetics data of
        a drug interacting with protein targets
    """
    chebi_id = drug.db_refs.get('CHEBI')
    mesh_id = drug.db_refs.get('MESH')
    if chebi_id:
        drug_chembl_id = chebi_client.get_chembl_id(chebi_id)
    elif mesh_id:
        drug_chembl_id = get_chembl_id(mesh_id)
    else:
        logger.error('Drug missing ChEBI or MESH grounding.')
        return None
    logger.info('Drug: %s' % (drug_chembl_id))
    query_dict = {'query': 'activity',
                  'params': {'molecule_chembl_id': drug_chembl_id,
                             'limit': 10000}
                  }
    res = send_query(query_dict)
    activities = res['activities']
    targ_act_dict = activities_by_target(activities)
    target_chembl_ids = [x for x in targ_act_dict]
    protein_targets = get_protein_targets_only(target_chembl_ids)
    filtered_targ_act_dict = {t: targ_act_dict[t]
                              for t in [x for x in protein_targets]}
    stmts = []
    for target_chembl_id in filtered_targ_act_dict:
        target_activity_ids = filtered_targ_act_dict[target_chembl_id]
        target_activites = [x for x in activities
                            if x['activity_id'] in target_activity_ids]
        target_upids = []
        targ_comp = protein_targets[target_chembl_id]['target_components']
        for t_c in targ_comp:
            target_upids.append(t_c['accession'])
        evidence = []
        for assay in target_activites:
            ev = get_evidence(assay)
            if not ev:
                continue
            evidence.append(ev)
        if len(evidence) > 0:
            for target_upid in target_upids:
                agent_name = uniprot_client.get_gene_name(target_upid)
                target_agent = Agent(agent_name, db_refs={'UP': target_upid})
                st = Inhibition(drug, target_agent, evidence=evidence)
                stmts.append(st)
    return stmts

def send_query(query_dict):
    """Query ChEMBL API

    Parameters
    ----------
    query_dict : dict
        'query' : string of the endpoint to query
        'params' : dict of params for the query
    Returns
    -------
    js : dict
        dict parsed from json that is unique to the submitted query

    Example
    -------
    >>> query_dict = {'query': 'target',
    >>>               'params': {'target_chembl_id': 'CHEMBL5145',
    >>>               'limit': 1}}
    >>> send_query(query_dict)
    """
    query = query_dict['query']
    params = query_dict['params']
    url = 'https://www.ebi.ac.uk/chembl/api/data/' + query + '.json'
    r = requests.get(url, params=params)
    r.raise_for_status()
    js = r.json()
    return js

def query_target(target_chembl_id):
    """Query ChEMBL API target by id

    Parameters
    ----------
    target_chembl_id : str
    Returns
    -------
    target : dict
        dict parsed from json that is unique for the target
    """
    query_dict = {'query': 'target',
                  'params': {'target_chembl_id': target_chembl_id,
                             'limit': 1}}
    res = send_query(query_dict)
    assert(res['page_meta']['total_count'] == 1)
    target = res['targets'][0]
    return target

def activities_by_target(activities):
    """Get back lists of activities in a dict keyed by ChEMBL target id
    Parameters
    ----------
    activities : dict
        response from a query returning activities for a drug
    Returns
    -------
    targ_act_dict : dict
        dictionary keyed to ChEMBL target ids with lists of activity ids
    """
    targ_act_dict = defaultdict(lambda: [])
    for activity in activities:
        target_chembl_id = activity['target_chembl_id']
        activity_id = activity['activity_id']
        targ_act_dict[target_chembl_id].append(activity_id)
    for target_chembl_id in targ_act_dict:
        targ_act_dict[target_chembl_id] = \
            list(set(targ_act_dict[target_chembl_id]))
    return targ_act_dict

def get_protein_targets_only(target_chembl_ids):
    """Given list of ChEMBL target ids, return dict of only SINGLE PROTEIN targ
    Parameters
    ----------
    target_chembl_ids : list
        list of chembl_ids as strings
    Returns
    -------
    protein_targets : dict
        dictionary keyed to ChEMBL target ids with lists of activity ids
    """
    protein_targets = {}
    for target_chembl_id in target_chembl_ids:
        target = query_target(target_chembl_id)
        if 'SINGLE PROTEIN' in target['target_type']:
            protein_targets[target_chembl_id] = target
    return protein_targets

def get_evidence(assay):
    kin = get_kinetics(assay)
    source_id = assay.get('assay_chembl_id')
    if not kin:
        return None
    annotations = {'kinetics': kin}
    chembl_doc_id = str(assay.get('document_chembl_id'))
    pmid = get_pmid(chembl_doc_id)
    ev = Evidence(source_api='chembl', pmid=pmid, source_id=source_id,
                  annotations=annotations)
    return ev

def get_kinetics(assay):
    try:
        val = float(assay.get('standard_value'))
    except TypeError:
        logger.warning('Invalid assay value: %s' % assay.get('standard_value'))
        return None
    unit = assay.get('standard_units')
    if unit == 'nM':
        unit_sym = 1e-9 * units.mol / units.liter
    elif unit == 'uM':
        unit_sym = 1e-6 * units.mol / units.liter
    else:
        logger.warning('Unhandled unit: %s' % unit)
        return None
    param_type = assay.get('standard_type')
    if param_type not in ['IC50', 'Kd']:
        logger.warning('Unhandled parameter type: %s' % param_type)
        return None
    kin = {param_type: val * unit_sym}
    return kin

def get_pmid(doc_id):
    url_pmid = 'https://www.ebi.ac.uk/chembl/api/data/document.json'
    params = {'document_chembl_id': doc_id}
    res = requests.get(url_pmid, params=params)
    js = res.json()
    pmid = str(js['documents'][0]['pubmed_id'])
    return pmid

def get_target_chemblid(target_upid):
    url = 'https://www.ebi.ac.uk/chembl/api/data/target.json'
    params = {'target_components__accession': target_upid}
    r = requests.get(url, params=params)
    r.raise_for_status()
    js = r.json()
    target_chemblid = js['targets'][0]['target_chembl_id']
    return target_chemblid

def get_mesh_id(nlm_mesh):
    url_nlm2mesh = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi'
    params = {'db': 'mesh', 'term': nlm_mesh, 'retmode': 'JSON'}
    r = requests.get(url_nlm2mesh, params=params)
    res = r.json()
    mesh_id = res['esearchresult']['idlist'][0]
    return mesh_id

def get_pcid(mesh_id):
    url_mesh2pcid = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi'
    params = {'dbfrom': 'mesh', 'id': mesh_id,
              'db': 'pccompound', 'retmode': 'JSON'}
    r = requests.get(url_mesh2pcid, params=params)
    res = r.json()
    pcid = res['linksets'][0]['linksetdbs'][0]['links'][0]
    return pcid

def get_chembl_id(nlm_mesh):
    mesh_id = get_mesh_id(nlm_mesh)
    pcid = get_pcid(mesh_id)
    url_mesh2pcid = 'https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/' + \
                    'cid/%s/synonyms/JSON' % pcid
    r = requests.get(url_mesh2pcid)
    res = r.json()
    synonyms = res['InformationList']['Information'][0]['Synonym']
    chembl_id = [syn for syn in synonyms
                 if 'CHEMBL' in syn and 'SCHEMBL' not in syn][0]
    return chembl_id
<|endoftext|>


<commit_before>__version__ = '0.6.15'
<commit_msg>version bump<commit_after>__version__ = '0.6.16'
<|endoftext|>