<commit_before># Copyright 2014 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from six.moves import zip_longest

from touchdown.core.plan import Plan
from touchdown.core import argument

from .vpc import VPC
from touchdown.core import serializers
from touchdown.aws.common import Resource
from ..common import SimpleDescribe, SimpleApply, SimpleDestroy

class PortRange(Resource):

    resource_name = "port_range"

    start = argument.Integer(default=1, min=-1, max=65535, field="From")
    end = argument.Integer(default=65535, min=-1, max=65535, field="To")

    @classmethod
    def clean(cls, value):
        if isinstance(value, int):
            return super(PortRange, cls).clean({"start": value, "end": value})
        return super(PortRange, cls).clean(value)

class IcmpTypeCode(Resource):

    resource_name = "icmp_type_code"

    type = argument.Integer(default=-1, field="Type")
    code = argument.Integer(default=-1, field="Code")

class Rule(Resource):

    resource_name = "rule"
    dot_ignore = True

    network = argument.IPNetwork(field="CidrBlock")
    protocol = argument.String(default='tcp', choices=['tcp', 'udp', 'icmp'], field="Protocol")
    port = argument.Resource(PortRange, field="PortRange", serializer=serializers.Resource())
    icmp = argument.Resource(IcmpTypeCode, field="IcmpTypeCode", serializer=serializers.Resource())
    action = argument.String(default="allow", choices=["allow", "deny"], field="RuleAction")

    # def clean_port(self, value):
    #     if isinstance(value, (int, str)):
    #         return PortRange(None, from_port=value, to_port=value)
    #     return value

    def clean_protocol(self, protocol):
        # see https://github.com/aws/aws-cli/pull/532/files
        if protocol == 'tcp':
            return '6'
        elif protocol == 'udp':
            return '17'
        elif protocol == 'icmp':
            return '1'
        elif protocol == 'all':
            return '-1'

    def __str__(self):
        # if self.from_port == self.to_port:
        #     ports = "port {}".format(self.from_port)
        # else:
        #     ports = "ports {} to {}".format(self.from_port, self.to_port)
        ports = ""
        return "{} {} from {}".format(self.protocol, ports, self.network)

class NetworkACL(Resource):

    resource_name = "network_acl"

    name = argument.String(field="Name", group="tags")
    inbound = argument.ResourceList(Rule)
    outbound = argument.ResourceList(Rule)

    tags = argument.Dict()
    vpc = argument.Resource(VPC, field="VpcId")

class Describe(SimpleDescribe, Plan):

    resource = NetworkACL
    service_name = 'ec2'
    describe_action = "describe_network_acls"
    describe_envelope = "NetworkAcls"
    key = 'NetworkAclId'

    biggest_serial = 0

    def get_describe_filters(self):
        vpc = self.runner.get_plan(self.resource.vpc)
        if not vpc.resource_id:
            return None

        if self.key in self.object:
            return {
                "Filters": [
                    {'Name': 'network-acl-id', 'Values': [self.object[self.key]]}
                ]
            }

        return {
            "Filters": [
                {'Name': 'vpc-id', 'Values': [vpc.resource_id]}
            ],
        }

    def _check_rules(self, local, remote, egress):
        rules = zip_longest(
            local,
            filter(
                lambda r: r['Egress'] == egress and r['RuleNumber'] <= 32766,
                remote['Entries'],
            ),
        )
        for i, (left, right) in enumerate(rules, start=1):
            if not left or not right:
                return False
            if i != right['RuleNumber']:
                return False
            if not left.matches(self.runner, right):
                return False
        return True

    def _compare_rules(self, network_acl):
        if not self._check_rules(self.resource.inbound, network_acl, False):
            return False
        if not self._check_rules(self.resource.outbound, network_acl, True):
            return False
        return True

    def describe_object_matches(self, network_acl):
        if self.key in self.object and network_acl[self.key] == self.object[self.key]:
            return True

        tags = {tag['Key']: tag['Value'] for tag in network_acl.get("Tags", [])}
        name = tags.get('Name', '')

        if name != self.resource.name and not name.startswith(self.resource.name + "."):
            return False

        try:
            serial = name.rsplit(".", 1)[1]
            self.biggest_serial = max(int(serial), self.biggest_serial)
        except (IndexError, TypeError, ValueError):
            pass

        return self._compare_rules(network_acl)

    def get_possible_objects(self):
        vpc = self.runner.get_plan(self.resource.vpc)
        if not vpc.resource_id:
            return

        prefix = "{}.".format(self.resource.name)

        result = self.client.describe_network_acls(
            Filters=[
                {"Name": 'vpc-id', "Values": [vpc.resource_id]},
            ]
        )

        for network_acl in result.get('NetworkAcls', []):
            tags = {tag['Key']: tag['Value'] for tag in network_acl.get("Tags", [])}
            name = tags.get('Name', '')

            # Only ignore ACL's for the current resource
            if name != self.resource.name and not name.startswith(prefix):
                continue

            # Don't delete the default ACL
            if network_acl['IsDefault']:
                continue

            yield network_acl

class Apply(SimpleApply, Describe):

    create_action = "create_network_acl"
    waiter = "network_acl_available"
    waiter_eventual_consistency_threshold = 5

    def prepare_to_create(self):
        for network_acl in self.get_possible_objects():
            # Don't try and delete ACL's that are in use
            if len(network_acl['Associations']) != 0:
                continue

            yield self.generic_action(
                "Destroy stale network acl: {}".format(network_acl['NetworkAclId']),
                self.client.delete_network_acl,
                NetworkAclId=network_acl['NetworkAclId'],
            )

    def get_create_name(self):
        return "{}.{}".format(
            self.resource.name,
            self.biggest_serial + 1,
        )

    def _insert_rule(self, rule, rule_number, egress):
        return self.generic_action(
            "Add {direction} rule: {rule}".format(
                rule=rule,
                direction='egress' if egress else 'ingress',
            ),
            self.client.create_network_acl_entry,
            rule.serializer_with_args(
                NetworkAclId=self.resource.identifier(),
                Egress=egress,
                RuleNumber=rule_number,
            )
        )

    def insert_network_rules(self):
        for i, rule in enumerate(self.resource.inbound, start=1):
            yield self._insert_rule(rule, rule_number=i, egress=False)
        for i, rule in enumerate(self.resource.outbound, start=1):
            yield self._insert_rule(rule, rule_number=i, egress=True)

    def update_object(self):
        for action in super(Apply, self).update_object():
            yield action

        if not self.object:
            for action in self.insert_network_rules():
                yield action

class Destroy(SimpleDestroy, Describe):

    destroy_action = "delete_network_acl"

    def destroy_object(self):
        for network_acl in self.get_possible_objects():
            tags = {tag['Key']: tag['Value'] for tag in network_acl.get("Tags", [])}
            name = tags.get('Name', '')

            yield self.generic_action(
                "Destroy network acl: {} ({})".format(name, network_acl['NetworkAclId']),
                self.client.delete_network_acl,
                NetworkAclId=network_acl['NetworkAclId'],
            )
<commit_msg>Fix case where network acls are not removed on touchdown destroy<commit_after># Copyright 2014 Isotoma Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from six.moves import zip_longest

from touchdown.core.plan import Plan
from touchdown.core import argument

from .vpc import VPC
from touchdown.core import serializers
from touchdown.aws.common import Resource
from ..common import SimpleDescribe, SimpleApply, SimpleDestroy

class PortRange(Resource):

    resource_name = "port_range"

    start = argument.Integer(default=1, min=-1, max=65535, field="From")
    end = argument.Integer(default=65535, min=-1, max=65535, field="To")

    @classmethod
    def clean(cls, value):
        if isinstance(value, int):
            return super(PortRange, cls).clean({"start": value, "end": value})
        return super(PortRange, cls).clean(value)

class IcmpTypeCode(Resource):

    resource_name = "icmp_type_code"

    type = argument.Integer(default=-1, field="Type")
    code = argument.Integer(default=-1, field="Code")

class Rule(Resource):

    resource_name = "rule"
    dot_ignore = True

    network = argument.IPNetwork(field="CidrBlock")
    protocol = argument.String(default='tcp', choices=['tcp', 'udp', 'icmp'], field="Protocol")
    port = argument.Resource(PortRange, field="PortRange", serializer=serializers.Resource())
    icmp = argument.Resource(IcmpTypeCode, field="IcmpTypeCode", serializer=serializers.Resource())
    action = argument.String(default="allow", choices=["allow", "deny"], field="RuleAction")

    # def clean_port(self, value):
    #     if isinstance(value, (int, str)):
    #         return PortRange(None, from_port=value, to_port=value)
    #     return value

    def clean_protocol(self, protocol):
        # see https://github.com/aws/aws-cli/pull/532/files
        if protocol == 'tcp':
            return '6'
        elif protocol == 'udp':
            return '17'
        elif protocol == 'icmp':
            return '1'
        elif protocol == 'all':
            return '-1'

    def __str__(self):
        # if self.from_port == self.to_port:
        #     ports = "port {}".format(self.from_port)
        # else:
        #     ports = "ports {} to {}".format(self.from_port, self.to_port)
        ports = ""
        return "{} {} from {}".format(self.protocol, ports, self.network)

class NetworkACL(Resource):

    resource_name = "network_acl"

    name = argument.String(field="Name", group="tags")
    inbound = argument.ResourceList(Rule)
    outbound = argument.ResourceList(Rule)

    tags = argument.Dict()
    vpc = argument.Resource(VPC, field="VpcId")

class Describe(SimpleDescribe, Plan):

    resource = NetworkACL
    service_name = 'ec2'
    describe_action = "describe_network_acls"
    describe_envelope = "NetworkAcls"
    key = 'NetworkAclId'

    biggest_serial = 0

    def get_describe_filters(self):
        vpc = self.runner.get_plan(self.resource.vpc)
        if not vpc.resource_id:
            return None

        if self.key in self.object:
            return {
                "Filters": [
                    {'Name': 'network-acl-id', 'Values': [self.object[self.key]]}
                ]
            }

        return {
            "Filters": [
                {'Name': 'vpc-id', 'Values': [vpc.resource_id]}
            ],
        }

    def _check_rules(self, local, remote, egress):
        rules = zip_longest(
            local,
            filter(
                lambda r: r['Egress'] == egress and r['RuleNumber'] <= 32766,
                remote['Entries'],
            ),
        )
        for i, (left, right) in enumerate(rules, start=1):
            if not left or not right:
                return False
            if i != right['RuleNumber']:
                return False
            if not left.matches(self.runner, right):
                return False
        return True

    def _compare_rules(self, network_acl):
        if not self._check_rules(self.resource.inbound, network_acl, False):
            return False
        if not self._check_rules(self.resource.outbound, network_acl, True):
            return False
        return True

    def describe_object_matches(self, network_acl):
        if self.key in self.object and network_acl[self.key] == self.object[self.key]:
            return True

        tags = {tag['Key']: tag['Value'] for tag in network_acl.get("Tags", [])}
        name = tags.get('Name', '')

        if name != self.resource.name and not name.startswith(self.resource.name + "."):
            return False

        try:
            serial = name.rsplit(".", 1)[1]
            self.biggest_serial = max(int(serial), self.biggest_serial)
        except (IndexError, TypeError, ValueError):
            pass

        return self._compare_rules(network_acl)

    def get_possible_objects(self):
        vpc = self.runner.get_plan(self.resource.vpc)
        if not vpc.resource_id:
            return

        prefix = "{}.".format(self.resource.name)

        result = self.client.describe_network_acls(
            Filters=[
                {"Name": 'vpc-id', "Values": [vpc.resource_id]},
            ]
        )

        for network_acl in result.get('NetworkAcls', []):
            tags = {tag['Key']: tag['Value'] for tag in network_acl.get("Tags", [])}
            name = tags.get('Name', '')

            # Only ignore ACL's for the current resource
            if name != self.resource.name and not name.startswith(prefix):
                continue

            # Don't delete the default ACL
            if network_acl['IsDefault']:
                continue

            yield network_acl

class Apply(SimpleApply, Describe):

    create_action = "create_network_acl"
    waiter = "network_acl_available"
    waiter_eventual_consistency_threshold = 5

    def prepare_to_create(self):
        for network_acl in self.get_possible_objects():
            # Don't try and delete ACL's that are in use
            if len(network_acl['Associations']) != 0:
                continue

            yield self.generic_action(
                "Destroy stale network acl: {}".format(network_acl['NetworkAclId']),
                self.client.delete_network_acl,
                NetworkAclId=network_acl['NetworkAclId'],
            )

    def get_create_name(self):
        return "{}.{}".format(
            self.resource.name,
            self.biggest_serial + 1,
        )

    def _insert_rule(self, rule, rule_number, egress):
        return self.generic_action(
            "Add {direction} rule: {rule}".format(
                rule=rule,
                direction='egress' if egress else 'ingress',
            ),
            self.client.create_network_acl_entry,
            rule.serializer_with_args(
                NetworkAclId=self.resource.identifier(),
                Egress=egress,
                RuleNumber=rule_number,
            )
        )

    def insert_network_rules(self):
        for i, rule in enumerate(self.resource.inbound, start=1):
            yield self._insert_rule(rule, rule_number=i, egress=False)
        for i, rule in enumerate(self.resource.outbound, start=1):
            yield self._insert_rule(rule, rule_number=i, egress=True)

    def update_object(self):
        for action in super(Apply, self).update_object():
            yield action

        if not self.object:
            for action in self.insert_network_rules():
                yield action

class Destroy(SimpleDestroy, Describe):

    destroy_action = "delete_network_acl"

    def get_actions(self):
        for network_acl in self.get_possible_objects():
            tags = {tag['Key']: tag['Value'] for tag in network_acl.get("Tags", [])}
            name = tags.get('Name', '')

            yield self.generic_action(
                "Destroy network acl: {} ({})".format(name, network_acl['NetworkAclId']),
                self.client.delete_network_acl,
                NetworkAclId=network_acl['NetworkAclId'],
            )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2004-2009 Edgewall Software
# Copyright (C) 2004 Francois Harvey <fharvey@securiweb.net>
# Copyright (C) 2005 Matthew Good <trac@matt-good.net>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Francois Harvey <fharvey@securiweb.net>
#         Matthew Good <trac@matt-good.net>

import os.path

from trac.config import Option, PathOption
from trac.core import *
from trac.perm import IPermissionPolicy
from trac.resource import Resource
from trac.util import read_file
from trac.util.compat import any
from trac.util.text import exception_to_unicode, to_unicode
from trac.util.translation import _
from trac.versioncontrol.api import RepositoryManager

def parent_iter(path):
    path = path.strip('/')
    if path:
        path = '/' + path + '/'
    else:
        path = '/'

    while 1:
        yield path
        if path == '/':
            return
        path = path[:-1]
        yield path
        idx = path.rfind('/')
        path = path[:idx + 1]

class ParseError(Exception):
    """Exception thrown for parse errors in authz files"""

def parse(authz):
    """Parse a Subversion authorization file.

    Return a dict of modules, each containing a dict of paths, each containing
    a dict mapping users to permissions.
    """
    groups = {}
    aliases = {}
    sections = {}
    section = None
    lineno = 0
    for line in authz.splitlines():
        lineno += 1
        line = to_unicode(line.strip())
        if not line or line.startswith('#') or line.startswith(';'):
            continue
        if line.startswith('[') and line.endswith(']'):
            section = line[1:-1]
            continue
        if section is None:
            raise ParseError(_('Line %(lineno)d: Entry before first '
                               'section header', lineno=lineno))
        parts = line.split('=', 1)
        if len(parts) != 2:
            raise ParseError(_('Line %(lineno)d: Invalid entry',
                               lineno=lineno))
        name, value = parts
        name = name.strip()
        if section == 'groups':
            group = groups.setdefault(name, set())
            group.update(each.strip() for each in value.split(','))
        elif section == 'aliases':
            aliases[name] = value.strip()
        else:
            sections.setdefault(section, []).append((name.strip(), value))

    def resolve(subject, done):
        if subject.startswith('@'):
            done.add(subject)
            for members in groups[subject[1:]] - done:
                for each in resolve(members, done):
                    yield each
        elif subject.startswith('&'):
            yield aliases[subject[1:]]
        else:
            yield subject

    authz = {}
    for name, items in sections.iteritems():
        parts = name.split(':', 1)
        module = authz.setdefault(len(parts) > 1 and parts[0] or '', {})
        section = module.setdefault(parts[-1], {})
        for subject, perms in items:
            for user in resolve(subject, set()):
                section.setdefault(user, 'r' in perms)  # The first match wins

    return authz

class AuthzSourcePolicy(Component):
    """Permission policy for `source:` and `changeset:` resources using a
    Subversion authz file.

    `FILE_VIEW` and `BROWSER_VIEW` permissions are granted as specified in the
    authz file.

    `CHANGESET_VIEW` permission is granted for changesets where `FILE_VIEW` is
    granted on at least one modified file, as well as empty for changesets.
    """

    implements(IPermissionPolicy)

    authz_file = PathOption('trac', 'authz_file', '',
        """The path to the Subversion
        [http://svnbook.red-bean.com/en/1.5/svn.serverconfig.pathbasedauthz.html authorization (authz) file].
        To enable authz permission checking, the `AuthzSourcePolicy` permission
        policy must be added to `[trac] permission_policies`.
        """)

    authz_module_name = Option('trac', 'authz_module_name', '',
        """The module prefix used in the `authz_file` for the default
        repository. If left empty, the global sections will be used.
        """)

    _mtime = 0
    _authz = {}
    _users = set()

    # IPermissionPolicy methods

    def check_permission(self, action, username, resource, perm):
        if action == 'FILE_VIEW' or action == 'BROWSER_VIEW':
            authz, users = self._get_authz_info()
            if authz is None:
                return False
            if resource is None:
                return users is True or username in users
            if resource.realm == 'source':
                modules = [resource.parent.id or self.authz_module_name]
                if modules[0]:
                    modules.append('')
                for p in parent_iter(resource.id):
                    for module in modules:
                        section = authz.get(module, {}).get(p, {})
                        result = section.get(username)
                        if result is not None:
                            return result
                        result = section.get('*')
                        if result is not None:
                            return result
                return False
        elif action == 'CHANGESET_VIEW':
            authz, users = self._get_authz_info()
            if authz is None:
                return False
            if resource is None:
                return users is True or username in users
            if resource.realm == 'changeset':
                rm = RepositoryManager(self.env)
                repos = rm.get_repository(resource.parent.id)
                changes = list(repos.get_changeset(resource.id).get_changes())
                if not changes:
                    return True
                source = Resource('source', version=resource.id,
                                  parent=resource.parent)
                return any('FILE_VIEW' in perm(source(id=change[0]))
                           for change in changes)

    def _get_authz_info(self):
        try:
            mtime = os.path.getmtime(self.authz_file)
        except OSError, e:
            if self._authz is not None:
                self.log.error('Error accessing authz file: %s',
                               exception_to_unicode(e))
            self._mtime = mtime = 0
            self._authz = None
            self._users = set()
        if mtime > self._mtime:
            self._mtime = mtime
            self.log.info('Parsing authz file: %s' % self.authz_file)
            try:
                self._authz = parse(read_file(self.authz_file))
                users = set(user for module in self._authz.itervalues()
                            for path in module.itervalues()
                            for user, result in path.iteritems() if result)
                self._users = '*' in users or users
            except Exception, e:
                self._authz = None
                self._users = set()
                self.log.error('Error parsing authz file: %s',
                               exception_to_unicode(e))
        return self._authz, self._users
<commit_msg>authz: Fixed a typo in a docstring.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2004-2009 Edgewall Software
# Copyright (C) 2004 Francois Harvey <fharvey@securiweb.net>
# Copyright (C) 2005 Matthew Good <trac@matt-good.net>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Francois Harvey <fharvey@securiweb.net>
#         Matthew Good <trac@matt-good.net>

import os.path

from trac.config import Option, PathOption
from trac.core import *
from trac.perm import IPermissionPolicy
from trac.resource import Resource
from trac.util import read_file
from trac.util.compat import any
from trac.util.text import exception_to_unicode, to_unicode
from trac.util.translation import _
from trac.versioncontrol.api import RepositoryManager

def parent_iter(path):
    path = path.strip('/')
    if path:
        path = '/' + path + '/'
    else:
        path = '/'

    while 1:
        yield path
        if path == '/':
            return
        path = path[:-1]
        yield path
        idx = path.rfind('/')
        path = path[:idx + 1]

class ParseError(Exception):
    """Exception thrown for parse errors in authz files"""

def parse(authz):
    """Parse a Subversion authorization file.

    Return a dict of modules, each containing a dict of paths, each containing
    a dict mapping users to permissions.
    """
    groups = {}
    aliases = {}
    sections = {}
    section = None
    lineno = 0
    for line in authz.splitlines():
        lineno += 1
        line = to_unicode(line.strip())
        if not line or line.startswith('#') or line.startswith(';'):
            continue
        if line.startswith('[') and line.endswith(']'):
            section = line[1:-1]
            continue
        if section is None:
            raise ParseError(_('Line %(lineno)d: Entry before first '
                               'section header', lineno=lineno))
        parts = line.split('=', 1)
        if len(parts) != 2:
            raise ParseError(_('Line %(lineno)d: Invalid entry',
                               lineno=lineno))
        name, value = parts
        name = name.strip()
        if section == 'groups':
            group = groups.setdefault(name, set())
            group.update(each.strip() for each in value.split(','))
        elif section == 'aliases':
            aliases[name] = value.strip()
        else:
            sections.setdefault(section, []).append((name.strip(), value))

    def resolve(subject, done):
        if subject.startswith('@'):
            done.add(subject)
            for members in groups[subject[1:]] - done:
                for each in resolve(members, done):
                    yield each
        elif subject.startswith('&'):
            yield aliases[subject[1:]]
        else:
            yield subject

    authz = {}
    for name, items in sections.iteritems():
        parts = name.split(':', 1)
        module = authz.setdefault(len(parts) > 1 and parts[0] or '', {})
        section = module.setdefault(parts[-1], {})
        for subject, perms in items:
            for user in resolve(subject, set()):
                section.setdefault(user, 'r' in perms)  # The first match wins

    return authz

class AuthzSourcePolicy(Component):
    """Permission policy for `source:` and `changeset:` resources using a
    Subversion authz file.

    `FILE_VIEW` and `BROWSER_VIEW` permissions are granted as specified in the
    authz file.

    `CHANGESET_VIEW` permission is granted for changesets where `FILE_VIEW` is
    granted on at least one modified file, as well as for empty changesets.
    """

    implements(IPermissionPolicy)

    authz_file = PathOption('trac', 'authz_file', '',
        """The path to the Subversion
        [http://svnbook.red-bean.com/en/1.5/svn.serverconfig.pathbasedauthz.html authorization (authz) file].
        To enable authz permission checking, the `AuthzSourcePolicy` permission
        policy must be added to `[trac] permission_policies`.
        """)

    authz_module_name = Option('trac', 'authz_module_name', '',
        """The module prefix used in the `authz_file` for the default
        repository. If left empty, the global sections will be used.
        """)

    _mtime = 0
    _authz = {}
    _users = set()

    # IPermissionPolicy methods

    def check_permission(self, action, username, resource, perm):
        if action == 'FILE_VIEW' or action == 'BROWSER_VIEW':
            authz, users = self._get_authz_info()
            if authz is None:
                return False
            if resource is None:
                return users is True or username in users
            if resource.realm == 'source':
                modules = [resource.parent.id or self.authz_module_name]
                if modules[0]:
                    modules.append('')
                for p in parent_iter(resource.id):
                    for module in modules:
                        section = authz.get(module, {}).get(p, {})
                        result = section.get(username)
                        if result is not None:
                            return result
                        result = section.get('*')
                        if result is not None:
                            return result
                return False
        elif action == 'CHANGESET_VIEW':
            authz, users = self._get_authz_info()
            if authz is None:
                return False
            if resource is None:
                return users is True or username in users
            if resource.realm == 'changeset':
                rm = RepositoryManager(self.env)
                repos = rm.get_repository(resource.parent.id)
                changes = list(repos.get_changeset(resource.id).get_changes())
                if not changes:
                    return True
                source = Resource('source', version=resource.id,
                                  parent=resource.parent)
                return any('FILE_VIEW' in perm(source(id=change[0]))
                           for change in changes)

    def _get_authz_info(self):
        try:
            mtime = os.path.getmtime(self.authz_file)
        except OSError, e:
            if self._authz is not None:
                self.log.error('Error accessing authz file: %s',
                               exception_to_unicode(e))
            self._mtime = mtime = 0
            self._authz = None
            self._users = set()
        if mtime > self._mtime:
            self._mtime = mtime
            self.log.info('Parsing authz file: %s' % self.authz_file)
            try:
                self._authz = parse(read_file(self.authz_file))
                users = set(user for module in self._authz.itervalues()
                            for path in module.itervalues()
                            for user, result in path.iteritems() if result)
                self._users = '*' in users or users
            except Exception, e:
                self._authz = None
                self._users = set()
                self.log.error('Error parsing authz file: %s',
                               exception_to_unicode(e))
        return self._authz, self._users
<|endoftext|>


<commit_before>import sys
from contextlib import contextmanager
from functools import wraps

from twisted.internet.task import Clock

def deferred(f):
    """This is almost exactly the same as nose.twistedtools.deferred, except it allows one to have simulated time in
    their test code by passing in a `Clock` instance. That `Clock` instance will be `advance`d as long as there are
    any deferred calls bound to it.

    """
    @wraps(f)
    def ret():
        error = [None]

        clock = Clock()

        d = f(clock)
        @d.addErrback
        def on_error(f):
            error[0] = sys.exc_info()

        while True:
            time_to_wait = max([0] + [call.getTime() - clock.seconds() for call in clock.getDelayedCalls()])
            if time_to_wait == 0:
                break
            else:
                clock.advance(time_to_wait)

        if error[0]:
            exc_info = error[0]
            raise exc_info[0], exc_info[1], exc_info[2]

    return ret

@contextmanager
def assert_not_raises(exc_class=Exception, message=None):
    try:
        yield
    except exc_class as e:
        raise AssertionError(message or "No exception should have been raised but instead %s was raised" % repr(e))
<commit_msg>Added util.testing.assert_raises--just a thin convenience wrapper around nose.tools.assert_raises<commit_after>import sys
from contextlib import contextmanager
from functools import wraps

from nose.tools import assert_raises as nose_assert_raises
from twisted.internet.task import Clock

__all__ = ['deferred', 'assert_raises', 'assert_not_raises']

def deferred(f):
    """This is almost exactly the same as nose.twistedtools.deferred, except it allows one to have simulated time in
    their test code by passing in a `Clock` instance. That `Clock` instance will be `advance`d as long as there are
    any deferred calls bound to it.

    """
    @wraps(f)
    def ret():
        error = [None]

        clock = Clock()

        d = f(clock)
        @d.addErrback
        def on_error(f):
            error[0] = sys.exc_info()

        while True:
            time_to_wait = max([0] + [call.getTime() - clock.seconds() for call in clock.getDelayedCalls()])
            if time_to_wait == 0:
                break
            else:
                clock.advance(time_to_wait)

        if error[0]:
            exc_info = error[0]
            raise exc_info[0], exc_info[1], exc_info[2]

    return ret

@contextmanager
def assert_not_raises(exc_class=Exception, message=None):
    try:
        yield
    except exc_class as e:
        raise AssertionError(message or "No exception should have been raised but instead %s was raised" % repr(e))

def assert_raises(exc_class=Exception, message=None):
    return nose_assert_raises(exc_class, message)
<|endoftext|>


<commit_before>import sys, requests, random, optparse
from Queue import Queue
from threading import Thread
from collections import Counter
from tempfile import NamedTemporaryFile
from splash2.tests.utils import TestServers

class StressTest():

    ok_urls = 0.5
    error_urls = 0.3
    timeout_urls = 0.2

    def __init__(self, total=1000, concurrency=50):
        self.requests = total
        self.concurrency = concurrency

    def _ok_urls(self):
        url = ["http://localhost:8050/render.html?url=http://localhost:8998/jsrender"]
        return int(self.requests * self.ok_urls) * url

    def _error_urls(self):
        url = ["http://localhost:8050/render.html?url=http://non-existent-host/"]
        return int(self.requests * self.error_urls) * url

    def _timeout_urls(self):
        url = ["http://localhost:8050/render.html?url=http://localhost:8998/delay?n=10&timeout=0.5"]
        return int(self.requests * self.timeout_urls) * url

    def run(self):
        f = NamedTemporaryFile(prefix="splash-stress-", suffix=".log", delete=False)
        # we need to use log file to prevent deadlocks on heavy loads
        with TestServers(logfile=f.name):
            ok_urls = self._ok_urls()
            error_urls = self._error_urls()
            timeout_urls = self._timeout_urls()
            urls = ok_urls + error_urls + timeout_urls
            random.shuffle(urls)

            print "Total requests: %d" % len(urls)
            print "Concurrency   : %d" % self.concurrency
            print "Log file      : %s" % f.name

            q, p = Queue(), Queue()
            for _ in xrange(self.concurrency):
                t = Thread(target=worker, args=(q, p))
                t.daemon = True
                t.start()
            for url in urls:
                q.put(url)
            q.join()

            outputs = []
            for _ in xrange(self.requests):
                outputs.append(p.get())

            expected = {
                200: len(ok_urls),
                502: len(error_urls),
                504: len(timeout_urls),
            }
            print
            print "Received/Expected (per status code or error):"
            for c, n in Counter(outputs).items():
                print "  %s: %d/%d" % (c, n, expected.get(c, 0))

def worker(q, p):
    while True:
        try:
            url = q.get()
            r = requests.get(url)
            p.put(r.status_code)
            sys.stdout.write(".")
            sys.stdout.flush()
        except Exception as e:
            p.put(type(e))
            sys.stdout.write("E")
            sys.stdout.flush()
        finally:
            q.task_done()

def parse_opts():
    op = optparse.OptionParser()
    op.add_option("-c", dest="concurrency", type="int", default=50,
            help="concurrency (default: %default)")
    op.add_option("-n", dest="requests", type="int", default=1000,
            help="number of requests (default: %default)")
    return op.parse_args()

def main():
    opts, _ = parse_opts()
    t = StressTest(opts.requests, opts.concurrency)
    t.run()

if __name__ == "__main__":
    main()
<commit_msg>show total elapsed time and avg time per request on stress test report<commit_after>import sys, requests, random, optparse, time
from Queue import Queue
from threading import Thread
from collections import Counter
from tempfile import NamedTemporaryFile
from splash2.tests.utils import TestServers

class StressTest():

    ok_urls = 0.5
    error_urls = 0.3
    timeout_urls = 0.2

    def __init__(self, total=1000, concurrency=50):
        self.requests = total
        self.concurrency = concurrency

    def _ok_urls(self):
        url = ["http://localhost:8050/render.html?url=http://localhost:8998/jsrender"]
        return int(self.requests * self.ok_urls) * url

    def _error_urls(self):
        url = ["http://localhost:8050/render.html?url=http://non-existent-host/"]
        return int(self.requests * self.error_urls) * url

    def _timeout_urls(self):
        url = ["http://localhost:8050/render.html?url=http://localhost:8998/delay?n=10&timeout=0.5"]
        return int(self.requests * self.timeout_urls) * url

    def run(self):
        f = NamedTemporaryFile(prefix="splash-stress-", suffix=".log", delete=False)
        # we need to use log file to prevent deadlocks on heavy loads
        with TestServers(logfile=f.name):
            ok_urls = self._ok_urls()
            error_urls = self._error_urls()
            timeout_urls = self._timeout_urls()
            urls = ok_urls + error_urls + timeout_urls
            random.shuffle(urls)

            print "Total requests: %d" % len(urls)
            print "Concurrency   : %d" % self.concurrency
            print "Log file      : %s" % f.name

            starttime = time.time()
            q, p = Queue(), Queue()
            for _ in xrange(self.concurrency):
                t = Thread(target=worker, args=(q, p))
                t.daemon = True
                t.start()
            for url in urls:
                q.put(url)
            q.join()

            outputs = []
            for _ in xrange(self.requests):
                outputs.append(p.get())

            elapsed = time.time() - starttime
            expected = {
                200: len(ok_urls),
                502: len(error_urls),
                504: len(timeout_urls),
            }
            print
            print "Total requests: %d" % len(urls)
            print "Concurrency   : %d" % self.concurrency
            print "Log file      : %s" % f.name
            print "Elapsed time  : %.3fs" % elapsed
            print "Avg time p/req: %.3fs" % (elapsed/len(urls))
            print "Received/Expected (per status code or error):"
            for c, n in Counter(outputs).items():
                print "  %s: %d/%d" % (c, n, expected.get(c, 0))

def worker(q, p):
    while True:
        try:
            url = q.get()
            r = requests.get(url)
            p.put(r.status_code)
            sys.stdout.write(".")
            sys.stdout.flush()
        except Exception as e:
            p.put(type(e))
            sys.stdout.write("E")
            sys.stdout.flush()
        finally:
            q.task_done()

def parse_opts():
    op = optparse.OptionParser()
    op.add_option("-c", dest="concurrency", type="int", default=50,
            help="concurrency (default: %default)")
    op.add_option("-n", dest="requests", type="int", default=1000,
            help="number of requests (default: %default)")
    return op.parse_args()

def main():
    opts, _ = parse_opts()
    t = StressTest(opts.requests, opts.concurrency)
    t.run()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import subprocess
import json
import time
import os

# this must be a round number
MINUTES_UNTIL_PAUSE = 25
SPOTIFY_PATH = '/usr/local/bin/spotify'

class SpotifyHandler(object):
	"""docstring for SpotifyHandler"""
	def __init__(self):
		super(SpotifyHandler, self).__init__()
	def poll_spotify(self):
		return subprocess.check_output(SPOTIFY_PATH + ' status', shell=True)

	def is_spotify_playing(self):
		result = self.poll_spotify()
		if 'Spotify is currently playing.' in result:
			return True
		elif 'Spotify is currently paused.' in result:
			return False
		else:
			raise Exception()
	def is_command_line_installed(self):
		result = subprocess.call('which spotify', shell=True)
		if result == 0:
			return True
		else:
			return False

	def pause_spotify(self):
		return subprocess.check_output(SPOTIFY_PATH + ' pause', shell=True)

	def play_spotify(self):
		return subprocess.check_output(SPOTIFY_PATH + ' play', shell=True)

	def toggle_spotify_playing(self, should_play):
		if should_play:
			return self.play_spotify()
		else:
			return self.pause_spotify()

class TimeChecker(object):
	"""docstring for TimeChecker"""
	def get_number_of_checks_file(self):
		current_directory = os.path.dirname(os.path.realpath(__file__))
		return os.path.join(current_directory, 'number_of_checks.json')
	def get_number_of_checks(self):
		with open(self.get_number_of_checks_file()) as checks_file:
			data = json.load(checks_file)
			self.checks = data
	def __init__(self):
		super(TimeChecker, self).__init__()
		self.current_time = time.time()
		self.get_number_of_checks()
	def update_number_of_checks(self):
		current_number_of_checks = self.checks['checks']
		updated_number_of_checks = current_number_of_checks
		if current_number_of_checks >= (MINUTES_UNTIL_PAUSE - 1):
			updated_number_of_checks = 0
		else:
			updated_number_of_checks += 1
		with open(self.get_number_of_checks_file(), 'w') as checks_file:
			checks = {'checks' : updated_number_of_checks, 'last_updated' : self.current_time}
			json.dump(checks, checks_file)
	def reset_number_of_checks(self):
		with open(self.get_number_of_checks_file(), 'w') as checks_file:
			checks = {'checks' : 0, 'last_updated' : self.current_time}
			json.dump(checks, checks_file)
	def should_pause(self):
		current_number_of_checks = self.checks['checks']
		last_check_time = self.checks['last_updated']
		if self.current_time - last_check_time >= (MINUTES_UNTIL_PAUSE * 60):
			return True
		elif current_number_of_checks == (MINUTES_UNTIL_PAUSE - 1):
			return True
		else:
			return False

def main():
	spotify = SpotifyHandler()
	if not spotify.is_command_line_installed():
		print 'Spotify is not installed or in crontab PATH'
		print 'Install with "brew install shpotify"'
		return
	checker = TimeChecker()
	# don't check or update number of checks if spotify is not playing
	if not spotify.is_spotify_playing():
		checker.reset_number_of_checks()
		return
	if checker.should_pause():
		if spotify.is_spotify_playing():
			spotify.toggle_spotify_playing(False)
			# if we toggle off, make sure to reset number of checks!
			checker.reset_number_of_checks()
	# don't forget to update number of checks afterwards
	checker.update_number_of_checks()

if __name__ == '__main__':
	main()<commit_msg>Fixed spotify script and cronjob<commit_after>#!/usr/bin/env python

import subprocess
import json
import time
import os

# this must be a round number
MINUTES_UNTIL_PAUSE = 25
SPOTIFY_PATH = '/usr/local/bin/spotify'

class SpotifyHandler(object):
	"""docstring for SpotifyHandler"""
	def __init__(self):
		super(SpotifyHandler, self).__init__()
	def poll_spotify(self):
		result = subprocess.Popen(SPOTIFY_PATH + ' status', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()
		return result[0]

	def is_spotify_playing(self):
		result = self.poll_spotify()
		if 'Spotify is currently playing.' in result:
			return True
		elif 'Spotify is currently paused.' in result:
			return False
		else:
			raise Exception()
	def is_command_line_installed(self):
		result = subprocess.Popen('which ' + SPOTIFY_PATH, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()
		if len(result[0]) > 0:
			return True
		else:
			return False

	def pause_spotify(self):
		result = subprocess.Popen(SPOTIFY_PATH + ' pause', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()
		return result[0]

	def play_spotify(self):
		result = subprocess.Popen(SPOTIFY_PATH + ' play', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE).communicate()
		return result[0]

	def toggle_spotify_playing(self, should_play):
		if should_play:
			return self.play_spotify()
		else:
			return self.pause_spotify()

class TimeChecker(object):
	"""docstring for TimeChecker"""
	def get_number_of_checks_file(self):
		current_directory = os.path.dirname(os.path.realpath(__file__))
		return os.path.join(current_directory, 'number_of_checks.json')
	def get_number_of_checks(self):
		with open(self.get_number_of_checks_file()) as checks_file:
			data = json.load(checks_file)
			self.checks = data
	def __init__(self):
		super(TimeChecker, self).__init__()
		self.current_time = time.time()
		self.get_number_of_checks()
	def update_number_of_checks(self):
		current_number_of_checks = self.checks['checks']
		updated_number_of_checks = current_number_of_checks
		if current_number_of_checks >= (MINUTES_UNTIL_PAUSE - 1):
			updated_number_of_checks = 0
		else:
			updated_number_of_checks += 1
		with open(self.get_number_of_checks_file(), 'w') as checks_file:
			checks = {'checks' : updated_number_of_checks, 'last_updated' : self.current_time}
			json.dump(checks, checks_file)
	def reset_number_of_checks(self):
		with open(self.get_number_of_checks_file(), 'w') as checks_file:
			checks = {'checks' : 0, 'last_updated' : self.current_time}
			json.dump(checks, checks_file)
	def should_pause(self):
		current_number_of_checks = self.checks['checks']
		last_check_time = self.checks['last_updated']
		if self.current_time - last_check_time >= (MINUTES_UNTIL_PAUSE * 60):
			return True
		elif current_number_of_checks == (MINUTES_UNTIL_PAUSE - 1):
			return True
		else:
			return False

def main():
	spotify = SpotifyHandler()
	if not spotify.is_command_line_installed():
		print 'Spotify is not installed or in crontab PATH'
		print 'Install with "brew install shpotify"'
		return
	checker = TimeChecker()
	# don't check or update number of checks if spotify is not playing
	if not spotify.is_spotify_playing():
		checker.reset_number_of_checks()
		return
	if checker.should_pause():
		if spotify.is_spotify_playing():
			spotify.toggle_spotify_playing(False)
			# if we toggle off, make sure to reset number of checks!
			checker.reset_number_of_checks()
	# don't forget to update number of checks afterwards
	checker.update_number_of_checks()

if __name__ == '__main__':
	main()<|endoftext|>


<commit_before># Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# TODO(akalin): Make it so that warnings are errors on Windows.
# TODO(akalin): Clean up warnings on Windows.

{
  'variables': {
    # The root directory for the proto files.
    'proto_dir_root': 'files/src',
    # The relative path of the cacheinvalidation proto files from
    # proto_dir_root.
    # TODO(akalin): Add a RULE_INPUT_DIR predefined variable to gyp so
    # we don't need this variable.
    'proto_dir_relpath': 'google/cacheinvalidation',
    # Where files generated from proto files are put.
    'protoc_out_dir': '<(SHARED_INTERMEDIATE_DIR)/protoc_out',
  },
  'targets': [
    # The rule/action to generate files from the cacheinvalidation proto
    # files.
    {
      'target_name': 'cacheinvalidation_proto',
      'type': 'none',
      'sources': [
        '<(proto_dir_root)/google/cacheinvalidation/types.proto',
      ],
      # TODO(akalin): This block was copied from the sync_proto target
      # from chrome.gyp.  Decomp the shared blocks out somehow.
      'rules': [
        {
          'rule_name': 'genproto',
          'extension': 'proto',
          'inputs': [
            '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)protoc<(EXECUTABLE_SUFFIX)',
          ],
          'outputs': [
            '<(protoc_out_dir)/<(proto_dir_relpath)/<(RULE_INPUT_ROOT).pb.h',
            '<(protoc_out_dir)/<(proto_dir_relpath)/<(RULE_INPUT_ROOT).pb.cc',
          ],
          'action': [
            '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)protoc<(EXECUTABLE_SUFFIX)',
            '--proto_path=<(proto_dir_root)',
            # This path needs to be prefixed by proto_path, so we can't
            # use RULE_INPUT_PATH (which is an absolute path).
            '<(proto_dir_root)/<(proto_dir_relpath)/<(RULE_INPUT_NAME)',
            '--cpp_out=<(protoc_out_dir)',
          ],
          'message': 'Generating C++ code from <(RULE_INPUT_PATH)',
        },
      ],
      'dependencies': [
        '../../third_party/protobuf2/protobuf.gyp:protoc#host',
      ],
    },
    # The library created from the generated c++ cacheinvalidation proto
    # files.
    {
      'target_name': 'cacheinvalidation_proto_cc',
      'type': '<(library)',
      'sources': [
        '<(protoc_out_dir)/<(proto_dir_relpath)/types.pb.h',
        '<(protoc_out_dir)/<(proto_dir_relpath)/types.pb.cc',
      ],
      'include_dirs': [
        '<(protoc_out_dir)',
      ],
      'dependencies': [
        '../../third_party/protobuf2/protobuf.gyp:protobuf_lite',
        'cacheinvalidation_proto',
      ],
      'direct_dependent_settings': {
        'include_dirs': [
          '<(protoc_out_dir)',
        ],
      },
      'export_dependent_settings': [
        '../../third_party/protobuf2/protobuf.gyp:protobuf_lite',
      ],
    },
    # The main cache invalidation library.  External clients should depend
    # only on this.
    {
      'target_name': 'cacheinvalidation',
      'type': '<(library)',
      'sources': [
        'overrides/google/cacheinvalidation/callback.h',
        'overrides/google/cacheinvalidation/compiler-specific.h',
        'overrides/google/cacheinvalidation/googletest.h',
        'overrides/google/cacheinvalidation/logging.h',
        'overrides/google/cacheinvalidation/mutex.h',
        'overrides/google/cacheinvalidation/stl-namespace.h'
        'overrides/google/cacheinvalidation/string_util.h'
        'overrides/google/cacheinvalidation/time.h',
        'files/src/google/cacheinvalidation/invalidation-client-impl.cc',
        'files/src/google/cacheinvalidation/invalidation-client-impl.h',
        'files/src/google/cacheinvalidation/invalidation-client.cc',
        'files/src/google/cacheinvalidation/invalidation-client.h',
        'files/src/google/cacheinvalidation/log-macro.h',
        'files/src/google/cacheinvalidation/network-manager.cc',
        'files/src/google/cacheinvalidation/network-manager.h',
        'files/src/google/cacheinvalidation/registration-update-manager.cc',
        'files/src/google/cacheinvalidation/registration-update-manager.h',
        'files/src/google/cacheinvalidation/session-manager.cc',
        'files/src/google/cacheinvalidation/session-manager.h',
        'files/src/google/cacheinvalidation/throttle.cc',
        'files/src/google/cacheinvalidation/throttle.h',
      ],
      'include_dirs': [
        './overrides',
        './files/src',
      ],
      'dependencies': [
        '../../base/base.gyp:base',
        'cacheinvalidation_proto',
        'cacheinvalidation_proto_cc',
      ],
      'direct_dependent_settings': {
        'include_dirs': [
          './overrides',
          './files/src',
        ],
      },
      'export_dependent_settings': [
        'cacheinvalidation_proto',
        'cacheinvalidation_proto_cc',
      ],
    },
    # Unittests for the cache invalidation library.
    {
      'target_name': 'cacheinvalidation_unittests',
      'type': 'executable',
      'sources': [
        'files/src/google/cacheinvalidation/system-resources-for-test.h',
        'files/src/google/cacheinvalidation/invalidation-client-impl_test.cc',
        'files/src/google/cacheinvalidation/throttle_test.cc',
      ],
      'dependencies': [
        '../../base/base.gyp:base',
        '../../testing/gtest.gyp:gtest',
        '../../testing/gtest.gyp:gtestmain',
        'cacheinvalidation',
      ],
    },
  ],
}

# Local Variables:
# tab-width:2
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=2 shiftwidth=2:
<commit_msg>Fixed cacheinvalidation.gyp so dependents compile properly and consistently.<commit_after># Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# TODO(akalin): Make it so that warnings are errors on Windows.
# TODO(akalin): Clean up warnings on Windows.

{
  'variables': {
    # The root directory for the proto files.
    'proto_dir_root': 'files/src',
    # The relative path of the cacheinvalidation proto files from
    # proto_dir_root.
    # TODO(akalin): Add a RULE_INPUT_DIR predefined variable to gyp so
    # we don't need this variable.
    'proto_dir_relpath': 'google/cacheinvalidation',
    # Where files generated from proto files are put.
    'protoc_out_dir': '<(SHARED_INTERMEDIATE_DIR)/protoc_out',
  },
  'targets': [
    # The rule/action to generate files from the cacheinvalidation proto
    # files.
    {
      'target_name': 'cacheinvalidation_proto',
      'type': 'none',
      'sources': [
        '<(proto_dir_root)/google/cacheinvalidation/types.proto',
      ],
      # TODO(akalin): This block was copied from the sync_proto target
      # from chrome.gyp.  Decomp the shared blocks out somehow.
      'rules': [
        {
          'rule_name': 'genproto',
          'extension': 'proto',
          'inputs': [
            '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)protoc<(EXECUTABLE_SUFFIX)',
          ],
          'outputs': [
            '<(protoc_out_dir)/<(proto_dir_relpath)/<(RULE_INPUT_ROOT).pb.h',
            '<(protoc_out_dir)/<(proto_dir_relpath)/<(RULE_INPUT_ROOT).pb.cc',
          ],
          'action': [
            '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)protoc<(EXECUTABLE_SUFFIX)',
            '--proto_path=<(proto_dir_root)',
            # This path needs to be prefixed by proto_path, so we can't
            # use RULE_INPUT_PATH (which is an absolute path).
            '<(proto_dir_root)/<(proto_dir_relpath)/<(RULE_INPUT_NAME)',
            '--cpp_out=<(protoc_out_dir)',
          ],
          'message': 'Generating C++ code from <(RULE_INPUT_PATH)',
        },
      ],
      'dependencies': [
        '../../third_party/protobuf2/protobuf.gyp:protoc#host',
      ],
      # This target exports a hard dependency because it generates header
      # files.
      'hard_dependency': 1,
    },
    # The main cache invalidation library.  External clients should depend
    # only on this.
    {
      'target_name': 'cacheinvalidation',
      'type': '<(library)',
      'sources': [
        '<(protoc_out_dir)/<(proto_dir_relpath)/types.pb.h',
        '<(protoc_out_dir)/<(proto_dir_relpath)/types.pb.cc',
        'overrides/google/cacheinvalidation/callback.h',
        'overrides/google/cacheinvalidation/compiler-specific.h',
        'overrides/google/cacheinvalidation/googletest.h',
        'overrides/google/cacheinvalidation/logging.h',
        'overrides/google/cacheinvalidation/mutex.h',
        'overrides/google/cacheinvalidation/stl-namespace.h'
        'overrides/google/cacheinvalidation/string_util.h'
        'overrides/google/cacheinvalidation/time.h',
        'files/src/google/cacheinvalidation/invalidation-client-impl.cc',
        'files/src/google/cacheinvalidation/invalidation-client-impl.h',
        'files/src/google/cacheinvalidation/invalidation-client.cc',
        'files/src/google/cacheinvalidation/invalidation-client.h',
        'files/src/google/cacheinvalidation/log-macro.h',
        'files/src/google/cacheinvalidation/network-manager.cc',
        'files/src/google/cacheinvalidation/network-manager.h',
        'files/src/google/cacheinvalidation/registration-update-manager.cc',
        'files/src/google/cacheinvalidation/registration-update-manager.h',
        'files/src/google/cacheinvalidation/session-manager.cc',
        'files/src/google/cacheinvalidation/session-manager.h',
        'files/src/google/cacheinvalidation/throttle.cc',
        'files/src/google/cacheinvalidation/throttle.h',
      ],
      'include_dirs': [
        '<(protoc_out_dir)',
        './overrides',
        './files/src',
      ],
      'dependencies': [
        '../../base/base.gyp:base',
        '../../third_party/protobuf2/protobuf.gyp:protobuf_lite',
        'cacheinvalidation_proto',
      ],
      # This target exports a hard dependency because depedents require
      # cacheinvalidation_proto to compile.
      'hard_dependency': 1,
      'direct_dependent_settings': {
        'include_dirs': [
          '<(protoc_out_dir)',
          './overrides',
          './files/src',
        ],
      },
      'export_dependent_settings': [
        '../../third_party/protobuf2/protobuf.gyp:protobuf_lite',
        'cacheinvalidation_proto',
      ],
    },
    # Unittests for the cache invalidation library.
    {
      'target_name': 'cacheinvalidation_unittests',
      'type': 'executable',
      'sources': [
        'files/src/google/cacheinvalidation/system-resources-for-test.h',
        'files/src/google/cacheinvalidation/invalidation-client-impl_test.cc',
        'files/src/google/cacheinvalidation/throttle_test.cc',
      ],
      'dependencies': [
        '../../base/base.gyp:base',
        '../../testing/gtest.gyp:gtest',
        '../../testing/gtest.gyp:gtestmain',
        'cacheinvalidation',
      ],
    },
  ],
}

# Local Variables:
# tab-width:2
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=2 shiftwidth=2:
<|endoftext|>


<commit_before>info = (0, 3, 2)
version = '.'.join(map(str, info))
<commit_msg>bump version<commit_after>info = (0, 4, 0)
version = '.'.join(map(str, info))
<|endoftext|>


<commit_before>""" Functions for helping with serialization and deserialization of
Bokeh objects.

"""
from __future__ import absolute_import

from six import iterkeys
import numpy as np
try:
    import pandas as pd
    is_pandas = True
except ImportError:
    is_pandas = False

import logging
log = logging.getLogger(__name__)

_simple_id = 1000

def make_id():
    """ Return a new unique ID for a Bokeh object.

    Normally this function will return UUIDs to use for identifying Bokeh
    objects. This is especally important for Bokeh objects stored on a
    Bokeh server. However, it is convenient to have more human-readable
    IDs during development, so this behavior can be overridden by
    setting the environment variable ``BOKEH_SIMPLE_IDS=yes``.

    """
    global _simple_id

    import uuid
    from ..settings import settings

    if settings.simple_ids(False):
        _simple_id += 1
        new_id = _simple_id
    else:
        new_id = uuid.uuid4()
    return str(new_id)

def urljoin(*args):
    """ Construct an absolute URL from several URL components.

    Args:
        *args (str) : URL components to join

    Returns:
        str : joined URL

    """
    from six.moves.urllib.parse import urljoin as sys_urljoin
    from functools import reduce
    return reduce(sys_urljoin, args)

def get_json(response):
    """ Unify retrieving JSON responses from different sources.

    Works correctly for HTTP responses from requests <=1.0, >1.0, and
    the Flask test client.

    Args:
        response (Flask or requests response) : a response to process

    Returns:
        JSON

    """
    import json
    try:
        import flask
    except ImportError:
        flask = None
    if flask and isinstance(response, flask.Response):
        # flask testing
        return json.loads(response.data.decode('utf-8'))
    else:
        # requests
        if hasattr(response.json, '__call__'):
            return response.json()
        else:
            return response.json

def dump(objs, docid, changed_only=True):
    """ Serialize a sequence of Bokeh objects into JSON

        Args:
            objs (seq[obj]) : a sequence of Bokeh object to dump
            docid (str) : an ID for a Bokeh Document to dump relative to
            changed_only (bool, optional) : whether to dump only attributes
                that have had their values changed at some point (default: True)

        Returns:
            list[json]
    """
    json_objs = []
    for obj in objs:
        ref = obj.ref
        ref["attributes"] = obj.vm_serialize(changed_only=changed_only)
        ref["attributes"].update({"id": ref["id"], "doc" : docid})
        json_objs.append(ref)
    return json_objs

def is_ref(frag):
    """ Test whether a given Bokeh object graph fragment is a reference.

    A Bokeh "reference" is a ``dict`` with ``"type"`` and ``"id"`` keys.

    Args:
        frag (dict) : a fragment of a Bokeh object graph

    Returns:
        True, if the fragment is a reference, otherwise False

    """
    return isinstance(frag, dict) and \
           frag.get('type') and \
           frag.get('id')

def json_apply(fragment, check_func, func):
    """ Apply a function to JSON fragments that match the given predicate
    and return the collected results.

    Recursively traverses a nested collection of ``dict`` and ``list``,
    applying ``check_func`` to each fragment. If True, then collect
    ``func(fragment)`` in the final output

    Args:
        fragment (JSON-like) : the fragment to apply ``func`` to recursively
        check_func (callable) : the predicate to test fragments with
        func (callable) : the conversion function to apply

    Returns:
        converted fragments

    """
    if check_func(fragment):
        return func(fragment)
    elif isinstance(fragment, list):
        output = []
        for val in fragment:
            output.append(json_apply(val, check_func, func))
        return output
    elif isinstance(fragment, dict):
        output = {}
        for k, val in fragment.items():
            output[k] = json_apply(val, check_func, func)
        return output
    else:
        return fragment

def transform_series(obj):
    """transforms pandas series into array of values
    """
    vals = obj.values
    return transform_array(vals)

def transform_array(obj):
    """Transform arrays into lists of json safe types
    also handles pandas series, and replacing
    nans and infs with strings
    """
    # Check for astype failures (putative Numpy < 1.7)
    dt2001 = np.datetime64('2001')
    legacy_datetime64 = (dt2001.astype('int64') ==
                         dt2001.astype('datetime64[ms]').astype('int64'))
    ## not quite correct, truncates to ms..
    if obj.dtype.kind == 'M':
        if legacy_datetime64:
            if obj.dtype == np.dtype('datetime64[ns]'):
                return (obj.astype('int64') / 10**6.0).tolist()
        else:
            return (obj.astype('datetime64[us]').astype('int64') / 1000.).tolist()
    elif obj.dtype.kind in ('u', 'i', 'f'):
        return transform_numerical_array(obj)
    return obj.tolist()

def transform_numerical_array(obj):
    """handles nans/inf conversion
    """
    if isinstance(obj, np.ma.MaskedArray):
        obj = obj.filled(np.nan)  # Set masked values to nan
    if not np.isnan(obj).any() and not np.isinf(obj).any():
        return obj.tolist()
    else:
        transformed = obj.astype('object')
        transformed[np.isnan(obj)] = 'NaN'
        transformed[np.isposinf(obj)] = 'Infinity'
        transformed[np.isneginf(obj)] = '-Infinity'
        return transformed.tolist()

def traverse_data(datum):
    """recursively dig until a flat array is found
    convert it to a numpy array and send off to transform_array()
    to handle nan, inf, -inf
    """
    if not any(isinstance(el, (list, tuple)) for el in datum):
        return transform_array(np.asarray(datum))
    datum_copy = []
    for item in datum:
        if isinstance(item, (list, tuple)):
            datum_copy.append(traverse_data(item))
    return datum_copy

def transform_column_source_data(data):
    """iterate through the data of a ColumnSourceData object replacing
    non-JSON-compliant objects with compliant ones
    """
    data_copy = {}
    for key in iterkeys(data):
        if is_pandas and isinstance(data[key], (pd.Series, pd.Index)):
            data_copy[key] = transform_series(data[key])
        elif isinstance(data[key], np.ndarray):
            data_copy[key] = transform_array(data[key])
        else:
            data_copy[key] = traverse_data(data[key])
    return data_copy
<commit_msg>optional numpy<commit_after>""" Functions for helping with serialization and deserialization of
Bokeh objects.

"""
from __future__ import absolute_import

from six import iterkeys

try:
    import numpy as np
    is_numpy = True
except ImportError:
    is_numpy = False

try:
    import pandas as pd
    is_pandas = True
except ImportError:
    is_pandas = False

import logging
log = logging.getLogger(__name__)

_simple_id = 1000

def make_id():
    """ Return a new unique ID for a Bokeh object.

    Normally this function will return UUIDs to use for identifying Bokeh
    objects. This is especally important for Bokeh objects stored on a
    Bokeh server. However, it is convenient to have more human-readable
    IDs during development, so this behavior can be overridden by
    setting the environment variable ``BOKEH_SIMPLE_IDS=yes``.

    """
    global _simple_id

    import uuid
    from ..settings import settings

    if settings.simple_ids(False):
        _simple_id += 1
        new_id = _simple_id
    else:
        new_id = uuid.uuid4()
    return str(new_id)

def urljoin(*args):
    """ Construct an absolute URL from several URL components.

    Args:
        *args (str) : URL components to join

    Returns:
        str : joined URL

    """
    from six.moves.urllib.parse import urljoin as sys_urljoin
    from functools import reduce
    return reduce(sys_urljoin, args)

def get_json(response):
    """ Unify retrieving JSON responses from different sources.

    Works correctly for HTTP responses from requests <=1.0, >1.0, and
    the Flask test client.

    Args:
        response (Flask or requests response) : a response to process

    Returns:
        JSON

    """
    import json
    try:
        import flask
    except ImportError:
        flask = None
    if flask and isinstance(response, flask.Response):
        # flask testing
        return json.loads(response.data.decode('utf-8'))
    else:
        # requests
        if hasattr(response.json, '__call__'):
            return response.json()
        else:
            return response.json

def dump(objs, docid, changed_only=True):
    """ Serialize a sequence of Bokeh objects into JSON

        Args:
            objs (seq[obj]) : a sequence of Bokeh object to dump
            docid (str) : an ID for a Bokeh Document to dump relative to
            changed_only (bool, optional) : whether to dump only attributes
                that have had their values changed at some point (default: True)

        Returns:
            list[json]
    """
    json_objs = []
    for obj in objs:
        ref = obj.ref
        ref["attributes"] = obj.vm_serialize(changed_only=changed_only)
        ref["attributes"].update({"id": ref["id"], "doc" : docid})
        json_objs.append(ref)
    return json_objs

def is_ref(frag):
    """ Test whether a given Bokeh object graph fragment is a reference.

    A Bokeh "reference" is a ``dict`` with ``"type"`` and ``"id"`` keys.

    Args:
        frag (dict) : a fragment of a Bokeh object graph

    Returns:
        True, if the fragment is a reference, otherwise False

    """
    return isinstance(frag, dict) and \
           frag.get('type') and \
           frag.get('id')

def json_apply(fragment, check_func, func):
    """ Apply a function to JSON fragments that match the given predicate
    and return the collected results.

    Recursively traverses a nested collection of ``dict`` and ``list``,
    applying ``check_func`` to each fragment. If True, then collect
    ``func(fragment)`` in the final output

    Args:
        fragment (JSON-like) : the fragment to apply ``func`` to recursively
        check_func (callable) : the predicate to test fragments with
        func (callable) : the conversion function to apply

    Returns:
        converted fragments

    """
    if check_func(fragment):
        return func(fragment)
    elif isinstance(fragment, list):
        output = []
        for val in fragment:
            output.append(json_apply(val, check_func, func))
        return output
    elif isinstance(fragment, dict):
        output = {}
        for k, val in fragment.items():
            output[k] = json_apply(val, check_func, func)
        return output
    else:
        return fragment

def transform_series(obj):
    """transforms pandas series into array of values
    """
    vals = obj.values
    return transform_array(vals)

def transform_array(obj):
    """Transform arrays into lists of json safe types
    also handles pandas series, and replacing
    nans and infs with strings
    """
    # Check for astype failures (putative Numpy < 1.7)
    dt2001 = np.datetime64('2001')
    legacy_datetime64 = (dt2001.astype('int64') ==
                         dt2001.astype('datetime64[ms]').astype('int64'))
    ## not quite correct, truncates to ms..
    if obj.dtype.kind == 'M':
        if legacy_datetime64:
            if obj.dtype == np.dtype('datetime64[ns]'):
                return (obj.astype('int64') / 10**6.0).tolist()
        else:
            return (obj.astype('datetime64[us]').astype('int64') / 1000.).tolist()
    elif obj.dtype.kind in ('u', 'i', 'f'):
        return transform_numerical_array(obj)
    return obj.tolist()

def transform_numerical_array(obj):
    """handles nans/inf conversion
    """
    if isinstance(obj, np.ma.MaskedArray):
        obj = obj.filled(np.nan)  # Set masked values to nan
    if not np.isnan(obj).any() and not np.isinf(obj).any():
        return obj.tolist()
    else:
        transformed = obj.astype('object')
        transformed[np.isnan(obj)] = 'NaN'
        transformed[np.isposinf(obj)] = 'Infinity'
        transformed[np.isneginf(obj)] = '-Infinity'
        return transformed.tolist()

def traverse_data(datum):
    """recursively dig until a flat list is found
    if numpy is available convert the flat list to a numpy array
    and send off to transform_array() to handle nan, inf, -inf
    otherwise iterate through items in array converting non-json items
    """
    if not any(isinstance(el, (list, tuple)) for el in datum) and is_numpy:
        return transform_array(np.asarray(datum))
    datum_copy = []
    for item in datum:
        if isinstance(item, (list, tuple)):
            datum_copy.append(traverse_data(item))
        elif isinstance(item, float):
            if np.isnan(item):
                item = 'NaN'
            elif np.isposinf(item):
                item = 'Infinity'
            elif np.isneginf(item):
                item = '-Infinity'
            datum_copy.append(item)
        else:
            datum_copy.append(item)
    return datum_copy

def transform_column_source_data(data):
    """iterate through the data of a ColumnSourceData object replacing
    non-JSON-compliant objects with compliant ones
    """
    data_copy = {}
    for key in iterkeys(data):
        if is_pandas and isinstance(data[key], (pd.Series, pd.Index)):
            data_copy[key] = transform_series(data[key])
        elif isinstance(data[key], np.ndarray):
            data_copy[key] = transform_array(data[key])
        else:
            data_copy[key] = traverse_data(data[key])
    return data_copy
<|endoftext|>


<commit_before><commit_msg>Reorganized the code and added additional comments<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
import os, subprocess
from shlex import quote
from shutil import copytree
from fnmatch import fnmatch
from igf_data.utils.fileutils import get_temp_dir,remove_dir
from ehive.runnable.IGFBaseProcess import IGFBaseProcess
from igf_data.utils.tools.cellranger.cellranger_count_utils import get_cellranger_count_input_list
from igf_data.utils.tools.cellranger.cellranger_count_utils import check_cellranger_count_output
from igf_data.utils.tools.cellranger.cellranger_count_utils import get_cellranger_reference_genome

class RunCellrangerCount(IGFBaseProcess):
  '''
  A ehive process class for running cellranger count pipeline
  '''
  def param_defaults(self):
    params_dict=super(RunBcl2Fastq,self).param_defaults()
    params_dict.update({
        'force_overwrite':True,
        'cellranger_exe':None,
        'fastq_collection_type':'demultiplexed_fastq',
        'reference_type':'cellranger_reference_transcriptome',
        'cellranger_options':'{ "--jobmode":"pbspro","--localcores":"1","--localmem":"4","--mempercore":"4","--maxjobs":"20"}',
        'job_timeout':172800,
      })
    return params_dict

  def run(self):
    '''
    A method for running the cellranger count for a given sample using ehive pipeline

    :param project_igf_id: A project igf id
    :param experiment_igf_id: An experiment igf id
    :param sample_igf_id: A sample igf id
    :param igf_session_class: A database session class
    :param cellranger_exe: Cellranger executable path
    :param cellranger_options: Cellranger parameters

                               List of default parameters
                                 --jobmode=pbspro
                                 --localcores=1
                                 --localmem=4
                                 --mempercore=4
                                 --maxjobs=20

    :param base_work_dir: Base work directory path
    :param fastq_collection_type: Collection type name for input fastq files, default demultiplexed_fastq
    :param species_name: Reference genome collection name
    :param reference_type: Reference genome collection type, default cellranger_reference_transcriptome
    :param job_timeout: Timeout for cellranger job, default 48hrs
    :returns: Adding cellranger_output to the dataflow_params
    '''
    try:
      project_igf_id=self.param_required('project_igf_id')
      experiment_igf_id=self.param_required('experiment_igf_id')
      sample_igf_id=self.param_required('sample_igf_id')
      sample_submitter_id=self.param_required('sample_submitter_id')
      igf_session_class=self.param_required('igf_session_class')
      cellranger_exe=self.param_required('cellranger_exe')
      cellranger_options=self.param_required('cellranger_options')
      base_work_dir=self.param_required('base_work_dir')
      fastq_collection_type=self.param_required('fastq_collection_type')
      job_timeout=self.param_required('job_timeout')
      species_name=self.param('species_name')
      reference_type=self.param('reference_type')

      job_name=self.job_name()
      work_dir=os.path.join(base_work_dir,seqrun_igf_id,job_name)               # get work directory name
      if not os.path.exists(work_dir):
        os.makedirs(work_dir,mode=0o770)                                        # create work directory

      os.chdir(work_dir)                                                        # move to work dir
      os.environ['PATH'] += '{0}{1}'.format(os.pathsep,
                                            os.path.basename(cellranger_exe))   # add cellranger path to env
      reference_genome=get_cellranger_reference_genome(\
                         collection_name=species_name,
                         collection_type=reference_type,
                         session_class=igf_session_class)                       # fetch reference genome for cellranger run
      input_fastq_dirs=get_cellranger_count_input_list(\
                         db_session_class=igf_session_class,
                         experiment_igf_id=experiment_igf_id,
                         fastq_collection_type=fastq_collection_type)           # fetch fastq dir paths as list for run
      cellranger_options=self.format_tool_options(cellranger_options,
                                                  separator='=')
      cellranger_cmd=[cellranger_exe,
                      'count'
                      '{0}={1}'.format('--fastqs=',
                                       quote(','.join(input_fastq_dirs))),
                      '{0}={1}'.format('--id=',quote(experiment_igf_id)),
                      '{0}={1}'.format('--sample=',quote(sample_submitter_id)),
                      '{0}={1}'.format('--transcriptome',quote(reference_genome)),
                     ]                                                          # set initial parameters
      cellranger_cmd.extend(cellranger_options)                                 # add optional parameters
      message='started cellranger count for {0}, {1} {2}'.\
              format(project_igf_id,
                     sample_igf_id,
                     experiment_igf_id)
      self.post_message_to_slack(message,reaction='pass')                       # send log to slack
      self.comment_asana_task(task_name=seqrun_igf_id, comment=message)         # send comment to Asana
      message=' '.join(cellranger_cmd)
      self.comment_asana_task(task_name=seqrun_igf_id, comment=message)         # send cellranger command to Asana

      subprocess.check_call(cellranger_cmd,
                            timeout=job_timeout)                                # run cellranger count
      cellranger_output=os.path.join(work_dir,
                                     experiment_igf_id,
                                     'outs')                                    # get cellranger output path
      message='finished cellranger count for {0}, {1} {2} : {3}'.\
              format(project_igf_id,
                     sample_igf_id,
                     experiment_igf_id,
                     cellranger_output)
      self.post_message_to_slack(message,reaction='pass')                       # send log to slack
      self.comment_asana_task(task_name=seqrun_igf_id, comment=message)         # send comment to Asana
      check_cellranger_count_output(output_path=cellranger_output)              # check output file
      bam_list=list()                                                           # define empty bamfile list
      for file in os.listdir(check_cellranger_count_output):
        if fnmatch(file, '*.bam'):
          bam_list.append(file)                                                 # add all bams to bam_list

      if len(bam_list)>1:
        raise ValueError('More than one bam found for cellranger count run:{0}'.\
                         format(check_cellranger_count_output))                 # check number of bams, presence of one bam is already validated by check method

      reference_type=reference_type.replace('transcriptome','fasta')            # changing reference type for cram conversion
      self.param('dataflow_params',{'cellranger_output':cellranger_output,
                                    'bam_file':bam_list[0],
                                    'reference_type':reference_type})           # pass on cellranger output path
    except:
      message='project: {2}, sample:{3}, Error in {0}: {1}'.format(self.__class__.__name__, \
                                                      e, \
                                                      project_igf_id,
                                                      sample_igf_id)
      self.warning(message)
      self.post_message_to_slack(message,reaction='fail')                       # post msg to slack for failed jobs
      raise<commit_msg>fixed job specific work dir fetching<commit_after>#!/usr/bin/env python
import os, subprocess
from shlex import quote
from shutil import copytree
from fnmatch import fnmatch
from igf_data.utils.fileutils import get_temp_dir,remove_dir
from ehive.runnable.IGFBaseProcess import IGFBaseProcess
from igf_data.utils.tools.cellranger.cellranger_count_utils import get_cellranger_count_input_list
from igf_data.utils.tools.cellranger.cellranger_count_utils import check_cellranger_count_output
from igf_data.utils.tools.cellranger.cellranger_count_utils import get_cellranger_reference_genome

class RunCellrangerCount(IGFBaseProcess):
  '''
  A ehive process class for running cellranger count pipeline
  '''
  def param_defaults(self):
    params_dict=super(RunBcl2Fastq,self).param_defaults()
    params_dict.update({
        'force_overwrite':True,
        'cellranger_exe':None,
        'fastq_collection_type':'demultiplexed_fastq',
        'reference_type':'cellranger_reference_transcriptome',
        'cellranger_options':'{ "--jobmode":"pbspro","--localcores":"1","--localmem":"4","--mempercore":"4","--maxjobs":"20"}',
        'job_timeout':172800,
      })
    return params_dict

  def run(self):
    '''
    A method for running the cellranger count for a given sample using ehive pipeline

    :param project_igf_id: A project igf id
    :param experiment_igf_id: An experiment igf id
    :param sample_igf_id: A sample igf id
    :param igf_session_class: A database session class
    :param cellranger_exe: Cellranger executable path
    :param cellranger_options: Cellranger parameters

                               List of default parameters
                                 --jobmode=pbspro
                                 --localcores=1
                                 --localmem=4
                                 --mempercore=4
                                 --maxjobs=20

    :param base_work_dir: Base work directory path
    :param fastq_collection_type: Collection type name for input fastq files, default demultiplexed_fastq
    :param species_name: Reference genome collection name
    :param reference_type: Reference genome collection type, default cellranger_reference_transcriptome
    :param job_timeout: Timeout for cellranger job, default 48hrs
    :returns: Adding cellranger_output to the dataflow_params
    '''
    try:
      project_igf_id=self.param_required('project_igf_id')
      experiment_igf_id=self.param_required('experiment_igf_id')
      sample_igf_id=self.param_required('sample_igf_id')
      sample_submitter_id=self.param_required('sample_submitter_id')
      igf_session_class=self.param_required('igf_session_class')
      cellranger_exe=self.param_required('cellranger_exe')
      cellranger_options=self.param_required('cellranger_options')
      base_work_dir=self.param_required('base_work_dir')
      fastq_collection_type=self.param_required('fastq_collection_type')
      job_timeout=self.param_required('job_timeout')
      species_name=self.param('species_name')
      reference_type=self.param('reference_type')

      work_dir=os.path.join(base_work_dir,project_igf_id,sample_igf_id,experiment_igf_id)
      work_dir=self.get_job_work_dir(work_dir,work_dir)

      os.chdir(work_dir)                                                        # move to work dir
      os.environ['PATH'] += '{0}{1}'.format(os.pathsep,
                                            os.path.basename(cellranger_exe))   # add cellranger path to env
      reference_genome=get_cellranger_reference_genome(\
                         collection_name=species_name,
                         collection_type=reference_type,
                         session_class=igf_session_class)                       # fetch reference genome for cellranger run
      input_fastq_dirs=get_cellranger_count_input_list(\
                         db_session_class=igf_session_class,
                         experiment_igf_id=experiment_igf_id,
                         fastq_collection_type=fastq_collection_type)           # fetch fastq dir paths as list for run
      cellranger_options=self.format_tool_options(cellranger_options,
                                                  separator='=')
      cellranger_cmd=[cellranger_exe,
                      'count'
                      '{0}={1}'.format('--fastqs=',
                                       quote(','.join(input_fastq_dirs))),
                      '{0}={1}'.format('--id=',quote(experiment_igf_id)),
                      '{0}={1}'.format('--sample=',quote(sample_submitter_id)),
                      '{0}={1}'.format('--transcriptome',quote(reference_genome)),
                     ]                                                          # set initial parameters
      cellranger_cmd.extend(cellranger_options)                                 # add optional parameters
      message='started cellranger count for {0}, {1} {2}'.\
              format(project_igf_id,
                     sample_igf_id,
                     experiment_igf_id)
      self.post_message_to_slack(message,reaction='pass')                       # send log to slack
      self.comment_asana_task(task_name=seqrun_igf_id, comment=message)         # send comment to Asana
      message=' '.join(cellranger_cmd)
      self.comment_asana_task(task_name=seqrun_igf_id, comment=message)         # send cellranger command to Asana

      subprocess.check_call(cellranger_cmd,
                            timeout=job_timeout)                                # run cellranger count
      cellranger_output=os.path.join(work_dir,
                                     experiment_igf_id,
                                     'outs')                                    # get cellranger output path
      message='finished cellranger count for {0}, {1} {2} : {3}'.\
              format(project_igf_id,
                     sample_igf_id,
                     experiment_igf_id,
                     cellranger_output)
      self.post_message_to_slack(message,reaction='pass')                       # send log to slack
      self.comment_asana_task(task_name=seqrun_igf_id, comment=message)         # send comment to Asana
      check_cellranger_count_output(output_path=cellranger_output)              # check output file
      bam_list=list()                                                           # define empty bamfile list
      for file in os.listdir(check_cellranger_count_output):
        if fnmatch(file, '*.bam'):
          bam_list.append(file)                                                 # add all bams to bam_list

      if len(bam_list)>1:
        raise ValueError('More than one bam found for cellranger count run:{0}'.\
                         format(check_cellranger_count_output))                 # check number of bams, presence of one bam is already validated by check method

      reference_type=reference_type.replace('transcriptome','fasta')            # changing reference type for cram conversion
      self.param('dataflow_params',{'cellranger_output':cellranger_output,
                                    'bam_file':bam_list[0],
                                    'reference_type':reference_type})           # pass on cellranger output path
    except:
      message='project: {2}, sample:{3}, Error in {0}: {1}'.format(self.__class__.__name__, \
                                                      e, \
                                                      project_igf_id,
                                                      sample_igf_id)
      self.warning(message)
      self.post_message_to_slack(message,reaction='fail')                       # post msg to slack for failed jobs
      raise<|endoftext|>