<commit_before>from casexml.apps.case.sharedmodels import CommCareCaseIndex
from casexml.apps.case.const import CASE_INDEX_CHILD, CASE_INDEX_EXTENSION
from corehq.form_processor.interfaces.dbaccessors import CaseIndexInfo
from dimagi.utils.couch.database import iter_docs

def get_indexed_case_ids(domain, case_ids):
    """
    Given a base list of case ids, gets all ids of cases they reference (parent and host cases)
    """
    from casexml.apps.case.models import CommCareCase
    keys = [[domain, case_id, 'index'] for case_id in case_ids]
    return [r['value']['referenced_id'] for r in CommCareCase.get_db().view(
        'case_indices/related',
        keys=keys,
        reduce=False,
    )]

def get_extension_case_ids(domain, case_ids):
    """
    Given a base list of case ids,  get all ids of all extension cases that reference them
    """
    return [r.case_id for r in get_all_reverse_indices_info(domain, case_ids, CASE_INDEX_EXTENSION)]

def get_reverse_indexed_case_ids(domain, case_ids):
    """
    Given a base list of case ids, gets all ids of cases that reference them (child cases)
    """
    return [r.case_id for r in get_all_reverse_indices_info(domain, case_ids, CASE_INDEX_CHILD)]

def get_reverse_indexed_cases(domain, case_ids, relationship=None):
    """
    Gets all reverse indexed cases of a case (including child cases and extensions).
    """
    from casexml.apps.case.models import CommCareCase
    return CommCareCase.view(
        'case_indices/related',
        keys=_get_keys_for_reverse_index_view(domain, case_ids, relationship),
        reduce=False,
        include_docs=True,
    )

def get_all_reverse_indices_info(domain, case_ids, relationship=None):
    from casexml.apps.case.models import CommCareCase

    def _row_to_index_info(row):
        return CaseIndexInfo(
            case_id=row['id'],
            identifier=row['value']['identifier'],
            referenced_id=row['key'][1],
            referenced_type=row['value']['referenced_type'],
            relationship=row['value']['relationship']
        )

    return map(_row_to_index_info, CommCareCase.get_db().view(
        'case_indices/related',
        keys=_get_keys_for_reverse_index_view(domain, case_ids, relationship),
        reduce=False,
    ))

def _get_keys_for_reverse_index_view(domain, case_ids, relationship=None):
    if relationship is None:
        return [[domain, case_id, 'reverse_index', reln]
                for case_id in case_ids for reln in [CASE_INDEX_CHILD, CASE_INDEX_EXTENSION]]
    else:
        return [[domain, case_id, 'reverse_index', relationship] for case_id in case_ids]

def get_reverse_indices_json(domain, case_id):
    from casexml.apps.case.models import CommCareCase
    return CommCareCase.get_db().view(
        "case_indices/related",
        startkey=[domain, case_id, "reverse_index"],
        endkey=[domain, case_id, "reverse_index", {}],
        reduce=False,
        wrapper=lambda r: r['value'],
    ).all()

def get_reverse_indices(case):
    return get_reverse_indices_for_case_id(case['domain'], case['_id'])

def get_reverse_indices_for_case_id(domain, case_id):
    return [CommCareCaseIndex.wrap(raw)
            for raw in get_reverse_indices_json(domain, case_id)]
<commit_msg>Assert case_ids is a list<commit_after>from casexml.apps.case.sharedmodels import CommCareCaseIndex
from casexml.apps.case.const import CASE_INDEX_CHILD, CASE_INDEX_EXTENSION
from corehq.form_processor.interfaces.dbaccessors import CaseIndexInfo
from dimagi.utils.couch.database import iter_docs

def get_indexed_case_ids(domain, case_ids):
    """
    Given a base list of case ids, gets all ids of cases they reference (parent and host cases)
    """
    from casexml.apps.case.models import CommCareCase
    keys = [[domain, case_id, 'index'] for case_id in case_ids]
    return [r['value']['referenced_id'] for r in CommCareCase.get_db().view(
        'case_indices/related',
        keys=keys,
        reduce=False,
    )]

def get_extension_case_ids(domain, case_ids):
    """
    Given a base list of case ids,  get all ids of all extension cases that reference them
    """
    return [r.case_id for r in get_all_reverse_indices_info(domain, case_ids, CASE_INDEX_EXTENSION)]

def get_reverse_indexed_case_ids(domain, case_ids):
    """
    Given a base list of case ids, gets all ids of cases that reference them (child cases)
    """
    return [r.case_id for r in get_all_reverse_indices_info(domain, case_ids, CASE_INDEX_CHILD)]

def get_reverse_indexed_cases(domain, case_ids, relationship=None):
    """
    Gets all reverse indexed cases of a case (including child cases and extensions).
    """
    from casexml.apps.case.models import CommCareCase
    return CommCareCase.view(
        'case_indices/related',
        keys=_get_keys_for_reverse_index_view(domain, case_ids, relationship),
        reduce=False,
        include_docs=True,
    )

def get_all_reverse_indices_info(domain, case_ids, relationship=None):
    from casexml.apps.case.models import CommCareCase

    def _row_to_index_info(row):
        return CaseIndexInfo(
            case_id=row['id'],
            identifier=row['value']['identifier'],
            referenced_id=row['key'][1],
            referenced_type=row['value']['referenced_type'],
            relationship=row['value']['relationship']
        )

    return map(_row_to_index_info, CommCareCase.get_db().view(
        'case_indices/related',
        keys=_get_keys_for_reverse_index_view(domain, case_ids, relationship),
        reduce=False,
    ))

def _get_keys_for_reverse_index_view(domain, case_ids, relationship=None):
    assert isinstance(case_ids, list)
    if relationship is None:
        return [[domain, case_id, 'reverse_index', reln]
                for case_id in case_ids for reln in [CASE_INDEX_CHILD, CASE_INDEX_EXTENSION]]
    else:
        return [[domain, case_id, 'reverse_index', relationship] for case_id in case_ids]

def get_reverse_indices_json(domain, case_id):
    from casexml.apps.case.models import CommCareCase
    return CommCareCase.get_db().view(
        "case_indices/related",
        startkey=[domain, case_id, "reverse_index"],
        endkey=[domain, case_id, "reverse_index", {}],
        reduce=False,
        wrapper=lambda r: r['value'],
    ).all()

def get_reverse_indices(case):
    return get_reverse_indices_for_case_id(case['domain'], case['_id'])

def get_reverse_indices_for_case_id(domain, case_id):
    return [CommCareCaseIndex.wrap(raw)
            for raw in get_reverse_indices_json(domain, case_id)]
<|endoftext|>


<commit_before>'''
    flask-HMAC
    ----------

    Use HMAC tokens and a decorator to authenticate access to routes
'''

# Standard Libs
import base64
import binascii
import hashlib
import hmac
from functools import wraps

# Third Party Libs
import six
from flask import abort, request

from .exceptions import InvalidSignature, SecretKeyIsNotSet, UnknownKeyName

def encode_string(value):
    """ Encode unicode to string: unicode -> str, str -> str
    Arguments:
        value (str/unicode): string to encode
    Returns:
        encoded value (string)
    """
    return value.encode('utf-8') if isinstance(value, six.text_type) else value

def decode_string(value):
    """ Decode string: bytes -> str, str -> str
    Arguments:
        value (bytes/str): string to decode
    Returns:
        decoded value (strings)
    """
    return value if isinstance(value, six.string_types) else value.decode('utf-8')

class Hmac(object):

    def __init__(self, app=None, header=None, digestmod=None):
        self.header = header or 'Signature'
        self.digestmod = digestmod or hashlib.sha256
        if app:
            self.init_app(app)

    def get_signature(self, request):
        try:
            return six.b(request.headers[self.header])
        except KeyError:
            raise SecretKeyIsNotSet()

    def init_app(self, app):
        self.hmac_key = app.config.get('HMAC_KEY', '')
        self.hmac_keys = app.config.get('HMAC_KEYS', {})
        self.hmac_disarm = app.config.get('HMAC_DISARM', False)

    def auth(self, route):
        ''' Route decorator

        .. sourcecode:: python

            @app.route("/hmac_auth_view")
            @hmac.auth  # decorate view
            def hmac_auth_view():
                return "hmac_auth_view"
        '''
        @wraps(route)
        def decorated_view_function(*args, **kwargs):
            try:
                self.validate_signature(request)
            except (SecretKeyIsNotSet, InvalidSignature):
                return self.abort()
            return route(*args, **kwargs)
        return decorated_view_function

    def abort(self):
        abort(403)

    def _hmac_factory(self, data, key=None):
        key = key if key else self.hmac_key
        return hmac.new(six.b(key), data, digestmod=self.digestmod)

    def make_hmac(self, data='', key=None):
        hmac_token_server = self._hmac_factory(encode_string(data), key).digest()
        hmac_token_server = base64.urlsafe_b64encode(hmac_token_server)
        return hmac_token_server

    def make_hmac_for(self, name, data=''):
        ''' Generates HMAC key for named key
        Arguments:
            name (str): key name from HMAC_SECRETS dict
            data (str): HMAC message
        '''
        try:
            key = self.hmac_keys[name]
        except KeyError as ex:
            raise UnknownKeyName(ex)
        valuekey = '{0}:{1}'.format(name, decode_string(self.make_hmac(data, key)))
        token = base64.urlsafe_b64encode(six.b(valuekey))
        return token

    def validate_signature(self, request):
        ''' Generates HMAC key

        Arguments:
            request (Request): flask request

        Raise:
            InvalidSignature: when signatures don't match
        '''
        if self.hmac_disarm:
            return True
        hmac_token_client = self.get_signature(request)
        hmac_token_server = self.make_hmac(request.data)
        if hmac_token_client != hmac_token_server:
            raise InvalidSignature('Signatures are different: {0} {1}'.format(
                hmac_token_client, hmac_token_server
            ))
        return True

    def _parse_multiple_signature(self, signature):
        try:
            valuekey = base64.urlsafe_b64decode(encode_string(signature))
            return decode_string(valuekey).split(':')
        except (TypeError, binascii.Error):
            raise InvalidSignature()

    def validate_service_signature(self, request):
        if self.hmac_disarm:
            return True
        signature = self.get_signature(request)
        key_name, hmac_token_client = self._parse_multiple_signature(signature)

        hmac_token_server = self.make_hmac_for(key_name, request.data)
        if signature != hmac_token_server:
            raise InvalidSignature('Signatures are different: {0} {1}'.format(
                signature, hmac_token_server
            ))
        return True
<commit_msg>Don't use urlsafe_base64encode<commit_after>'''
    flask-HMAC
    ----------

    Use HMAC tokens and a decorator to authenticate access to routes
'''

# Standard Libs
import base64
import binascii
import hashlib
import hmac
from functools import wraps

# Third Party Libs
import six
from flask import abort, request

from .exceptions import InvalidSignature, SecretKeyIsNotSet, UnknownKeyName

def encode_string(value):
    """ Encode unicode to string: unicode -> str, str -> str
    Arguments:
        value (str/unicode): string to encode
    Returns:
        encoded value (string)
    """
    return value.encode('utf-8') if isinstance(value, six.text_type) else value

def decode_string(value):
    """ Decode string: bytes -> str, str -> str
    Arguments:
        value (bytes/str): string to decode
    Returns:
        decoded value (strings)
    """
    return value if isinstance(value, six.string_types) else value.decode('utf-8')

class Hmac(object):

    def __init__(self, app=None, header=None, digestmod=None):
        self.header = header or 'Signature'
        self.digestmod = digestmod or hashlib.sha256
        if app:
            self.init_app(app)

    def get_signature(self, request):
        try:
            return six.b(request.headers[self.header])
        except KeyError:
            raise SecretKeyIsNotSet()

    def init_app(self, app):
        self.hmac_key = app.config.get('HMAC_KEY', '')
        self.hmac_keys = app.config.get('HMAC_KEYS', {})
        self.hmac_disarm = app.config.get('HMAC_DISARM', False)

    def auth(self, route):
        ''' Route decorator

        .. sourcecode:: python

            @app.route("/hmac_auth_view")
            @hmac.auth  # decorate view
            def hmac_auth_view():
                return "hmac_auth_view"
        '''
        @wraps(route)
        def decorated_view_function(*args, **kwargs):
            try:
                self.validate_signature(request)
            except (SecretKeyIsNotSet, InvalidSignature):
                return self.abort()
            return route(*args, **kwargs)
        return decorated_view_function

    def abort(self):
        abort(403)

    def _hmac_factory(self, data, key=None):
        key = key if key else self.hmac_key
        return hmac.new(six.b(key), data, digestmod=self.digestmod)

    def make_hmac(self, data='', key=None):
        hmac_token_server = self._hmac_factory(encode_string(data), key).digest()
        hmac_token_server = base64.b64encode(hmac_token_server)
        return hmac_token_server

    def make_hmac_for(self, name, data=''):
        ''' Generates HMAC key for named key
        Arguments:
            name (str): key name from HMAC_SECRETS dict
            data (str): HMAC message
        '''
        try:
            key = self.hmac_keys[name]
        except KeyError as ex:
            raise UnknownKeyName(ex)
        valuekey = '{0}:{1}'.format(name, decode_string(self.make_hmac(data, key)))
        token = base64.b64encode(six.b(valuekey))
        return token

    def validate_signature(self, request):
        ''' Generates HMAC key

        Arguments:
            request (Request): flask request

        Raise:
            InvalidSignature: when signatures don't match
        '''
        if self.hmac_disarm:
            return True
        hmac_token_client = self.get_signature(request)
        hmac_token_server = self.make_hmac(request.data)
        if hmac_token_client != hmac_token_server:
            raise InvalidSignature('Signatures are different: {0} {1}'.format(
                hmac_token_client, hmac_token_server
            ))
        return True

    def _parse_multiple_signature(self, signature):
        try:
            valuekey = base64.urlsafe_b64decode(encode_string(signature))
            return decode_string(valuekey).split(':')
        except (TypeError, binascii.Error):
            raise InvalidSignature()

    def validate_service_signature(self, request):
        if self.hmac_disarm:
            return True
        signature = self.get_signature(request)
        key_name, hmac_token_client = self._parse_multiple_signature(signature)

        hmac_token_server = self.make_hmac_for(key_name, request.data)
        if signature != hmac_token_server:
            raise InvalidSignature('Signatures are different: {0} {1}'.format(
                signature, hmac_token_server
            ))
        return True
<|endoftext|>


<commit_before>'''
Copyright (C) 2015 Dato, Inc.
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
'''
import sys
import os
import logging
from distutils.util import get_platform as _get_platform
import glob as _glob
import subprocess as _subprocess

def make_unity_server_env():
    """
    Returns the environemnt for unity_server.

    The environment is necessary to start the unity_server
    by setting the proper enviornments for shared libraries,
    hadoop classpath, and module search paths for python lambda workers.

    The environment has 3 components:
    1. CLASSPATH, contains hadoop class path
    2. LD_LIBRARY_PATH: contains path to shared libraries of jvm and libpython2.7. This is essential for unity_server to get started.
    3. __GL_SYS_PATH__: contains the python sys.path of the interpreter. This is essential for pylambda worker to behave properly.
    """
    env = os.environ.copy()

    # Add hadoop class path
    classpath = get_hadoop_class_path()
    if ("CLASSPATH" in env):
        env["CLASSPATH"] = env['CLASSPATH'] + (os.path.pathsep + classpath if classpath != '' else '')
    else:
        env["CLASSPATH"] = classpath

    # Add JVM path
    libjvm_path = get_libjvm_path()

    # Add libpython path
    libpython_path = get_libpython_path()
    libpath_varname = None
    if sys.platform == 'win32':
        libpath_varname = 'PATH'
    else:
        libpath_varname = 'LD_LIBRARY_PATH'
    if libpath_varname in env:
        env[libpath_varname] = env[libpath_varname] + os.path.pathsep + libpython_path + (os.path.pathsep + libjvm_path if libjvm_path != '' else '')
    else:
        env[libpath_varname] = libpython_path + os.path.pathsep + libjvm_path

    # Add python syspath
    env['__GL_SYS_PATH__'] = (os.path.pathsep).join(sys.path)

    #### Remove PYTHONEXECUTABLE ####
    # Anaconda overwrites this environment variable
    # which forces all python sub-processes to use the same binary.
    # When using virtualenv with ipython (which is outside virtualenv),
    # all subprocess launched under unity_server will use the
    # conda binary outside of virtualenv, which lacks the access
    # to all packeages installed inside virtualenv.
    if 'PYTHONEXECUTABLE' in env:
        del env['PYTHONEXECUTABLE']

    ## set local to be c standard so that unity_server will run ##
    env['LC_ALL']='C'
    # add certificate file
    try:
        import certifi
        env['GRAPHLAB_FILEIO_ALTERNATIVE_SSL_CERT_FILE'] = certifi.where()
    except:
        pass
    return env

def get_libpython_path():
    """
    Return the path of libpython2.7.so in the current python environment.

    The path is a guess based on the heuristics that most of the time the library
    is under sys.exec_prefix + "/lib".
    """
    libpython_path = os.path.join(sys.exec_prefix, "lib")
    return libpython_path

__jvm_lib_warned = False

def get_libjvm_path():
    """
    Return path of libjvm.so for LD_LIBRARY_PATH variable.
    This is only required for Linux platform, and by looking in a bunch of 'known' paths.
    """
    cur_platform = _get_platform()
    path_prefixes = []
    path_suffixes = []
    jvm_lib_name = ''

    if cur_platform.startswith("macosx"):
        return ''
    else:
        if sys.platform == 'win32':
            path_suffixes = ['/jre/bin/server','/bin/server']
            jvm_lib_name = 'jvm.dll'
        else:
            path_suffixes = ['/jre/lib/amd64/server']
            path_prefixes = [
                    '/usr/lib/jvm/default-java',               # ubuntu / debian distros
                    '/usr/lib/jvm/java',                       # rhel6
                    '/usr/lib/jvm',                            # centos6
                    '/usr/lib64/jvm',                          # opensuse 13
                    '/usr/local/lib/jvm/default-java',         # alt ubuntu / debian distros
                    '/usr/local/lib/jvm/java',                 # alt rhel6
                    '/usr/local/lib/jvm',                      # alt centos6
                    '/usr/local/lib64/jvm',                    # alt opensuse 13
                    '/usr/local/lib/jvm/java-7-openjdk-amd64', # alt ubuntu / debian distros
                    '/usr/lib/jvm/java-7-openjdk-amd64',       # alt ubuntu / debian distros
                    '/usr/local/lib/jvm/java-6-openjdk-amd64', # alt ubuntu / debian distros
                    '/usr/lib/jvm/java-6-openjdk-amd64',       # alt ubuntu / debian distros
                    '/usr/lib/jvm/java-7-oracle',              # alt ubuntu
                    '/usr/lib/jvm/java-8-oracle',              # alt ubuntu
                    '/usr/lib/jvm/java-6-oracle',              # alt ubuntu
                    '/usr/local/lib/jvm/java-7-oracle',        # alt ubuntu
                    '/usr/local/lib/jvm/java-8-oracle',        # alt ubuntu
                    '/usr/local/lib/jvm/java-6-oracle',        # alt ubuntu
                    '/usr/lib/jvm/default',                    # alt centos
                    '/usr/java/latest',                        # alt centos
                    ]
            jvm_lib_name = 'libjvm.so'

        # First check GRAPHLAB_LIBJVM_DIRECTORY which is expected to be a
        # directory that libjvm.so resides (the path_suffix will not be used)
        # Then, check GRAPHLAB_JAVA_HOME, which will override JAVA_HOME.
        potential_paths = []
        if os.environ.has_key('JAVA_HOME'):
            path_prefixes.insert(0, os.environ['JAVA_HOME'])
        if os.environ.has_key('GRAPHLAB_JAVA_HOME'):
            path_prefixes.insert(0, os.environ['GRAPHLAB_JAVA_HOME'])

        # Construct the full paths to search
        for i in path_suffixes:
            for j in path_prefixes:
                potential_paths.append(j + i)

        if os.environ.has_key('GRAPHLAB_LIBJVM_DIRECTORY'):
            potential_paths.insert(0, os.environ['GRAPHLAB_LIBJVM_DIRECTORY'])
        for path in potential_paths:
            if sys.platform == 'win32':
                path = path.replace('/','\\')
            file_to_try = os.path.join(path, jvm_lib_name)
            if os.path.isfile(file_to_try):
                logging.getLogger(__name__).debug('%s path used: %s' % (jvm_lib_name, path))
                return path

        global __jvm_lib_warned
        if not __jvm_lib_warned:
            __jvm_lib_warned = True
            logging.getLogger(__name__).debug(jvm_lib_name + ' not found. Operations '
            'using HDFS may not work.\nSet GRAPHLAB_JAVA_HOME environment variable'
            ' before starting GraphLab Create to specify preferred java '
            'installation.')
        return ''

__hadoop_class_warned = False

def get_hadoop_class_path():
    # Try get the classpath directly from executing hadoop
    env = os.environ.copy()
    hadoop_exe_name = 'hadoop'
    if sys.platform == 'win32':
        hadoop_exe_name += '.cmd'
    output = None
    try:
        try:
            output = _subprocess.check_output([hadoop_exe_name, 'classpath'])
        except:
            output = _subprocess.check_output(['/'.join([env['HADOOP_HOME'],'bin',hadoop_exe_name]), 'classpath'])

        output = (os.path.pathsep).join(os.path.realpath(path) for path in output.split(os.path.pathsep))
        return _get_expanded_classpath(output)

    except Exception as e:
        global __hadoop_class_warned
        if not __hadoop_class_warned:
            __hadoop_class_warned = True
            logging.getLogger(__name__).debug("Exception trying to retrieve Hadoop classpath: %s" % e)

    logging.getLogger(__name__).debug("Hadoop not found. HDFS url is not supported. Please make hadoop available from PATH or set the environment variable HADOOP_HOME.")
    return ""

def _get_expanded_classpath(classpath):
    """
    Take a classpath of the form:
      /etc/hadoop/conf:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/.//*: ...

    and return it expanded to all the JARs (and nothing else):
      /etc/hadoop/conf:/usr/lib/hadoop/lib/netty-3.6.2.Final.jar:/usr/lib/hadoop/lib/jaxb-api-2.2.2.jar: ...

    mentioned in the path
    """
    if classpath is None or classpath == '':
        return ''

    #  so this set comprehension takes paths that end with * to be globbed to find the jars, and then
    #  recombined back into a colon separated list of jar paths, removing dupes and using full file paths
    jars = (os.path.pathsep).join((os.path.pathsep).join([os.path.abspath(jarpath) for jarpath in _glob.glob(path)])
                    for path in classpath.split(os.path.pathsep))
    logging.getLogger(__name__).debug('classpath being used: %s' % jars)
    return jars
<commit_msg>Minor change. Do not set environment variable if already set.<commit_after>'''
Copyright (C) 2015 Dato, Inc.
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
'''
import sys
import os
import logging
from distutils.util import get_platform as _get_platform
import glob as _glob
import subprocess as _subprocess

def make_unity_server_env():
    """
    Returns the environemnt for unity_server.

    The environment is necessary to start the unity_server
    by setting the proper enviornments for shared libraries,
    hadoop classpath, and module search paths for python lambda workers.

    The environment has 3 components:
    1. CLASSPATH, contains hadoop class path
    2. LD_LIBRARY_PATH: contains path to shared libraries of jvm and libpython2.7. This is essential for unity_server to get started.
    3. __GL_SYS_PATH__: contains the python sys.path of the interpreter. This is essential for pylambda worker to behave properly.
    """
    env = os.environ.copy()

    # Add hadoop class path
    classpath = get_hadoop_class_path()
    if ("CLASSPATH" in env):
        env["CLASSPATH"] = env['CLASSPATH'] + (os.path.pathsep + classpath if classpath != '' else '')
    else:
        env["CLASSPATH"] = classpath

    # Add JVM path
    libjvm_path = get_libjvm_path()

    # Add libpython path
    libpython_path = get_libpython_path()
    libpath_varname = None
    if sys.platform == 'win32':
        libpath_varname = 'PATH'
    else:
        libpath_varname = 'LD_LIBRARY_PATH'
    if libpath_varname in env:
        env[libpath_varname] = env[libpath_varname] + os.path.pathsep + libpython_path + (os.path.pathsep + libjvm_path if libjvm_path != '' else '')
    else:
        env[libpath_varname] = libpython_path + os.path.pathsep + libjvm_path

    # Add python syspath
    env['__GL_SYS_PATH__'] = (os.path.pathsep).join(sys.path)

    #### Remove PYTHONEXECUTABLE ####
    # Anaconda overwrites this environment variable
    # which forces all python sub-processes to use the same binary.
    # When using virtualenv with ipython (which is outside virtualenv),
    # all subprocess launched under unity_server will use the
    # conda binary outside of virtualenv, which lacks the access
    # to all packeages installed inside virtualenv.
    if 'PYTHONEXECUTABLE' in env:
        del env['PYTHONEXECUTABLE']

    ## set local to be c standard so that unity_server will run ##
    env['LC_ALL']='C'
    # add certificate file
    if 'GRAPHLAB_FILEIO_ALTERNATIVE_SSL_CERT_FILE' not in env:
        try:
            import certifi
            env['GRAPHLAB_FILEIO_ALTERNATIVE_SSL_CERT_FILE'] = certifi.where()
        except:
            pass
    return env

def get_libpython_path():
    """
    Return the path of libpython2.7.so in the current python environment.

    The path is a guess based on the heuristics that most of the time the library
    is under sys.exec_prefix + "/lib".
    """
    libpython_path = os.path.join(sys.exec_prefix, "lib")
    return libpython_path

__jvm_lib_warned = False

def get_libjvm_path():
    """
    Return path of libjvm.so for LD_LIBRARY_PATH variable.
    This is only required for Linux platform, and by looking in a bunch of 'known' paths.
    """
    cur_platform = _get_platform()
    path_prefixes = []
    path_suffixes = []
    jvm_lib_name = ''

    if cur_platform.startswith("macosx"):
        return ''
    else:
        if sys.platform == 'win32':
            path_suffixes = ['/jre/bin/server','/bin/server']
            jvm_lib_name = 'jvm.dll'
        else:
            path_suffixes = ['/jre/lib/amd64/server']
            path_prefixes = [
                    '/usr/lib/jvm/default-java',               # ubuntu / debian distros
                    '/usr/lib/jvm/java',                       # rhel6
                    '/usr/lib/jvm',                            # centos6
                    '/usr/lib64/jvm',                          # opensuse 13
                    '/usr/local/lib/jvm/default-java',         # alt ubuntu / debian distros
                    '/usr/local/lib/jvm/java',                 # alt rhel6
                    '/usr/local/lib/jvm',                      # alt centos6
                    '/usr/local/lib64/jvm',                    # alt opensuse 13
                    '/usr/local/lib/jvm/java-7-openjdk-amd64', # alt ubuntu / debian distros
                    '/usr/lib/jvm/java-7-openjdk-amd64',       # alt ubuntu / debian distros
                    '/usr/local/lib/jvm/java-6-openjdk-amd64', # alt ubuntu / debian distros
                    '/usr/lib/jvm/java-6-openjdk-amd64',       # alt ubuntu / debian distros
                    '/usr/lib/jvm/java-7-oracle',              # alt ubuntu
                    '/usr/lib/jvm/java-8-oracle',              # alt ubuntu
                    '/usr/lib/jvm/java-6-oracle',              # alt ubuntu
                    '/usr/local/lib/jvm/java-7-oracle',        # alt ubuntu
                    '/usr/local/lib/jvm/java-8-oracle',        # alt ubuntu
                    '/usr/local/lib/jvm/java-6-oracle',        # alt ubuntu
                    '/usr/lib/jvm/default',                    # alt centos
                    '/usr/java/latest',                        # alt centos
                    ]
            jvm_lib_name = 'libjvm.so'

        # First check GRAPHLAB_LIBJVM_DIRECTORY which is expected to be a
        # directory that libjvm.so resides (the path_suffix will not be used)
        # Then, check GRAPHLAB_JAVA_HOME, which will override JAVA_HOME.
        potential_paths = []
        if os.environ.has_key('JAVA_HOME'):
            path_prefixes.insert(0, os.environ['JAVA_HOME'])
        if os.environ.has_key('GRAPHLAB_JAVA_HOME'):
            path_prefixes.insert(0, os.environ['GRAPHLAB_JAVA_HOME'])

        # Construct the full paths to search
        for i in path_suffixes:
            for j in path_prefixes:
                potential_paths.append(j + i)

        if os.environ.has_key('GRAPHLAB_LIBJVM_DIRECTORY'):
            potential_paths.insert(0, os.environ['GRAPHLAB_LIBJVM_DIRECTORY'])
        for path in potential_paths:
            if sys.platform == 'win32':
                path = path.replace('/','\\')
            file_to_try = os.path.join(path, jvm_lib_name)
            if os.path.isfile(file_to_try):
                logging.getLogger(__name__).debug('%s path used: %s' % (jvm_lib_name, path))
                return path

        global __jvm_lib_warned
        if not __jvm_lib_warned:
            __jvm_lib_warned = True
            logging.getLogger(__name__).debug(jvm_lib_name + ' not found. Operations '
            'using HDFS may not work.\nSet GRAPHLAB_JAVA_HOME environment variable'
            ' before starting GraphLab Create to specify preferred java '
            'installation.')
        return ''

__hadoop_class_warned = False

def get_hadoop_class_path():
    # Try get the classpath directly from executing hadoop
    env = os.environ.copy()
    hadoop_exe_name = 'hadoop'
    if sys.platform == 'win32':
        hadoop_exe_name += '.cmd'
    output = None
    try:
        try:
            output = _subprocess.check_output([hadoop_exe_name, 'classpath'])
        except:
            output = _subprocess.check_output(['/'.join([env['HADOOP_HOME'],'bin',hadoop_exe_name]), 'classpath'])

        output = (os.path.pathsep).join(os.path.realpath(path) for path in output.split(os.path.pathsep))
        return _get_expanded_classpath(output)

    except Exception as e:
        global __hadoop_class_warned
        if not __hadoop_class_warned:
            __hadoop_class_warned = True
            logging.getLogger(__name__).debug("Exception trying to retrieve Hadoop classpath: %s" % e)

    logging.getLogger(__name__).debug("Hadoop not found. HDFS url is not supported. Please make hadoop available from PATH or set the environment variable HADOOP_HOME.")
    return ""

def _get_expanded_classpath(classpath):
    """
    Take a classpath of the form:
      /etc/hadoop/conf:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/.//*: ...

    and return it expanded to all the JARs (and nothing else):
      /etc/hadoop/conf:/usr/lib/hadoop/lib/netty-3.6.2.Final.jar:/usr/lib/hadoop/lib/jaxb-api-2.2.2.jar: ...

    mentioned in the path
    """
    if classpath is None or classpath == '':
        return ''

    #  so this set comprehension takes paths that end with * to be globbed to find the jars, and then
    #  recombined back into a colon separated list of jar paths, removing dupes and using full file paths
    jars = (os.path.pathsep).join((os.path.pathsep).join([os.path.abspath(jarpath) for jarpath in _glob.glob(path)])
                    for path in classpath.split(os.path.pathsep))
    logging.getLogger(__name__).debug('classpath being used: %s' % jars)
    return jars
<|endoftext|>


<commit_before>import unittest
from math import sqrt
"""
Find number of ways to represent n as a sum of squares of integers.
For eg:
Input: 4
Output: 2 (1^2+1^2+1^2+1^2, 2^2)

The problem can be converted into coin change problem where available denomination of coins are:
[1^2,2^2,...,sqrt(N)*sqrt(N)]
"""

def n_as_sum_of_squares(n):
    ways = [0] * (n+1)
    ways[0] = 1
    s_list = [x*x for x in range(1, int(sqrt(n))+1)]

    for s in s_list:
        for i in range(s, n+1):
            ways[i] += ways[i-s]

    return ways[n]

class TestSumOfSquares(unittest.TestCase):

    def test_sum_of_squares(self):
        self.assertEqual(n_as_sum_of_squares(4), 2)
        self.assertEqual(n_as_sum_of_squares(5), 2)
        self.assertEqual(n_as_sum_of_squares(16), 8)
<commit_msg>N as sum of integers<commit_after>import unittest
from math import sqrt
"""
Find number of ways to represent n as a sum of squares of integers.
For eg:
Input: 4
Output: 2 (1^2+1^2+1^2+1^2, 2^2)

The problem can be converted into coin change problem where available denomination of coins are:
[1^2,2^2,...,sqrt(N)*sqrt(N)]
"""

def n_as_sum_of_squares(n):
    ways = [0] * (n+1)
    ways[0] = 1
    s_list = [x*x for x in range(1, int(sqrt(n))+1)]

    for s in s_list:
        for i in range(s, n+1):
            ways[i] += ways[i-s]

    return ways[n]

def n_as_sum_of_integers(n):
    smaller = [x for x in range(1, n)]
    # table[i][j] stores number of ways to denote i as sum of integers using only smaller[0...j]
    # table[n][len(smaller)-1] stores the final result.
    table = [[0] * len(smaller) for _ in range(n+1)]

    for i in range(len(smaller)):
        table[0][i] = 1

    for i in range(1, n+1):
        for j in range(len(smaller)):
            if i - smaller[j] >= 0:
                table[i][j] += table[i - smaller[j]][j]
            if j > 0:
                table[i][j] += table[i][j-1]

    return table[n][len(smaller)-1]

class TestSumOfSquares(unittest.TestCase):

    def test_sum_of_squares(self):
        self.assertEqual(n_as_sum_of_squares(4), 2)
        self.assertEqual(n_as_sum_of_squares(5), 2)
        self.assertEqual(n_as_sum_of_squares(16), 8)

class TestSumOfInts(unittest.TestCase):

    def test_sum_of_ints(self):
        self.assertEqual(n_as_sum_of_integers(5), 6)
        self.assertEqual(n_as_sum_of_integers(4), 4)
<|endoftext|>


<commit_before>#########################################################################
# Developed in the framework of INDIGO project,
#
# Author: Mario David <mariojmdavid@gmail.com>
#########################################################################

import os
import h5py
import numpy as np

if __name__ == '__main__':

    root_dir = "/home/david/Dropbox/AA-work/bench-run4"
    fout = root_dir + "/bench.hdf"
    haddocks = ['disvis', 'powerfit']
    cases = {'disvis': ['RNA-polymerase-II', 'PRE5-PUP2-complex'],
             'powerfit': ['GroEL-GroES', 'RsgA-ribosome']}

    angles = ['5.0', '10.0']
    voxspac = ['1', '2']
    type = 'GPU'
    nvidia = ['QK5200', 'TK40']

    machs = {'QK5200': ['Phys-C7-QK5200', 'Dock-C7-QK5200', 'Dock-U16-QK5200',
                       'UDock-C7-QK5200', 'UDock-U16-QK5200'],
            'TK40': ['VM-U16-TK40', 'Dock-C7-TK40', 'Dock-U16-TK40',
                     'UDock-C7-TK40', 'UDock-U16-TK40']}

    lrunsG = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10']
    initName = 'time-'

    with h5py.File(fout, 'w') as f:
        for case in cases['disvis']:
            for angle in angles:
                for vs in voxspac:
                    for nv in nvidia:
                        for mach in machs[nv]:
                            grp_name = 'disvis' + '/' +\
                                       case + '-' + angle + '-' + vs + '/' +\
                                       nv + '/' + mach
                            grp1 = f.create_group(grp_name)
                            npres = np.arange(10)
                            for n in lrunsG:
                                tr_file = root_dir + os.sep + mach + os.sep + \
                                          initName + case + os.sep + 'tr_ang-' + angle + \
                                          '-vs-' + vs + '-type-GPU' + '-n-' + n + '.txt'
                                try:
                                    fraw = open(tr_file)
                                    npres[int(n)-1] = float(fraw.readline())
                                except IOError:
                                    print 'Cannot open', tr_file
                            res = grp1.create_dataset("runtime", data=npres)

        for case in cases['powerfit']:
            for nv in nvidia:
                for mach in machs[nv]:
                    grp_name = 'powerfit' + '/' + case + '/' + nv + '/' + mach
                    grp1 = f.create_group(grp_name)
                    npres = np.arange(10)
                    for n in lrunsG:
                        tr_file = root_dir + os.sep + mach + os.sep + initName + case + os.sep + 'tr_ang-4.71' + \
                                  '-type-GPU' + '-n-' + n + '.txt'
                        try:
                            fraw = open(tr_file)
                            npres[int(n) - 1] = float(fraw.readline())
                        except IOError:
                            print 'Cannot open', tr_file
                    res = grp1.create_dataset("runtime", data=npres)

<commit_msg>add attrs<commit_after>#########################################################################
# Developed in the framework of INDIGO project,
#
# Author: Mario David <mariojmdavid@gmail.com>
#########################################################################

import os
import h5py
import numpy as np

if __name__ == '__main__':

    root_dir = "/home/david/Dropbox/AA-work/bench-run4"
    fout = root_dir + "/bench.hdf"
    haddocks = ['disvis', 'powerfit']
    cases = {'disvis': ['RNA-polymerase-II', 'PRE5-PUP2-complex'],
             'powerfit': ['GroEL-GroES', 'RsgA-ribosome']}

    angles = ['5.0', '10.0']
    voxspac = ['1', '2']
    type = 'GPU'
    nvidia = ['QK5200', 'TK40']

    machs = {'QK5200': ['Phys-C7-QK5200', 'Dock-C7-QK5200', 'Dock-U16-QK5200',
                       'UDock-C7-QK5200', 'UDock-U16-QK5200'],
            'TK40': ['VM-U16-TK40', 'Dock-C7-TK40', 'Dock-U16-TK40',
                     'UDock-C7-TK40', 'UDock-U16-TK40']}

    lrunsG = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10']
    initName = 'time-'

    with h5py.File(fout, 'w') as f:
        for case in cases['disvis']:
            for angle in angles:
                for vs in voxspac:
                    for nv in nvidia:
                        for mach in machs[nv]:
                            grp_name = 'disvis' + '/' +\
                                       case + '-' + angle + '-' + vs + '/' +\
                                       nv + '/' + mach
                            grp1 = f.create_group(grp_name)
                            npres = np.arange(10)
                            for n in lrunsG:
                                tr_file = root_dir + os.sep + mach + os.sep + \
                                          initName + case + os.sep + 'tr_ang-' + angle + \
                                          '-vs-' + vs + '-type-GPU' + '-n-' + n + '.txt'
                                try:
                                    fraw = open(tr_file)
                                    npres[int(n)-1] = float(fraw.readline())
                                except IOError:
                                    print 'Cannot open', tr_file
                            res = grp1.create_dataset("runtime", data=npres)
                            res.attrs['mean'] = np.mean(npres)
                            res.attrs['stdev'] = np.std(npres)

        for case in cases['powerfit']:
            for nv in nvidia:
                for mach in machs[nv]:
                    grp_name = 'powerfit' + '/' + case + '/' + nv + '/' + mach
                    grp1 = f.create_group(grp_name)
                    npres = np.arange(10)
                    for n in lrunsG:
                        tr_file = root_dir + os.sep + mach + os.sep + initName + case + os.sep + 'tr_ang-4.71' + \
                                  '-type-GPU' + '-n-' + n + '.txt'
                        try:
                            fraw = open(tr_file)
                            npres[int(n) - 1] = float(fraw.readline())
                        except IOError:
                            print 'Cannot open', tr_file
                    res = grp1.create_dataset("runtime", data=npres)
                    res.attrs['mean'] = np.mean(npres)
                    res.attrs['stdev'] = np.std(npres)

<|endoftext|>


<commit_before>import os
import time
from slackclient import SlackClient
from redmine import Redmine

# read environment variables
REDMINE_HOST = os.environ.get('REDMINE_HOST')
REDMINE_EXT_HOST = os.environ.get('REDMINE_EXT_HOST')
REDMINE_VERSION = os.environ.get('REDMINE_VERSION')
REDMINE_TOKEN = os.environ.get('REDMINE_TOKEN')
REDMINE_CLOSE_ID = os.environ.get('REDMINE_CLOSE_ID')
REDMINE_PROJECT = os.environ.get('REDMINE_PROJECT')
REDMINE_TRACKER_ID = os.environ.get('REDMINE_TRACKER_ID')
BOT_ID = os.environ.get('BOT_ID')
BOT_TOKEN = os.environ.get('BOT_TOKEN')

# constants
AT_BOT = "<@" + BOT_ID + ">"

# instantiate Slack & Redmine clients
sc = SlackClient(BOT_TOKEN)
rc = Redmine(REDMINE_HOST, version=REDMINE_VERSION, key=REDMINE_TOKEN)

def handle_command(command, channel, username):
    """
        Receives commands directed at the bot and determines if they
        are valid commands. If so, then acts on the commands. If not,
        returns back what it needs for clarification.
    """
    response = ":question: Unknown/invalid command - Try `help` for a list of supported commands"
    commands = command.split()
    if commands:
        operator = commands[0].lower()
        s = " "
        msg = s.join(commands[1:])
        if operator == "issueto" and len(commands) > 2:
            assigneduser = commands[1]
            newmsg = s.join(commands[2:])
            response = create_issue(newmsg, username, assigneduser)
        elif operator == "issue" and len(commands) > 1:
            response = create_issue(msg, username, username)
        elif operator == "update" and len(commands) > 2:
            issue = commands[1]
            newmsg = s.join(commands[2:])
            response = update_issue(newmsg, issue, username)
        elif operator == "close" and len(commands) > 2:
            issue = commands[1]
            newmsg = s.join(commands[2:])
            response = close_issue(newmsg, issue, username)
        elif operator == "list":
            response = list_issues(username)
        elif operator == "listfor" and len(commands) > 1:
            listuser = commands[1]
            response = list_issues(listuser)
        elif operator == "help":
            response = show_commands()
    message = "<@" + username + "> " + response
    sc.api_call("chat.postMessage", channel=channel, \
                          text=message, as_user=True)

def parse_slack_output(slack_rtm_output):
    """
        The Slack Real Time Messaging API is an events firehose.
        this parsing function returns None unless a message is
        directed at the Bot, based on its ID.
    """
    output_list = slack_rtm_output
    if output_list and len(output_list) > 0:
        for output in output_list:
            if output and 'text' in output and AT_BOT in output['text']:
                profile = sc.api_call("users.info", user=output['user'])
                username = profile['user']['name']
                # return text after the @ mention, whitespace removed
                return output['text'].split(AT_BOT)[1].strip(), \
                       output['channel'], username
    return None, None, None

def show_commands():
    """
        Return ist of commands that bot can handle
    """
    return ":wrench: List of supported commands:\n" \
           "`issue <subject>` - creates new issue and assigns it to you\n" \
           "`issueto <name> <subject>` - creates new issue and assigns it to `<name>`\n" \
           "`update <issue #> <comment>` - updates an issue with the following `comment`\n" \
           "`close <issue #> <comment>` - closes an issue with the following comment\n" \
           "`list` - list all open issues assigned to you\n" \
           "`listfor <name>` - list all open issues assigned to `<name>`"

"""
    Redmine commands
"""
def get_user(username):
    users = rc.user.filter(name=username)
    if len(users) == 0:
        return None
    return users[0]

def impersonate_redmine(userlogin):
    return Redmine(REDMINE_HOST, version=REDMINE_VERSION, key=REDMINE_TOKEN, impersonate=userlogin)

def issue_url(issueid):
    return "<"+REDMINE_EXT_HOST+"/issues/"+str(issueid)+"|#"+str(issueid)+">"

def issue_subject_url(issueid, subject):
    return "<"+REDMINE_EXT_HOST+"/issues/"+str(issueid)+"|#"+str(issueid)+" "+subject+">"

def list_issues(username):
    user = get_user(username)
    if not user:
        return ":x: Failed to find user `"+username+"` in Redmine"
    result = rc.issue.filter(sort='project', assigned_to_id=user.id, status_id='open')
    response = ""
    if len(result) > 0:
        response = ":book: Open issues assigned to "+user.firstname+" "+user.lastname+":\n"
        for issue in result:
            response += ""+issue.project.name+" "+issue_subject_url(issue.id, issue.subject)+"\n"
    else:
        response = ":thumbsup_all: No open issues assigned to "+user.firstname+" "+user.lastname
    return response

def update_issue(text, issue, username):
    user = get_user(username)
    if not user:
        return ":x: Failed to find user `"+username+"` in Redmine"
    # impersonate user so it looks like the update is from them
    rcn = impersonate_redmine(user.login)
    result = rcn.issue.update(issue, notes=text)
    if result:
        return ":memo: Updated Issue "+issue_url(issue)+" with comment `"+text+"`"
    else:
        return ":x: Issue update failed"

def close_issue(text, issue, username):
    user = get_user(username)
    if not user:
        return ":x: Failed to find user `"+username+"` in Redmine"
    # impersonate user so it looks like the update is from them
    rcn = impersonate_redmine(user.login)
    result = rcn.issue.update(issue, status_id=REDMINE_CLOSE_ID, notes=text, done_ratio=100)
    if result:
        return ":white_check_mark: Closed Issue "+issue_url(issue)+" with comment `"+text+"`"
    else:
        return ":x: Issue closing failed"

def create_issue(text, username, assigneduser):
    user = get_user(username)
    if not user:
        return ":x: Failed to find user `"+username+"` in Redmine"
    assigned = get_user(assigneduser)
    if not assigned:
        return ":x: Failed to find user `"+assigneduser+"` in Redmine"
    # impersonate user so it looks like the update is from them
    rcn = impersonate_redmine(user.login)
    issue = rcn.issue.create(project_id=REDMINE_PROJECT, subject=text, tracker_id=REDMINE_TRACKER_ID, assigned_to_id=assigned.id)
    if issue.id:
        return ":white_check_mark: Created Issue "+issue_subject_url(issue.id,issue.subject)+" assigned to "+assigned.firstname+" "+assigned.lastname
    else:
        return ":x: Issue creation failed"

if __name__ == "__main__":
    READ_WEBSOCKET_DELAY = 1 # 1 second delay between reading from firehose
    if sc.rtm_connect():
        print("RedmineBot connected and running!")
        while True:
            command, channel, username = parse_slack_output(sc.rtm_read())
            if command and channel and username:
                handle_command(command, channel, username)
            elif channel and username:
                handle_command("help", channel, username)
            time.sleep(READ_WEBSOCKET_DELAY)
    else:
        print("Connection failed. Invalid Slack token or bot ID?")
<commit_msg>adopted try/except for error handling and issues with invalid users/ids<commit_after>import os
import time
from slackclient import SlackClient
from redmine import Redmine

# read environment variables
REDMINE_HOST = os.environ.get('REDMINE_HOST')
REDMINE_EXT_HOST = os.environ.get('REDMINE_EXT_HOST')
REDMINE_VERSION = os.environ.get('REDMINE_VERSION')
REDMINE_TOKEN = os.environ.get('REDMINE_TOKEN')
REDMINE_CLOSE_ID = os.environ.get('REDMINE_CLOSE_ID')
REDMINE_PROJECT = os.environ.get('REDMINE_PROJECT')
REDMINE_TRACKER_ID = os.environ.get('REDMINE_TRACKER_ID')
BOT_ID = os.environ.get('BOT_ID')
BOT_TOKEN = os.environ.get('BOT_TOKEN')

# constants
AT_BOT = "<@" + BOT_ID + ">"

# instantiate Slack & Redmine clients
sc = SlackClient(BOT_TOKEN)
rc = Redmine(REDMINE_HOST, version=REDMINE_VERSION, key=REDMINE_TOKEN)

def handle_command(command, channel, username):
    """
        Receives commands directed at the bot and determines if they
        are valid commands. If so, then acts on the commands. If not,
        returns back what it needs for clarification.
    """
    response = ":question: Unknown/invalid command - Try `help` for a list of supported commands"
    commands = command.split()
    try:
        if commands:
            operator = commands[0].lower()
            s = " "
            msg = s.join(commands[1:])
            if operator == "issueto" and len(commands) > 2:
                assigneduser = commands[1]
                newmsg = s.join(commands[2:])
                response = create_issue(newmsg, username, assigneduser)
            elif operator == "issue" and len(commands) > 1:
                response = create_issue(msg, username, username)
            elif operator == "update" and len(commands) > 2:
                issue = commands[1]
                newmsg = s.join(commands[2:])
                response = update_issue(newmsg, issue, username)
            elif operator == "close" and len(commands) > 2:
                issue = commands[1]
                newmsg = s.join(commands[2:])
                response = close_issue(newmsg, issue, username)
            elif operator == "list":
                response = list_issues(username)
            elif operator == "listfor" and len(commands) > 1:
                listuser = commands[1]
                response = list_issues(listuser)
            elif operator == "help":
                response = show_commands()
    except RuntimeError as e:
        response = e[0]
    message = "<@" + username + "> " + response
    sc.api_call("chat.postMessage", channel=channel, \
                          text=message, as_user=True)

def parse_slack_output(slack_rtm_output):
    """
        The Slack Real Time Messaging API is an events firehose.
        this parsing function returns None unless a message is
        directed at the Bot, based on its ID.
    """
    output_list = slack_rtm_output
    if output_list and len(output_list) > 0:
        for output in output_list:
            if output and 'text' in output and AT_BOT in output['text']:
                profile = sc.api_call("users.info", user=output['user'])
                username = profile['user']['name']
                # return text after the @ mention, whitespace removed
                return output['text'].split(AT_BOT)[1].strip(), \
                       output['channel'], username
    return None, None, None

def show_commands():
    """
        Return ist of commands that bot can handle
    """
    return ":wrench: List of supported commands:\n" \
            "`issue <subject>` - creates new issue and assigns it to you\n" \
            "`issueto <name> <subject>` - creates new issue and assigns it to `<name>`\n" \
            "`update <issue #> <comment>` - updates an issue with the following `comment`\n" \
            "`close <issue #> <comment>` - closes an issue with the following comment\n" \
            "`list` - list all open issues assigned to you\n" \
            "`listfor <name>` - list all open issues assigned to `<name>`"

"""
    Redmine commands
"""
def get_user(username):
    try:
        return rc.user.filter(name=username)[0]
    except:
        return RuntimeError(":x: Failed to find user `"+username+"` in Redmine")

def get_issue(issueid):
    try:
        return rc.issue.get(int(issueid)).id
    except:
        raise RuntimeError(":x: Failed to find issue ID `"+issueid+"` in Redmine")

def impersonate_redmine(userlogin):
    try:
        return Redmine(REDMINE_HOST, version=REDMINE_VERSION, key=REDMINE_TOKEN, impersonate=userlogin)
    except:
        raise RuntimeError(":x: Failed impersonate user `"+userlogin+"` in Redmine")

def issue_url(issueid):
    return "<"+REDMINE_EXT_HOST+"/issues/"+str(issueid)+"|#"+str(issueid)+">"

def issue_subject_url(issueid, subject):
    return "<"+REDMINE_EXT_HOST+"/issues/"+str(issueid)+"|#"+str(issueid)+" "+subject+">"

def list_issues(username):
    user = get_user(username)
    try:
        result = rc.issue.filter(sort='project', assigned_to_id=user.id, status_id='open')
        response = ""
        if len(result) > 0:
            response = ":book: Open issues assigned to "+user.firstname+" "+user.lastname+":\n"
            for issue in result:
                response += ""+issue.project.name+" "+issue_subject_url(issue.id, issue.subject)+"\n"
        else:
            response = ":thumbsup_all: No open issues assigned to "+user.firstname+" "+user.lastname
        return response
    except:
        raise RuntimeError(":x: List operation failed")

def update_issue(text, issue, username):
    user = get_user(username)
    issueid = get_issue(issue)
    # impersonate user so it looks like the update is from them
    rcn = impersonate_redmine(user.login)
    try:
        result = rcn.issue.update(issueid, notes=text)
        return ":memo: Updated Issue "+issue_url(issueid)+" with comment `"+text+"`"
    except:
        raise RuntimeError(":x: Issue update failed")

def close_issue(text, issue, username):
    user = get_user(username)
    issueid = get_issue(issue)
    # impersonate user so it looks like the update is from them
    rcn = impersonate_redmine(user.login)
    try:
        result = rcn.issue.update(issueid, status_id=REDMINE_CLOSE_ID, notes=text, done_ratio=100)
        return ":white_check_mark: Closed Issue "+issue_url(issueid)+" with comment `"+text+"`"
    except:
        raise RuntimeError(":x: Issue closing failed")

def create_issue(text, username, assigneduser):
    user = get_user(username)
    assigned = get_user(assigneduser)
    # impersonate user so it looks like the update is from them
    rcn = impersonate_redmine(user.login)
    try:
        issue = rcn.issue.create(project_id=REDMINE_PROJECT, subject=text, tracker_id=REDMINE_TRACKER_ID, assigned_to_id=assigned.id)
        return ":white_check_mark: Created Issue "+issue_subject_url(issue.id,issue.subject)+" assigned to "+assigned.firstname+" "+assigned.lastname
    except:
        raise RuntimeError(":x: Issue creation failed")

if __name__ == "__main__":
    READ_WEBSOCKET_DELAY = 1 # 1 second delay between reading from firehose
    if sc.rtm_connect():
        print("RedmineBot connected and running!")
        while True:
            command, channel, username = parse_slack_output(sc.rtm_read())
            if command and channel and username:
                handle_command(command, channel, username)
            elif channel and username:
                handle_command("help", channel, username)
            time.sleep(READ_WEBSOCKET_DELAY)
    else:
        print("Connection failed. Invalid Slack token or bot ID?")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    flaskext.wtf
    ~~~~~~~~~~~~

    Flask-WTF extension

    :copyright: (c) 2010 by Dan Jacob.
    :license: BSD, see LICENSE for more details.
"""

import warnings
import uuid

from wtforms.fields import BooleanField, DecimalField, DateField, \
    DateTimeField, FieldList, FloatField, FormField, \
    HiddenField, IntegerField, PasswordField, RadioField, SelectField, \
    SelectMultipleField, StringField, SubmitField, TextField, TextAreaField

from wtforms.validators import Email, email, EqualTo, equal_to, \
    IPAddress, ip_address, Length, length, MacAddress, mac_address, \
    NumberRange, number_range, Optional, optional, Required, required, Regexp, \
    regexp, UUID, URL, url, AnyOf, any_of, NoneOf, none_of

from wtforms.widgets import CheckboxInput, FileInput, HiddenInput, \
    ListWidget, Option, PasswordInput, RadioInput, Select, SubmitInput, \
    TableWidget, TextArea, TextInput

from wtforms.fields import FileField as _FileField

try:
    import sqlalchemy
    _is_sqlalchemy = True
except ImportError:
    _is_sqlalchemy = False

from wtforms import Form as BaseForm
from wtforms import fields, widgets, validators, ValidationError

from flask import request, session, current_app

from jinja2 import Markup

from flaskext.wtf import html5

from flaskext.wtf import recaptcha

from flaskext.wtf.recaptcha.fields import RecaptchaField
from flaskext.wtf.recaptcha.widgets import RecaptchaWidget
from flaskext.wtf.recaptcha.validators import Recaptcha

fields.RecaptchaField = RecaptchaField
widgets.RecaptchaWidget = RecaptchaWidget
validators.Recaptcha = Recaptcha

from flaskext.wtf.file import FileField
from flaskext.wtf.file import FileAllowed, FileRequired, file_allowed, \
        file_required

fields.FileField = FileField

validators.file_allowed = file_allowed
validators.file_required = file_required
validators.FileAllowed = FileAllowed
validators.FileRequired = FileRequired

__all__  = ['Form', 'ValidationError',
            'fields', 'validators', 'widgets', 'html5']

__all__ += fields.core.__all__
__all__ += validators.__all__
__all__ += widgets.core.__all__
__all__ += recaptcha.__all__

if _is_sqlalchemy:
    from wtforms.ext.sqlalchemy.fields import QuerySelectField, \
        QuerySelectMultipleField

    __all__ += ['QuerySelectField',
                'QuerySelectMultipleField']

    for field in (QuerySelectField,
                  QuerySelectMultipleField):

        setattr(fields, field.__name__, field)

def _generate_csrf_token():
    return str(uuid.uuid4())

class Form(BaseForm):

    """
    Subclass of WTForms **Form** class. The main difference is that
    **request.form** is passed as `formdata` argument to constructor
    so can handle request data implicitly.

    In addition this **Form** implementation has automatic CSRF handling.
    """

    csrf = fields.HiddenField()

    def __init__(self, formdata=None, *args, **kwargs):

        csrf_enabled = kwargs.pop('csrf_enabled', None)

        if csrf_enabled is None:
            csrf_enabled = current_app.config.get('CSRF_ENABLED', True)

        self.csrf_enabled = csrf_enabled

        self.csrf_session_key = kwargs.pop('csrf_session_key', None)

        if self.csrf_session_key is None:
            self.csrf_session_key = \
                current_app.config.get('CSRF_SESSION_KEY', '_csrf_token')

        csrf_token = session.get(self.csrf_session_key, None)

        if csrf_token is None:
            csrf_token = self.reset_csrf()

        super(Form, self).__init__(formdata, csrf=csrf_token, *args, **kwargs)

    def is_submitted(self):
        """
        Checks if form has been submitted. The default case is if the HTTP
        method is **PUT** or **POST**.
        """

        return request and request.method in ("PUT", "POST")

    def process(self, formdata=None, obj=None, **kwargs):

        if self.is_submitted():

            if formdata is None:
                formdata = request.form

            # ensure csrf validation occurs ONLY when formdata is passed
            # in case "csrf" is the only field in the form

            if not formdata and not request.files:
                self.csrf_is_valid = False
            else:
                self.csrf_is_valid = None

        super(Form, self).process(formdata, obj, **kwargs)

    @property
    def csrf_token(self):
        """
        Renders CSRF field inside a hidden DIV.

        :deprecated: Use **hidden_tag** instead.
        """
        warnings.warn("csrf_token is deprecated. Use hidden_tag instead",
                      DeprecationWarning)

        return self.hidden_tag('csrf')

    def reset_csrf(self):
        """
        Resets the CSRF token in the session. If you are reusing the form
        in the same view (i.e. you are not redirecting somewhere else)
        it's recommended you call this before rendering the form.
        """

        csrf_token = _generate_csrf_token()
        session[self.csrf_session_key] = csrf_token
        return csrf_token

    def validate_csrf(self, field):
        if not self.csrf_enabled:
            return

        csrf_token = session.pop(self.csrf_session_key, None)
        is_valid = field.data and \
                   field.data == csrf_token and \
                   self.csrf_is_valid is not False

        # reset this field, otherwise stale token is displayed
        field.data = self.reset_csrf()

        # we set this flag to ensure consistent behaviour when
        # calling validate() more than once

        self.csrf_is_valid = bool(is_valid)

        if not is_valid:
            raise ValidationError, "Missing or invalid CSRF token"

    def hidden_tag(self, *fields):
        """
        Wraps hidden fields in a hidden DIV tag, in order to keep XHTML
        compliance.

        .. versionadded:: 0.3

        :param fields: list of hidden field names. If not provided will render
                       all hidden fields, including the CSRF field.
        """

        if not fields:
            fields = [f for f in self if isinstance(f, HiddenField)]

        rv = [u'
']
        for field in fields:
            if isinstance(field, basestring):
                field = getattr(self, field)
            rv.append(unicode(field))
        rv.append(u"
")

        return Markup(u"".join(rv))

    def validate_on_submit(self):
        """
        Checks if form has been submitted and if so runs validate. This is
        a shortcut, equivalent to ``form.is_submitted() and form.validate()``
        """
        return self.is_submitted() and self.validate()

<commit_msg>Add FieldBase, to allow users to create custom fields.<commit_after># -*- coding: utf-8 -*-
"""
    flaskext.wtf
    ~~~~~~~~~~~~

    Flask-WTF extension

    :copyright: (c) 2010 by Dan Jacob.
    :license: BSD, see LICENSE for more details.
"""

import warnings
import uuid

from wtforms.fields import BooleanField, DecimalField, DateField, \
    DateTimeField, FieldList, FloatField, FormField, \
    HiddenField, IntegerField, PasswordField, RadioField, SelectField, \
    SelectMultipleField, StringField, SubmitField, TextField, TextAreaField

from wtforms.validators import Email, email, EqualTo, equal_to, \
    IPAddress, ip_address, Length, length, MacAddress, mac_address, \
    NumberRange, number_range, Optional, optional, Required, required, Regexp, \
    regexp, UUID, URL, url, AnyOf, any_of, NoneOf, none_of

from wtforms.widgets import CheckboxInput, FileInput, HiddenInput, \
    ListWidget, Option, PasswordInput, RadioInput, Select, SubmitInput, \
    TableWidget, TextArea, TextInput

from wtforms.fields import Field as FieldBase
from wtforms.fields import FileField as _FileField

try:
    import sqlalchemy
    _is_sqlalchemy = True
except ImportError:
    _is_sqlalchemy = False

from wtforms import Form as BaseForm
from wtforms import fields, widgets, validators, ValidationError

from flask import request, session, current_app

from jinja2 import Markup

from flaskext.wtf import html5

from flaskext.wtf import recaptcha

from flaskext.wtf.recaptcha.fields import RecaptchaField
from flaskext.wtf.recaptcha.widgets import RecaptchaWidget
from flaskext.wtf.recaptcha.validators import Recaptcha

fields.RecaptchaField = RecaptchaField
widgets.RecaptchaWidget = RecaptchaWidget
validators.Recaptcha = Recaptcha

from flaskext.wtf.file import FileField
from flaskext.wtf.file import FileAllowed, FileRequired, file_allowed, \
        file_required

fields.FileField = FileField

validators.file_allowed = file_allowed
validators.file_required = file_required
validators.FileAllowed = FileAllowed
validators.FileRequired = FileRequired

__all__  = ['Form', 'ValidationError',
            'fields', 'validators', 'widgets', 'html5']

__all__ += fields.core.__all__
__all__ += validators.__all__
__all__ += widgets.core.__all__
__all__ += recaptcha.__all__

if _is_sqlalchemy:
    from wtforms.ext.sqlalchemy.fields import QuerySelectField, \
        QuerySelectMultipleField

    __all__ += ['QuerySelectField',
                'QuerySelectMultipleField']

    for field in (QuerySelectField,
                  QuerySelectMultipleField):

        setattr(fields, field.__name__, field)

def _generate_csrf_token():
    return str(uuid.uuid4())

class Form(BaseForm):

    """
    Subclass of WTForms **Form** class. The main difference is that
    **request.form** is passed as `formdata` argument to constructor
    so can handle request data implicitly.

    In addition this **Form** implementation has automatic CSRF handling.
    """

    csrf = fields.HiddenField()

    def __init__(self, formdata=None, *args, **kwargs):

        csrf_enabled = kwargs.pop('csrf_enabled', None)

        if csrf_enabled is None:
            csrf_enabled = current_app.config.get('CSRF_ENABLED', True)

        self.csrf_enabled = csrf_enabled

        self.csrf_session_key = kwargs.pop('csrf_session_key', None)

        if self.csrf_session_key is None:
            self.csrf_session_key = \
                current_app.config.get('CSRF_SESSION_KEY', '_csrf_token')

        csrf_token = session.get(self.csrf_session_key, None)

        if csrf_token is None:
            csrf_token = self.reset_csrf()

        super(Form, self).__init__(formdata, csrf=csrf_token, *args, **kwargs)

    def is_submitted(self):
        """
        Checks if form has been submitted. The default case is if the HTTP
        method is **PUT** or **POST**.
        """

        return request and request.method in ("PUT", "POST")

    def process(self, formdata=None, obj=None, **kwargs):

        if self.is_submitted():

            if formdata is None:
                formdata = request.form

            # ensure csrf validation occurs ONLY when formdata is passed
            # in case "csrf" is the only field in the form

            if not formdata and not request.files:
                self.csrf_is_valid = False
            else:
                self.csrf_is_valid = None

        super(Form, self).process(formdata, obj, **kwargs)

    @property
    def csrf_token(self):
        """
        Renders CSRF field inside a hidden DIV.

        :deprecated: Use **hidden_tag** instead.
        """
        warnings.warn("csrf_token is deprecated. Use hidden_tag instead",
                      DeprecationWarning)

        return self.hidden_tag('csrf')

    def reset_csrf(self):
        """
        Resets the CSRF token in the session. If you are reusing the form
        in the same view (i.e. you are not redirecting somewhere else)
        it's recommended you call this before rendering the form.
        """

        csrf_token = _generate_csrf_token()
        session[self.csrf_session_key] = csrf_token
        return csrf_token

    def validate_csrf(self, field):
        if not self.csrf_enabled:
            return

        csrf_token = session.pop(self.csrf_session_key, None)
        is_valid = field.data and \
                   field.data == csrf_token and \
                   self.csrf_is_valid is not False

        # reset this field, otherwise stale token is displayed
        field.data = self.reset_csrf()

        # we set this flag to ensure consistent behaviour when
        # calling validate() more than once

        self.csrf_is_valid = bool(is_valid)

        if not is_valid:
            raise ValidationError, "Missing or invalid CSRF token"

    def hidden_tag(self, *fields):
        """
        Wraps hidden fields in a hidden DIV tag, in order to keep XHTML
        compliance.

        .. versionadded:: 0.3

        :param fields: list of hidden field names. If not provided will render
                       all hidden fields, including the CSRF field.
        """

        if not fields:
            fields = [f for f in self if isinstance(f, HiddenField)]

        rv = [u'
']
        for field in fields:
            if isinstance(field, basestring):
                field = getattr(self, field)
            rv.append(unicode(field))
        rv.append(u"
")

        return Markup(u"".join(rv))

    def validate_on_submit(self):
        """
        Checks if form has been submitted and if so runs validate. This is
        a shortcut, equivalent to ``form.is_submitted() and form.validate()``
        """
        return self.is_submitted() and self.validate()

<|endoftext|>


<commit_before>### Calculates the net emissions over the study period, with units of CO2/ha on a pixel-by-pixel basis

import utilities
import os
import datetime
import rasterio
import subprocess
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

def net_calc(tile_id):

    print "Calculating net emissions for", tile_id

    # Start time
    start = datetime.datetime.now()

    # Names of the gain and emissions tiles
    gain_in = '{0}_{1}.tif'.format(tile_id, cn.pattern_cumul_gain_combo)
    loss_in = '{0}_{1}.tif'.format(tile_id, cn.pattern_gross_emis_all_drivers)

    # Output net emissions file
    net_flux = '{0}_{1}.tif'.format(tile_id, cn.pattern_net_flux)

    # Opens cumulative gain input tile
    gain_src = rasterio.open(gain_in)

    # Grabs metadata about the tif, like its location/projection/cellsize
    kwargs = gain_src.meta

    # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
    windows = gain_src.block_windows(1)

    # Opens loss tile
    loss_src = rasterio.open(loss_in)

    kwargs.update(
        driver='GTiff',
        count=1,
        compress='lzw',
        nodata=0,
        dtype='float32'
    )

    # Opens the output tile, giving it the arguments of the input tiles
    net_flux_dst = rasterio.open(net_flux, 'w', **kwargs)

    # Iterates across the windows (1 pixel strips) of the input tile
    for idx, window in windows:

        # Creates windows for each input tile
        gain = gain_src.read(1, window=window)
        loss = loss_src.read(1, window=window)

        # Converts gain from C to CO2 and subtracts that from loss
        dst_data = loss - (gain*float(cn.c_to_co2))

        net_flux_dst.write_band(1, dst_data, window=window)

    # Need to include these or the spot machine will run out of memory otherwise
    os.remove(gain_in)
    os.remove(loss_in)

    # Prints information about the tile that was just processed
    uu.end_of_fx_summary(start, tile_id, os.path.join(cn.pattern_net_flux))<commit_msg>Testing the net flux calculation script now.<commit_after>### Calculates the net emissions over the study period, with units of CO2/ha on a pixel-by-pixel basis

import os
import datetime
import rasterio
import sys
sys.path.append('../')
import constants_and_names as cn
import universal_util as uu

def net_calc(tile_id):

    print "Calculating net emissions for", tile_id

    # Start time
    start = datetime.datetime.now()

    # Names of the gain and emissions tiles
    gain_in = '{0}_{1}.tif'.format(tile_id, cn.pattern_cumul_gain_combo)
    loss_in = '{0}_{1}.tif'.format(tile_id, cn.pattern_gross_emis_all_drivers)

    # Output net emissions file
    net_flux = '{0}_{1}.tif'.format(tile_id, cn.pattern_net_flux)

    # Opens cumulative gain input tile
    gain_src = rasterio.open(gain_in)

    # Grabs metadata about the tif, like its location/projection/cellsize
    kwargs = gain_src.meta

    # Grabs the windows of the tile (stripes) so we can iterate over the entire tif without running out of memory
    windows = gain_src.block_windows(1)

    # Opens loss tile
    loss_src = rasterio.open(loss_in)

    kwargs.update(
        driver='GTiff',
        count=1,
        compress='lzw',
        nodata=0,
        dtype='float32'
    )

    # Opens the output tile, giving it the arguments of the input tiles
    net_flux_dst = rasterio.open(net_flux, 'w', **kwargs)

    # Iterates across the windows (1 pixel strips) of the input tile
    for idx, window in windows:

        # Creates windows for each input tile
        gain = gain_src.read(1, window=window)
        loss = loss_src.read(1, window=window)

        print gain
        print loss
        print gain*float(cn.c_to_co2)
        print loss - (gain*float(cn.c_to_co2))

        # Converts gain from C to CO2 and subtracts that from loss
        dst_data = loss - (gain*float(cn.c_to_co2))

        net_flux_dst.write_band(1, dst_data, window=window)

        sys.quit()

    # Need to include these or the spot machine will run out of memory otherwise
    os.remove(gain_in)
    os.remove(loss_in)

    # Prints information about the tile that was just processed
    uu.end_of_fx_summary(start, tile_id, os.path.join(cn.pattern_net_flux))<|endoftext|>