<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""BUILD rules for tfrt."""

# Sanitize a dependency so that it works correctly from code that includes
# TensorFlow as a submodule.
def clean_dep(dep):
    return str(Label(dep))

def if_oss(oss_value, google_value = []):
    """Returns one of the arguments based on the non-configurable build env.

    Specifically, it does not return a `select`, and can be used to e.g.
    compute elements of list attributes.
    """
    return oss_value  # copybara:comment_replace return google_value

def if_google(google_value, oss_value = []):
    """Returns one of the arguments based on the non-configurable build env.

    Specifically, it does not return a `select`, and can be used to e.g.
    compute elements of list attributes.
    """
    return oss_value  # copybara:comment_replace return google_value

TFRT_COPTS = select({
    "@tf_runtime//:windows": [
        "-Zc:inline",
        "-Zc:strictStrings",
        "-Zc:rvalueCast",
        "-Oi",
        "-wd4141",
        "-wd4146",
        "-wd4180",
        "-wd4244",
        "-wd4258",
        "-wd4267",
        "-wd4291",
        "-wd4345",
        "-wd4351",
        "-wd4355",
        "-wd4456",
        "-wd4457",
        "-wd4458",
        "-wd4459",
        "-wd4503",
        "-wd4624",
        "-wd4722",
        "-wd4800",
        "-wd4100",
        "-wd4127",
        "-wd4512",
        "-wd4505",
        "-wd4610",
        "-wd4510",
        "-wd4702",
        "-wd4245",
        "-wd4706",
        "-wd4310",
        "-wd4701",
        "-wd4703",
        "-wd4389",
        "-wd4611",
        "-wd4805",
        "-wd4204",
        "-wd4577",
        "-wd4091",
        "-wd4592",
        "-wd4319",
        "-wd4324",
        "-w14062",
        "-we4238",
    ],
    "//conditions:default": ["-Wno-unused-local-typedef"],
    "@tf_runtime//:disable_rtti_and_exceptions": [
        # Disable RTTI.
        "-fno-rtti",
        # Disable exceptions.
        "-fno-exceptions",
    ],
})

TFRT_LINKOPTS = select({
    "@tf_runtime//:windows": [],
    "//conditions:default": ["-ldl", "-lm", "-lpthread"],  # copybara:comment_replace "//conditions:default": ["-ldl", "-lm"],
})

TFRT_FEATURES = select({
    "//conditions:default": [],
    "@tf_runtime//:disable_rtti_and_exceptions": [
        # Precompiled header modules do not work with fno-rtti or
        # fno-exceptions. See b/137799263.
        "-use_header_modules",
    ],
})

# Use relative include path.
TFRT_INCLUDES = ["include"]

def tfrt_cc_library(
        name = "",
        srcs = [],
        deps = [],
        includes = [],
        copts = [],
        linkopts = [],
        features = [],
        alwayslink_static_registration_src = "",
        alwayslink_static_registration_deps = [],
        **kwargs):
    """A cc_library with tfrt-specific options."""
    for tfrt_inc in TFRT_INCLUDES:
        for inc in includes:
            if tfrt_inc == inc:
                fail(
                    "Found include path '" + inc + "' that is already in TFRT_INCLUDES",
                    "includes",
                )

    native.cc_library(
        name = name,
        srcs = srcs,
        deps = deps,
        includes = TFRT_INCLUDES + includes,
        copts = TFRT_COPTS + copts,
        linkopts = TFRT_LINKOPTS + linkopts,
        features = TFRT_FEATURES + features,
        **kwargs
    )

    # Generate a second target with "_alwayslink" suffix in its name.
    if alwayslink_static_registration_src != "":
        native.cc_library(
            name = name + "_alwayslink",
            srcs = [alwayslink_static_registration_src],
            # Depend on non-alwayslink target to avoid duplicate symbol linker
            # error.
            deps = deps + alwayslink_static_registration_deps + [":" + name],
            includes = TFRT_INCLUDES + includes,
            copts = TFRT_COPTS + copts,
            linkopts = TFRT_LINKOPTS + linkopts,
            features = TFRT_FEATURES + features,
            alwayslink = 1,
            **kwargs
        )

def tfrt_cc_binary(
        includes = [],
        copts = [],
        linkopts = [],
        features = [],
        **kwargs):
    """A cc_binary with tfrt-specific options."""
    native.cc_binary(
        copts = TFRT_COPTS + copts,
        includes = TFRT_INCLUDES + includes,
        linkopts = TFRT_LINKOPTS + linkopts,
        features = TFRT_FEATURES,
        **kwargs
    )

def tfrt_cc_test(
        includes = [],
        copts = ["-Wno-private-header"],
        linkopts = [],
        features = [],
        **kwargs):
    """A cc_test with tfrt-specific options."""
    native.cc_test(
        includes = TFRT_INCLUDES + includes,
        copts = TFRT_COPTS + copts,
        linkopts = TFRT_LINKOPTS + linkopts,
        features = TFRT_FEATURES + features,
        **kwargs
    )

def tfrt_py_binary(
        tags = [],
        **kwargs):
    """A py_binary with tfrt-specific options."""
    native.py_binary(
        tags = tags + ["do_not_disable_rtti"],
        **kwargs
    )
<commit_msg>Fix a small test build issue on Windows<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""BUILD rules for tfrt."""

# Sanitize a dependency so that it works correctly from code that includes
# TensorFlow as a submodule.
def clean_dep(dep):
    return str(Label(dep))

def if_oss(oss_value, google_value = []):
    """Returns one of the arguments based on the non-configurable build env.

    Specifically, it does not return a `select`, and can be used to e.g.
    compute elements of list attributes.
    """
    return oss_value  # copybara:comment_replace return google_value

def if_google(google_value, oss_value = []):
    """Returns one of the arguments based on the non-configurable build env.

    Specifically, it does not return a `select`, and can be used to e.g.
    compute elements of list attributes.
    """
    return oss_value  # copybara:comment_replace return google_value

TFRT_COPTS = select({
    "@tf_runtime//:windows": [
        "-Zc:inline",
        "-Zc:strictStrings",
        "-Zc:rvalueCast",
        "-Oi",
        "-wd4141",
        "-wd4146",
        "-wd4180",
        "-wd4244",
        "-wd4258",
        "-wd4267",
        "-wd4291",
        "-wd4345",
        "-wd4351",
        "-wd4355",
        "-wd4456",
        "-wd4457",
        "-wd4458",
        "-wd4459",
        "-wd4503",
        "-wd4624",
        "-wd4722",
        "-wd4800",
        "-wd4100",
        "-wd4127",
        "-wd4512",
        "-wd4505",
        "-wd4610",
        "-wd4510",
        "-wd4702",
        "-wd4245",
        "-wd4706",
        "-wd4310",
        "-wd4701",
        "-wd4703",
        "-wd4389",
        "-wd4611",
        "-wd4805",
        "-wd4204",
        "-wd4577",
        "-wd4091",
        "-wd4592",
        "-wd4319",
        "-wd4324",
        "-w14062",
        "-we4238",
    ],
    "//conditions:default": ["-Wno-unused-local-typedef"],
    "@tf_runtime//:disable_rtti_and_exceptions": [
        # Disable RTTI.
        "-fno-rtti",
        # Disable exceptions.
        "-fno-exceptions",
    ],
})

TFRT_LINKOPTS = select({
    "@tf_runtime//:windows": [],
    "//conditions:default": ["-ldl", "-lm", "-lpthread"],  # copybara:comment_replace "//conditions:default": ["-ldl", "-lm"],
})

TFRT_FEATURES = select({
    "//conditions:default": [],
    "@tf_runtime//:disable_rtti_and_exceptions": [
        # Precompiled header modules do not work with fno-rtti or
        # fno-exceptions. See b/137799263.
        "-use_header_modules",
    ],
})

# Use relative include path.
TFRT_INCLUDES = ["include"]

def tfrt_cc_library(
        name = "",
        srcs = [],
        deps = [],
        includes = [],
        copts = [],
        linkopts = [],
        features = [],
        alwayslink_static_registration_src = "",
        alwayslink_static_registration_deps = [],
        **kwargs):
    """A cc_library with tfrt-specific options."""
    for tfrt_inc in TFRT_INCLUDES:
        for inc in includes:
            if tfrt_inc == inc:
                fail(
                    "Found include path '" + inc + "' that is already in TFRT_INCLUDES",
                    "includes",
                )

    native.cc_library(
        name = name,
        srcs = srcs,
        deps = deps,
        includes = TFRT_INCLUDES + includes,
        copts = TFRT_COPTS + copts,
        linkopts = TFRT_LINKOPTS + linkopts,
        features = TFRT_FEATURES + features,
        **kwargs
    )

    # Generate a second target with "_alwayslink" suffix in its name.
    if alwayslink_static_registration_src != "":
        native.cc_library(
            name = name + "_alwayslink",
            srcs = [alwayslink_static_registration_src],
            # Depend on non-alwayslink target to avoid duplicate symbol linker
            # error.
            deps = deps + alwayslink_static_registration_deps + [":" + name],
            includes = TFRT_INCLUDES + includes,
            copts = TFRT_COPTS + copts,
            linkopts = TFRT_LINKOPTS + linkopts,
            features = TFRT_FEATURES + features,
            alwayslink = 1,
            **kwargs
        )

def tfrt_cc_binary(
        includes = [],
        copts = [],
        linkopts = [],
        features = [],
        **kwargs):
    """A cc_binary with tfrt-specific options."""
    native.cc_binary(
        copts = TFRT_COPTS + copts,
        includes = TFRT_INCLUDES + includes,
        linkopts = TFRT_LINKOPTS + linkopts,
        features = TFRT_FEATURES,
        **kwargs
    )

def tfrt_cc_test(
        includes = [],
        copts = select({
            "@tf_runtime//:windows": [],
            "//conditions:default": [
                "-Wno-private-header",
            ],
        }),
        linkopts = [],
        features = [],
        **kwargs):
    """A cc_test with tfrt-specific options."""
    native.cc_test(
        includes = TFRT_INCLUDES + includes,
        copts = TFRT_COPTS + copts,
        linkopts = TFRT_LINKOPTS + linkopts,
        features = TFRT_FEATURES + features,
        **kwargs
    )

def tfrt_py_binary(
        tags = [],
        **kwargs):
    """A py_binary with tfrt-specific options."""
    native.py_binary(
        tags = tags + ["do_not_disable_rtti"],
        **kwargs
    )
<|endoftext|>


<commit_before>'''
utilities (:mod:`calour.util`)
==============================

.. currentmodule:: calour.util

Functions
^^^^^^^^^
.. autosummary::
   :toctree: generated

   compute_prevalence
   register_functions
   set_log_level
'''

# ----------------------------------------------------------------------------
# Copyright (c) 2016--,  Calour development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

import os
import hashlib
import inspect
import re
import configparser
import warnings
from types import FunctionType
from functools import wraps, update_wrapper
from importlib import import_module
from collections.abc import Sequence
from logging import getLogger
from numbers import Real
from pkg_resources import resource_filename

import numpy as np
import scipy

logger = getLogger(__name__)

def _iter_fasta(file_name):
    '''Iterate over fasta file.
    Fasta file must contain header line (starting with ">") and one or more sequence lines.

    Parameters
    ----------
    file_name: str
        name of the fasta file

    Yields
    ------
    (header, sequence)
    header: str
        the header line (without ">")
    sequence: str
        the sequence ('ACGT')
    NOTE: header and sequence are stripped (no new line/spaces at beginning or end)
    '''
    # skip non-header lines at beginning of file
    with open(file_name, 'r') as fl:
        for cline in fl:
            if cline[0] == ">":
                title = cline[1:].rstrip()
                break
            logger.warning('Fasta file %s has no headers' % file_name)
            return

        lines = []
        for cline in fl:
            if cline[0] == ">":
                yield title, ''.join(lines)
                lines = []
                title = cline[1:].strip()
                continue
            lines.append(cline.strip())
        yield title, "".join(lines)

def compute_prevalence(abundance):
    '''Return the prevalence at each abundance cutoffs.

    Each sample that has the OTU above the cutoff (exclusive) will
    be counted.

    Parameters
    ----------
    abundance : iterable of numeric
        The abundance of a species across samples.

    Examples
    --------
    >>> abund = [0, 0, 1, 2, 4]
    >>> x, y = compute_prevalence(abund)
    >>> x   #doctest: +SKIP
    array([0, 1, 2, 4])
    >>> y   #doctest: +SKIP
    array([0.6, 0.4, 0.2, 0.])
    '''
    # unique values are sorted
    cutoffs, counts = np.unique(abundance, return_counts=True)
    cum_counts = np.cumsum(counts)
    prevalences = 1 - cum_counts / counts.sum()
    return cutoffs, prevalences

def _transition_index(obj):
    '''Return the transition index and current value of the list.

    Examples
    -------
    >>> obj = ['a', 'a', 'b']
    >>> list(_transition_index(obj))
    [(2, 'a'), (3, 'b')]
    >>> obj = ['a', 'a', 'b', 1, 2, None, None]
    >>> list(_transition_index(obj))
    [(2, 'a'), (3, 'b'), (4, 1), (5, 2), (7, None)]

    Parameters
    ----------
    obj : Iterable of arbitrary objects

    Yields
    ------
    tuple of (int, arbitrary)
        the transition index, the item value
    '''
    it = enumerate(obj)
    i, item = next(it)
    item = str(type(item)), item
    for i, current in it:
        current = str(type(current)), current
        if item != current:
            yield i, item[1]
            item = current
    yield i + 1, item[1]

def _convert_axis_name(func):
    '''Convert str value of axis to 0/1.

    This allows the decorated function with ``axis`` parameter
    to accept "sample" and "feature" as value for ``axis`` parameter.

    This should be always the closest decorator to the function if
    you have multiple decorators for this function.
    '''
    conversion = {'sample': 0,
                  's': 0,
                  'samples': 0,
                  'feature': 1,
                  'f': 1,
                  'features': 1}

    @wraps(func)
    def inner(*args, **kwargs):
        sig = inspect.signature(func)
        ba = sig.bind(*args, **kwargs)
        param = ba.arguments
        v = param.get('axis', None)
        if v is None:
            return func(*args, **kwargs)
        if isinstance(v, str):
            param['axis'] = conversion[v.lower()]
        elif v not in {0, 1}:
            raise ValueError('unknown axis `%r`' % v)

        return func(*ba.args, **ba.kwargs)
    return inner

def _get_taxonomy_string(exp, sep=';', remove_underscore=True, to_lower=False):
    '''Get a nice taxonomy string.

    Convert the taxonomy list stored (from biom.read_table) to a single string per feature

    Parameters
    ----------
    exp : Experiment
        with the taxonomy entry in the feature_metadata
    sep : str, optional
        the output separator to use between the taxonomic levels
    remove_underscore : bool, optional
        True (default) to remove the 'g__' entries and missing values
        False to keep them
    to_lower : bool, optional
        False (default) to keep case
        True to convert to lowercase

    Returns
    -------
    taxonomy : list of str
        list of taxonomy string per feature
    '''
    # test if we have taxonomy in the feature metadata
    logger.debug('getting taxonomy string')
    if 'taxonomy' not in exp.feature_metadata.columns:
        raise ValueError('No taxonomy field in experiment')

    # if it is not a list - just return it
    if not isinstance(exp.feature_metadata['taxonomy'][0], list):
        return list(exp.feature_metadata['taxonomy'].values)

    if not remove_underscore:
        taxonomy = [sep.join(x) for x in exp.feature_metadata['taxonomy']]
    else:
        taxonomy = []
        for ctax in exp.feature_metadata['taxonomy']:
            taxstr = ''
            for clevel in ctax:
                clevel = clevel.strip()
                if len(clevel) > 3:
                    if clevel[1:3] == '__':
                        clevel = clevel[3:]
                    taxstr += clevel + sep
            if len(taxstr) == 0:
                taxstr = 'na'
            taxonomy.append(taxstr)

    if to_lower:
        taxonomy = [x.lower() for x in taxonomy]
    return taxonomy

def get_file_md5(f, encoding='utf-8'):
    '''get the md5 of the text file.

    Parameters
    ----------
    f : str
        name of the file to calculate md5 on
    encoding : str or None, optional
        encoding of the text file (see python str.encode() ). None to use 'utf-8'

    Returns
    -------
    flmd5: str
        the md5 of the file f
    '''
    logger.debug('getting file md5 for file %s' % f)
    if f is None:
        return None
    with open(f, 'rb') as fl:
        flmd5 = hashlib.md5()
        chunk_size = 4096
        for chunk in iter(lambda: fl.read(chunk_size), b""):
            flmd5.update(chunk)
        flmd5 = flmd5.hexdigest()
        logger.debug('md5 of %s: %s' % (f, flmd5))
        return flmd5

def get_data_md5(data):
    '''Calculate the md5 of a dense/sparse matrix

    Calculat matrix md5 based on row by row order

    Parameters
    ----------
    data : dense or sparse matrix

    Returns
    -------
    datmd5 : str
        the md5 of the data
    '''
    logger.debug('caculating data md5')
    if scipy.sparse.issparse(data):
        # if sparse need to convert to numpy array
        data = data.toarray()
    # convert to string of raw data since hashlib.md5 does not take numpy array as input
    datmd5 = hashlib.md5(data.tobytes())
    datmd5 = datmd5.hexdigest()
    logger.debug('data md5 is: %s' % datmd5)
    return datmd5

def get_config_file():
    '''Get the calour config file location

    If the environment CALOUR_CONFIG_FILE is set, take the config file from it
    otherwise return CALOUR_PACKAGE_LOCATION/calour/calour.config

    Returns
    -------
    config_file_name : str
        the full path to the calour config file
    '''
    if 'CALOUR_CONFIG_FILE' in os.environ:
        config_file_name = os.environ['CALOUR_CONFIG_FILE']
        logger.debug('Using calour config file %s from CALOUR_CONFIG_FILE variable' % config_file_name)
    else:
        config_file_name = resource_filename(__package__, 'calour.config')
    return config_file_name

def set_config_value(key, value, section='DEFAULT', config_file_name=None):
    '''Set the value in the calour config file

    Parameters
    ----------
    key : str
        the key to get the value for
    value : str
        the value to store
    section : str, optional
        the section to get the value from
    config_file_name : str, optional
        the full path to the config file or None to use default config file
    '''
    if config_file_name is None:
        config_file_name = get_config_file()

    config = configparser.ConfigParser()
    config.read(config_file_name)
    if section not in config:
        config.add_section(section)
    config.set(section, key, value)
    with open(config_file_name, 'w') as config_file:
        config.write(config_file)
    logger.debug('wrote key %s value %s to config file' % (key, value))

def get_config_sections(config_file_name=None):
    '''Get a list of the sections in the config file

    Parameters
    ----------
     config_file_name : str, optional
        the full path to the config file or None to use default config file

    Returns
    -------
    list of str
        List of the sections in the config file
    '''
    if config_file_name is None:
        config_file_name = get_config_file()
    logger.debug('getting sections from config file %s' % config_file_name)
    config = configparser.ConfigParser()
    config.read(config_file_name)
    return config.sections()

def get_config_value(key, fallback=None, section='DEFAULT', config_file_name=None):
    '''Get the value from the calour config file

    Parameters
    ----------
    key : str
        the key to get the value for
    fallback : str, optional
        the fallback value if the key/section/file does not exist
    section : str, optional
        the section to get the value from
    config_file_name : str, optional
        the full path to the config file or None to use default config file

    Returns
    -------
    value : str
        value of the key or fallback if file/section/key does not exist
    '''
    if config_file_name is None:
        config_file_name = get_config_file()

    config = configparser.ConfigParser()
    config.read(config_file_name)

    if section not in config:
        logger.debug('section %s not in config file %s' % (section, config_file_name))
        return fallback

    if key not in config[section]:
        logger.debug('key %s not in config file %s section %s' % (key, config_file_name, section))
        return fallback

    value = config[section][key]
    return value

def set_log_level(level):
    '''Set the debug level for calour

    You can see the logging levels at:
    https://docs.python.org/3.5/library/logging.html#levels

    Parameters
    ----------
    level : int or str
        10 for debug, 20 for info, 30 for warn, etc.
        It is passing to :func:`logging.Logger.setLevel`

    '''
    clog = getLogger('calour')
    clog.setLevel(level)

def _to_list(x):
    '''if x is non iterable or string, convert to iterable.

    See the expected behavior in the examples below.

    Examples
    --------
    >>> _to_list('a')
    ['a']
    >>> _to_list({})
    [{}]
    >>> _to_list(['a'])
    ['a']
    >>> _to_list(set(['a']))
    [{'a'}]
    '''
    if isinstance(x, str):
        return [x]
    if isinstance(x, Sequence):
        return x
    return [x]

def _argsort(values, reverse=False):
    '''Sort a sequence of values of heterogeneous variable types.

    Used to overcome the problem when using numpy.argsort on a pandas
    series values with missing values

    Examples
    --------
    >>> l = [10, 'b', np.nan, 2.5, 'a']
    >>> idx = _argsort(l)
    >>> idx
    [3, 0, 2, 4, 1]
    >>> l_sorted = [l[i] for i in idx]
    >>> l_sorted
    [2.5, 10, nan, 'a', 'b']
    >>> l_sorted_reverse = [l[i] for i in _argsort(l, True)]
    >>> l_sorted_reverse
    ['b', 'a', nan, 10, 2.5]

    Parameters
    ----------
    values : iterable
        the values to sort

    Returns
    -------
    list of ints
        the positions of the sorted values

    '''
    pairs = []
    for cval in values:
        if isinstance(cval, Real):
            if np.isnan(cval):
                cval = np.inf
            else:
                cval = float(cval)
        pairs.append((str(type(cval)), cval))

    # # convert all numbers to float otherwise int will be sorted different place
    # values = [float(x) if isinstance(x, Real) else x for x in values]
    # # make values ordered by type and sort inside each var type
    # values = [(str(type(x)), x) if not np.isnan(x) else (str(type(x)), np.inf) for x in values]
    # return sorted(range(len(values)), key=values.__getitem__)
    return sorted(range(len(pairs)), key=pairs.__getitem__, reverse=reverse)

def _clone_function(f):
    '''Make a copy of a function'''
    # based on http://stackoverflow.com/a/13503277/2289509
    new_f = FunctionType(f.__code__, f.__globals__,
                         name=f.__name__,
                         argdefs=f.__defaults__,
                         closure=f.__closure__)
    new_f = update_wrapper(new_f, f)
    new_f.__kwdefaults__ = f.__kwdefaults__
    return new_f

def register_functions(cls, modules=None):
    '''Dynamically register functions to the class as methods.

    Parameters
    ----------
    cls : ``class`` object
        The class that the functions will be added to
    modules : iterable of str, optional
        The module names where the functions are defined. ``None`` means all public
        modules in `calour`.
    '''
    # pattern to recognize the Parameters section
    p = re.compile(r"(\n +Parameters\n +-+ *)")
    if modules is None:
        modules = ['calour.' + i for i in
                   ['io', 'sorting', 'filtering', 'analysis', 'training', 'transforming',
                    'heatmap.heatmap', 'plotting', 'manipulation', 'database', 'export_html']]
    for module_name in modules:
        module = import_module(module_name)
        functions = inspect.getmembers(module, inspect.isfunction)
        for fn, f in functions:
            # skip private functions
            if not fn.startswith('_'):
                params = inspect.signature(f).parameters
                if params:
                    # if the func accepts parameters, ie params is not empty
                    first = next(iter(params.values()))
                    if first.annotation is cls:
                        # make a copy of the function because we want
                        # to update the docstring of the original
                        # function but not that of the registered
                        # version
                        setattr(cls, fn, _clone_function(f))
                        updated = ('\n    .. note:: This function is also available as a class method :meth:`.{0}.{1}`\n'
                                   '\\1'
                                   '\n    exp : {0}'
                                   '\n        Input experiment object.'
                                   '\n')

                        if not f.__doc__:
                            f.__doc__ = ''

                        f.__doc__ = p.sub(updated.format(cls.__name__, fn), f.__doc__)

def deprecated(message):
    '''Deprecation decorator.

    Parameters
    ----------
    message : str
        the message to print together with deprecation info.
    '''
    def deprecated_decorator(func):
        @wraps(func)
        def deprecated_func(*args, **kwargs):
            warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
                          category=DeprecationWarning,
                          stacklevel=2)
            warnings.simplefilter('default', DeprecationWarning)
            return func(*args, **kwargs)
        return deprecated_func
    return deprecated_decorator

def format_docstring(*args, **kwargs):
    '''Format the docstring of the decorated function.'''
    def dec(obj):
        obj.__doc__ = obj.__doc__.format(*args, **kwargs)
        return obj
    return dec
<commit_msg>pr fixes<commit_after>'''
utilities (:mod:`calour.util`)
==============================

.. currentmodule:: calour.util

Functions
^^^^^^^^^
.. autosummary::
   :toctree: generated

   compute_prevalence
   register_functions
   set_log_level
'''

# ----------------------------------------------------------------------------
# Copyright (c) 2016--,  Calour development team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
# ----------------------------------------------------------------------------

import os
import hashlib
import inspect
import re
import configparser
import warnings
from types import FunctionType
from functools import wraps, update_wrapper
from importlib import import_module
from collections.abc import Sequence
from logging import getLogger
from numbers import Real
from pkg_resources import resource_filename

import numpy as np
import scipy

logger = getLogger(__name__)

def _iter_fasta(fp):
    '''Iterate over fasta file.

    Fasta file must contain header line (starting with ">") and one or more sequence lines.

    Parameters
    ----------
    fp: str
        name of the fasta file

    Yields
    ------
    header: str
        the header line (without ">")
    sequence: str
        the sequence ('ACGT'). Both header and sequence are whitespace stripped.
    '''
    # skip non-header lines at beginning of file
    with open(fp, 'r') as fl:
        for cline in fl:
            if cline[0] == ">":
                title = cline[1:].rstrip()
                break
            logger.warning('Fasta file %s has no headers' % fp)
            return

        lines = []
        for cline in fl:
            if cline[0] == ">":
                yield title, ''.join(lines)
                lines = []
                title = cline[1:].strip()
                continue
            lines.append(cline.strip())
        yield title, "".join(lines)

def compute_prevalence(abundance):
    '''Return the prevalence at each abundance cutoffs.

    Each sample that has the OTU above the cutoff (exclusive) will
    be counted.

    Parameters
    ----------
    abundance : iterable of numeric
        The abundance of a species across samples.

    Examples
    --------
    >>> abund = [0, 0, 1, 2, 4]
    >>> x, y = compute_prevalence(abund)
    >>> x   #doctest: +SKIP
    array([0, 1, 2, 4])
    >>> y   #doctest: +SKIP
    array([0.6, 0.4, 0.2, 0.])
    '''
    # unique values are sorted
    cutoffs, counts = np.unique(abundance, return_counts=True)
    cum_counts = np.cumsum(counts)
    prevalences = 1 - cum_counts / counts.sum()
    return cutoffs, prevalences

def _transition_index(obj):
    '''Return the transition index and current value of the list.

    Examples
    -------
    >>> obj = ['a', 'a', 'b']
    >>> list(_transition_index(obj))
    [(2, 'a'), (3, 'b')]
    >>> obj = ['a', 'a', 'b', 1, 2, None, None]
    >>> list(_transition_index(obj))
    [(2, 'a'), (3, 'b'), (4, 1), (5, 2), (7, None)]

    Parameters
    ----------
    obj : Iterable of arbitrary objects

    Yields
    ------
    tuple of (int, arbitrary)
        the transition index, the item value
    '''
    it = enumerate(obj)
    i, item = next(it)
    item = str(type(item)), item
    for i, current in it:
        current = str(type(current)), current
        if item != current:
            yield i, item[1]
            item = current
    yield i + 1, item[1]

def _convert_axis_name(func):
    '''Convert str value of axis to 0/1.

    This allows the decorated function with ``axis`` parameter
    to accept "sample" and "feature" as value for ``axis`` parameter.

    This should be always the closest decorator to the function if
    you have multiple decorators for this function.
    '''
    conversion = {'sample': 0,
                  's': 0,
                  'samples': 0,
                  'feature': 1,
                  'f': 1,
                  'features': 1}

    @wraps(func)
    def inner(*args, **kwargs):
        sig = inspect.signature(func)
        ba = sig.bind(*args, **kwargs)
        param = ba.arguments
        v = param.get('axis', None)
        if v is None:
            return func(*args, **kwargs)
        if isinstance(v, str):
            param['axis'] = conversion[v.lower()]
        elif v not in {0, 1}:
            raise ValueError('unknown axis `%r`' % v)

        return func(*ba.args, **ba.kwargs)
    return inner

def _get_taxonomy_string(exp, sep=';', remove_underscore=True, to_lower=False):
    '''Get a nice taxonomy string.

    Convert the taxonomy list stored (from biom.read_table) to a single string per feature

    Parameters
    ----------
    exp : Experiment
        with the taxonomy entry in the feature_metadata
    sep : str, optional
        the output separator to use between the taxonomic levels
    remove_underscore : bool, optional
        True (default) to remove the 'g__' entries and missing values
        False to keep them
    to_lower : bool, optional
        False (default) to keep case
        True to convert to lowercase

    Returns
    -------
    taxonomy : list of str
        list of taxonomy string per feature
    '''
    # test if we have taxonomy in the feature metadata
    logger.debug('getting taxonomy string')
    if 'taxonomy' not in exp.feature_metadata.columns:
        raise ValueError('No taxonomy field in experiment')

    # if it is not a list - just return it
    if not isinstance(exp.feature_metadata['taxonomy'][0], list):
        return list(exp.feature_metadata['taxonomy'].values)

    if not remove_underscore:
        taxonomy = [sep.join(x) for x in exp.feature_metadata['taxonomy']]
    else:
        taxonomy = []
        for ctax in exp.feature_metadata['taxonomy']:
            taxstr = ''
            for clevel in ctax:
                clevel = clevel.strip()
                if len(clevel) > 3:
                    if clevel[1:3] == '__':
                        clevel = clevel[3:]
                    taxstr += clevel + sep
            if len(taxstr) == 0:
                taxstr = 'na'
            taxonomy.append(taxstr)

    if to_lower:
        taxonomy = [x.lower() for x in taxonomy]
    return taxonomy

def get_file_md5(f, encoding='utf-8'):
    '''get the md5 of the text file.

    Parameters
    ----------
    f : str
        name of the file to calculate md5 on
    encoding : str or None, optional
        encoding of the text file (see python str.encode() ). None to use 'utf-8'

    Returns
    -------
    flmd5: str
        the md5 of the file f
    '''
    logger.debug('getting file md5 for file %s' % f)
    if f is None:
        return None
    with open(f, 'rb') as fl:
        flmd5 = hashlib.md5()
        chunk_size = 4096
        for chunk in iter(lambda: fl.read(chunk_size), b""):
            flmd5.update(chunk)
        flmd5 = flmd5.hexdigest()
        logger.debug('md5 of %s: %s' % (f, flmd5))
        return flmd5

def get_data_md5(data):
    '''Calculate the md5 of a dense/sparse matrix

    Calculat matrix md5 based on row by row order

    Parameters
    ----------
    data : dense or sparse matrix

    Returns
    -------
    datmd5 : str
        the md5 of the data
    '''
    logger.debug('caculating data md5')
    if scipy.sparse.issparse(data):
        # if sparse need to convert to numpy array
        data = data.toarray()
    # convert to string of raw data since hashlib.md5 does not take numpy array as input
    datmd5 = hashlib.md5(data.tobytes())
    datmd5 = datmd5.hexdigest()
    logger.debug('data md5 is: %s' % datmd5)
    return datmd5

def get_config_file():
    '''Get the calour config file location

    If the environment CALOUR_CONFIG_FILE is set, take the config file from it
    otherwise return CALOUR_PACKAGE_LOCATION/calour/calour.config

    Returns
    -------
    config_file_name : str
        the full path to the calour config file
    '''
    if 'CALOUR_CONFIG_FILE' in os.environ:
        config_file_name = os.environ['CALOUR_CONFIG_FILE']
        logger.debug('Using calour config file %s from CALOUR_CONFIG_FILE variable' % config_file_name)
    else:
        config_file_name = resource_filename(__package__, 'calour.config')
    return config_file_name

def set_config_value(key, value, section='DEFAULT', config_file_name=None):
    '''Set the value in the calour config file

    Parameters
    ----------
    key : str
        the key to get the value for
    value : str
        the value to store
    section : str, optional
        the section to get the value from
    config_file_name : str, optional
        the full path to the config file or None to use default config file
    '''
    if config_file_name is None:
        config_file_name = get_config_file()

    config = configparser.ConfigParser()
    config.read(config_file_name)
    if section not in config:
        config.add_section(section)
    config.set(section, key, value)
    with open(config_file_name, 'w') as config_file:
        config.write(config_file)
    logger.debug('wrote key %s value %s to config file' % (key, value))

def get_config_sections(config_file_name=None):
    '''Get a list of the sections in the config file

    Parameters
    ----------
     config_file_name : str, optional
        the full path to the config file or None to use default config file

    Returns
    -------
    list of str
        List of the sections in the config file
    '''
    if config_file_name is None:
        config_file_name = get_config_file()
    logger.debug('getting sections from config file %s' % config_file_name)
    config = configparser.ConfigParser()
    config.read(config_file_name)
    return config.sections()

def get_config_value(key, fallback=None, section='DEFAULT', config_file_name=None):
    '''Get the value from the calour config file

    Parameters
    ----------
    key : str
        the key to get the value for
    fallback : str, optional
        the fallback value if the key/section/file does not exist
    section : str, optional
        the section to get the value from
    config_file_name : str, optional
        the full path to the config file or None to use default config file

    Returns
    -------
    value : str
        value of the key or fallback if file/section/key does not exist
    '''
    if config_file_name is None:
        config_file_name = get_config_file()

    config = configparser.ConfigParser()
    config.read(config_file_name)

    if section not in config:
        logger.debug('section %s not in config file %s' % (section, config_file_name))
        return fallback

    if key not in config[section]:
        logger.debug('key %s not in config file %s section %s' % (key, config_file_name, section))
        return fallback

    value = config[section][key]
    return value

def set_log_level(level):
    '''Set the debug level for calour

    You can see the logging levels at:
    https://docs.python.org/3.5/library/logging.html#levels

    Parameters
    ----------
    level : int or str
        10 for debug, 20 for info, 30 for warn, etc.
        It is passing to :func:`logging.Logger.setLevel`

    '''
    clog = getLogger('calour')
    clog.setLevel(level)

def _to_list(x):
    '''if x is non iterable or string, convert to iterable.

    See the expected behavior in the examples below.

    Examples
    --------
    >>> _to_list('a')
    ['a']
    >>> _to_list({})
    [{}]
    >>> _to_list(['a'])
    ['a']
    >>> _to_list(set(['a']))
    [{'a'}]
    '''
    if isinstance(x, str):
        return [x]
    if isinstance(x, Sequence):
        return x
    return [x]

def _argsort(values, reverse=False):
    '''Sort a sequence of values of heterogeneous variable types.

    Used to overcome the problem when using numpy.argsort on a pandas
    series values with missing values

    Examples
    --------
    >>> l = [10, 'b', np.nan, 2.5, 'a']
    >>> idx = _argsort(l)
    >>> idx
    [3, 0, 2, 4, 1]
    >>> l_sorted = [l[i] for i in idx]
    >>> l_sorted
    [2.5, 10, nan, 'a', 'b']
    >>> l_sorted_reverse = [l[i] for i in _argsort(l, True)]
    >>> l_sorted_reverse
    ['b', 'a', nan, 10, 2.5]

    Parameters
    ----------
    values : iterable
        the values to sort

    Returns
    -------
    list of ints
        the positions of the sorted values

    '''
    pairs = []
    for cval in values:
        if isinstance(cval, Real):
            if np.isnan(cval):
                cval = np.inf
            else:
                cval = float(cval)
        pairs.append((str(type(cval)), cval))

    # # convert all numbers to float otherwise int will be sorted different place
    # values = [float(x) if isinstance(x, Real) else x for x in values]
    # # make values ordered by type and sort inside each var type
    # values = [(str(type(x)), x) if not np.isnan(x) else (str(type(x)), np.inf) for x in values]
    # return sorted(range(len(values)), key=values.__getitem__)
    return sorted(range(len(pairs)), key=pairs.__getitem__, reverse=reverse)

def _clone_function(f):
    '''Make a copy of a function'''
    # based on http://stackoverflow.com/a/13503277/2289509
    new_f = FunctionType(f.__code__, f.__globals__,
                         name=f.__name__,
                         argdefs=f.__defaults__,
                         closure=f.__closure__)
    new_f = update_wrapper(new_f, f)
    new_f.__kwdefaults__ = f.__kwdefaults__
    return new_f

def register_functions(cls, modules=None):
    '''Dynamically register functions to the class as methods.

    Parameters
    ----------
    cls : ``class`` object
        The class that the functions will be added to
    modules : iterable of str, optional
        The module names where the functions are defined. ``None`` means all public
        modules in `calour`.
    '''
    # pattern to recognize the Parameters section
    p = re.compile(r"(\n +Parameters\n +-+ *)")
    if modules is None:
        modules = ['calour.' + i for i in
                   ['io', 'sorting', 'filtering', 'analysis', 'training', 'transforming',
                    'heatmap.heatmap', 'plotting', 'manipulation', 'database', 'export_html']]
    for module_name in modules:
        module = import_module(module_name)
        functions = inspect.getmembers(module, inspect.isfunction)
        for fn, f in functions:
            # skip private functions
            if not fn.startswith('_'):
                params = inspect.signature(f).parameters
                if params:
                    # if the func accepts parameters, ie params is not empty
                    first = next(iter(params.values()))
                    if first.annotation is cls:
                        # make a copy of the function because we want
                        # to update the docstring of the original
                        # function but not that of the registered
                        # version
                        setattr(cls, fn, _clone_function(f))
                        updated = ('\n    .. note:: This function is also available as a class method :meth:`.{0}.{1}`\n'
                                   '\\1'
                                   '\n    exp : {0}'
                                   '\n        Input experiment object.'
                                   '\n')

                        if not f.__doc__:
                            f.__doc__ = ''

                        f.__doc__ = p.sub(updated.format(cls.__name__, fn), f.__doc__)

def deprecated(message):
    '''Deprecation decorator.

    Parameters
    ----------
    message : str
        the message to print together with deprecation info.
    '''
    def deprecated_decorator(func):
        @wraps(func)
        def deprecated_func(*args, **kwargs):
            warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
                          category=DeprecationWarning,
                          stacklevel=2)
            warnings.simplefilter('default', DeprecationWarning)
            return func(*args, **kwargs)
        return deprecated_func
    return deprecated_decorator

def format_docstring(*args, **kwargs):
    '''Format the docstring of the decorated function.'''
    def dec(obj):
        obj.__doc__ = obj.__doc__.format(*args, **kwargs)
        return obj
    return dec
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Utilities: Agave wrapper for sherlock
"""
import os
from os import path as op
from subprocess import check_output, CalledProcessError, STDOUT
import re
from time import sleep
import logging
from pprint import pprint
import pkg_resources as pkgr

from cappat import AGAVE_JOB_LOGS, AGAVE_JOB_OUTPUT
from cappat.tpl import Template
from cappat.utils import check_folder, getsystemname

SLURM_FAIL_STATUS = ['CA', 'F', 'TO', 'NF', 'SE']
SLURM_WAIT_STATUS = ['R', 'PD', 'CF', 'CG']
SLEEP_SECONDS = 5

JOB_LOG = logging.getLogger('taskmanager')

class TaskManager(object):
    """
    A task manager factory class
    """
    def __init__(self):
        raise RuntimeError('This class cannot be instatiated.')

    @staticmethod
    def build(task_list, settings=None, work_dir=None,
              hostname=None):
        """
        Get the appropriate TaskManager object
        """
        hostname = settings.get('execution_system', None)

        if hostname is None:
            hostname = getsystemname()

        JOB_LOG.info('Identified host: "%s"', hostname)

        if not hostname:
            raise RuntimeError('Could not identify execution system')

        if hostname.endswith('ls5.tacc.utexas.edu'):
            return Lonestar5Submission(task_list, settings, work_dir)
        elif hostname.endswith('stanford.edu'):
            return SherlockSubmission(task_list, settings, work_dir)
        elif hostname.endswith('stampede.tacc.utexas.edu'):
            raise NotImplementedError
        elif hostname == 'test.circleci':
            return CircleCISubmission(task_list, settings, work_dir)
        elif hostname == 'test.local':
            return TestSubmission(task_list, settings, work_dir)
        else:
            raise RuntimeError(
                'Could not identify "{}" as a valid execution system'.format(hostname))

class TaskSubmissionBase(object):
    """
    A base class for task submission
    """
    settings = {}
    jobexp = re.compile(r'Submitted batch job (?P<jobid>\d*)')
    _cmd_prefix = []

    SLURM_TEMPLATE = pkgr.resource_filename('cappat', 'tpl/sherlock-sbatch.jnj2')

    def __init__(self, task_list, settings=None, work_dir=None):

        if not task_list:
            raise RuntimeError('a list of tasks is required')

        self.task_list = task_list

        if settings is not None:
            self.settings.update(settings)

        if work_dir is None:
            work_dir = os.getcwd()

        self.settings['child_runtime'] = _secs2time(
            int(0.85 * _time2secs(self.settings['max_runtime'])))

        self.work_dir = check_folder(op.abspath(work_dir))
        self.aux_dir = check_folder(op.join(self.work_dir, AGAVE_JOB_LOGS))
        self.settings.update(
            {'work_dir': self.work_dir, 'aux_dir': self.aux_dir}
        )
        self.sbatch_files = self._generate_sbatch()
        self._jobs = {}
        self._group_cmd = [self.settings['executable'], self.settings['bids_dir'],
                           AGAVE_JOB_OUTPUT, 'group']

        # if self.settings.get('modules', []):
        #     modules_list = []
        #     for m in self.settings['modules']:
        #         modules_list += ['module'] + m.split(' ')
        #         modules_list[-1] += ';'

        #     self._group_cmd = modules_list + self._group_cmd
        JOB_LOG.info('Automatically inferred group level command: "%s"',
                      ' '.join(self.group_cmd))

        JOB_LOG.info('Created TaskManager type "%s" with default settings: \n\t%s',
                     self.__class__.__name__, pprint(self.settings))

    @property
    def job_ids(self):
        return list(self._jobs.keys())

    @property
    def jobs(self):
        return self._jobs

    @property
    def group_cmd(self):
        return self._group_cmd

    @group_cmd.setter
    def group_cmd(self, value):
        if not isinstance(value, list):
            value = [value]

        self._group_cmd = value

    def _parse_jobid(self, slurm_msg):
        if isinstance(slurm_msg, (list, tuple)):
            slurm_msg = '\n'.join(slurm_msg)

        jobid = self.jobexp.search(slurm_msg).group('jobid')
        if jobid:
            self._jobs[jobid] = 'SUBMITTED'
        else:
            raise RuntimeError('Job ID could not extracted. Slurm message:\n{}'.format(
                slurm_msg))
        return jobid

    def _generate_sbatch(self):
        raise NotImplementedError

    def _submit_sbatch(self, task):
        return _run_cmd(self._cmd_prefix + ['sbatch', task])

    def _run_sacct(self):
        # sacct -n -X -j 10016750,10016749 -o JobID,State,ExitCode
        return _run_cmd(self._cmd_prefix + [
            'sacct', '-n', '-X', '-j', ','.join(self.job_ids),
            '-o', 'JobID,State,ExitCode'])

    def _get_job_acct(self):
        JOB_LOG.info('Checking exit code of jobs %s', ' '.join(self.job_ids))
        results = self._run_sacct()

        if results is None:
            JOB_LOG.critical('Running sacct over jobs %s did not produce any output',
                             ', '.join(self.job_ids))
            raise RuntimeError('sacct command output is empty')

        #parse results
        exit_codes = []
        for line in results.split('\n'):
            fields = line.split()
            self._jobs[fields[0]] = fields[1]
            exit_codes.append(int(fields[2].split(':')[0]))
        return exit_codes

    def _get_jobs_status(self):
        squeue = _run_cmd(self._cmd_prefix + [
            'squeue', '-j', ','.join(self.job_ids), '-o', '%i,%t', '-h'])

        # Jobs are not in the queue anymore
        if squeue is None:
            JOB_LOG.warn('Command "squeue" was empty: jobs are completed.')
            return True

        if 'Invalid job id specified' in squeue:
            JOB_LOG.warn('Jobs completed - squeue: %s', squeue)
            return True

        pending = []
        statuses = squeue.split('\n')
        for line in statuses:
            status = line.split(',')
            if len(status) < 2:
                JOB_LOG.error('Error parsing squeue output: \n%s\n', line)
                raise RuntimeError('Error parsing squeue output: {}'.format(
                    line))

            self._jobs[status[0]] = status[1]

            if status in SLURM_FAIL_STATUS:
                JOB_LOG.warn('Job id %s failed (%s).', *status)
            else:
                pending.append(status[0])

        if pending:
            JOB_LOG.info('There are pending jobs: %s', ' '.join(pending))
            return False

        JOB_LOG.info('Jobs %s not present in squeue list, finishing polling.',
                     ', '.join(self.job_ids))
        return True

    def map_participant(self):
        """
        Submits a list of sbatch files and returns the assigned job ids
        """
        for i, task in enumerate(self.sbatch_files):
            JOB_LOG.info('Submitting sbatch/launcher file %s (%d)', task, i)
            # run sbatch
            sresult = self._submit_sbatch(task)
            # parse output and get job id
            jobid = self._parse_jobid(sresult)
            JOB_LOG.info(
                'Submitted task %d, job ID %s was assigned', i, jobid)

    def wait_participant(self):
        """
        Busy wait until all jobs in the list are done
        """
        JOB_LOG.info('Starting busy wait on jobs %s',
                     ' '.join(self.job_ids))
        all_finished = False

        while not all_finished:
            all_finished = self._get_jobs_status()
            sleep(SLEEP_SECONDS)

        JOB_LOG.info('Finished wait on jobs %s', ', '.join(self.job_ids))

        # Run sacct to check the exit code of jobs
        overall_exit = sum(self._get_job_acct())

        JOB_LOG.info('Final status of jobs: %s', ', '.join([
            '%s (%s)' % (k, v) for k, v in list(self._jobs.items())]))

        if overall_exit > 0:
            failed_jobs = ['{0} (logfiles: log/bidsapp-{0}.{{err,out}}).'.format(k) for k, v in list(
                self._jobs.items()) if v != 'COMPLETED']
            JOB_LOG.critical('One or more tasks finished with non-zero code:\n'
                             '\t%s', '\n\t'.join(failed_jobs))
            raise RuntimeError('One or more tasks finished with non-zero code')
        return self.job_ids

    def run_grouplevel(self):
        """
        Run the reduce operation over the participant map
        """
        if not self.group_cmd:
            JOB_LOG.warning('Group level command not set, skipping reduce operation.')
            return True

        JOB_LOG.info('Kicking off reduce operation')
        if _run_cmd(self.group_cmd, shell=True):
            JOB_LOG.info('Group level finished successfully.')
            return True
        return False

class Lonestar5Submission(TaskSubmissionBase):
    """
    The LS5 submission manager
    """

    def _generate_sbatch(self):
        """
        Generates one launcher file
        """
        launcher_file = op.join(self.aux_dir, 'launch_script.sh')
        with open(launcher_file, 'w') as lfh:
            lfh.write('\n'.join(self.task_list) + '\n')
        return [launcher_file]

    def _submit_sbatch(self, task):
        with open(task, 'r') as tfh:
            nodes = sum(1 for line in tfh if line.strip() and not line.strip().startswith('#'))
        values = {
            'cwd': os.getcwd(),
            'launcher_file': task,
            'nodes': nodes,
            'ncpus': 1,
            'jobname': self.settings['job_name'],
            'runtime': self.settings['child_runtime']
        }
        launcher_cmd = """\
export LAUNCHER_WORKDIR={cwd}; \
/corral-repl/utexas/poldracklab/users/wtriplet/external/ls5_launch/launch -s {launcher_file} \
-n {ncpus} -N {nodes} -d {cwd} -r {runtime} -j {jobname} -f {cwd}/{jobname}.qsub \
""".format(**values)
        return _run_cmd(['ssh', '-oStrictHostKeyChecking=no', 'login2', launcher_cmd])

class SherlockSubmission(TaskSubmissionBase):
    """
    The Sherlock submission
    """

    def __init__(self, task_list, settings=None, work_dir=None):
        if not settings is None:
            self.settings.update(settings)
        self.settings['qos'] = self.settings['partition']
        super(SherlockSubmission, self).__init__(
            task_list, work_dir=work_dir)

    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        settings = self.settings.copy()
        sbatch_files = []
        for i, task in enumerate(self.task_list):
            sbatch_files.append(op.join(self.aux_dir, 'slurm-%06d.sbatch' % i))
            settings['commandline'] = task
            conf = Template(self.SLURM_TEMPLATE)
            conf.generate_conf(settings, sbatch_files[-1])
        return sbatch_files

class CircleCISubmission(SherlockSubmission):
    """
    A CircleCI submission manager to work with the slurm docker image
    """
    settings = {
        'nodes': 1,
        'time': '01:00:00',
        'partition': 'debug',
        'job_name': 'crn-bidsapp'
    }
    _cmd_prefix = ['sshpass', '-p', 'testpass',
                   'ssh', '-p', '10022', 'circleci@localhost']

    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        # Remove default settings of Sherlock which are unsupported
        self.settings.pop('qos', None)
        self.settings.pop('mincpus', None)
        self.settings.pop('mem_per_cpu', None)
        self.settings.pop('modules', None)
        self.settings['work_dir'] = self.settings['work_dir'].replace(
            op.expanduser('~/'), '/')
        self.settings['work_dir'] = self.settings['work_dir'].replace(
            '~/', '/')
        return super(CircleCISubmission, self)._generate_sbatch()

    def _submit_sbatch(self, task):
        # Fix paths for docker image in CircleCI
        task = task.replace(op.expanduser('~/'), '/')
        task = task.replace('~/', '/')
        return super(CircleCISubmission, self)._submit_sbatch(task)

class TestSubmission(SherlockSubmission):
    """
    A Test submission manager to work with the slurm docker image
    """
    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        # Remove default settings of Sherlock not supported
        self.settings.pop('qos', None)
        self.settings.pop('mincpus', None)
        self.settings.pop('mem_per_cpu', None)
        self.settings.pop('modules', None)
        self.settings.pop('srun_cmd', None)
        return super(TestSubmission, self)._generate_sbatch()

    def _submit_sbatch(self, task):
        return _run_cmd(['/bin/bash', task])

    def _get_jobs_status(self):
        jobs = ['%s,COMPLETED' % j for j in self.job_ids]
        return _run_cmd(['echo', '\n'.join(jobs)]).strip()

    def _run_sacct(self):
        return '\n'.join(['%s  COMPLETED  0:0' % j for j in self.job_ids])

def _run_cmd(cmd, shell=False):
    JOB_LOG.info('Executing command line: %s', ' '.join(cmd))
    try:
        result = check_output(cmd, stderr=STDOUT, shell=shell)
    except CalledProcessError as error:
        JOB_LOG.critical('Error submitting (exit code %d): \n\tCmdline: %s\n\tOutput:\n\t%s',
                         error.returncode, ' '.join(cmd), error.output)
        raise
    result = '\n'.join([line for line in result.split('\n') if line.strip()])
    if not result:
        JOB_LOG.info('Command output was empty')
        return None

    JOB_LOG.info('Command output: \n%s', result)
    return result

def _time2secs(timestr):
    return sum((60**i) * int(t) for i, t in enumerate(reversed(timestr.split(':'))))

def _secs2time(seconds):
    m, s = divmod(seconds, 60)
    h, m = divmod(m, 60)
    return "%02d:%02d:%02d" % (h, m, s)

<commit_msg>setting up environment for group level<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Utilities: Agave wrapper for sherlock
"""
import os
from os import path as op
import subprocess as sp
import re
from time import sleep
import logging
from pprint import pprint
import pkg_resources as pkgr
from io import open

from cappat import AGAVE_JOB_LOGS, AGAVE_JOB_OUTPUT
from cappat.tpl import Template
from cappat.utils import check_folder, getsystemname

SLURM_FAIL_STATUS = ['CA', 'F', 'TO', 'NF', 'SE']
SLURM_WAIT_STATUS = ['R', 'PD', 'CF', 'CG']
SLEEP_SECONDS = 5

JOB_LOG = logging.getLogger('taskmanager')

class TaskManager(object):
    """
    A task manager factory class
    """
    def __init__(self):
        raise RuntimeError('This class cannot be instatiated.')

    @staticmethod
    def build(task_list, settings=None, work_dir=None,
              hostname=None):
        """
        Get the appropriate TaskManager object
        """
        hostname = settings.get('execution_system', None)

        if hostname is None:
            hostname = getsystemname()

        JOB_LOG.info('Identified host: "%s"', hostname)

        if not hostname:
            raise RuntimeError('Could not identify execution system')

        if hostname.endswith('ls5.tacc.utexas.edu'):
            return Lonestar5Submission(task_list, settings, work_dir)
        elif hostname.endswith('stanford.edu'):
            return SherlockSubmission(task_list, settings, work_dir)
        elif hostname.endswith('stampede.tacc.utexas.edu'):
            raise NotImplementedError
        elif hostname == 'test.circleci':
            return CircleCISubmission(task_list, settings, work_dir)
        elif hostname == 'test.local':
            return TestSubmission(task_list, settings, work_dir)
        else:
            raise RuntimeError(
                'Could not identify "{}" as a valid execution system'.format(hostname))

class TaskSubmissionBase(object):
    """
    A base class for task submission
    """
    settings = {}
    jobexp = re.compile(r'Submitted batch job (?P<jobid>\d*)')
    _cmd_prefix = []

    SLURM_TEMPLATE = pkgr.resource_filename('cappat', 'tpl/sherlock-sbatch.jnj2')

    def __init__(self, task_list, settings=None, work_dir=None):

        if not task_list:
            raise RuntimeError('a list of tasks is required')

        self.task_list = task_list

        if settings is not None:
            self.settings.update(settings)

        if work_dir is None:
            work_dir = os.getcwd()

        self.settings['child_runtime'] = _secs2time(
            int(0.85 * _time2secs(self.settings['max_runtime'])))

        self.work_dir = check_folder(op.abspath(work_dir))
        self.aux_dir = check_folder(op.join(self.work_dir, AGAVE_JOB_LOGS))
        self.settings.update(
            {'work_dir': self.work_dir, 'aux_dir': self.aux_dir}
        )
        self.sbatch_files = self._generate_sbatch()
        self._jobs = {}
        self._group_cmd = [self.settings['executable'], self.settings['bids_dir'],
                           AGAVE_JOB_OUTPUT, 'group']
        JOB_LOG.info('Automatically inferred group level command: "%s"',
                      ' '.join(self.group_cmd))

        JOB_LOG.info('Created TaskManager type "%s" with default settings: \n\t%s',
                     self.__class__.__name__, pprint(self.settings))

    @property
    def job_ids(self):
        return list(self._jobs.keys())

    @property
    def jobs(self):
        return self._jobs

    @property
    def group_cmd(self):
        return self._group_cmd

    @group_cmd.setter
    def group_cmd(self, value):
        if not isinstance(value, list):
            value = [value]

        self._group_cmd = value

    def _parse_jobid(self, slurm_msg):
        if isinstance(slurm_msg, (list, tuple)):
            slurm_msg = '\n'.join(slurm_msg)

        jobid = self.jobexp.search(slurm_msg).group('jobid')
        if jobid:
            self._jobs[jobid] = 'SUBMITTED'
        else:
            raise RuntimeError('Job ID could not extracted. Slurm message:\n{}'.format(
                slurm_msg))
        return jobid

    def _generate_sbatch(self):
        raise NotImplementedError

    def _submit_sbatch(self, task):
        return _run_cmd(self._cmd_prefix + ['sbatch', task])

    def _run_sacct(self):
        # sacct -n -X -j 10016750,10016749 -o JobID,State,ExitCode
        return _run_cmd(self._cmd_prefix + [
            'sacct', '-n', '-X', '-j', ','.join(self.job_ids),
            '-o', 'JobID,State,ExitCode'])

    def _get_job_acct(self):
        JOB_LOG.info('Checking exit code of jobs %s', ' '.join(self.job_ids))
        results = self._run_sacct()

        if results is None:
            JOB_LOG.critical('Running sacct over jobs %s did not produce any output',
                             ', '.join(self.job_ids))
            raise RuntimeError('sacct command output is empty')

        #parse results
        exit_codes = []
        for line in results.split('\n'):
            fields = line.split()
            self._jobs[fields[0]] = fields[1]
            exit_codes.append(int(fields[2].split(':')[0]))
        return exit_codes

    def _get_jobs_status(self):
        squeue = _run_cmd(self._cmd_prefix + [
            'squeue', '-j', ','.join(self.job_ids), '-o', '%i,%t', '-h'])

        # Jobs are not in the queue anymore
        if squeue is None:
            JOB_LOG.warn('Command "squeue" was empty: jobs are completed.')
            return True

        if 'Invalid job id specified' in squeue:
            JOB_LOG.warn('Jobs completed - squeue: %s', squeue)
            return True

        pending = []
        statuses = squeue.split('\n')
        for line in statuses:
            status = line.split(',')
            if len(status) < 2:
                JOB_LOG.error('Error parsing squeue output: \n%s\n', line)
                raise RuntimeError('Error parsing squeue output: {}'.format(
                    line))

            self._jobs[status[0]] = status[1]

            if status in SLURM_FAIL_STATUS:
                JOB_LOG.warn('Job id %s failed (%s).', *status)
            else:
                pending.append(status[0])

        if pending:
            JOB_LOG.info('There are pending jobs: %s', ' '.join(pending))
            return False

        JOB_LOG.info('Jobs %s not present in squeue list, finishing polling.',
                     ', '.join(self.job_ids))
        return True

    def map_participant(self):
        """
        Submits a list of sbatch files and returns the assigned job ids
        """
        for i, task in enumerate(self.sbatch_files):
            JOB_LOG.info('Submitting sbatch/launcher file %s (%d)', task, i)
            # run sbatch
            sresult = self._submit_sbatch(task)
            # parse output and get job id
            jobid = self._parse_jobid(sresult)
            JOB_LOG.info(
                'Submitted task %d, job ID %s was assigned', i, jobid)

    def wait_participant(self):
        """
        Busy wait until all jobs in the list are done
        """
        JOB_LOG.info('Starting busy wait on jobs %s',
                     ' '.join(self.job_ids))
        all_finished = False

        while not all_finished:
            all_finished = self._get_jobs_status()
            sleep(SLEEP_SECONDS)

        JOB_LOG.info('Finished wait on jobs %s', ', '.join(self.job_ids))

        # Run sacct to check the exit code of jobs
        overall_exit = sum(self._get_job_acct())

        JOB_LOG.info('Final status of jobs: %s', ', '.join([
            '%s (%s)' % (k, v) for k, v in list(self._jobs.items())]))

        if overall_exit > 0:
            failed_jobs = ['{0} (logfiles: log/bidsapp-{0}.{{err,out}}).'.format(k) for k, v in list(
                self._jobs.items()) if v != 'COMPLETED']
            JOB_LOG.critical('One or more tasks finished with non-zero code:\n'
                             '\t%s', '\n\t'.join(failed_jobs))
            raise RuntimeError('One or more tasks finished with non-zero code')
        return self.job_ids

    def run_grouplevel(self):
        """
        Run the reduce operation over the participant map
        """
        if not self.group_cmd:
            JOB_LOG.warning('Group level command not set, skipping reduce operation.')
            return True

        JOB_LOG.info('Kicking off reduce operation')

        envdict = None

        if self.settings.get('modules', []):
            JOB_LOG.info('Group level command depends on module, checking environment')
            modules_load = []
            modules_use = []
            for m in self.settings['modules']:
                if m.startswith('load '):
                    modules_load += m[5:].split(' ')
                elif m.startswith('module load '):
                    modules_load += m[12:].split(' ')
                elif m.startswith('use '):
                    modules_use += m[4:].split(' ')
                elif m.startswith('module use '):
                    modules_use += m[11:].split(' ')

            with open('group-env.sh', 'w') as f:
                f.write('module unload crnenv')
                f.write('module use ' + ' '.join(modules_use))
                f.write('module load ' + ' '.join(modules_load))

            proc = sp.Popen(['bash', '-c', 'source group-env.sh && env'],
                            stdout=sp.PIPE)

            envdict = {}
            for line in proc.stdout:
                key, _, value = line.partition('=')
                envdict[key] = value
            proc.communicate()

        if _run_cmd(self.group_cmd, env=envdict):
            JOB_LOG.info('Group level finished successfully.')
            return True
        return False

class Lonestar5Submission(TaskSubmissionBase):
    """
    The LS5 submission manager
    """

    def _generate_sbatch(self):
        """
        Generates one launcher file
        """
        launcher_file = op.join(self.aux_dir, 'launch_script.sh')
        with open(launcher_file, 'w') as lfh:
            lfh.write('\n'.join(self.task_list) + '\n')
        return [launcher_file]

    def _submit_sbatch(self, task):
        with open(task, 'r') as tfh:
            nodes = sum(1 for line in tfh if line.strip() and not line.strip().startswith('#'))
        values = {
            'cwd': os.getcwd(),
            'launcher_file': task,
            'nodes': nodes,
            'ncpus': 1,
            'jobname': self.settings['job_name'],
            'runtime': self.settings['child_runtime']
        }
        launcher_cmd = """\
export LAUNCHER_WORKDIR={cwd}; \
/corral-repl/utexas/poldracklab/users/wtriplet/external/ls5_launch/launch -s {launcher_file} \
-n {ncpus} -N {nodes} -d {cwd} -r {runtime} -j {jobname} -f {cwd}/{jobname}.qsub \
""".format(**values)
        return _run_cmd(['ssh', '-oStrictHostKeyChecking=no', 'login2', launcher_cmd])

class SherlockSubmission(TaskSubmissionBase):
    """
    The Sherlock submission
    """

    def __init__(self, task_list, settings=None, work_dir=None):
        if not settings is None:
            self.settings.update(settings)
        self.settings['qos'] = self.settings['partition']
        super(SherlockSubmission, self).__init__(
            task_list, work_dir=work_dir)

    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        settings = self.settings.copy()
        sbatch_files = []
        for i, task in enumerate(self.task_list):
            sbatch_files.append(op.join(self.aux_dir, 'slurm-%06d.sbatch' % i))
            settings['commandline'] = task
            conf = Template(self.SLURM_TEMPLATE)
            conf.generate_conf(settings, sbatch_files[-1])
        return sbatch_files

class CircleCISubmission(SherlockSubmission):
    """
    A CircleCI submission manager to work with the slurm docker image
    """
    settings = {
        'nodes': 1,
        'time': '01:00:00',
        'partition': 'debug',
        'job_name': 'crn-bidsapp'
    }
    _cmd_prefix = ['sshpass', '-p', 'testpass',
                   'ssh', '-p', '10022', 'circleci@localhost']

    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        # Remove default settings of Sherlock which are unsupported
        self.settings.pop('qos', None)
        self.settings.pop('mincpus', None)
        self.settings.pop('mem_per_cpu', None)
        self.settings.pop('modules', None)
        self.settings['work_dir'] = self.settings['work_dir'].replace(
            op.expanduser('~/'), '/')
        self.settings['work_dir'] = self.settings['work_dir'].replace(
            '~/', '/')
        return super(CircleCISubmission, self)._generate_sbatch()

    def _submit_sbatch(self, task):
        # Fix paths for docker image in CircleCI
        task = task.replace(op.expanduser('~/'), '/')
        task = task.replace('~/', '/')
        return super(CircleCISubmission, self)._submit_sbatch(task)

class TestSubmission(SherlockSubmission):
    """
    A Test submission manager to work with the slurm docker image
    """
    def _generate_sbatch(self):
        """
        Generates one sbatch file per task
        """
        # Remove default settings of Sherlock not supported
        self.settings.pop('qos', None)
        self.settings.pop('mincpus', None)
        self.settings.pop('mem_per_cpu', None)
        self.settings.pop('modules', None)
        self.settings.pop('srun_cmd', None)
        return super(TestSubmission, self)._generate_sbatch()

    def _submit_sbatch(self, task):
        return _run_cmd(['/bin/bash', task])

    def _get_jobs_status(self):
        jobs = ['%s,COMPLETED' % j for j in self.job_ids]
        return _run_cmd(['echo', '\n'.join(jobs)]).strip()

    def _run_sacct(self):
        return '\n'.join(['%s  COMPLETED  0:0' % j for j in self.job_ids])

def _run_cmd(cmd, shell=False, env=None):
    JOB_LOG.info('Executing command line: %s', ' '.join(cmd))
    try:
        result = sp.check_output(cmd, stderr=sp.STDOUT, shell=shell, env=env)
    except sp.CalledProcessError as error:
        JOB_LOG.critical('Error submitting (exit code %d): \n\tCmdline: %s\n\tOutput:\n\t%s',
                         error.returncode, ' '.join(cmd), error.output)
        raise
    result = '\n'.join([line for line in result.split('\n') if line.strip()])
    if not result:
        JOB_LOG.info('Command output was empty')
        return None

    JOB_LOG.info('Command output: \n%s', result)
    return result

def _time2secs(timestr):
    return sum((60**i) * int(t) for i, t in enumerate(reversed(timestr.split(':'))))

def _secs2time(seconds):
    m, s = divmod(seconds, 60)
    h, m = divmod(m, 60)
    return "%02d:%02d:%02d" % (h, m, s)

<|endoftext|>


<commit_before>from django.shortcuts import render, get_object_or_404
from django.http import HttpResponseRedirect

from .forms import DistrictForm, BinderForm, CaseForm, EventForm, VictimFormset, SuspectFormset
from .models import District, Binder, Case, Event, Person

def homepage(request):
    cases = Case.objects.all()

    return render(request, 'homepage.html', {'cases': cases})

def add_entry(request):
    if request.method == 'GET':

        district_form = DistrictForm(prefix='district')
        binder_form = BinderForm(prefix='binder')
        case_form = CaseForm(prefix='case')
        event_form = EventForm(prefix='event')
        victim_formset = VictimFormset(prefix='victim')
        suspect_formset = SuspectFormset(prefix='suspect')

        # print(victim_formset)
        # print(suspect_formset)

        return render(request, 'add-entry.html', {
            'district_form': district_form,
            'binder_form': binder_form,
            'case_form': case_form,
            'event_form': event_form,
            'victim_formset': victim_formset,
            'suspect_formset': suspect_formset,
        })

    elif request.method == 'POST':
        district_form = DistrictForm(request.POST, prefix='district')
        binder_form = BinderForm(request.POST, prefix='binder')
        case_form = CaseForm(request.POST, prefix='case')
        event_form = EventForm(request.POST, prefix='event')
        victim_formset = VictimFormset(request.POST, prefix='victim')
        suspect_formset = SuspectFormset(request.POST, prefix='suspect')

        district_valid = district_form.is_valid()
        binder_valid = binder_form.is_valid()
        case_valid = case_form.is_valid()
        event_valid = event_form.is_valid()
        victims_valid = victim_formset.is_valid()
        suspects_valid = suspect_formset.is_valid()

        print(district_form.errors)
        print(binder_form.errors)
        print(case_form.errors)
        print(event_form.errors)
        print(victim_formset.errors)
        print(suspect_formset.errors)

        if district_valid and case_valid:
            # victim = victim_form.save()
            # suspect = suspect_form.save()
            district = district_form.save()
            binder = binder_form.save()
            case = case_form.save(commit=False)
            event = event_form.save(commit=False)

            case.district = district
            # Need to save case before adding ManyToMany relationships

            case.save()
            if binder_valid:
                case.binders.add(binder)

            for victim in victim_formset:
                # if victim.is_valid():
                victim = victim.save()
                print(victim)
                case.victims.add(victim)

            for suspect in suspect_formset:
                # if suspect.is_valid():
                suspect = suspect.save()
                case.suspects.add(suspect)

            if event_valid:
                event.case = case
                event.save()

            return HttpResponseRedirect('/detail/' + case.dr_nbr)

def detail(request, case_id):
    if request.method == 'GET':
        district_form = DistrictForm(prefix='district')
        binder_form = BinderForm(prefix='binder')
        case_form = CaseForm(prefix='case')
        event_form = EventForm(prefix='event')
        victim_formset = VictimFormset(prefix='victim')
        suspect_formset = SuspectFormset(prefix='suspect')
        case = get_object_or_404(Case, dr_nbr=case_id)
        event = get_object_or_404(Event, case=case)

        return render(request, 'detail-entry.html', {
            'district_form': district_form,
            'binder_form': binder_form,
            'case_form': case_form,
            'event_form': event_form,
            'victim_formset': victim_formset,
            'suspect_formset': suspect_formset,
            'case': case,
            'event': event,
        })

    elif request.method == 'POST':
        print('f')

    # if request.method == 'GET':
    #     the_case = get_object_or_404(Case, dr_nbr=case_id)
    #     district = the_case.district
    #     binder = the_case.binders
    #     # suspect = the_case.suspects.all()
    #     print(the_case.suspects)
    #     event = get_object_or_404(Event, case=the_case)
    #     return render(request, 'detail-entry.html', {
    #         'case_form': the_case,
    #         'district_form': district,
    #         'binder_form': binder,
    #         'event_form': event,
    #         'suspect_form': suspect,
    #     })
    # elif request.method == 'POST':
    #     print('hey')
    #

def advanced_search(request):
    if request.method == 'GET':

        district_form = DistrictForm(prefix='district')
        binder_form = BinderForm(prefix='binder')
        case_form = CaseForm(prefix='case')
        event_form = EventForm(prefix='event')
        victim_formset = VictimFormset(prefix='victim')
        suspect_formset = SuspectFormset(prefix='suspect')

        # print(victim_formset)
        # print(suspect_formset)

        return render(request, 'advanced-search.html', {
            'district_form': district_form,
            'binder_form': binder_form,
            'case_form': case_form,
            'event_form': event_form,
            'victim_formset': victim_formset,
            'suspect_formset': suspect_formset,
        })
    elif request.method == 'POST':
        district_form = DistrictForm(prefix='district')
        binder_form = BinderForm(prefix='binder')
        case_form = CaseForm(prefix='case')
        event_form = EventForm(prefix='event')
        victim_formset = VictimFormset(prefix='victim')
        suspect_formset = SuspectFormset(prefix='suspect')

        cases = Case.objects.all()

        print("original length ", len(cases))

        for field in case_form.fields:
            value = case_form[field].value()
            if value not in {'', False, None}:
                print(field, value)
                cases = cases.filter(**{field: value})

        print("new length ", len(cases))

def district_detail(request, district_id):
    office = get_object_or_404(District, id=district_id)
    return render(request, 'district/detail.html', {'district': district})
<commit_msg>filter case by binders<commit_after>from django.shortcuts import render, get_object_or_404
from django.http import HttpResponseRedirect

from .forms import DistrictForm, BinderForm, CaseForm, EventForm, VictimFormset, SuspectFormset
from .models import District, Binder, Case, Event, Person

def homepage(request):
    cases = Case.objects.all()

    return render(request, 'homepage.html', {'cases': cases})

def add_entry(request):
    if request.method == 'GET':

        district_form = DistrictForm(prefix='district')
        binder_form = BinderForm(prefix='binder')
        case_form = CaseForm(prefix='case')
        event_form = EventForm(prefix='event')
        victim_formset = VictimFormset(prefix='victim')
        suspect_formset = SuspectFormset(prefix='suspect')

        # print(victim_formset)
        # print(suspect_formset)

        return render(request, 'add-entry.html', {
            'district_form': district_form,
            'binder_form': binder_form,
            'case_form': case_form,
            'event_form': event_form,
            'victim_formset': victim_formset,
            'suspect_formset': suspect_formset,
        })

    elif request.method == 'POST':
        district_form = DistrictForm(request.POST, prefix='district')
        binder_form = BinderForm(request.POST, prefix='binder')
        case_form = CaseForm(request.POST, prefix='case')
        event_form = EventForm(request.POST, prefix='event')
        victim_formset = VictimFormset(request.POST, prefix='victim')
        suspect_formset = SuspectFormset(request.POST, prefix='suspect')

        district_valid = district_form.is_valid()
        binder_valid = binder_form.is_valid()
        case_valid = case_form.is_valid()
        event_valid = event_form.is_valid()
        victims_valid = victim_formset.is_valid()
        suspects_valid = suspect_formset.is_valid()

        print(district_form.errors)
        print(binder_form.errors)
        print(case_form.errors)
        print(event_form.errors)
        print(victim_formset.errors)
        print(suspect_formset.errors)

        if district_valid and case_valid:
            # victim = victim_form.save()
            # suspect = suspect_form.save()
            district = district_form.save()
            binder = binder_form.save()
            case = case_form.save(commit=False)
            event = event_form.save(commit=False)

            case.district = district
            # Need to save case before adding ManyToMany relationships

            case.save()
            if binder_valid:
                case.binders.add(binder)

            for victim in victim_formset:
                # if victim.is_valid():
                victim = victim.save()
                print(victim)
                case.victims.add(victim)

            for suspect in suspect_formset:
                # if suspect.is_valid():
                suspect = suspect.save()
                case.suspects.add(suspect)

            if event_valid:
                event.case = case
                event.save()

            return HttpResponseRedirect('/detail/' + case.dr_nbr)

def detail(request, case_id):
    if request.method == 'GET':
        district_form = DistrictForm(prefix='district')
        binder_form = BinderForm(prefix='binder')
        case_form = CaseForm(prefix='case')
        event_form = EventForm(prefix='event')
        victim_formset = VictimFormset(prefix='victim')
        suspect_formset = SuspectFormset(prefix='suspect')
        case = get_object_or_404(Case, dr_nbr=case_id)
        event = get_object_or_404(Event, case=case)

        return render(request, 'detail-entry.html', {
            'district_form': district_form,
            'binder_form': binder_form,
            'case_form': case_form,
            'event_form': event_form,
            'victim_formset': victim_formset,
            'suspect_formset': suspect_formset,
            'case': case,
            'event': event,
        })

    elif request.method == 'POST':
        print('f')

    # if request.method == 'GET':
    #     the_case = get_object_or_404(Case, dr_nbr=case_id)
    #     district = the_case.district
    #     binder = the_case.binders
    #     # suspect = the_case.suspects.all()
    #     print(the_case.suspects)
    #     event = get_object_or_404(Event, case=the_case)
    #     return render(request, 'detail-entry.html', {
    #         'case_form': the_case,
    #         'district_form': district,
    #         'binder_form': binder,
    #         'event_form': event,
    #         'suspect_form': suspect,
    #     })
    # elif request.method == 'POST':
    #     print('hey')
    #

def advanced_search(request):
    if request.method == 'GET':

        district_form = DistrictForm(prefix='district')
        binder_form = BinderForm(prefix='binder')
        case_form = CaseForm(prefix='case')
        event_form = EventForm(prefix='event')
        victim_formset = VictimFormset(prefix='victim')
        suspect_formset = SuspectFormset(prefix='suspect')

        # print(victim_formset)
        # print(suspect_formset)

        return render(request, 'advanced-search.html', {
            'district_form': district_form,
            'binder_form': binder_form,
            'case_form': case_form,
            'event_form': event_form,
            'victim_formset': victim_formset,
            'suspect_formset': suspect_formset,
        })
    elif request.method == 'POST':
        district_form = DistrictForm(prefix='district')
        binder_form = BinderForm(prefix='binder')
        case_form = CaseForm(prefix='case')
        event_form = EventForm(prefix='event')
        victim_formset = VictimFormset(prefix='victim')
        suspect_formset = SuspectFormset(prefix='suspect')

        cases = Case.objects.all()

        print("original length ", len(cases))
        print(len(Case.objects.filter(binders__master_dr=891038582)))

        for field in case_form.fields:
            value = case_form[field].value()
            if value not in {'', False, None}:
                print(field, value)
                cases = cases.filter(**{field: value})

        print("new length ", len(cases))

def district_detail(request, district_id):
    office = get_object_or_404(District, id=district_id)
    return render(request, 'district/detail.html', {'district': district})
<|endoftext|>