<commit_before>from functools import reduce, partial
import inspect
import operator

def identity(x):
    return x

def thread_first(val, *forms):
    """ Thread value through a sequence of functions/forms

    >>> def double(x): return 2*x
    >>> def inc(x):    return x + 1
    >>> thread_first(1, inc, double)
    4

    If the function expects more than one input you can specify those inputs
    in a tuple.  The value is used as the first input.

    >>> def add(x, y): return x + y
    >>> def pow(x, y): return x**y
    >>> thread_first(1, (add, 4), (pow, 2))  # pow(add(1, 4), 2)
    25

    So in general
        thread_first(x, f, (g, y, z))
    expands to
        g(f(x), y, z)

    See Also:
        thread_last
    """
    def evalform_front(val, form):
        if callable(form):
            return form(val)
        if isinstance(form, tuple):
            func, args = form[0], form[1:]
            args = (val,) + args
            return func(*args)
    return reduce(evalform_front, forms, val)

def thread_last(val, *forms):
    """ Thread value through a sequence of functions/forms

    >>> def double(x): return 2*x
    >>> def inc(x):    return x + 1
    >>> thread_last(1, inc, double)
    4

    If the function expects more than one input you can specify those inputs
    in a tuple.  The value is used as the last input.

    >>> def add(x, y): return x + y
    >>> def pow(x, y): return x**y
    >>> thread_last(1, (add, 4), (pow, 2))  # pow(2, add(4, 1))
    32

    So in general
        thread_last(x, f, (g, y, z))
    expands to
        g(y, z, f(x))

    >>> def iseven(x):
    ...     return x % 2 == 0
    >>> list(thread_last([1, 2, 3], (map, inc), (filter, iseven)))
    [2, 4]

    See Also:
        thread_first
    """
    def evalform_back(val, form):
        if callable(form):
            return form(val)
        if isinstance(form, tuple):
            func, args = form[0], form[1:]
            args = args + (val,)
            return func(*args)
    return reduce(evalform_back, forms, val)

def memoize(func, cache=None):
    """ Cache a function's result for speedy future evaluation

    Considerations:
        Trades memory for speed.
        Only use on pure functions.

    >>> def add(x, y):  return x + y
    >>> add = memoize(add)

    Or use as a decorator

    >>> @memoize
    ... def add(x, y):
    ...     return x + y
    """
    if cache is None:
        cache = {}

    try:
        spec = inspect.getargspec(func)
        if spec and not spec.keywords and not spec.defaults:
            may_have_kwargs = False
        else:
            may_have_kwargs = True
    except TypeError:
        may_have_kwargs = True

    def memof(*args, **kwargs):
        try:
            if may_have_kwargs:
                key = (args, frozenset(kwargs.items()))
            else:
                key = args
            in_cache = key in cache
        except TypeError:
            raise TypeError("Arguments to memoized function must be hashable")

        if in_cache:
            return cache[key]
        else:
            result = func(*args, **kwargs)
            cache[key] = result
            return result

    try:
        memof.__name__ = func.__name__
    except AttributeError:
        pass
    memof.__doc__ = func.__doc__
    return memof

def _num_required_args(func):
    """ Number of args for func

    >>> def foo(a, b, c=None):
    ...     return a + b + c

    >>> _num_required_args(foo)
    2

    >>> def bar(*args):
    ...     return sum(args)

    >>> print(_num_required_args(bar))
    None
    """
    try:
        spec = inspect.getargspec(func)
        if spec.varargs:
            return None
        num_defaults = len(spec.defaults) if spec.defaults else 0
        return len(spec.args) - num_defaults
    except TypeError:
        return None

class curry(object):
    """ Curry a callable function

    Enables partial application of arguments through calling a function with an
    incomplete set of arguments.

    >>> def mul(x, y):
    ...     return x * y
    >>> mul = curry(mul)

    >>> double = mul(2)
    >>> double(10)
    20

    Also supports keyword arguments

    >>> @curry                  # Can use curry as a decorator
    ... def f(x, y, a=10):
    ...     return a * (x + y)

    >>> add = f(a=1)
    >>> add(2, 3)
    5

    See Also:
        toolz.curried - namespace of curried functions
                        http://toolz.readthedocs.org/en/latest/curry.html
    """
    def __init__(self, func, *args, **kwargs):
        if not callable(func):
            raise TypeError("Input must be callable")

        self.func = func
        self.args = args
        self.keywords = kwargs if kwargs else None
        self.__doc__ = self.func.__doc__
        try:
            self.func_name = self.func.func_name
        except AttributeError:
            pass

    def __str__(self):
        return str(self.func)

    def __repr__(self):
        return repr(self.func)

    def __call__(self, *args, **_kwargs):
        args = self.args + args
        if _kwargs:
            kwargs = {}
            if self.keywords:
                kwargs.update(self.keywords)
            kwargs.update(_kwargs)
        elif self.keywords:
            kwargs = self.keywords
        else:
            kwargs = {}

        try:
            return self.func(*args, **kwargs)
        except TypeError as e:
            required_args = _num_required_args(self.func)

            # If there was a genuine TypeError
            if required_args is not None and len(args) >= required_args:
                raise e

            # If we only need one more argument
            if (required_args is not None and required_args - len(args) == 1):
                if kwargs:
                    return partial(self.func, *args, **kwargs)
                else:
                    return partial(self.func, *args)

            return curry(self.func, *args, **kwargs)

def compose(*funcs):
    """ Compose functions to operate in series.

    Returns a function that applies other functions in sequence.

    Functions are applied from right to left so that
    ``compose(f, g, h)(x, y)`` is the same as ``f(g(h(x, y)))``.

    If no arguments are provided, the identity function (f(x) = x) is returned.

    >>> inc = lambda i: i + 1
    >>> compose(str, inc)(3)
    '4'

    See Also:
        pipe
    """
    if not funcs:
        return identity
    if len(funcs) == 1:
        return funcs[0]
    else:
        funcs = list(reversed(funcs))

        def composed(*args, **kwargs):
            ret = funcs[0](*args, **kwargs)
            for func in funcs[1:]:
                ret = func(ret)
            return ret

        return composed

def pipe(data, *funcs):
    """ Pipe a value through a sequence of functions

    I.e. ``pipe(data, f, g, h)`` is equivalent to ``h(g(f(data)))``

    We think of the value as progressing through a pipe of several
    transformations, much like pipes in UNIX

    ``$ cat data | f | g | h``

    >>> double = lambda i: 2 * i
    >>> pipe(3, double, str)
    '6'

    See Also:
        compose
        thread_first
        thread_last
    """
    for func in funcs:
        data = func(data)
    return data

def complement(func):
    """ Convert a predicate function to its logical complement.

    In other words, return a function that, for inputs that normally
    yield True, yields False, and vice-versa.

    >>> def iseven(n): return n % 2 == 0
    >>> isodd = complement(iseven)
    >>> iseven(2)
    True
    >>> isodd(2)
    False
    """
    return compose(operator.not_, func)

def fold(binop, coll, default, map=map, chunksize=128):
    """ Reduce without guarantee of ordered reduction

    inputs:

    ``binop``     - associative operator. The associative property allows us to
                    leverage a parallel map to perform reductions in parallel.
    ``coll``      - a collection or sequence to be aggregated
    ``default``   - an identity element like 0 for ``add`` or 1 for mul

    ``map``       - an implementation of ``map``. This may be parallel and
                    determines how work is distributed.
    ``chunksize`` - Number of elements of ``coll`` that should be handled
                    within a single function call

    Fold chunks up the collection into blocks of size ``chunksize`` and then
    feeds each of these to calls to `builtin.reduce`. This work is distributed
    with a call to ``map``, gathered back and then refolded to finish the
    computation. In this way ``fold`` specifies only how to chunk up data but
    leaves the distribution of this work to an externally provided ``map``
    function. This function can be sequential or rely on multithreading,
    multiprocessing, or even distributed solutions.

    If ``map`` intends to serialize functions it should be prepared to accept
    and serialize lambdas. Note that the standard ``pickle`` module fails
    here.
    """
    from toolz import partition_all

    combine = binop    # For future generality
    coll = list(coll)  # TODO: Support laziness
    if len(coll) < chunksize:
        return reduce(binop, coll, default)
    else:
        chunks = partition_all(chunksize, coll)
        results = list(map(lambda chunk: reduce(binop, chunk, default),
                           chunks))
        return fold(combine, results, default, map=map)
<commit_msg>propagate chunksize to recursive fold call<commit_after>from functools import reduce, partial
import inspect
import operator

def identity(x):
    return x

def thread_first(val, *forms):
    """ Thread value through a sequence of functions/forms

    >>> def double(x): return 2*x
    >>> def inc(x):    return x + 1
    >>> thread_first(1, inc, double)
    4

    If the function expects more than one input you can specify those inputs
    in a tuple.  The value is used as the first input.

    >>> def add(x, y): return x + y
    >>> def pow(x, y): return x**y
    >>> thread_first(1, (add, 4), (pow, 2))  # pow(add(1, 4), 2)
    25

    So in general
        thread_first(x, f, (g, y, z))
    expands to
        g(f(x), y, z)

    See Also:
        thread_last
    """
    def evalform_front(val, form):
        if callable(form):
            return form(val)
        if isinstance(form, tuple):
            func, args = form[0], form[1:]
            args = (val,) + args
            return func(*args)
    return reduce(evalform_front, forms, val)

def thread_last(val, *forms):
    """ Thread value through a sequence of functions/forms

    >>> def double(x): return 2*x
    >>> def inc(x):    return x + 1
    >>> thread_last(1, inc, double)
    4

    If the function expects more than one input you can specify those inputs
    in a tuple.  The value is used as the last input.

    >>> def add(x, y): return x + y
    >>> def pow(x, y): return x**y
    >>> thread_last(1, (add, 4), (pow, 2))  # pow(2, add(4, 1))
    32

    So in general
        thread_last(x, f, (g, y, z))
    expands to
        g(y, z, f(x))

    >>> def iseven(x):
    ...     return x % 2 == 0
    >>> list(thread_last([1, 2, 3], (map, inc), (filter, iseven)))
    [2, 4]

    See Also:
        thread_first
    """
    def evalform_back(val, form):
        if callable(form):
            return form(val)
        if isinstance(form, tuple):
            func, args = form[0], form[1:]
            args = args + (val,)
            return func(*args)
    return reduce(evalform_back, forms, val)

def memoize(func, cache=None):
    """ Cache a function's result for speedy future evaluation

    Considerations:
        Trades memory for speed.
        Only use on pure functions.

    >>> def add(x, y):  return x + y
    >>> add = memoize(add)

    Or use as a decorator

    >>> @memoize
    ... def add(x, y):
    ...     return x + y
    """
    if cache is None:
        cache = {}

    try:
        spec = inspect.getargspec(func)
        if spec and not spec.keywords and not spec.defaults:
            may_have_kwargs = False
        else:
            may_have_kwargs = True
    except TypeError:
        may_have_kwargs = True

    def memof(*args, **kwargs):
        try:
            if may_have_kwargs:
                key = (args, frozenset(kwargs.items()))
            else:
                key = args
            in_cache = key in cache
        except TypeError:
            raise TypeError("Arguments to memoized function must be hashable")

        if in_cache:
            return cache[key]
        else:
            result = func(*args, **kwargs)
            cache[key] = result
            return result

    try:
        memof.__name__ = func.__name__
    except AttributeError:
        pass
    memof.__doc__ = func.__doc__
    return memof

def _num_required_args(func):
    """ Number of args for func

    >>> def foo(a, b, c=None):
    ...     return a + b + c

    >>> _num_required_args(foo)
    2

    >>> def bar(*args):
    ...     return sum(args)

    >>> print(_num_required_args(bar))
    None
    """
    try:
        spec = inspect.getargspec(func)
        if spec.varargs:
            return None
        num_defaults = len(spec.defaults) if spec.defaults else 0
        return len(spec.args) - num_defaults
    except TypeError:
        return None

class curry(object):
    """ Curry a callable function

    Enables partial application of arguments through calling a function with an
    incomplete set of arguments.

    >>> def mul(x, y):
    ...     return x * y
    >>> mul = curry(mul)

    >>> double = mul(2)
    >>> double(10)
    20

    Also supports keyword arguments

    >>> @curry                  # Can use curry as a decorator
    ... def f(x, y, a=10):
    ...     return a * (x + y)

    >>> add = f(a=1)
    >>> add(2, 3)
    5

    See Also:
        toolz.curried - namespace of curried functions
                        http://toolz.readthedocs.org/en/latest/curry.html
    """
    def __init__(self, func, *args, **kwargs):
        if not callable(func):
            raise TypeError("Input must be callable")

        self.func = func
        self.args = args
        self.keywords = kwargs if kwargs else None
        self.__doc__ = self.func.__doc__
        try:
            self.func_name = self.func.func_name
        except AttributeError:
            pass

    def __str__(self):
        return str(self.func)

    def __repr__(self):
        return repr(self.func)

    def __call__(self, *args, **_kwargs):
        args = self.args + args
        if _kwargs:
            kwargs = {}
            if self.keywords:
                kwargs.update(self.keywords)
            kwargs.update(_kwargs)
        elif self.keywords:
            kwargs = self.keywords
        else:
            kwargs = {}

        try:
            return self.func(*args, **kwargs)
        except TypeError as e:
            required_args = _num_required_args(self.func)

            # If there was a genuine TypeError
            if required_args is not None and len(args) >= required_args:
                raise e

            # If we only need one more argument
            if (required_args is not None and required_args - len(args) == 1):
                if kwargs:
                    return partial(self.func, *args, **kwargs)
                else:
                    return partial(self.func, *args)

            return curry(self.func, *args, **kwargs)

def compose(*funcs):
    """ Compose functions to operate in series.

    Returns a function that applies other functions in sequence.

    Functions are applied from right to left so that
    ``compose(f, g, h)(x, y)`` is the same as ``f(g(h(x, y)))``.

    If no arguments are provided, the identity function (f(x) = x) is returned.

    >>> inc = lambda i: i + 1
    >>> compose(str, inc)(3)
    '4'

    See Also:
        pipe
    """
    if not funcs:
        return identity
    if len(funcs) == 1:
        return funcs[0]
    else:
        funcs = list(reversed(funcs))

        def composed(*args, **kwargs):
            ret = funcs[0](*args, **kwargs)
            for func in funcs[1:]:
                ret = func(ret)
            return ret

        return composed

def pipe(data, *funcs):
    """ Pipe a value through a sequence of functions

    I.e. ``pipe(data, f, g, h)`` is equivalent to ``h(g(f(data)))``

    We think of the value as progressing through a pipe of several
    transformations, much like pipes in UNIX

    ``$ cat data | f | g | h``

    >>> double = lambda i: 2 * i
    >>> pipe(3, double, str)
    '6'

    See Also:
        compose
        thread_first
        thread_last
    """
    for func in funcs:
        data = func(data)
    return data

def complement(func):
    """ Convert a predicate function to its logical complement.

    In other words, return a function that, for inputs that normally
    yield True, yields False, and vice-versa.

    >>> def iseven(n): return n % 2 == 0
    >>> isodd = complement(iseven)
    >>> iseven(2)
    True
    >>> isodd(2)
    False
    """
    return compose(operator.not_, func)

def fold(binop, coll, default, map=map, chunksize=128):
    """ Reduce without guarantee of ordered reduction

    inputs:

    ``binop``     - associative operator. The associative property allows us to
                    leverage a parallel map to perform reductions in parallel.
    ``coll``      - a collection or sequence to be aggregated
    ``default``   - an identity element like 0 for ``add`` or 1 for mul

    ``map``       - an implementation of ``map``. This may be parallel and
                    determines how work is distributed.
    ``chunksize`` - Number of elements of ``coll`` that should be handled
                    within a single function call

    Fold chunks up the collection into blocks of size ``chunksize`` and then
    feeds each of these to calls to ``reduce``. This work is distributed
    with a call to ``map``, gathered back and then refolded to finish the
    computation. In this way ``fold`` specifies only how to chunk up data but
    leaves the distribution of this work to an externally provided ``map``
    function. This function can be sequential or rely on multithreading,
    multiprocessing, or even distributed solutions.

    If ``map`` intends to serialize functions it should be prepared to accept
    and serialize lambdas. Note that the standard ``pickle`` module fails
    here.
    """
    from toolz import partition_all

    combine = binop    # For future generality
    coll = list(coll)  # TODO: Support laziness
    if len(coll) < chunksize:
        return reduce(binop, coll, default)
    else:
        chunks = partition_all(chunksize, coll)
        results = list(map(lambda chunk: reduce(binop, chunk, default),
                           chunks))
        return fold(combine, results, default, map=map, chunksize=chunksize)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from django.shortcuts import render
from django.conf import settings
from django.contrib import messages
from django.core.mail import send_mail, BadHeaderError
from django.http import HttpResponse
from .forms import ContactForm

# check if form data is valid
def contactview(request):
    if request.method == 'GET':
        form = ContactForm()
    else:
        form = ContactForm(request.POST)
        if form.is_valid():
            name = form.cleaned_data['name']
            email = form.cleaned_data['email']
            message = form.cleaned_data['message']
            organisation = form.cleaned_data['organisation']
            type = form.cleaned_data['type']

            # collate and send filled form data in one email
            message_from = 'Name: ' + name + '\n' + 'From: ' + organisation
            message_details = 'Email: ' + email + '\n' + 'Message: ' + message
            email_message = message_from + '\n' + message_details
            email_subject = type + ' Enquiry from ' + name

            # specify where form data is sent, depending on the type of enquiry
            if type == 'Service':
                recepients = settings.SERVICE_EMAIL_RECEPIENTS
            else:
                recepients = settings.PRESS_EMAIL_RECEPIENTS

            # flatten recepients list:
            # (('name1', 'email1'), ...) -> ('email1', ...)
            recepients = map(lambda recepient: recepient[1], recepients)

            # all form data will originate from the same email address
            try:
                send_mail(
                    email_subject, email_message,
                    settings.CONTACT_EMAIL_SENDER, recepients)
            except BadHeaderError:
                return HttpResponse('Invalid header found.')

            # pop up message once form data has been sent successfully
            messages.info(
                request,
                'Thank you for your message. ' +
                'Someone from Open Knowledge Foundation' +
                ' will be in touch soon.')

    # reload the contact page after form data has been sent successfully
    return render(request, "cms_contact.html", {'form': form})
<commit_msg>drop unicode_literals import<commit_after>from django.shortcuts import render
from django.conf import settings
from django.contrib import messages
from django.core.mail import send_mail, BadHeaderError
from django.http import HttpResponse
from .forms import ContactForm

# check if form data is valid
def contactview(request):
    if request.method == 'GET':
        form = ContactForm()
    else:
        form = ContactForm(request.POST)
        if form.is_valid():
            name = form.cleaned_data['name']
            email = form.cleaned_data['email']
            message = form.cleaned_data['message']
            organisation = form.cleaned_data['organisation']
            type = form.cleaned_data['type']

            # collate and send filled form data in one email
            message_from = 'Name: ' + name + '\n' + 'From: ' + organisation
            message_details = 'Email: ' + email + '\n' + 'Message: ' + message
            email_message = message_from + '\n' + message_details
            email_subject = type + ' Enquiry from ' + name

            # specify where form data is sent, depending on the type of enquiry
            if type == 'Service':
                recepients = settings.SERVICE_EMAIL_RECEPIENTS
            else:
                recepients = settings.PRESS_EMAIL_RECEPIENTS

            # flatten recepients list:
            # (('name1', 'email1'), ...) -> ('email1', ...)
            recepients = map(lambda recepient: recepient[1], recepients)

            # all form data will originate from the same email address
            try:
                send_mail(
                    email_subject, email_message,
                    settings.CONTACT_EMAIL_SENDER, recepients)
            except BadHeaderError:
                return HttpResponse('Invalid header found.')

            # pop up message once form data has been sent successfully
            messages.info(
                request,
                'Thank you for your message. ' +
                'Someone from Open Knowledge Foundation' +
                ' will be in touch soon.')

    # reload the contact page after form data has been sent successfully
    return render(request, "cms_contact.html", {'form': form})
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2010  Romain Bignon, Christophe Benz
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

from __future__ import with_statement

from threading import Timer, Event, RLock, _Timer
from weboob.tools.log import getLogger

__all__ = ['Scheduler']

class IScheduler(object):
    def schedule(self, interval, function, *args):
        raise NotImplementedError()

    def repeat(self, interval, function, *args):
        raise NotImplementedError()

    def cancel(self, ev):
        raise NotImplementedError()

    def run(self):
        raise NotImplementedError()

    def want_stop(self):
        raise NotImplementedError()

class RepeatedTimer(_Timer):
    def run(self):
        while not self.finished.is_set():
            self.function(*self.args, **self.kwargs)
            self.finished.wait(self.interval)
        self.finished.set()

class Scheduler(IScheduler):
    def __init__(self):
        self.logger = getLogger('scheduler')
        self.mutex = RLock()
        self.stop_event = Event()
        self.count = 0
        self.queue = {}

    def schedule(self, interval, function, *args):
        return self._schedule(Timer, interval, self._schedule_callback, function, *args)

    def repeat(self, interval, function, *args):
        return self._schedule(RepeatedTimer, interval, self._repeat_callback, function, *args)

    def _schedule(self, klass, interval, meta_func, function, *args):
        if self.stop_event.is_set():
            return

        with self.mutex:
            self.count += 1
            self.logger.debug('function "%s" will be called in %s seconds' % (function.__name__, interval))
            timer = klass(interval, meta_func, (self.count, interval, function, args))
            self.queue[self.count] = timer
            timer.start()
            return self.count

    def _schedule_callback(self, count, interval, function, args):
        with self.mutex:
            self.queue.pop(count)
        return function(*args)

    def _repeat_callback(self, count, interval, function, args):
        function(*args)
        with self.mutex:
            try:
                e = self.queue[count]
            except KeyError:
                return
            else:
                self.logger.debug('function "%s" will be called in %s seconds' % (function.__name__, e.interval))

    def cancel(self, ev):
        with self.mutex:
            try:
                e = self.queue.pop(ev)
            except KeyError:
                return False
            e.cancel()
            self.logger.debug('scheduled function "%s" is canceled' % e.function.__name__)
            return True

    def _wait_to_stop(self):
        self.want_stop()
        with self.mutex:
            for e in self.queue.itervalues():
                e.cancel()
                e.join()
            self.queue = {}

    def run(self):
        try:
            while 1:
                self.stop_event.wait(0.1)
        except KeyboardInterrupt:
            self._wait_to_stop()
            raise
        else:
            self._wait_to_stop()
        return True

    def want_stop(self):
        self.stop_event.set()
        with self.mutex:
            for t in self.queue.itervalues():
                t.cancel()
                # Contrary to _wait_to_stop(), don't call t.join
                # because want_stop() have to be non-blocking.
            self.queue = {}
<commit_msg>python2.5 compatibility<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2010  Romain Bignon, Christophe Benz
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

from __future__ import with_statement

from threading import Timer, Event, RLock, _Timer
from weboob.tools.log import getLogger

__all__ = ['Scheduler']

class IScheduler(object):
    def schedule(self, interval, function, *args):
        raise NotImplementedError()

    def repeat(self, interval, function, *args):
        raise NotImplementedError()

    def cancel(self, ev):
        raise NotImplementedError()

    def run(self):
        raise NotImplementedError()

    def want_stop(self):
        raise NotImplementedError()

class RepeatedTimer(_Timer):
    def run(self):
        while not self.finished.isSet():
            self.function(*self.args, **self.kwargs)
            self.finished.wait(self.interval)
        self.finished.set()

class Scheduler(IScheduler):
    def __init__(self):
        self.logger = getLogger('scheduler')
        self.mutex = RLock()
        self.stop_event = Event()
        self.count = 0
        self.queue = {}

    def schedule(self, interval, function, *args):
        return self._schedule(Timer, interval, self._schedule_callback, function, *args)

    def repeat(self, interval, function, *args):
        return self._schedule(RepeatedTimer, interval, self._repeat_callback, function, *args)

    def _schedule(self, klass, interval, meta_func, function, *args):
        if self.stop_event.isSet():
            return

        with self.mutex:
            self.count += 1
            self.logger.debug('function "%s" will be called in %s seconds' % (function.__name__, interval))
            timer = klass(interval, meta_func, (self.count, interval, function, args))
            self.queue[self.count] = timer
            timer.start()
            return self.count

    def _schedule_callback(self, count, interval, function, args):
        with self.mutex:
            self.queue.pop(count)
        return function(*args)

    def _repeat_callback(self, count, interval, function, args):
        function(*args)
        with self.mutex:
            try:
                e = self.queue[count]
            except KeyError:
                return
            else:
                self.logger.debug('function "%s" will be called in %s seconds' % (function.__name__, e.interval))

    def cancel(self, ev):
        with self.mutex:
            try:
                e = self.queue.pop(ev)
            except KeyError:
                return False
            e.cancel()
            self.logger.debug('scheduled function "%s" is canceled' % e.function.__name__)
            return True

    def _wait_to_stop(self):
        self.want_stop()
        with self.mutex:
            for e in self.queue.itervalues():
                e.cancel()
                e.join()
            self.queue = {}

    def run(self):
        try:
            while 1:
                self.stop_event.wait(0.1)
        except KeyboardInterrupt:
            self._wait_to_stop()
            raise
        else:
            self._wait_to_stop()
        return True

    def want_stop(self):
        self.stop_event.set()
        with self.mutex:
            for t in self.queue.itervalues():
                t.cancel()
                # Contrary to _wait_to_stop(), don't call t.join
                # because want_stop() have to be non-blocking.
            self.queue = {}
<|endoftext|>


<commit_before>#
# Copyright 2014 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""

Risk Report
===========

    +-----------------+----------------------------------------------------+
    | key             | value                                              |
    +=================+====================================================+
    | trading_days    | The number of trading days between self.start_date |
    |                 | and self.end_date                                  |
    +-----------------+----------------------------------------------------+
    | benchmark_volat\| The volatility of the benchmark between            |
    | ility           | self.start_date and self.end_date.                 |
    +-----------------+----------------------------------------------------+
    | algo_volatility | The volatility of the algo between self.start_date |
    |                 | and self.end_date.                                 |
    +-----------------+----------------------------------------------------+
    | treasury_period\| The return of treasuries over the period. Treasury |
    | _return         | maturity is chosen to match the duration of the    |
    |                 | test period.                                       |
    +-----------------+----------------------------------------------------+
    | sharpe          | The sharpe ratio based on the _algorithm_ (rather  |
    |                 | than the static portfolio) returns.                |
    +-----------------+----------------------------------------------------+
    | information     | The information ratio based on the _algorithm_     |
    |                 | (rather than the static portfolio) returns.        |
    +-----------------+----------------------------------------------------+
    | beta            | The _algorithm_ beta to the benchmark.             |
    +-----------------+----------------------------------------------------+
    | alpha           | The _algorithm_ alpha to the benchmark.            |
    +-----------------+----------------------------------------------------+
    | excess_return   | The excess return of the algorithm over the        |
    |                 | treasuries.                                        |
    +-----------------+----------------------------------------------------+
    | max_drawdown    | The largest relative peak to relative trough move  |
    |                 | for the portfolio returns between self.start_date  |
    |                 | and self.end_date.                                 |
    +-----------------+----------------------------------------------------+

"""

import logbook
import math
import numpy as np

from zipline.finance import trading
import zipline.utils.math_utils as zp_math

log = logbook.Logger('Risk')

TREASURY_DURATIONS = [
    '1month', '3month', '6month',
    '1year', '2year', '3year', '5year',
    '7year', '10year', '30year'
]

# check if a field in rval is nan, and replace it with
# None.
def check_entry(key, value):
    if key != 'period_label':
        return np.isnan(value) or np.isinf(value)
    else:
        return False

############################
# Risk Metric Calculations #
############################

def sharpe_ratio(algorithm_volatility, algorithm_return, treasury_return):
    """
    http://en.wikipedia.org/wiki/Sharpe_ratio

    Args:
        algorithm_volatility (float): Algorithm volatility.
        algorithm_return (float): Algorithm return percentage.
        treasury_return (float): Treasury return percentage.

    Returns:
        float. The Sharpe ratio.
    """
    if zp_math.tolerant_equals(algorithm_volatility, 0):
        return np.nan

    return (algorithm_return - treasury_return) / algorithm_volatility

def downside_risk(algorithm_returns, mean_returns, normalization_factor):
    rets = algorithm_returns.round(8)
    mar = mean_returns.round(8)
    downside_diff = (rets[rets < mar] - mar[rets < mar])
    if len(downside_diff) <= 1:
        return 0.0
    return np.std(downside_diff, ddof=1) * math.sqrt(normalization_factor)

def sortino_ratio(algorithm_period_return, treasury_period_return, mar):
    """
    http://en.wikipedia.org/wiki/Sortino_ratio

    Args:
        algorithm_returns (np.array-like):
            Returns from algorithm lifetime.
        algorithm_period_return (float):
            Algorithm return percentage from latest period.
        mar (float): Minimum acceptable return.

    Returns:
        float. The Sortino ratio.
    """
    if zp_math.tolerant_equals(mar, 0):
        return 0.0

    return (algorithm_period_return - treasury_period_return) / mar

def information_ratio(algorithm_returns, benchmark_returns):
    """
    http://en.wikipedia.org/wiki/Information_ratio

    Args:
        algorithm_returns (np.array-like):
            All returns during algorithm lifetime.
        benchmark_returns (np.array-like):
            All benchmark returns during algo lifetime.

    Returns:
        float. Information ratio.
    """
    relative_returns = algorithm_returns - benchmark_returns

    relative_deviation = relative_returns.std(ddof=1)

    if (
        zp_math.tolerant_equals(relative_deviation, 0)
        or
        np.isnan(relative_deviation)
    ):
        return 0.0

    return np.mean(relative_returns) / relative_deviation

def alpha(algorithm_period_return, treasury_period_return,
          benchmark_period_returns, beta):
    """
    http://en.wikipedia.org/wiki/Alpha_(investment)

    Args:
        algorithm_period_return (float):
            Return percentage from algorithm period.
        treasury_period_return (float):
            Return percentage for treasury period.
        benchmark_period_return (float):
            Return percentage for benchmark period.
        beta (float):
            beta value for the same period as all other values

    Returns:
        float. The alpha of the algorithm.
    """
    return algorithm_period_return - \
        (treasury_period_return + beta *
         (benchmark_period_returns - treasury_period_return))

###########################
# End Risk Metric Section #
###########################

def get_treasury_rate(treasury_curves, treasury_duration, day):
    rate = None

    curve = treasury_curves.ix[day]
    # 1month note data begins in 8/2001,
    # so we can use 3month instead.
    idx = TREASURY_DURATIONS.index(treasury_duration)
    for duration in TREASURY_DURATIONS[idx:]:
        rate = curve[duration]
        if rate is not None:
            break

    return rate

def search_day_distance(end_date, dt):
    tdd = trading.environment.trading_day_distance(dt, end_date)
    if tdd is None:
        return None
    assert tdd >= 0
    return tdd

def select_treasury_duration(start_date, end_date):
    td = end_date - start_date
    if td.days <= 31:
        treasury_duration = '1month'
    elif td.days <= 93:
        treasury_duration = '3month'
    elif td.days <= 186:
        treasury_duration = '6month'
    elif td.days <= 366:
        treasury_duration = '1year'
    elif td.days <= 365 * 2 + 1:
        treasury_duration = '2year'
    elif td.days <= 365 * 3 + 1:
        treasury_duration = '3year'
    elif td.days <= 365 * 5 + 2:
        treasury_duration = '5year'
    elif td.days <= 365 * 7 + 2:
        treasury_duration = '7year'
    elif td.days <= 365 * 10 + 2:
        treasury_duration = '10year'
    else:
        treasury_duration = '30year'

    return treasury_duration

def choose_treasury(select_treasury, treasury_curves, start_date, end_date,
                    compound=True):
    treasury_duration = select_treasury(start_date, end_date)
    end_day = end_date.replace(hour=0, minute=0, second=0, microsecond=0)
    search_day = None

    if end_day in treasury_curves.index:
        rate = get_treasury_rate(treasury_curves,
                                 treasury_duration,
                                 end_day)
        if rate is not None:
            search_day = end_day

    if not search_day:
        # in case end date is not a trading day or there is no treasury
        # data, search for the previous day with an interest rate.
        search_days = treasury_curves.index

        # Find rightmost value less than or equal to end_day
        i = search_days.searchsorted(end_day)
        for prev_day in search_days[i - 1::-1]:
            rate = get_treasury_rate(treasury_curves,
                                     treasury_duration,
                                     prev_day)
            if rate is not None:
                search_day = prev_day
                search_dist = search_day_distance(end_date, prev_day)
                break

        if search_day:
            if (search_dist is None or search_dist > 1) and \
                    search_days[0] <= end_day <= search_days[-1]:
                message = "No rate within 1 trading day of end date = \
{dt} and term = {term}. Using {search_day}. Check that date doesn't exceed \
treasury history range."
                message = message.format(dt=end_date,
                                         term=treasury_duration,
                                         search_day=search_day)
                log.warn(message)

    if search_day:
        td = end_date - start_date
        if compound:
            return rate * (td.days + 1) / 365
        else:
            return rate

    message = "No rate for end date = {dt} and term = {term}. Check \
that date doesn't exceed treasury history range."
    message = message.format(
        dt=end_date,
        term=treasury_duration
    )
    raise Exception(message)
<commit_msg>PERF: Reduce number of times the downside mask is created.<commit_after>#
# Copyright 2014 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""

Risk Report
===========

    +-----------------+----------------------------------------------------+
    | key             | value                                              |
    +=================+====================================================+
    | trading_days    | The number of trading days between self.start_date |
    |                 | and self.end_date                                  |
    +-----------------+----------------------------------------------------+
    | benchmark_volat\| The volatility of the benchmark between            |
    | ility           | self.start_date and self.end_date.                 |
    +-----------------+----------------------------------------------------+
    | algo_volatility | The volatility of the algo between self.start_date |
    |                 | and self.end_date.                                 |
    +-----------------+----------------------------------------------------+
    | treasury_period\| The return of treasuries over the period. Treasury |
    | _return         | maturity is chosen to match the duration of the    |
    |                 | test period.                                       |
    +-----------------+----------------------------------------------------+
    | sharpe          | The sharpe ratio based on the _algorithm_ (rather  |
    |                 | than the static portfolio) returns.                |
    +-----------------+----------------------------------------------------+
    | information     | The information ratio based on the _algorithm_     |
    |                 | (rather than the static portfolio) returns.        |
    +-----------------+----------------------------------------------------+
    | beta            | The _algorithm_ beta to the benchmark.             |
    +-----------------+----------------------------------------------------+
    | alpha           | The _algorithm_ alpha to the benchmark.            |
    +-----------------+----------------------------------------------------+
    | excess_return   | The excess return of the algorithm over the        |
    |                 | treasuries.                                        |
    +-----------------+----------------------------------------------------+
    | max_drawdown    | The largest relative peak to relative trough move  |
    |                 | for the portfolio returns between self.start_date  |
    |                 | and self.end_date.                                 |
    +-----------------+----------------------------------------------------+

"""

import logbook
import math
import numpy as np

from zipline.finance import trading
import zipline.utils.math_utils as zp_math

log = logbook.Logger('Risk')

TREASURY_DURATIONS = [
    '1month', '3month', '6month',
    '1year', '2year', '3year', '5year',
    '7year', '10year', '30year'
]

# check if a field in rval is nan, and replace it with
# None.
def check_entry(key, value):
    if key != 'period_label':
        return np.isnan(value) or np.isinf(value)
    else:
        return False

############################
# Risk Metric Calculations #
############################

def sharpe_ratio(algorithm_volatility, algorithm_return, treasury_return):
    """
    http://en.wikipedia.org/wiki/Sharpe_ratio

    Args:
        algorithm_volatility (float): Algorithm volatility.
        algorithm_return (float): Algorithm return percentage.
        treasury_return (float): Treasury return percentage.

    Returns:
        float. The Sharpe ratio.
    """
    if zp_math.tolerant_equals(algorithm_volatility, 0):
        return np.nan

    return (algorithm_return - treasury_return) / algorithm_volatility

def downside_risk(algorithm_returns, mean_returns, normalization_factor):
    rets = algorithm_returns.round(8)
    mar = mean_returns.round(8)
    mask = rets < mar
    downside_diff = rets[mask] - mar[mask]
    if len(downside_diff) <= 1:
        return 0.0
    return np.std(downside_diff, ddof=1) * math.sqrt(normalization_factor)

def sortino_ratio(algorithm_period_return, treasury_period_return, mar):
    """
    http://en.wikipedia.org/wiki/Sortino_ratio

    Args:
        algorithm_returns (np.array-like):
            Returns from algorithm lifetime.
        algorithm_period_return (float):
            Algorithm return percentage from latest period.
        mar (float): Minimum acceptable return.

    Returns:
        float. The Sortino ratio.
    """
    if zp_math.tolerant_equals(mar, 0):
        return 0.0

    return (algorithm_period_return - treasury_period_return) / mar

def information_ratio(algorithm_returns, benchmark_returns):
    """
    http://en.wikipedia.org/wiki/Information_ratio

    Args:
        algorithm_returns (np.array-like):
            All returns during algorithm lifetime.
        benchmark_returns (np.array-like):
            All benchmark returns during algo lifetime.

    Returns:
        float. Information ratio.
    """
    relative_returns = algorithm_returns - benchmark_returns

    relative_deviation = relative_returns.std(ddof=1)

    if (
        zp_math.tolerant_equals(relative_deviation, 0)
        or
        np.isnan(relative_deviation)
    ):
        return 0.0

    return np.mean(relative_returns) / relative_deviation

def alpha(algorithm_period_return, treasury_period_return,
          benchmark_period_returns, beta):
    """
    http://en.wikipedia.org/wiki/Alpha_(investment)

    Args:
        algorithm_period_return (float):
            Return percentage from algorithm period.
        treasury_period_return (float):
            Return percentage for treasury period.
        benchmark_period_return (float):
            Return percentage for benchmark period.
        beta (float):
            beta value for the same period as all other values

    Returns:
        float. The alpha of the algorithm.
    """
    return algorithm_period_return - \
        (treasury_period_return + beta *
         (benchmark_period_returns - treasury_period_return))

###########################
# End Risk Metric Section #
###########################

def get_treasury_rate(treasury_curves, treasury_duration, day):
    rate = None

    curve = treasury_curves.ix[day]
    # 1month note data begins in 8/2001,
    # so we can use 3month instead.
    idx = TREASURY_DURATIONS.index(treasury_duration)
    for duration in TREASURY_DURATIONS[idx:]:
        rate = curve[duration]
        if rate is not None:
            break

    return rate

def search_day_distance(end_date, dt):
    tdd = trading.environment.trading_day_distance(dt, end_date)
    if tdd is None:
        return None
    assert tdd >= 0
    return tdd

def select_treasury_duration(start_date, end_date):
    td = end_date - start_date
    if td.days <= 31:
        treasury_duration = '1month'
    elif td.days <= 93:
        treasury_duration = '3month'
    elif td.days <= 186:
        treasury_duration = '6month'
    elif td.days <= 366:
        treasury_duration = '1year'
    elif td.days <= 365 * 2 + 1:
        treasury_duration = '2year'
    elif td.days <= 365 * 3 + 1:
        treasury_duration = '3year'
    elif td.days <= 365 * 5 + 2:
        treasury_duration = '5year'
    elif td.days <= 365 * 7 + 2:
        treasury_duration = '7year'
    elif td.days <= 365 * 10 + 2:
        treasury_duration = '10year'
    else:
        treasury_duration = '30year'

    return treasury_duration

def choose_treasury(select_treasury, treasury_curves, start_date, end_date,
                    compound=True):
    treasury_duration = select_treasury(start_date, end_date)
    end_day = end_date.replace(hour=0, minute=0, second=0, microsecond=0)
    search_day = None

    if end_day in treasury_curves.index:
        rate = get_treasury_rate(treasury_curves,
                                 treasury_duration,
                                 end_day)
        if rate is not None:
            search_day = end_day

    if not search_day:
        # in case end date is not a trading day or there is no treasury
        # data, search for the previous day with an interest rate.
        search_days = treasury_curves.index

        # Find rightmost value less than or equal to end_day
        i = search_days.searchsorted(end_day)
        for prev_day in search_days[i - 1::-1]:
            rate = get_treasury_rate(treasury_curves,
                                     treasury_duration,
                                     prev_day)
            if rate is not None:
                search_day = prev_day
                search_dist = search_day_distance(end_date, prev_day)
                break

        if search_day:
            if (search_dist is None or search_dist > 1) and \
                    search_days[0] <= end_day <= search_days[-1]:
                message = "No rate within 1 trading day of end date = \
{dt} and term = {term}. Using {search_day}. Check that date doesn't exceed \
treasury history range."
                message = message.format(dt=end_date,
                                         term=treasury_duration,
                                         search_day=search_day)
                log.warn(message)

    if search_day:
        td = end_date - start_date
        if compound:
            return rate * (td.days + 1) / 365
        else:
            return rate

    message = "No rate for end date = {dt} and term = {term}. Check \
that date doesn't exceed treasury history range."
    message = message.format(
        dt=end_date,
        term=treasury_duration
    )
    raise Exception(message)
<|endoftext|>


<commit_before># Copyright (c) 2012 Peter Kuma
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from django.template import Context, RequestContext, loader
from django.http import HttpResponse, HttpResponseNotFound, \
                        HttpResponseForbidden, HttpResponseRedirect, \
                        HttpResponseServerError, HttpResponseBadRequest
from django.shortcuts import render_to_response
from django.views.decorators.cache import never_cache
from django.utils.translation import ugettext_lazy as _
from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from django.shortcuts import render_to_response, get_object_or_404
from django.http import Http404
from django.conf import settings
from django.core.servers.basehttp import FileWrapper

from datetime import datetime as dt
import os
import json
import urllib
import mimetypes
import time
import binascii

from models import *

class JSONEncoder(json.JSONEncoder):
    def __init__(self, **kwargs):
        defaults = {
            "sort_keys": True,
            "indent": 4,
        }
        defaults.update(**kwargs)
        return json.JSONEncoder.__init__(self, **defaults)

    def default(self, obj):
        if isinstance(obj, dt):
            return obj.isoformat()
        return json.JSONEncoder.default(self, obj)

def require_store(view):
    def store_wrapper(*args, **kwargs):
        if not kwargs.has_key("store_path"):
            raise Http404()
        store_path = kwargs.pop("store_path")
        if  len(store_path) > 0 and store_path[-1] == "/":
            store_path = store_path[:-1]
        store = get_object_or_404(Store, path=store_path)
        return view(*args, store=store, **kwargs)

    return store_wrapper

def require_login(view):
    def login_wrapper(request, *args, **kwargs):
        store = kwargs.get("store")
        if not store: return Http404()

        if store.accesscode == "":
            return view(request, *args, **kwargs)

        if request.session.has_key("fileshack_stores") and \
           store.id in request.session["fileshack_stores"]:
            return view(request, *args, **kwargs)

        return HttpResponseNotAllowed()

    return login_wrapper

@require_store
def logout(request, store):
    try:
        request.session["fileshack_stores"].remove(store.id)
    except (KeyError, ValueError):
        pass
    request.session.save()
    return HttpResponseRedirect(store.get_absolute_url())

@require_store
def index(request, store):
    if (not request.session.has_key("fileshack_stores") or \
        not store.id in request.session["fileshack_stores"]) \
        and store.accesscode != "":

        if request.method == "POST":
            accesscode = request.POST.get("accesscode")
            if accesscode != "" and store.accesscode == accesscode:
                if not request.session.has_key("fileshack_stores"):
                    request.session["fileshack_stores"] = [store.id]
                else:
                   request.session["fileshack_stores"].append(store.id)
                request.session.set_expiry(3600)
                return HttpResponseRedirect(store.get_absolute_url())
            else:
                t = loader.get_template("fileshack/accesscode.html")
                c = RequestContext(request, {
                    "accesscode": accesscode,
                    "error_label": _("Wrong access code"),
                    "error_message": _("Please try again."),
                })
                return HttpResponse(t.render(c))
        else:
            t = loader.get_template("fileshack/accesscode.html")
            c = RequestContext(request)
            return HttpResponse(t.render(c))

    items = Item.objects.filter(store=store)
    t = loader.get_template("fileshack/index.html")
    c = RequestContext(request, {
        "store": store,
        "items": items,
        "item_size_limit": store.item_limit,
        "items_json": JSONEncoder().encode([i.simple() for i in items])
    })
    return HttpResponse(t.render(c))

@require_store
@require_login
def iframe(request, store):
    if request.method == "POST" and request.FILES.has_key("file"):
        f = request.FILES["file"]
        item = Item()
        item.store = store
        item.fileobject.save(urllib.unquote(f.name), f)
        item.size = f.size
        item.size_total = f.size
        item.save()

    t = loader.get_template("fileshack/iframe.html")
    c = RequestContext(request)
    return HttpResponse(t.render(c))

@never_cache
@require_store
@require_login
def upload(request, store, id):
    if request.method != "POST" or not request.FILES.has_key("file"):
        data = {
            "status": "failed",
            "error_label": "Upload failed",
            "error_message": "Invalid HTTP request",
        }
        return HttpResponseBadRequest(JSONEncoder().encode(data))

    if request.FILES.has_key("file"):
        f = request.FILES["file"]
        name = urllib.unquote(f.name)
        try: size_total = int(request.META["HTTP_X_FILE_SIZE"])
        except (ValueError, KeyError): size_total = f.size
    else:
        name = ''
        size_total = 0 # Unknown.

    try: name = request.META["HTTP_X_FILE_NAME"]
    except KeyError: name = ''

    name = os.path.basename(name)

    try: offset = int(request.META["HTTP_X_FILE_OFFSET"])
    except (ValueError, KeyError): offset = 0

    if store.item_limit and size_total and size_total > store.item_limit*1024*1024:
        data = {
            "status": "itemlimitreached",
            "error_label": "Upload failed",
            "error_message": "Item size is limited to %d MB" % store.item_limit,
            "item": None,
        }
        return HttpResponseServerError(JSONEncoder().encode(data))

    if store.store_limit and size_total and store.total() + size_total > store.store_limit*1024*1024:
        data = {
            "status": "storelimitreached",
            "error_label": "Upload failed",
            "error_message": "The store size limit of %d MB has been reached" % store.store_limit,
            "item": None,
        }
        return HttpResponseServerError(JSONEncoder().encode(data))

    # If the item exists, open the file for append.
    try:
        try: id = int(id)
        except ValueError: raise Item.DoesNotExist
        item = Item.objects.get(pk=id)
        if item.fileobject.size < offset:
            data = {
                "status": "outoforder",
                "error_label": "Chunk out of order",
                "error_message": "Application sent a chunk out of order",
                "item": item.simple(),
            }
            return HttpResponseServerError(JSONEncoder().encode(data))
        fp = default_storage.open(item.fileobject.path, "ab")
        fp.truncate(offset)

    # This is a new item.
    except Item.DoesNotExist:
        if offset != 0:
            data = {
                "status": "outoforder",
                "error_label": "Chunk out of order",
                "error_message": "Application sent a chunk of an item that does not exist",
                "item": None,
            }
            return HttpResponseServerError(JSONEncoder().encode(data))
        item = Item()
        item.store = store
        item.fileobject.save(name, ContentFile(""))
        item.fileobject.close()
        item.size_total = size_total
        item.save()
        fp = default_storage.open(item.fileobject.path, "wb")

    chunks = f.chunks().__iter__()
    while True:
        try: chunk = chunks.next()
        except StopIteration: break
        except IOError:
            fp.close()
            data = {
                "status": "failed",
                "error_label": "Upload failed",
                "error_message": "Server-side I/O error",
                "item": item.simple(),
            }
            return HttpResponseServerError(JSONEncoder().encode(data))
        else:
            try:
                if request.META.get("HTTP_X_FILE_ENCODING") == "base64":
                    fp.write(chunk.decode("base64"))
                else:
                    fp.write(chunk)
            except binascii.Error:
                fp.close()
                data = {
                    "status": "failed",
                    "error_label": "Upload failed",
                    "error_message": "The browser sent an invalid chunk",
                    "item": item.simple(),
                }
                return HttpResponseServerError(JSONEncoder().encode(data))

    item.size = fp.tell()
    fp.close()

    if item.size_total < item.size:
        item.size_total = item.size

    if item.size >= item.size_total:
        item.uploaded = dt.now()

    item.save()
    data = {
        "status": "success",
        "item": Item.objects.get(pk=item.pk).simple()
    }
    return HttpResponse(JSONEncoder().encode(data))

@require_store
@require_login
def simple_upload(request, store):
    if request.method != "POST" or not request.FILES.has_key("file"):
        return HttpResponseRedirect(store.get_absolute_url())

    #if store.item_limit and f.size > store.item_limit*1024*1024:
    #if store.store_limit and store.total() + f.size > store.store_limit*1024*1024:

    f = request.FILES["file"]
    item = Item()
    item.store = store
    item.fileobject.save(urllib.unquote(f.name), f)
    item.size = f.size
    item.size_total = f.size
    item.save()
    return HttpResponseRedirect(store.get_absolute_url())

@require_store
@require_login
def delete(request, store, item_id):
    if request.method != "POST":
        return HttpResponseForbidden()

    item = get_object_or_404(Item, pk=item_id, store=store)
    item.delete()

    return HttpResponse("Item has been deleted")

@require_store
@require_login
def update(request, store, since=None):
    since_dt = None
    if since != None:
        try:
            since_dt = dt.strptime(since, "%Y-%m-%d_%H:%M:%S")
        except ValueError:
            pass

    all_items = Item.objects.filter(store=store)
    item_ids = [item.id for item in all_items]

    if since_dt != None:
        items = Item.objects.filter(store=store, modified__gt=since_dt)
    else:
        items = all_items

    items_simple = []
    for item in items:
        items_simple.append(item.simple())

    dthandler = lambda obj: obj.isoformat() if isinstance(obj, datetime.datetime) else None
    data = JSONEncoder(sort_keys=True, indent=4).encode(dict(
            time=dt.now().strftime("%Y-%m-%d_%H:%M:%S"),
            item_ids=item_ids, items=items_simple))
    return HttpResponse(data)

class ItemFileWrapper(FileWrapper):
    def __init__(self, item, *args, **kwargs):
        self._item = item
        self._counter = 0
        self._stale = 0
        self._throttle = kwargs.get("throttle", 100)
        self._stale_limit = kwargs.get("stale_limit", 10000)
        return FileWrapper.__init__(self, *args, **kwargs)
    def next(self):
        try:
            data = FileWrapper.next(self)
            self._counter += len(data)
            if len(data) > 0:
                self._stale = 0
            return data
        except StopIteration:
            if self._counter >= self._item.size_total:
                raise StopIteration
            if self._stale_limit and self._stale >= self._stale_limit:
                raise StopIteration
            start = time.time()
            time.sleep(self._throttle/1000)
            end = time.time()
            self._stale += (end - start)*1000
            return ""

@require_store
@require_login
def download(request, store, item_id):
    item = get_object_or_404(Item, pk=item_id)

    if item.status() == "READY":
        return HttpResponseRedirect(item.get_absolute_url())

    f = default_storage.open(item.fileobject.path, "rb")
    wrapper = ItemFileWrapper(item, f)
    response = HttpResponse(wrapper, content_type=mimetypes.guess_type(item.fileobject.name))
    response["Content-Length"] = "%d" % item.size_total
    response["Content-Disposition"] = 'attachment; name="file"; filename="%s"' % urllib.quote(item.name())
    return response

def page_not_found(request):
    stores = Store.objects.all()
    store = None
    for s in stores:
        if request.path.startswith(s.get_absolute_url()):
            store = s
    if not store:
        try: store = Store.objects.get(path="")
        except Store.DoesNotExist: pass

    t = loader.get_template("fileshack/404.html")
    return HttpResponseNotFound(t.render(RequestContext(request, {
        "request_path": request.path,
        "store": store,
        "stores_number": len(stores),
        "admin_url": reverse("admin:index"),
    })))

def server_error(request):
    t = loader.get_template("fileshack/500.html")
    return HttpResponseServerError(t.render(RequestContext(request, { "request_path": request.path, })))
<commit_msg>Prevent browsers from caching the update view.<commit_after># Copyright (c) 2012 Peter Kuma
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from django.template import Context, RequestContext, loader
from django.http import HttpResponse, HttpResponseNotFound, \
                        HttpResponseForbidden, HttpResponseRedirect, \
                        HttpResponseServerError, HttpResponseBadRequest
from django.shortcuts import render_to_response
from django.views.decorators.cache import never_cache
from django.utils.translation import ugettext_lazy as _
from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from django.shortcuts import render_to_response, get_object_or_404
from django.http import Http404
from django.conf import settings
from django.core.servers.basehttp import FileWrapper

from datetime import datetime as dt
import os
import json
import urllib
import mimetypes
import time
import binascii

from models import *

class JSONEncoder(json.JSONEncoder):
    def __init__(self, **kwargs):
        defaults = {
            "sort_keys": True,
            "indent": 4,
        }
        defaults.update(**kwargs)
        return json.JSONEncoder.__init__(self, **defaults)

    def default(self, obj):
        if isinstance(obj, dt):
            return obj.isoformat()
        return json.JSONEncoder.default(self, obj)

def require_store(view):
    def store_wrapper(*args, **kwargs):
        if not kwargs.has_key("store_path"):
            raise Http404()
        store_path = kwargs.pop("store_path")
        if  len(store_path) > 0 and store_path[-1] == "/":
            store_path = store_path[:-1]
        store = get_object_or_404(Store, path=store_path)
        return view(*args, store=store, **kwargs)

    return store_wrapper

def require_login(view):
    def login_wrapper(request, *args, **kwargs):
        store = kwargs.get("store")
        if not store: return Http404()

        if store.accesscode == "":
            return view(request, *args, **kwargs)

        if request.session.has_key("fileshack_stores") and \
           store.id in request.session["fileshack_stores"]:
            return view(request, *args, **kwargs)

        return HttpResponseNotAllowed()

    return login_wrapper

@require_store
def logout(request, store):
    try:
        request.session["fileshack_stores"].remove(store.id)
    except (KeyError, ValueError):
        pass
    request.session.save()
    return HttpResponseRedirect(store.get_absolute_url())

@require_store
def index(request, store):
    if (not request.session.has_key("fileshack_stores") or \
        not store.id in request.session["fileshack_stores"]) \
        and store.accesscode != "":

        if request.method == "POST":
            accesscode = request.POST.get("accesscode")
            if accesscode != "" and store.accesscode == accesscode:
                if not request.session.has_key("fileshack_stores"):
                    request.session["fileshack_stores"] = [store.id]
                else:
                   request.session["fileshack_stores"].append(store.id)
                request.session.set_expiry(3600)
                return HttpResponseRedirect(store.get_absolute_url())
            else:
                t = loader.get_template("fileshack/accesscode.html")
                c = RequestContext(request, {
                    "accesscode": accesscode,
                    "error_label": _("Wrong access code"),
                    "error_message": _("Please try again."),
                })
                return HttpResponse(t.render(c))
        else:
            t = loader.get_template("fileshack/accesscode.html")
            c = RequestContext(request)
            return HttpResponse(t.render(c))

    items = Item.objects.filter(store=store)
    t = loader.get_template("fileshack/index.html")
    c = RequestContext(request, {
        "store": store,
        "items": items,
        "item_size_limit": store.item_limit,
        "items_json": JSONEncoder().encode([i.simple() for i in items])
    })
    return HttpResponse(t.render(c))

@require_store
@require_login
def iframe(request, store):
    if request.method == "POST" and request.FILES.has_key("file"):
        f = request.FILES["file"]
        item = Item()
        item.store = store
        item.fileobject.save(urllib.unquote(f.name), f)
        item.size = f.size
        item.size_total = f.size
        item.save()

    t = loader.get_template("fileshack/iframe.html")
    c = RequestContext(request)
    return HttpResponse(t.render(c))

@never_cache
@require_store
@require_login
def upload(request, store, id):
    if request.method != "POST" or not request.FILES.has_key("file"):
        data = {
            "status": "failed",
            "error_label": "Upload failed",
            "error_message": "Invalid HTTP request",
        }
        return HttpResponseBadRequest(JSONEncoder().encode(data))

    if request.FILES.has_key("file"):
        f = request.FILES["file"]
        name = urllib.unquote(f.name)
        try: size_total = int(request.META["HTTP_X_FILE_SIZE"])
        except (ValueError, KeyError): size_total = f.size
    else:
        name = ''
        size_total = 0 # Unknown.

    try: name = request.META["HTTP_X_FILE_NAME"]
    except KeyError: name = ''

    name = os.path.basename(name)

    try: offset = int(request.META["HTTP_X_FILE_OFFSET"])
    except (ValueError, KeyError): offset = 0

    if store.item_limit and size_total and size_total > store.item_limit*1024*1024:
        data = {
            "status": "itemlimitreached",
            "error_label": "Upload failed",
            "error_message": "Item size is limited to %d MB" % store.item_limit,
            "item": None,
        }
        return HttpResponseServerError(JSONEncoder().encode(data))

    if store.store_limit and size_total and store.total() + size_total > store.store_limit*1024*1024:
        data = {
            "status": "storelimitreached",
            "error_label": "Upload failed",
            "error_message": "The store size limit of %d MB has been reached" % store.store_limit,
            "item": None,
        }
        return HttpResponseServerError(JSONEncoder().encode(data))

    # If the item exists, open the file for append.
    try:
        try: id = int(id)
        except ValueError: raise Item.DoesNotExist
        item = Item.objects.get(pk=id)
        if item.fileobject.size < offset:
            data = {
                "status": "outoforder",
                "error_label": "Chunk out of order",
                "error_message": "Application sent a chunk out of order",
                "item": item.simple(),
            }
            return HttpResponseServerError(JSONEncoder().encode(data))
        fp = default_storage.open(item.fileobject.path, "ab")
        fp.truncate(offset)

    # This is a new item.
    except Item.DoesNotExist:
        if offset != 0:
            data = {
                "status": "outoforder",
                "error_label": "Chunk out of order",
                "error_message": "Application sent a chunk of an item that does not exist",
                "item": None,
            }
            return HttpResponseServerError(JSONEncoder().encode(data))
        item = Item()
        item.store = store
        item.fileobject.save(name, ContentFile(""))
        item.fileobject.close()
        item.size_total = size_total
        item.save()
        fp = default_storage.open(item.fileobject.path, "wb")

    chunks = f.chunks().__iter__()
    while True:
        try: chunk = chunks.next()
        except StopIteration: break
        except IOError:
            fp.close()
            data = {
                "status": "failed",
                "error_label": "Upload failed",
                "error_message": "Server-side I/O error",
                "item": item.simple(),
            }
            return HttpResponseServerError(JSONEncoder().encode(data))
        else:
            try:
                if request.META.get("HTTP_X_FILE_ENCODING") == "base64":
                    fp.write(chunk.decode("base64"))
                else:
                    fp.write(chunk)
            except binascii.Error:
                fp.close()
                data = {
                    "status": "failed",
                    "error_label": "Upload failed",
                    "error_message": "The browser sent an invalid chunk",
                    "item": item.simple(),
                }
                return HttpResponseServerError(JSONEncoder().encode(data))

    item.size = fp.tell()
    fp.close()

    if item.size_total < item.size:
        item.size_total = item.size

    if item.size >= item.size_total:
        item.uploaded = dt.now()

    item.save()
    data = {
        "status": "success",
        "item": Item.objects.get(pk=item.pk).simple()
    }
    return HttpResponse(JSONEncoder().encode(data))

@require_store
@require_login
def simple_upload(request, store):
    if request.method != "POST" or not request.FILES.has_key("file"):
        return HttpResponseRedirect(store.get_absolute_url())

    #if store.item_limit and f.size > store.item_limit*1024*1024:
    #if store.store_limit and store.total() + f.size > store.store_limit*1024*1024:

    f = request.FILES["file"]
    item = Item()
    item.store = store
    item.fileobject.save(urllib.unquote(f.name), f)
    item.size = f.size
    item.size_total = f.size
    item.save()
    return HttpResponseRedirect(store.get_absolute_url())

@require_store
@require_login
def delete(request, store, item_id):
    if request.method != "POST":
        return HttpResponseForbidden()

    item = get_object_or_404(Item, pk=item_id, store=store)
    item.delete()

    return HttpResponse("Item has been deleted")

@never_cache
@require_store
@require_login
def update(request, store, since=None):
    since_dt = None
    if since != None:
        try:
            since_dt = dt.strptime(since, "%Y-%m-%d_%H:%M:%S")
        except ValueError:
            pass

    all_items = Item.objects.filter(store=store)
    item_ids = [item.id for item in all_items]

    if since_dt != None:
        items = Item.objects.filter(store=store, modified__gt=since_dt)
    else:
        items = all_items

    items_simple = []
    for item in items:
        items_simple.append(item.simple())

    dthandler = lambda obj: obj.isoformat() if isinstance(obj, datetime.datetime) else None
    data = JSONEncoder(sort_keys=True, indent=4).encode(dict(
            time=dt.now().strftime("%Y-%m-%d_%H:%M:%S"),
            item_ids=item_ids, items=items_simple))
    return HttpResponse(data)

class ItemFileWrapper(FileWrapper):
    def __init__(self, item, *args, **kwargs):
        self._item = item
        self._counter = 0
        self._stale = 0
        self._throttle = kwargs.get("throttle", 100)
        self._stale_limit = kwargs.get("stale_limit", 10000)
        return FileWrapper.__init__(self, *args, **kwargs)
    def next(self):
        try:
            data = FileWrapper.next(self)
            self._counter += len(data)
            if len(data) > 0:
                self._stale = 0
            return data
        except StopIteration:
            if self._counter >= self._item.size_total:
                raise StopIteration
            if self._stale_limit and self._stale >= self._stale_limit:
                raise StopIteration
            start = time.time()
            time.sleep(self._throttle/1000)
            end = time.time()
            self._stale += (end - start)*1000
            return ""

@require_store
@require_login
def download(request, store, item_id):
    item = get_object_or_404(Item, pk=item_id)

    if item.status() == "READY":
        return HttpResponseRedirect(item.get_absolute_url())

    f = default_storage.open(item.fileobject.path, "rb")
    wrapper = ItemFileWrapper(item, f)
    response = HttpResponse(wrapper, content_type=mimetypes.guess_type(item.fileobject.name))
    response["Content-Length"] = "%d" % item.size_total
    response["Content-Disposition"] = 'attachment; name="file"; filename="%s"' % urllib.quote(item.name())
    return response

def page_not_found(request):
    stores = Store.objects.all()
    store = None
    for s in stores:
        if request.path.startswith(s.get_absolute_url()):
            store = s
    if not store:
        try: store = Store.objects.get(path="")
        except Store.DoesNotExist: pass

    t = loader.get_template("fileshack/404.html")
    return HttpResponseNotFound(t.render(RequestContext(request, {
        "request_path": request.path,
        "store": store,
        "stores_number": len(stores),
        "admin_url": reverse("admin:index"),
    })))

def server_error(request):
    t = loader.get_template("fileshack/500.html")
    return HttpResponseServerError(t.render(RequestContext(request, { "request_path": request.path, })))
<|endoftext|>


<commit_before>import numpy as np
import pandas as pb
# Use the motion parameters to fid the bad brains
# Use the qc_csv to find the bad brains

#  Motion Based outliers

def read_par_file(motion_params_file):
    trans_x = []
    trans_y = []
    trans_z = []
    rot_x = []
    rot_y = []
    rot_z = []

    with open(motion_params_file) as f:
        for line in f:
            line = line.split(' ')
            # print(line)
            trans_x.append(float(line[6]))
            trans_y.append(float(line[8]))
            trans_z.append(float(line[10]))
            rot_x.append(float(line[0]))
            rot_y.append(float(line[2]))
            rot_z.append(float(line[4]))
    return trans_x, trans_y, trans_z, rot_x, rot_y, rot_z

def motion_outliers(motion_params_file, threshold):
    motion_params_paths = np.load(motion_params_npy)
    outliers_sub_ids = []
    for subject_param_path in motion_params_paths:
        sub_id = subject_param_path.split('/')[-1].split('_')[0].split('-')[1]
        trans_x, trans_y, trans_z, rot_x, rot_y, rot_z = read_par_file(subject_param_path)
        params = np.array([trans_x, trans_y, trans_z, rot_x, rot_y, rot_z]).T
        outlier_idx = np.where(np.abs(params) > thresh)
        num_outlier_entries = len(set(outlier_idx[0]))
        if num_outlier_entries >= 10:
            print('Subject %s with %s Outliers'%(sub_id, num_outlier_entries ))
            outliers_sub_ids.append(sub_id)
    return outliers_sub_ids

def csf_wm_outliers(qc_csv_file, csf_thresh, wm_thresh):
    outliers_sub_id = []
    wm_csf_qc_df = pb.read_csv(qc_csv_file)
    wm_csf_qc_mat = wm_csf_qc_df[['sub_id','quality_csf','quality_wm']].values
    for i in range(wm_csf_qc_mat.shape[0]):
        if wm_csf_qc_mat[1] < csf_thresh:
            print('Subject %s has a very poor CSF quality of %s'%(wm_csf_qc_mat[0],wm_csf_qc_mat[1]))
            outliers_sub_id.append(wm_csf_qc_mat[0])
        if wm_csf_qc_mat[1] < wm_thresh:
            print('Subject %s has a very poor WM quality of %s'%(wm_csf_qc_mat[0],wm_csf_qc_mat[2]))
            if wm_csf_qc_mat[0] not in outliers_sub_id:
                outliers_sub_id.append(wm_csf_qc_mat[0])
    return outliers_sub_id

if __name__ == "__main__":
    motion_params_npy = '/mnt/project2/home/varunk/fMRI/results/resultsABIDE1_2/'+\
    'Preprocess_Datasink/motion_params_paths/motion_params_file_list.npy'

    qc_csv_file = '/mnt/project2/home/varunk/fMRI/results/resultsABIDE1_2/'+\
    'Preprocess_Datasink/qc_csv/qc.csv'

    motion_thresh = 2.5
    outliers_sub_ids = motion_outliers(motion_params_npy, motion_thresh)
    print(outliers_sub_ids)
        # print(num_outlier_entries)

    csf_thresh = 0.015
    wm_thresh =
<commit_msg>Added threshold to exclude subjects based on percentage of outlier volumes<commit_after>import numpy as np
import pandas as pb
# Use the motion parameters to fid the bad brains
# Use the qc_csv to find the bad brains

#  Motion Based outliers

def read_par_file(motion_params_file):
    volumes_count = 0
    trans_x = []
    trans_y = []
    trans_z = []
    rot_x = []
    rot_y = []
    rot_z = []

    with open(motion_params_file) as f:
        for line in f:
            line = line.split(' ')
            # print(line)
            trans_x.append(float(line[6]))
            trans_y.append(float(line[8]))
            trans_z.append(float(line[10]))
            rot_x.append(float(line[0]))
            rot_y.append(float(line[2]))
            rot_z.append(float(line[4]))
            volumes_count = volumes_count + 1
    return trans_x, trans_y, trans_z, rot_x, rot_y, rot_z, volumes_count

def motion_outliers(motion_params_file, motion_threshold,\
                        outliers_percentage_threshold = 30):
    motion_params_paths = np.load(motion_params_npy)
    outliers_sub_ids = []
    for subject_param_path in motion_params_paths:
        sub_id = subject_param_path.split('/')[-1].split('_')[0].split('-')[1]
        trans_x, trans_y, trans_z, rot_x, rot_y, rot_z, volumes_count = \
                                            read_par_file(subject_param_path)

        params = np.array([trans_x, trans_y, trans_z, rot_x, rot_y, rot_z]).T
        outlier_idx = np.where(np.abs(params) > motion_threshold)
        num_outlier_entries = len(set(outlier_idx[0]))
        percentage_outlier_entries = num_outlier_entries * 100 / volumes_count
        # if num_outlier_entries >= 10:
        if percentage_outlier_entries > outliers_percentage_threshold:
            print('Subject %s with %s Outliers has %s percent outliers'%(sub_id,\
             num_outlier_entries,percentage_outlier_entries ))
            outliers_sub_ids.append(sub_id)
    return outliers_sub_ids

def csf_wm_outliers(qc_csv_file, csf_thresh, wm_thresh):
    outliers_sub_id = []
    wm_csf_qc_df = pb.read_csv(qc_csv_file)
    wm_csf_qc_mat = wm_csf_qc_df[['sub_id','quality_csf','quality_wm']].values
    for i in range(wm_csf_qc_mat.shape[0]):
        if wm_csf_qc_mat[1] < csf_thresh:
            print('Subject %s has a very poor CSF quality of %s'%(wm_csf_qc_mat[0],wm_csf_qc_mat[1]))
            outliers_sub_id.append(wm_csf_qc_mat[0])
        if wm_csf_qc_mat[1] < wm_thresh:
            print('Subject %s has a very poor WM quality of %s'%(wm_csf_qc_mat[0],wm_csf_qc_mat[2]))
            if wm_csf_qc_mat[0] not in outliers_sub_id:
                outliers_sub_id.append(wm_csf_qc_mat[0])
    return outliers_sub_id

if __name__ == "__main__":
    motion_params_npy = '/mnt/project2/home/varunk/fMRI/results/resultsABIDE1_2/'+\
    'Preprocess_Datasink/motion_params_paths/motion_params_file_list.npy'

    qc_csv_file = '/mnt/project2/home/varunk/fMRI/results/resultsABIDE1_2/'+\
    'Preprocess_Datasink/qc_csv/qc.csv'

    motion_thresh = 2.5
    outliers_percentage_threshold = 30
    outliers_sub_ids = motion_outliers(motion_params_npy, motion_thresh, \
    outliers_percentage_threshold)
    print(outliers_sub_ids)
    # print(num_outlier_entries)

    # csf_thresh = 0.015
    # wm_thresh =
<|endoftext|>