<commit_before>#!/usr/bin/env python

import sys, os, time, atexit
import traceback
import locale
from signal import SIGTERM,SIGKILL
from zstacklib.utils import linux
from zstacklib.utils import log

logger = log.get_logger(__name__)

class Daemon(object):
    """
    A generic daemon class.

    Usage: subclass the Daemon class and override the run() method
    """
    atexit_hooks = []

    def __init__(self, pidfile, py_process_name, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.pidfile = pidfile
        self.py_process_name = py_process_name

    @staticmethod
    def register_atexit_hook(hook):
        Daemon.atexit_hooks.append(hook)

    @staticmethod
    def _atexit():
        for hook in Daemon.atexit_hooks:
            try:
                hook()
            except Exception:
                content = traceback.format_exc()
                err = 'Exception when calling atexit hook[%s]\n%s' % (hook.__name__, content)
                logger.error(err)

    def daemonize(self):
        """
        do the UNIX double-fork magic, see Stevens' "Advanced
        Programming in the UNIX Environment" for details (ISBN 0201563177)
        http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
        """
        try:
            pid = os.fork()
            if pid > 0:
                # exit first parent
                sys.exit(0)
        except OSError, e:
            sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
            sys.exit(1)

        # decouple from parent environment
        os.chdir("/")
        os.setsid()
        os.umask(0)

        # do second fork
        try:
            pid = os.fork()
            if pid > 0:
                # exit from second parent
                os._exit(0)
        except OSError, e:
            sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
            os._exit(1)

        # redirect standard file descriptors
        sys.stdout.flush()
        sys.stderr.flush()
        si = file(self.stdin, 'r')
        so = file(self.stdout, 'a+')
        se = file(self.stderr, 'a+', 0)
        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())

        # write pidfile
        Daemon.register_atexit_hook(self.delpid)
        atexit.register(Daemon._atexit)
        file(self.pidfile,'w').write("%d\n" % os.getpid())

    def delpid(self):
        os.remove(self.pidfile)

    def start(self):
        """
        Start the daemon
        """
        logger.debug("Start Daemon...")
        print "Start Daemon..."

        locale.setlocale(locale.LC_ALL, 'C')

        # Get the pid from the pidfile
        try:
            pf = file(self.pidfile, 'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None

        if pid and linux.process_exists(pid):
            message = "Daemon already running, pid is %s\n"
            sys.stderr.write(message % pid)
            sys.exit(0)
        else:
            message = "pidfile %s does not exist. Daemon not running?\n"
            sys.stderr.write(message % self.pidfile)
            self.get_start_agent_by_name()

        # Start the daemon
        self.daemonize()

        try:
            self.run()
        except Exception:
            content = traceback.format_exc()
            logger.error(content)
            sys.exit(1)

        print "Start Daemon Successfully"

    def stop(self):
        """
        Stop the daemon
        """
        logger.debug("Stop Daemon...")
        print "Stop Daemon..."
        #wait 2s for gracefully shutdown, then will force kill
        wait_stop = 2

        try:
            pf = file(self.pidfile, 'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None

        if pid and linux.process_exists(pid):
            self.stop_agent_by_pid(pid, wait_stop)
        else:
            message = "pidfile %s does not exist. Daemon not running?\n"
            sys.stderr.write(message % self.pidfile)

            pids = linux.get_agent_pid_by_name(self.py_process_name)

            if not pids:
                message = "Daemon not running?\n"
                sys.stderr.write(message)
                return # not an error in a restart

            # exclude self pid
            for pid in pids.split('\n'):
                if pid and int(pid) != os.getpid():
                    pid = int(pid)
                    self.stop_agent_by_pid(pid, wait_stop)

    def restart(self):
        """
        Restart the daemon
        """
        logger.debug("Restart Daemon...")
        self.stop()
        self.start()

    def run(self):
        """
        You should override this method when you subclass Daemon. It will be called after the process has been
        daemonized by start() or restart().
        """

    def get_start_agent_by_name(self):
        pids = linux.get_agent_pid_by_name(self.py_process_name)

        for pid in pids.split('\n'):
            if pid and int(pid) != os.getpid():
                pid = int(pid)
                if linux.process_exists(pid):
                    message = "Daemon already running, pid is %s\n"
                    sys.stderr.write(message % pid)
                    sys.exit(0)

    def stop_agent_by_pid(self, pid, wait_stop):
        # Try killing the daemon process
        start_time = time.time()
        while 1:
            if linux.process_exists(pid):
                curr_time = time.time()
                if (curr_time - start_time) > wait_stop:
                    os.kill(pid, SIGKILL)
                else:
                    os.kill(pid, SIGTERM)
                time.sleep(0.3)
            else:
                break

        print "Stop Daemon Successfully"
<commit_msg>add daemon log before writing pidfile<commit_after>#!/usr/bin/env python

import sys, os, time, atexit
import traceback
import locale
from signal import SIGTERM,SIGKILL
from zstacklib.utils import linux
from zstacklib.utils import log

logger = log.get_logger(__name__)

class Daemon(object):
    """
    A generic daemon class.

    Usage: subclass the Daemon class and override the run() method
    """
    atexit_hooks = []

    def __init__(self, pidfile, py_process_name, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.pidfile = pidfile
        self.py_process_name = py_process_name

    @staticmethod
    def register_atexit_hook(hook):
        Daemon.atexit_hooks.append(hook)

    @staticmethod
    def _atexit():
        for hook in Daemon.atexit_hooks:
            try:
                hook()
            except Exception:
                content = traceback.format_exc()
                err = 'Exception when calling atexit hook[%s]\n%s' % (hook.__name__, content)
                logger.error(err)

    def daemonize(self):
        """
        do the UNIX double-fork magic, see Stevens' "Advanced
        Programming in the UNIX Environment" for details (ISBN 0201563177)
        http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16
        """
        try:
            pid = os.fork()
            if pid > 0:
                # exit first parent
                sys.exit(0)
        except OSError, e:
            sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
            sys.exit(1)

        # decouple from parent environment
        os.chdir("/")
        os.setsid()
        os.umask(0)

        # do second fork
        try:
            pid = os.fork()
            if pid > 0:
                # exit from second parent
                os._exit(0)
        except OSError, e:
            sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
            os._exit(1)

        # redirect standard file descriptors
        sys.stdout.flush()
        sys.stderr.flush()
        si = file(self.stdin, 'r')
        so = file(self.stdout, 'a+')
        se = file(self.stderr, 'a+', 0)
        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())

        # write pidfile
        Daemon.register_atexit_hook(self.delpid)
        atexit.register(Daemon._atexit)

        # systemd needs this for non-native service
        pid = os.getpid()
        logger.info("writing pidfile (pid=%d)" % pid)
        try:
            file(self.pidfile,'w').write("%d\n" % pid)
        except IOError as e:
            logger.error(str(e))

    def delpid(self):
        os.remove(self.pidfile)

    def start(self):
        """
        Start the daemon
        """
        logger.debug("Start Daemon...")
        print "Start Daemon..."

        locale.setlocale(locale.LC_ALL, 'C')

        # Get the pid from the pidfile
        try:
            pf = file(self.pidfile, 'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None

        if pid and linux.process_exists(pid):
            message = "Daemon already running, pid is %s\n"
            sys.stderr.write(message % pid)
            sys.exit(0)
        else:
            message = "pidfile %s does not exist. Daemon not running?\n"
            sys.stderr.write(message % self.pidfile)
            self.get_start_agent_by_name()

        # Start the daemon
        self.daemonize()

        try:
            self.run()
        except Exception:
            content = traceback.format_exc()
            logger.error(content)
            sys.exit(1)

        print "Start Daemon Successfully"

    def stop(self):
        """
        Stop the daemon
        """
        logger.debug("Stop Daemon...")
        print "Stop Daemon..."
        #wait 2s for gracefully shutdown, then will force kill
        wait_stop = 2

        try:
            pf = file(self.pidfile, 'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None

        if pid and linux.process_exists(pid):
            self.stop_agent_by_pid(pid, wait_stop)
        else:
            message = "pidfile %s does not exist. Daemon not running?\n"
            sys.stderr.write(message % self.pidfile)

            pids = linux.get_agent_pid_by_name(self.py_process_name)

            if not pids:
                message = "Daemon not running?\n"
                sys.stderr.write(message)
                return # not an error in a restart

            # exclude self pid
            for pid in pids.split('\n'):
                if pid and int(pid) != os.getpid():
                    pid = int(pid)
                    self.stop_agent_by_pid(pid, wait_stop)

    def restart(self):
        """
        Restart the daemon
        """
        logger.debug("Restart Daemon...")
        self.stop()
        self.start()

    def run(self):
        """
        You should override this method when you subclass Daemon. It will be called after the process has been
        daemonized by start() or restart().
        """

    def get_start_agent_by_name(self):
        pids = linux.get_agent_pid_by_name(self.py_process_name)

        for pid in pids.split('\n'):
            if pid and int(pid) != os.getpid():
                pid = int(pid)
                if linux.process_exists(pid):
                    message = "Daemon already running, pid is %s\n"
                    sys.stderr.write(message % pid)
                    sys.exit(0)

    def stop_agent_by_pid(self, pid, wait_stop):
        # Try killing the daemon process
        start_time = time.time()
        while 1:
            if linux.process_exists(pid):
                curr_time = time.time()
                if (curr_time - start_time) > wait_stop:
                    os.kill(pid, SIGKILL)
                else:
                    os.kill(pid, SIGTERM)
                time.sleep(0.3)
            else:
                break

        print "Stop Daemon Successfully"
<|endoftext|>


<commit_before>from django.http.response import HttpResponseRedirect

from 臺灣言語平臺.tasks import 半瞑自sheets掠轉資料庫

def 正規化表馬上匯入資料庫(request):
    半瞑自sheets掠轉資料庫.delay()
    return HttpResponseRedirect('http://itaigi.tw')
<commit_msg>攏改做https<commit_after>from django.http.response import HttpResponseRedirect

from 臺灣言語平臺.tasks import 半瞑自sheets掠轉資料庫

def 正規化表馬上匯入資料庫(request):
    半瞑自sheets掠轉資料庫.delay()
    return HttpResponseRedirect('https://itaigi.tw')
<|endoftext|>


<commit_before>"""The Iteration/Expression hierarchy."""

from __future__ import absolute_import

from collections import Iterable, defaultdict
from itertools import chain
from operator import attrgetter

import cgen as c
from sympy import Eq, IndexedBase, Symbol, preorder_traversal

from devito.codeprinter import ccode
from devito.dimension import Dimension
from devito.dse.inspection import terminals
from devito.interfaces import SymbolicData
from devito.logger import warning
from devito.tools import as_tuple, filter_ordered

__all__ = ['Node', 'Block', 'Expression', 'Iteration']

class Node(object):

    is_Node = True
    is_Block = False
    is_Iteration = False
    is_Expression = False

    def __new__(cls, *args, **kwargs):
        obj = super(Node, cls).__new__(cls)
        obj._args = args + (kwargs,)
        return obj

    def _children(self):
        """Return the traversable children."""
        return []

    def _rebuild(self):
        """Reconstruct the Node."""
        raise NotImplementedError()

    def indexify(self):
        """Convert all enclosed nodes to "indexed" format"""
        for e in self._children():
            e.indexify()

    def substitute(self, substitutions):
        """Apply substitutions to children nodes

        :param substitutions: Dict containing the substitutions to apply.
        """
        candidates = [n for n in self._children() if isinstance(n, Node)]
        for n in candidates:
            n.substitute(substitutions)

    @property
    def index_offsets(self):
        """Collect all non-zero offsets used with each index in a map

        Note: This assumes we have indexified the stencil expression."""
        return defaultdict(list)

    @property
    def ccode(self):
        """Generate C code."""
        raise NotImplementedError()

    @property
    def signature(self):
        """List of data objects used by the Node."""
        raise NotImplementedError()

    @property
    def args(self):
        return self._args

class Block(Node):

    is_Block = True

    def __init__(self, header=None, body=None, footer=None):
        self.header = as_tuple(header)
        self.body = as_tuple(body)
        self.footer = as_tuple(footer)

    def __repr__(self):
        header = "".join([str(s) for s in self.header])
        body = "Block::\n\t%s" % "\n\t".join([str(s) for s in self.body])
        footer = "".join([str(s) for s in self.footer])
        return header + body + footer

    @property
    def ccode(self):
        body = tuple(s.ccode for s in self.body)
        return c.Block(self.header + body + self.footer)

    def _children(self):
        return self.header + self.body + self.footer

class Expression(Node):

    """Class encpasulating a single stencil expression"""

    is_Expression = True

    def __init__(self, stencil):
        assert isinstance(stencil, Eq)
        self.stencil = stencil

        self.dimensions = []
        self.functions = []
        # Traverse stencil to determine meta information
        for e in preorder_traversal(self.stencil):
            if isinstance(e, SymbolicData):
                self.dimensions += list(e.indices)
                self.functions += [e]
            if isinstance(e, IndexedBase):
                self.dimensions += list(e.function.indices)
                self.functions += [e.function]
        # Filter collected dimensions and functions
        self.dimensions = filter_ordered(self.dimensions)
        self.functions = filter_ordered(self.functions)

    def __repr__(self):
        return "Expression<%s = %s>" % (self.stencil.lhs, self.stencil.rhs)

    def substitute(self, substitutions):
        """Apply substitutions to the expression stencil

        :param substitutions: Dict containing the substitutions to apply to
                              the stored loop stencils.
        """
        self.stencil = self.stencil.xreplace(substitutions)

    @property
    def ccode(self):
        return c.Assign(ccode(self.stencil.lhs), ccode(self.stencil.rhs))

    @property
    def signature(self):
        """List of data objects used by the expression

        :returns: List of unique data objects required by the expression
        """
        return filter_ordered([f for f in self.functions],
                              key=attrgetter('name'))

    def indexify(self):
        """Convert stencil expression to "indexed" format"""
        self.stencil = indexify(self.stencil)

    @property
    def index_offsets(self):
        """Collect all non-zero offsets used with each index in a map

        Note: This assumes we have indexified the stencil expression."""
        offsets = defaultdict(list)
        for e in terminals(self.stencil):
            for a in e.indices:
                d = None
                off = []
                for idx in a.args:
                    if isinstance(idx, Dimension):
                        d = idx
                    else:
                        off += [idx]
                if d is not None:
                    offsets[d] += off
        return offsets

    def _rebuild(self, stencil):
        """Reconstruct the Expression as a plain new object. None of the
        embedded Sympy expressions are reconstructed."""
        return Expression(stencil)

class Iteration(Node):
    """Iteration object that encapsualtes a single loop over nodes, possibly
    just SymPy expressions.

    :param nodes: Single or list of :class:`Node` objects that
                        define the loop body.
    :param dimension: :class:`Dimension` object over which to iterate.
    :param limits: Limits for the iteration space, either the loop size or a
                   tuple of the form (start, finish, stepping).
    :param offsets: Optional map list of offsets to honour in the loop
    """

    is_Iteration = True

    def __init__(self, nodes, dimension, limits, offsets=None):
        # Ensure we deal with a list of Expression objects internally
        self.nodes = as_tuple(nodes)
        assert all(isinstance(i, Node) for i in self.nodes)

        # Generate index variable name and variable substitutions
        self.dim = dimension
        if isinstance(self.dim, Dimension):
            if self.dim.buffered is None:
                # Generate index variable from dimension
                self.index = str(self.dim.get_varname())
                self.subs = {self.dim: Symbol(self.index)}
            else:
                # Generate numbered indices for each buffer
                self.index = self.dim.name
                self.subs = {self.dim: Symbol(self.dim.get_varname())}
                for offset in self.index_offsets[self.dim]:
                    self.subs[self.dim + offset] = Symbol(self.dim.get_varname())
        else:
            warning("Generating Iteration without Dimension object")
            self.index = str(dimension)

        # Propagate variable names to the lower expressions
        self.substitute(self.subs)

        # Generate loop limits
        if isinstance(limits, Iterable):
            assert(len(limits) == 3)
            self.limits = list(limits)
        else:
            self.limits = list((0, limits, 1))

        # Replace open limits with variables names
        if self.limits[1] is None:
            # FIXME: Add dimension size as variable bound.
            # Needs further generalisation to support loop blocking.
            self.limits[1] = IterationBound("%s_size" % self.dim.name, self.dim)

        # Record offsets to later adjust loop limits accordingly
        self.offsets = [0, 0]
        for off in (offsets or {}):
            self.offsets[0] = min(self.offsets[0], int(off))
            self.offsets[1] = max(self.offsets[1], int(off))

    def __repr__(self):
        str_expr = "\n\t".join([str(s) for s in self.nodes])
        return "Iteration<%s; %s>::\n\t%s" % (self.index, self.limits,
                                              str_expr)

    @property
    def ccode(self):
        """Generate C code for the represented stencil loop

        :returns: :class:`cgen.For` object representing the loop
        """
        loop_body = [s.ccode for s in self.nodes]
        if self.dim.buffered is not None:
            modulo = self.dim.buffered
            v_subs = [c.Initializer(c.Value('int', v), "(%s) %% %d" % (s, modulo))
                      for s, v in self.subs.items()]
            loop_body = v_subs + loop_body
        loop_init = c.InlineInitializer(
            c.Value("int", self.index), "%d + %d" % (self.limits[0], -self.offsets[0]))
        loop_cond = '%s %s %s' % (self.index, '<' if self.limits[2] >= 0 else '>',
                                  "%s - %d" % (self.limits[1], self.offsets[1]))
        loop_inc = '%s += %s' % (self.index, self.limits[2])
        return c.For(loop_init, loop_cond, loop_inc, c.Block(loop_body))

    @property
    def signature(self):
        """List of data objects used by the loop and it's body

        :returns: List of unique data objects required by the loop
        """
        signature = [e.signature for e in self.nodes]
        signature = filter_ordered(chain(*signature))
        if isinstance(self.limits[1], IterationBound):
            signature += [self.dim]
        return signature

    @property
    def index_offsets(self):
        """Collect all non-zero offsets used with each index in a map

        Note: This assumes we have indexified the stencil expression."""
        offsets = defaultdict(list)
        for n in self.nodes:
            offsets.update(n.index_offsets)
        return offsets

    def _children(self):
        """Return the traversable children."""
        return self.nodes

    def _rebuild(self, nodes, dimension, limits, offsets):
        """Reconstruct the Iteration as a plain new object."""
        return Iteration(nodes, dimension, limits, offsets)

# Utilities

class IterationBound(object):
    """Utility class to encapsulate variable loop bounds and link them
    back to the respective Dimension object.

    :param name: Variable name for the open loop bound variable
    """

    def __init__(self, name, dim):
        self.name = name
        self.dim = dim

    def __repr__(self):
        return self.name

    @property
    def ccode(self):
        """C code for the variable declaration within a kernel signature"""
        return c.Value('const int', self.name)
<commit_msg>Add Timer class to nodes hierarchy<commit_after>"""The Iteration/Expression hierarchy."""

from __future__ import absolute_import

from collections import Iterable, defaultdict
from itertools import chain
from operator import attrgetter

import cgen as c
from sympy import Eq, IndexedBase, Symbol, preorder_traversal

from devito.codeprinter import ccode
from devito.dimension import Dimension
from devito.dse.inspection import terminals
from devito.interfaces import SymbolicData
from devito.logger import warning
from devito.tools import as_tuple, filter_ordered

__all__ = ['Node', 'Block', 'Expression', 'Iteration', 'Timer']

class Node(object):

    is_Node = True
    is_Block = False
    is_Iteration = False
    is_Expression = False

    def __new__(cls, *args, **kwargs):
        obj = super(Node, cls).__new__(cls)
        obj._args = args + (kwargs,)
        return obj

    def _children(self):
        """Return the traversable children."""
        return []

    def _rebuild(self):
        """Reconstruct the Node."""
        raise NotImplementedError()

    def indexify(self):
        """Convert all enclosed nodes to "indexed" format"""
        for e in self._children():
            e.indexify()

    def substitute(self, substitutions):
        """Apply substitutions to children nodes

        :param substitutions: Dict containing the substitutions to apply.
        """
        candidates = [n for n in self._children() if isinstance(n, Node)]
        for n in candidates:
            n.substitute(substitutions)

    @property
    def index_offsets(self):
        """Collect all non-zero offsets used with each index in a map

        Note: This assumes we have indexified the stencil expression."""
        return defaultdict(list)

    @property
    def ccode(self):
        """Generate C code."""
        raise NotImplementedError()

    @property
    def signature(self):
        """List of data objects used by the Node."""
        raise NotImplementedError()

    @property
    def args(self):
        return self._args

class Block(Node):

    is_Block = True

    def __init__(self, header=None, body=None, footer=None):
        self.header = as_tuple(header)
        self.body = as_tuple(body)
        self.footer = as_tuple(footer)

    def __repr__(self):
        header = "".join([str(s) for s in self.header])
        body = "Block::\n\t%s" % "\n\t".join([str(s) for s in self.body])
        footer = "".join([str(s) for s in self.footer])
        return header + body + footer

    @property
    def ccode(self):
        body = tuple(s.ccode for s in self.body)
        return c.Block(self.header + body + self.footer)

    def _children(self):
        return self.header + self.body + self.footer

class Expression(Node):

    """Class encpasulating a single stencil expression"""

    is_Expression = True

    def __init__(self, stencil):
        assert isinstance(stencil, Eq)
        self.stencil = stencil

        self.dimensions = []
        self.functions = []
        # Traverse stencil to determine meta information
        for e in preorder_traversal(self.stencil):
            if isinstance(e, SymbolicData):
                self.dimensions += list(e.indices)
                self.functions += [e]
            if isinstance(e, IndexedBase):
                self.dimensions += list(e.function.indices)
                self.functions += [e.function]
        # Filter collected dimensions and functions
        self.dimensions = filter_ordered(self.dimensions)
        self.functions = filter_ordered(self.functions)

    def __repr__(self):
        return "Expression<%s = %s>" % (self.stencil.lhs, self.stencil.rhs)

    def substitute(self, substitutions):
        """Apply substitutions to the expression stencil

        :param substitutions: Dict containing the substitutions to apply to
                              the stored loop stencils.
        """
        self.stencil = self.stencil.xreplace(substitutions)

    @property
    def ccode(self):
        return c.Assign(ccode(self.stencil.lhs), ccode(self.stencil.rhs))

    @property
    def signature(self):
        """List of data objects used by the expression

        :returns: List of unique data objects required by the expression
        """
        return filter_ordered([f for f in self.functions],
                              key=attrgetter('name'))

    def indexify(self):
        """Convert stencil expression to "indexed" format"""
        self.stencil = indexify(self.stencil)

    @property
    def index_offsets(self):
        """Collect all non-zero offsets used with each index in a map

        Note: This assumes we have indexified the stencil expression."""
        offsets = defaultdict(list)
        for e in terminals(self.stencil):
            for a in e.indices:
                d = None
                off = []
                for idx in a.args:
                    if isinstance(idx, Dimension):
                        d = idx
                    else:
                        off += [idx]
                if d is not None:
                    offsets[d] += off
        return offsets

    def _rebuild(self, stencil):
        """Reconstruct the Expression as a plain new object. None of the
        embedded Sympy expressions are reconstructed."""
        return Expression(stencil)

class Iteration(Node):
    """Iteration object that encapsualtes a single loop over nodes, possibly
    just SymPy expressions.

    :param nodes: Single or list of :class:`Node` objects that
                        define the loop body.
    :param dimension: :class:`Dimension` object over which to iterate.
    :param limits: Limits for the iteration space, either the loop size or a
                   tuple of the form (start, finish, stepping).
    :param offsets: Optional map list of offsets to honour in the loop
    """

    is_Iteration = True

    def __init__(self, nodes, dimension, limits, offsets=None):
        # Ensure we deal with a list of Expression objects internally
        self.nodes = as_tuple(nodes)
        assert all(isinstance(i, Node) for i in self.nodes)

        # Generate index variable name and variable substitutions
        self.dim = dimension
        if isinstance(self.dim, Dimension):
            if self.dim.buffered is None:
                # Generate index variable from dimension
                self.index = str(self.dim.get_varname())
                self.subs = {self.dim: Symbol(self.index)}
            else:
                # Generate numbered indices for each buffer
                self.index = self.dim.name
                self.subs = {self.dim: Symbol(self.dim.get_varname())}
                for offset in self.index_offsets[self.dim]:
                    self.subs[self.dim + offset] = Symbol(self.dim.get_varname())
        else:
            warning("Generating Iteration without Dimension object")
            self.index = str(dimension)

        # Propagate variable names to the lower expressions
        self.substitute(self.subs)

        # Generate loop limits
        if isinstance(limits, Iterable):
            assert(len(limits) == 3)
            self.limits = list(limits)
        else:
            self.limits = list((0, limits, 1))

        # Replace open limits with variables names
        if self.limits[1] is None:
            # FIXME: Add dimension size as variable bound.
            # Needs further generalisation to support loop blocking.
            self.limits[1] = IterationBound("%s_size" % self.dim.name, self.dim)

        # Record offsets to later adjust loop limits accordingly
        self.offsets = [0, 0]
        for off in (offsets or {}):
            self.offsets[0] = min(self.offsets[0], int(off))
            self.offsets[1] = max(self.offsets[1], int(off))

    def __repr__(self):
        str_expr = "\n\t".join([str(s) for s in self.nodes])
        return "Iteration<%s; %s>::\n\t%s" % (self.index, self.limits,
                                              str_expr)

    @property
    def ccode(self):
        """Generate C code for the represented stencil loop

        :returns: :class:`cgen.For` object representing the loop
        """
        loop_body = [s.ccode for s in self.nodes]
        if self.dim.buffered is not None:
            modulo = self.dim.buffered
            v_subs = [c.Initializer(c.Value('int', v), "(%s) %% %d" % (s, modulo))
                      for s, v in self.subs.items()]
            loop_body = v_subs + loop_body
        loop_init = c.InlineInitializer(
            c.Value("int", self.index), "%d + %d" % (self.limits[0], -self.offsets[0]))
        loop_cond = '%s %s %s' % (self.index, '<' if self.limits[2] >= 0 else '>',
                                  "%s - %d" % (self.limits[1], self.offsets[1]))
        loop_inc = '%s += %s' % (self.index, self.limits[2])
        return c.For(loop_init, loop_cond, loop_inc, c.Block(loop_body))

    @property
    def signature(self):
        """List of data objects used by the loop and it's body

        :returns: List of unique data objects required by the loop
        """
        signature = [e.signature for e in self.nodes]
        signature = filter_ordered(chain(*signature))
        if isinstance(self.limits[1], IterationBound):
            signature += [self.dim]
        return signature

    @property
    def index_offsets(self):
        """Collect all non-zero offsets used with each index in a map

        Note: This assumes we have indexified the stencil expression."""
        offsets = defaultdict(list)
        for n in self.nodes:
            offsets.update(n.index_offsets)
        return offsets

    def _children(self):
        """Return the traversable children."""
        return self.nodes

    def _rebuild(self, nodes, dimension, limits, offsets):
        """Reconstruct the Iteration as a plain new object."""
        return Iteration(nodes, dimension, limits, offsets)

# Utilities

class Timer(Block):

    """Wrap a Node with C-level timers."""

    def __init__(self, lname, gname, body):
        """
        Initialize a Timer object.

        :param lname: Timer name in the local scope.
        :param gname: Name of the global struct tracking all timers.
        :param body: Timed block of code.
        """
        self._name = lname
        # TODO: need omp master pragma to be thread safe
        header = [c.Statement("struct timeval start_%s, end_%s" % (lname, lname)),
                  c.Statement("gettimeofday(&start_%s, NULL)" % lname)]
        footer = [c.Statement("gettimeofday(&end_%s, NULL)" % lname),
                  c.Statement(("%(gn)s->%(ln)s += " +
                               "(double)(end_%(ln)s.tv_sec-start_%(ln)s.tv_sec)+" +
                               "(double)(end_%(ln)s.tv_usec-start_%(ln)s.tv_usec)" +
                               "/1000000") % {'gn': gname, 'ln': lname})]
        super(Timer, self).__init__(header, body, footer)

    @property
    def name(self):
        return self._name

class IterationBound(object):
    """Utility class to encapsulate variable loop bounds and link them
    back to the respective Dimension object.

    :param name: Variable name for the open loop bound variable
    """

    def __init__(self, name, dim):
        self.name = name
        self.dim = dim

    def __repr__(self):
        return self.name

    @property
    def ccode(self):
        """C code for the variable declaration within a kernel signature"""
        return c.Value('const int', self.name)
<|endoftext|>


<commit_before>from __future__ import absolute_import

import logging
from datetime import date, timedelta
from dateutil.parser import parse

from dexter.app import celery_app as app
from dexter.processing import DocumentProcessor

# force configs for API keys to be set
import dexter.core

# This is a collection of periodic tasks for Dexter, using
# Celery to drive task completion.

log = logging.getLogger(__name__)

@app.task
def fetch_yesterdays_feeds():
    """ Enqueue a task to fetch yesterday's feeds. """
    yesterday = date.today() - timedelta(days=1)
    fetch_daily_feeds.delay(yesterday.isoformat())

# retry after 30 minutes, retry for up to 7 days
@app.task(bind=True, default_retry_delay=30*60, max_retries=7*24*2)
def fetch_daily_feeds(self, day):
    """ Fetch feed of URLs to crawl and queue up a task to grab and process
    each url. """
    try:
        day = parse(day)

        dp = DocumentProcessor()
        count = 0
        for item in dp.fetch_daily_feed_items(day):
            get_feed_item.delay(item)
            count += 1
    except Exception as e:
        log.error("Error processing daily feeds for %s" % day, exc_info=e)
        self.retry(exc=e)

    if count == 0:
        # nothing to do, retry later
        self.retry()

# retry every minute, for up to 24 hours.
@app.task(bind=True, rate_limit="10/m", default_retry_delay=60, max_retries=24*60)
def get_feed_item(self, item):
    """ Fetch and process a document feed item. """
    try:
        dp = DocumentProcessor()
        dp.process_feed_item(item)
    except Exception as e:
        log.error("Error processing feed item: %s" % item, exc_info=e)
        self.retry(exc=e)
<commit_msg>Retry a task up to 10 times only<commit_after>from __future__ import absolute_import

import logging
from datetime import date, timedelta
from dateutil.parser import parse

from dexter.app import celery_app as app
from dexter.processing import DocumentProcessor

# force configs for API keys to be set
import dexter.core

# This is a collection of periodic tasks for Dexter, using
# Celery to drive task completion.

log = logging.getLogger(__name__)

@app.task
def fetch_yesterdays_feeds():
    """ Enqueue a task to fetch yesterday's feeds. """
    yesterday = date.today() - timedelta(days=1)
    fetch_daily_feeds.delay(yesterday.isoformat())

# retry after 30 minutes, retry for up to 7 days
@app.task(bind=True, default_retry_delay=30*60, max_retries=7*24*2)
def fetch_daily_feeds(self, day):
    """ Fetch feed of URLs to crawl and queue up a task to grab and process
    each url. """
    try:
        day = parse(day)

        dp = DocumentProcessor()
        count = 0
        for item in dp.fetch_daily_feed_items(day):
            get_feed_item.delay(item)
            count += 1
    except Exception as e:
        log.error("Error processing daily feeds for %s" % day, exc_info=e)
        self.retry(exc=e)

    if count == 0:
        # nothing to do, retry later
        self.retry()

# retry every minute, for up to 24 hours.
@app.task(bind=True, rate_limit="10/m", default_retry_delay=60, max_retries=10*60)
def get_feed_item(self, item):
    """ Fetch and process a document feed item. """
    try:
        dp = DocumentProcessor()
        dp.process_feed_item(item)
    except Exception as e:
        log.error("Error processing feed item: %s" % item, exc_info=e)
        self.retry()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
The MIT License (MIT)

Copyright (c) 2015-2016 Rapptz

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

import asyncio

from .permissions import Permissions
from .colour import Colour
from .mixins import Hashable
from .utils import snowflake_time

class Role(Hashable):
    """Represents a Discord role in a :class:`Guild`.

    Supported Operations:

    +-----------+------------------------------------------------------------------+
    | Operation |                           Description                            |
    +===========+==================================================================+
    | x == y    | Checks if two roles are equal.                                   |
    +-----------+------------------------------------------------------------------+
    | x != y    | Checks if two roles are not equal.                               |
    +-----------+------------------------------------------------------------------+
    | x > y     | Checks if a role is higher than another in the hierarchy.        |
    +-----------+------------------------------------------------------------------+
    | x < y     | Checks if a role is lower than another in the hierarchy.         |
    +-----------+------------------------------------------------------------------+
    | x >= y    | Checks if a role is higher or equal to another in the hierarchy. |
    +-----------+------------------------------------------------------------------+
    | x <= y    | Checks if a role is lower or equal to another in the hierarchy.  |
    +-----------+------------------------------------------------------------------+
    | hash(x)   | Return the role's hash.                                          |
    +-----------+------------------------------------------------------------------+
    | str(x)    | Returns the role's name.                                         |
    +-----------+------------------------------------------------------------------+

    Attributes
    ----------
    id: int
        The ID for the role.
    name: str
        The name of the role.
    permissions: :class:`Permissions`
        Represents the role's permissions.
    guild: :class:`Guild`
        The guild the role belongs to.
    colour: :class:`Colour`
        Represents the role colour. An alias exists under ``color``.
    hoist: bool
         Indicates if the role will be displayed separately from other members.
    position: int
        The position of the role. This number is usually positive. The bottom
        role has a position of 0.
    managed: bool
        Indicates if the role is managed by the guild through some form of
        integrations such as Twitch.
    mentionable: bool
        Indicates if the role can be mentioned by users.
    """

    __slots__ = ('id', 'name', 'permissions', 'color', 'colour', 'position',
                 'managed', 'mentionable', 'hoist', 'guild', '_state' )

    def __init__(self, *, guild, state, data):
        self.guild = guild
        self._state = state
        self.id = int(data['id'])
        self._update(data)

    def __str__(self):
        return self.name

    def __repr__(self):
        return '<Role id={0.id} name={0.name!r}>'.format(self)

    def __lt__(self, other):
        if not isinstance(other, Role) or  not isinstance(self, Role):
            return NotImplemented

        if self.guild != other.guild:
            raise RuntimeError('cannot compare roles from two different guilds.')

        if self.position < other.position:
            return True

        if self.position == other.position:
            return int(self.id) > int(other.id)

        return False

    def __le__(self, other):
        r = Role.__lt__(other, self)
        if r is NotImplemented:
            return NotImplemented
        return not r

    def __gt__(self, other):
        return Role.__lt__(other, self)

    def __ge__(self, other):
        r = Role.__lt__(self, other)
        if r is NotImplemented:
            return NotImplemented
        return not r

    def _update(self, data):
        self.name = data['name']
        self.permissions = Permissions(data.get('permissions', 0))
        self.position = data.get('position', 0)
        self.colour = Colour(data.get('color', 0))
        self.hoist = data.get('hoist', False)
        self.managed = data.get('managed', False)
        self.mentionable = data.get('mentionable', False)
        self.color = self.colour

    @property
    def is_everyone(self):
        """Checks if the role is the @everyone role."""
        return self.guild.id == self.id

    @property
    def created_at(self):
        """Returns the role's creation time in UTC."""
        return snowflake_time(self.id)

    @property
    def mention(self):
        """Returns a string that allows you to mention a role."""
        return '<@&{}>'.format(self.id)

    @asyncio.coroutine
    def _move(self, position):
        if position <= 0:
            raise InvalidArgument("Cannot move role to position 0 or below")

        if self.is_everyone:
            raise InvalidArgument("Cannot move default role")

        if self.position == position:
            return  # Save discord the extra request.

        http = self._state.http
        url = '{0}/{1}/roles'.format(http.GUILDS, self.guild.id)

        change_range = range(min(self.position, position), max(self.position, position) + 1)
        sorted_roles = sorted((x for x in self.guild.roles if x.position in change_range and x.id != self.id),
                              key=lambda x: x.position)

        roles = [r.id for r in sorted_roles]

        if self.position > position:
            roles.insert(0, self.id)
        else:
            roles.append(self.id)

        payload = [{"id": z[0], "position": z[1]} for z in zip(roles, change_range)]
        yield from http.patch(url, json=payload, bucket='move_role')

    @asyncio.coroutine
    def edit(self, **fields):
        """|coro|

        Edits the role.

        You must have the :attr:`Permissions.manage_roles` permission to
        use this.

        All fields are optional.

        Parameters
        -----------
        name: str
            The new role name to change to.
        permissions: :class:`Permissions`
            The new permissions to change to.
        colour: :class:`Colour`
            The new colour to change to. (aliased to color as well)
        hoist: bool
            Indicates if the role should be shown separately in the member list.
        mentionable: bool
            Indicates if the role should be mentionable by others.
        position: int
            The new role's position. This must be below your top role's
            position or it will fail.

        Raises
        -------
        Forbidden
            You do not have permissions to change the role.
        HTTPException
            Editing the role failed.
        InvalidArgument
            An invalid position was given or the default
            role was asked to be moved.
        """

        position = fields.get('position')
        if position is not None:
            yield from self._move(position)
            self.position = position

        try:
            colour = fields['colour']
        except KeyError:
            colour = fields.get('color', self.colour)

        payload = {
            'name': fields.get('name', self.name),
            'permissions': fields.get('permissions', self.permissions).value,
            'color': colour.value,
            'hoist': fields.get('hoist', self.hoist),
            'mentionable': fields.get('mentionable', self.mentionable)
        }

        data = yield from self._state.http.edit_role(self.guild.id, self.id, **payload)
        self._update(data)

    @asyncio.coroutine
    def delete(self):
        """|coro|

        Deletes the role.

        You must have the :attr:`Permissions.manage_roles` permission to
        use this.

        Raises
        --------
        Forbidden
            You do not have permissions to delete the role.
        HTTPException
            Deleting the role failed.
        """

        yield from self._state.http.delete_role(self.guild.id, self.id)
<commit_msg>Add Role.members to get every member with a role.<commit_after># -*- coding: utf-8 -*-

"""
The MIT License (MIT)

Copyright (c) 2015-2016 Rapptz

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

import asyncio

from .permissions import Permissions
from .colour import Colour
from .mixins import Hashable
from .utils import snowflake_time

class Role(Hashable):
    """Represents a Discord role in a :class:`Guild`.

    Supported Operations:

    +-----------+------------------------------------------------------------------+
    | Operation |                           Description                            |
    +===========+==================================================================+
    | x == y    | Checks if two roles are equal.                                   |
    +-----------+------------------------------------------------------------------+
    | x != y    | Checks if two roles are not equal.                               |
    +-----------+------------------------------------------------------------------+
    | x > y     | Checks if a role is higher than another in the hierarchy.        |
    +-----------+------------------------------------------------------------------+
    | x < y     | Checks if a role is lower than another in the hierarchy.         |
    +-----------+------------------------------------------------------------------+
    | x >= y    | Checks if a role is higher or equal to another in the hierarchy. |
    +-----------+------------------------------------------------------------------+
    | x <= y    | Checks if a role is lower or equal to another in the hierarchy.  |
    +-----------+------------------------------------------------------------------+
    | hash(x)   | Return the role's hash.                                          |
    +-----------+------------------------------------------------------------------+
    | str(x)    | Returns the role's name.                                         |
    +-----------+------------------------------------------------------------------+

    Attributes
    ----------
    id: int
        The ID for the role.
    name: str
        The name of the role.
    permissions: :class:`Permissions`
        Represents the role's permissions.
    guild: :class:`Guild`
        The guild the role belongs to.
    colour: :class:`Colour`
        Represents the role colour. An alias exists under ``color``.
    hoist: bool
         Indicates if the role will be displayed separately from other members.
    position: int
        The position of the role. This number is usually positive. The bottom
        role has a position of 0.
    managed: bool
        Indicates if the role is managed by the guild through some form of
        integrations such as Twitch.
    mentionable: bool
        Indicates if the role can be mentioned by users.
    """

    __slots__ = ('id', 'name', 'permissions', 'color', 'colour', 'position',
                 'managed', 'mentionable', 'hoist', 'guild', '_state' )

    def __init__(self, *, guild, state, data):
        self.guild = guild
        self._state = state
        self.id = int(data['id'])
        self._update(data)

    def __str__(self):
        return self.name

    def __repr__(self):
        return '<Role id={0.id} name={0.name!r}>'.format(self)

    def __lt__(self, other):
        if not isinstance(other, Role) or  not isinstance(self, Role):
            return NotImplemented

        if self.guild != other.guild:
            raise RuntimeError('cannot compare roles from two different guilds.')

        if self.position < other.position:
            return True

        if self.position == other.position:
            return int(self.id) > int(other.id)

        return False

    def __le__(self, other):
        r = Role.__lt__(other, self)
        if r is NotImplemented:
            return NotImplemented
        return not r

    def __gt__(self, other):
        return Role.__lt__(other, self)

    def __ge__(self, other):
        r = Role.__lt__(self, other)
        if r is NotImplemented:
            return NotImplemented
        return not r

    def _update(self, data):
        self.name = data['name']
        self.permissions = Permissions(data.get('permissions', 0))
        self.position = data.get('position', 0)
        self.colour = Colour(data.get('color', 0))
        self.hoist = data.get('hoist', False)
        self.managed = data.get('managed', False)
        self.mentionable = data.get('mentionable', False)
        self.color = self.colour

    @property
    def is_everyone(self):
        """Checks if the role is the @everyone role."""
        return self.guild.id == self.id

    @property
    def created_at(self):
        """Returns the role's creation time in UTC."""
        return snowflake_time(self.id)

    @property
    def mention(self):
        """Returns a string that allows you to mention a role."""
        return '<@&{}>'.format(self.id)

    @property
    def members(self):
        """Returns a list of :class:`Member` with this role."""
        all_members = self.guild.members
        if self.is_everyone:
            return all_members

        ret = []
        for member in all_members:
            if self in member.roles:
                ret.append(member)
        return ret

    @asyncio.coroutine
    def _move(self, position):
        if position <= 0:
            raise InvalidArgument("Cannot move role to position 0 or below")

        if self.is_everyone:
            raise InvalidArgument("Cannot move default role")

        if self.position == position:
            return  # Save discord the extra request.

        http = self._state.http
        url = '{0}/{1}/roles'.format(http.GUILDS, self.guild.id)

        change_range = range(min(self.position, position), max(self.position, position) + 1)
        sorted_roles = sorted((x for x in self.guild.roles if x.position in change_range and x.id != self.id),
                              key=lambda x: x.position)

        roles = [r.id for r in sorted_roles]

        if self.position > position:
            roles.insert(0, self.id)
        else:
            roles.append(self.id)

        payload = [{"id": z[0], "position": z[1]} for z in zip(roles, change_range)]
        yield from http.patch(url, json=payload, bucket='move_role')

    @asyncio.coroutine
    def edit(self, **fields):
        """|coro|

        Edits the role.

        You must have the :attr:`Permissions.manage_roles` permission to
        use this.

        All fields are optional.

        Parameters
        -----------
        name: str
            The new role name to change to.
        permissions: :class:`Permissions`
            The new permissions to change to.
        colour: :class:`Colour`
            The new colour to change to. (aliased to color as well)
        hoist: bool
            Indicates if the role should be shown separately in the member list.
        mentionable: bool
            Indicates if the role should be mentionable by others.
        position: int
            The new role's position. This must be below your top role's
            position or it will fail.

        Raises
        -------
        Forbidden
            You do not have permissions to change the role.
        HTTPException
            Editing the role failed.
        InvalidArgument
            An invalid position was given or the default
            role was asked to be moved.
        """

        position = fields.get('position')
        if position is not None:
            yield from self._move(position)
            self.position = position

        try:
            colour = fields['colour']
        except KeyError:
            colour = fields.get('color', self.colour)

        payload = {
            'name': fields.get('name', self.name),
            'permissions': fields.get('permissions', self.permissions).value,
            'color': colour.value,
            'hoist': fields.get('hoist', self.hoist),
            'mentionable': fields.get('mentionable', self.mentionable)
        }

        data = yield from self._state.http.edit_role(self.guild.id, self.id, **payload)
        self._update(data)

    @asyncio.coroutine
    def delete(self):
        """|coro|

        Deletes the role.

        You must have the :attr:`Permissions.manage_roles` permission to
        use this.

        Raises
        --------
        Forbidden
            You do not have permissions to delete the role.
        HTTPException
            Deleting the role failed.
        """

        yield from self._state.http.delete_role(self.guild.id, self.id)
<|endoftext|>


<commit_before>import numpy as np
from stingray.events import EventList
from stingray.varenergyspectrum import VarEnergySpectrum, RmsEnergySpectrum
from stingray.varenergyspectrum import LagEnergySpectrum
from stingray.varenergyspectrum import ExcessVarianceSpectrum
from stingray.lightcurve import Lightcurve

from astropy.tests.helper import pytest
np.random.seed(20150907)

class DummyVarEnergy(VarEnergySpectrum):
    def _spectrum_function(self):
        return None, None

class TestExcVarEnergySpectrum(object):
    @classmethod
    def setup_class(cls):
        from ..simulator.simulator import Simulator

        simulator = Simulator(0.1, 10000, rms=0.4, mean=200)
        test_lc = simulator.simulate(1)
        cls.test_ev1, cls.test_ev2 = EventList(), EventList()
        cls.test_ev1.simulate_times(test_lc)
        cls.test_ev1.energy = np.random.uniform(0.3, 12,
                                                len(cls.test_ev1.time))

    def test_allocate(self):
        exv = ExcessVarianceSpectrum(self.test_ev1, [0., 100],
                                     (0.3, 12, 5, "lin"),
                                     bin_time=1,
                                     segment_size=100)

class TestVarEnergySpectrum(object):

    @classmethod
    def setup_class(cls):
        tstart = 0.0
        tend = 100.0
        nphot = 1000
        alltimes = np.random.uniform(tstart, tend, nphot)
        alltimes.sort()
        cls.events = EventList(alltimes,
                               energy=np.random.uniform(0.3, 12, nphot),
                               gti = [[tstart, tend]])
        cls.vespec = DummyVarEnergy(cls.events, [0., 10000],
                                    (0.5, 5, 10, "lin"), [0.3, 10],
                                    bin_time=0.1)
        cls.vespeclog = \
            DummyVarEnergy(cls.events, [0., 10000],
                           (0.5, 5, 10, "log"), [0.3, 10])

    def test_intervals_overlapping(self):
        ref_int = self.vespec._decide_ref_intervals([0.5, 6], [0.3, 10])
        np.testing.assert_allclose(ref_int, [[0.3, 0.5], [6, 10]])
        ref_int = self.vespec._decide_ref_intervals([0.5, 11], [0.3, 10])
        np.testing.assert_allclose(ref_int, [[0.3, 0.5]])

    def test_intervals_non_overlapping(self):
        ref_int = self.vespec._decide_ref_intervals([6, 11], [0.3, 5])
        np.testing.assert_allclose(ref_int, [[0.3, 5]])

    def test_ref_band_none(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           energy=[0,0,0,0,1,1],
                           gti=[[0, 0.65]])
        vespec = DummyVarEnergy(events, [0., 10000],
                                (0, 1, 2, "lin"),
                                bin_time=0.1)
        assert np.all(vespec.ref_band == np.array([[0, np.inf]]))

    def test_energy_spec_wrong_list_not_tuple(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           energy=[0, 0, 0, 0, 1, 1],
                           gti=[[0, 0.65]])
        # Test using a list instead of tuple
        # with pytest.raises(ValueError):
        vespec = DummyVarEnergy(events, [0., 10000],
                                [0, 1, 2, "lin"],
                                bin_time=0.1)

    def test_energy_spec_wrong_str(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           energy=[0, 0, 0, 0, 1, 1],
                           gti=[[0, 0.65]])
        # Test using a list instead of tuple
        with pytest.raises(ValueError):
            vespec = DummyVarEnergy(events, [0., 10000],
                                    (0, 1, 2, "xxx"),
                                    bin_time=0.1)

    def test_construct_lightcurves(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           energy=[0,0,0,0,1,1],
                           gti=[[0, 0.65]])
        vespec = DummyVarEnergy(events, [0., 10000],
                                (0, 1, 2, "lin"), [0.5, 1.1],
                                bin_time=0.1)
        base_lc, ref_lc = \
            vespec._construct_lightcurves([0, 0.5],
                                          tstart=0, tstop=0.65)
        np.testing.assert_allclose(base_lc.counts, [1, 0, 2, 1, 0, 0])
        np.testing.assert_allclose(ref_lc.counts, [0, 0, 0, 1, 0, 1])

    def test_construct_lightcurves_no_exclude(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           energy=[0,0,0,0,1,1],
                           gti=[[0, 0.65]])

        vespec = DummyVarEnergy(events, [0., 10000],
                                (0, 1, 2, "lin"), [0, 0.5],
                                bin_time=0.1)
        base_lc, ref_lc = \
            vespec._construct_lightcurves([0, 0.5],
                                          tstart=0, tstop=0.65,
                                          exclude=False)
        np.testing.assert_equal(base_lc.counts, ref_lc.counts)

    def test_construct_lightcurves_pi(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           pi=np.asarray([0, 0, 0, 0, 1, 1]),
                           gti=[[0, 0.65]])
        vespec = DummyVarEnergy(events, [0., 10000],
                                (0, 1, 2, "lin"), [0.5, 1.1], use_pi=True,
                                   bin_time=0.1)
        base_lc, ref_lc = \
            vespec._construct_lightcurves([0, 0.5],
                                          tstart=0, tstop=0.65)
        np.testing.assert_allclose(base_lc.counts, [1, 0, 2, 1, 0, 0])
        np.testing.assert_allclose(ref_lc.counts, [0, 0, 0, 1, 0, 1])

class TestRMSEnergySpectrum(object):
    @classmethod
    def setup_class(cls):
        from ..simulator.simulator import Simulator

        simulator = Simulator(0.1, 10000, rms=0.4, mean=200)
        test_lc = simulator.simulate(1)
        test_ev1, test_ev2 = EventList(), EventList()
        test_ev1.simulate_times(test_lc)
        test_ev2.simulate_times(test_lc)
        test_ev1.energy = np.random.uniform(0.3, 12, len(test_ev1.time))
        test_ev2.energy = np.random.uniform(0.3, 12, len(test_ev2.time))

        cls.rms = RmsEnergySpectrum(test_ev1, [0., 100],
                                    (0.3, 12, 5, "lin"),
                                    bin_time=0.01,
                                    segment_size=100,
                                    events2=test_ev2)

    def test_correct_rms_values(self):
        # Assert that it is close to 0.4 (since we don't have infinite spectral
        # coverage, it will be a little less!)
        assert np.allclose(self.rms.spectrum, 0.4, 0.05)

    def test_correct_rms_errorbars(self):

        # Assert that the rms measured at all energies is the same
        assert np.all(
            np.abs(self.rms.spectrum - self.rms.spectrum[0]) < \
                self.rms.spectrum_error)

    def test_rms_invalid_evlist_warns(self):
        ev = EventList(time=[], energy=[], gti=self.rms.events1.gti)
        with pytest.warns(UserWarning) as record:
            rms = RmsEnergySpectrum(ev, [0., 100],
                                    (0.3, 12, 5, "lin"),
                                    bin_time=0.01,
                                    segment_size=100,
                                    events2=self.rms.events2)

        assert np.allclose(rms.spectrum, 0)
        assert np.allclose(rms.spectrum_error, 0)

class TestLagEnergySpectrum(object):
    @classmethod
    def setup_class(cls):
        from ..simulator.simulator import Simulator
        dt = 0.1
        simulator = Simulator(dt, 10000, rms=0.4, mean=200)
        test_lc1 = simulator.simulate(2)
        test_lc2 = Lightcurve(test_lc1.time,
                              np.array(np.roll(test_lc1.counts, 2)),
                              err_dist=test_lc1.err_dist,
                              dt=dt)

        test_ev1, test_ev2 = EventList(), EventList()
        test_ev1.simulate_times(test_lc1)
        test_ev2.simulate_times(test_lc2)
        test_ev1.energy = np.random.uniform(0.3, 9, len(test_ev1.time))
        test_ev2.energy = np.random.uniform(9, 12, len(test_ev2.time))

        cls.lag = LagEnergySpectrum(test_ev1, [0., 0.5],
                                    (0.3, 9, 4, "lin"), [9, 12],
                                    bin_time=0.1,
                                    segment_size=30,
                                    events2=test_ev2)

    def test_lagspectrum_values_and_errors(self):

        assert np.all(np.abs(self.lag.spectrum - 0.2) < \
                      3 * self.lag.spectrum_error)

    def test_lag_invalid_evlist_warns(self):
        ev = EventList(time=[], energy=[], gti=self.lag.events1.gti)
        with pytest.warns(UserWarning) as record:
            lag = LagEnergySpectrum(ev, [0., 0.5],
                                    (0.3, 9, 4, "lin"), [9, 12],
                                    bin_time=0.1,
                                    segment_size=30,
                                    events2=self.lag.events2)

        assert np.allclose(lag.spectrum, 0)
        assert np.allclose(lag.spectrum_error, 0)
<commit_msg>Revert "change value in test_corret_rms_values"<commit_after>import numpy as np
from stingray.events import EventList
from stingray.varenergyspectrum import VarEnergySpectrum, RmsEnergySpectrum
from stingray.varenergyspectrum import LagEnergySpectrum
from stingray.varenergyspectrum import ExcessVarianceSpectrum
from stingray.lightcurve import Lightcurve

from astropy.tests.helper import pytest
np.random.seed(20150907)

class DummyVarEnergy(VarEnergySpectrum):
    def _spectrum_function(self):
        return None, None

class TestExcVarEnergySpectrum(object):
    @classmethod
    def setup_class(cls):
        from ..simulator.simulator import Simulator

        simulator = Simulator(0.1, 10000, rms=0.4, mean=200)
        test_lc = simulator.simulate(1)
        cls.test_ev1, cls.test_ev2 = EventList(), EventList()
        cls.test_ev1.simulate_times(test_lc)
        cls.test_ev1.energy = np.random.uniform(0.3, 12,
                                                len(cls.test_ev1.time))

    def test_allocate(self):
        exv = ExcessVarianceSpectrum(self.test_ev1, [0., 100],
                                     (0.3, 12, 5, "lin"),
                                     bin_time=1,
                                     segment_size=100)

class TestVarEnergySpectrum(object):

    @classmethod
    def setup_class(cls):
        tstart = 0.0
        tend = 100.0
        nphot = 1000
        alltimes = np.random.uniform(tstart, tend, nphot)
        alltimes.sort()
        cls.events = EventList(alltimes,
                               energy=np.random.uniform(0.3, 12, nphot),
                               gti = [[tstart, tend]])
        cls.vespec = DummyVarEnergy(cls.events, [0., 10000],
                                    (0.5, 5, 10, "lin"), [0.3, 10],
                                    bin_time=0.1)
        cls.vespeclog = \
            DummyVarEnergy(cls.events, [0., 10000],
                           (0.5, 5, 10, "log"), [0.3, 10])

    def test_intervals_overlapping(self):
        ref_int = self.vespec._decide_ref_intervals([0.5, 6], [0.3, 10])
        np.testing.assert_allclose(ref_int, [[0.3, 0.5], [6, 10]])
        ref_int = self.vespec._decide_ref_intervals([0.5, 11], [0.3, 10])
        np.testing.assert_allclose(ref_int, [[0.3, 0.5]])

    def test_intervals_non_overlapping(self):
        ref_int = self.vespec._decide_ref_intervals([6, 11], [0.3, 5])
        np.testing.assert_allclose(ref_int, [[0.3, 5]])

    def test_ref_band_none(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           energy=[0,0,0,0,1,1],
                           gti=[[0, 0.65]])
        vespec = DummyVarEnergy(events, [0., 10000],
                                (0, 1, 2, "lin"),
                                bin_time=0.1)
        assert np.all(vespec.ref_band == np.array([[0, np.inf]]))

    def test_energy_spec_wrong_list_not_tuple(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           energy=[0, 0, 0, 0, 1, 1],
                           gti=[[0, 0.65]])
        # Test using a list instead of tuple
        # with pytest.raises(ValueError):
        vespec = DummyVarEnergy(events, [0., 10000],
                                [0, 1, 2, "lin"],
                                bin_time=0.1)

    def test_energy_spec_wrong_str(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           energy=[0, 0, 0, 0, 1, 1],
                           gti=[[0, 0.65]])
        # Test using a list instead of tuple
        with pytest.raises(ValueError):
            vespec = DummyVarEnergy(events, [0., 10000],
                                    (0, 1, 2, "xxx"),
                                    bin_time=0.1)

    def test_construct_lightcurves(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           energy=[0,0,0,0,1,1],
                           gti=[[0, 0.65]])
        vespec = DummyVarEnergy(events, [0., 10000],
                                (0, 1, 2, "lin"), [0.5, 1.1],
                                bin_time=0.1)
        base_lc, ref_lc = \
            vespec._construct_lightcurves([0, 0.5],
                                          tstart=0, tstop=0.65)
        np.testing.assert_allclose(base_lc.counts, [1, 0, 2, 1, 0, 0])
        np.testing.assert_allclose(ref_lc.counts, [0, 0, 0, 1, 0, 1])

    def test_construct_lightcurves_no_exclude(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           energy=[0,0,0,0,1,1],
                           gti=[[0, 0.65]])

        vespec = DummyVarEnergy(events, [0., 10000],
                                (0, 1, 2, "lin"), [0, 0.5],
                                bin_time=0.1)
        base_lc, ref_lc = \
            vespec._construct_lightcurves([0, 0.5],
                                          tstart=0, tstop=0.65,
                                          exclude=False)
        np.testing.assert_equal(base_lc.counts, ref_lc.counts)

    def test_construct_lightcurves_pi(self):
        events = EventList([0.09, 0.21, 0.23, 0.32, 0.4, 0.54],
                           pi=np.asarray([0, 0, 0, 0, 1, 1]),
                           gti=[[0, 0.65]])
        vespec = DummyVarEnergy(events, [0., 10000],
                                (0, 1, 2, "lin"), [0.5, 1.1], use_pi=True,
                                   bin_time=0.1)
        base_lc, ref_lc = \
            vespec._construct_lightcurves([0, 0.5],
                                          tstart=0, tstop=0.65)
        np.testing.assert_allclose(base_lc.counts, [1, 0, 2, 1, 0, 0])
        np.testing.assert_allclose(ref_lc.counts, [0, 0, 0, 1, 0, 1])

class TestRMSEnergySpectrum(object):
    @classmethod
    def setup_class(cls):
        from ..simulator.simulator import Simulator

        simulator = Simulator(0.1, 10000, rms=0.4, mean=200)
        test_lc = simulator.simulate(1)
        test_ev1, test_ev2 = EventList(), EventList()
        test_ev1.simulate_times(test_lc)
        test_ev2.simulate_times(test_lc)
        test_ev1.energy = np.random.uniform(0.3, 12, len(test_ev1.time))
        test_ev2.energy = np.random.uniform(0.3, 12, len(test_ev2.time))

        cls.rms = RmsEnergySpectrum(test_ev1, [0., 100],
                                    (0.3, 12, 5, "lin"),
                                    bin_time=0.01,
                                    segment_size=100,
                                    events2=test_ev2)

    def test_correct_rms_values(self):
        # Assert that it is close to 0.4 (since we don't have infinite spectral
        # coverage, it will be a little less!)
        assert np.allclose(self.rms.spectrum, 0.37, 0.05)

    def test_correct_rms_errorbars(self):

        # Assert that the rms measured at all energies is the same
        assert np.all(
            np.abs(self.rms.spectrum - self.rms.spectrum[0]) < \
                self.rms.spectrum_error)

    def test_rms_invalid_evlist_warns(self):
        ev = EventList(time=[], energy=[], gti=self.rms.events1.gti)
        with pytest.warns(UserWarning) as record:
            rms = RmsEnergySpectrum(ev, [0., 100],
                                    (0.3, 12, 5, "lin"),
                                    bin_time=0.01,
                                    segment_size=100,
                                    events2=self.rms.events2)

        assert np.allclose(rms.spectrum, 0)
        assert np.allclose(rms.spectrum_error, 0)

class TestLagEnergySpectrum(object):
    @classmethod
    def setup_class(cls):
        from ..simulator.simulator import Simulator
        dt = 0.1
        simulator = Simulator(dt, 10000, rms=0.4, mean=200)
        test_lc1 = simulator.simulate(2)
        test_lc2 = Lightcurve(test_lc1.time,
                              np.array(np.roll(test_lc1.counts, 2)),
                              err_dist=test_lc1.err_dist,
                              dt=dt)

        test_ev1, test_ev2 = EventList(), EventList()
        test_ev1.simulate_times(test_lc1)
        test_ev2.simulate_times(test_lc2)
        test_ev1.energy = np.random.uniform(0.3, 9, len(test_ev1.time))
        test_ev2.energy = np.random.uniform(9, 12, len(test_ev2.time))

        cls.lag = LagEnergySpectrum(test_ev1, [0., 0.5],
                                    (0.3, 9, 4, "lin"), [9, 12],
                                    bin_time=0.1,
                                    segment_size=30,
                                    events2=test_ev2)

    def test_lagspectrum_values_and_errors(self):

        assert np.all(np.abs(self.lag.spectrum - 0.2) < \
                      3 * self.lag.spectrum_error)

    def test_lag_invalid_evlist_warns(self):
        ev = EventList(time=[], energy=[], gti=self.lag.events1.gti)
        with pytest.warns(UserWarning) as record:
            lag = LagEnergySpectrum(ev, [0., 0.5],
                                    (0.3, 9, 4, "lin"), [9, 12],
                                    bin_time=0.1,
                                    segment_size=30,
                                    events2=self.lag.events2)

        assert np.allclose(lag.spectrum, 0)
        assert np.allclose(lag.spectrum_error, 0)
<|endoftext|>