<commit_before>"""The Iteration/Expression hierarchy."""

from __future__ import absolute_import

import inspect
from collections import Iterable, OrderedDict

import cgen as c
from sympy import Eq, IndexedBase, preorder_traversal

from devito.codeprinter import ccode
from devito.dimension import Dimension
from devito.dse.inspection import retrieve_indexed
from devito.interfaces import SymbolicData, TensorData
from devito.tools import DefaultOrderedDict, as_tuple, filter_ordered

__all__ = ['Node', 'Block', 'Expression', 'Function', 'Iteration', 'List',
           'TimedList']

class Node(object):

    is_Node = True
    is_Block = False
    is_Iteration = False
    is_Expression = False
    is_Function = False
    is_List = False
    is_Element = False

    """
    :attr:`_traversable`. A list of traversable objects (ie, traversed by
    :class:`Visitor` objects). A traversable object is intended as an argument
    of a Node constructor and is represented as a string.
    """
    _traversable = []

    def __new__(cls, *args, **kwargs):
        obj = super(Node, cls).__new__(cls)
        argnames = inspect.getargspec(cls.__init__).args
        obj._args = {k: v for k, v in zip(argnames[1:], args)}
        obj._args.update(kwargs.items())
        obj._args.update({k: None for k in argnames[1:] if k not in obj._args})
        return obj

    def _rebuild(self, *args, **kwargs):
        """Reconstruct self. None of the embedded Sympy expressions are rebuilt."""
        handle = self._args.copy()  # Original constructor arguments
        argnames = [i for i in self._traversable if i not in kwargs]
        handle.update(OrderedDict([(k, v) for k, v in zip(argnames, args)]))
        handle.update(kwargs)
        return type(self)(**handle)

    @property
    def ccode(self):
        """Generate C code."""
        raise NotImplementedError()

    @property
    def children(self):
        """Return the traversable children."""
        return ()

    @property
    def args(self):
        """Arguments used to construct the Node."""
        return self._args.copy()

    @property
    def args_frozen(self):
        """Arguments used to construct the Node that cannot be traversed."""
        return {k: v for k, v in self.args.items() if k not in self._traversable}

class Block(Node):

    is_Block = True

    _wrapper = c.Block

    _traversable = ['body']

    def __init__(self, header=None, body=None, footer=None):
        self.header = as_tuple(header)
        self.body = as_tuple(body)
        self.footer = as_tuple(footer)

    def __repr__(self):
        return "<%s (%d, %d, %d)>" % (self.__class__.__name__, len(self.header),
                                      len(self.body), len(self.footer))

    @property
    def ccode(self):
        body = tuple(s.ccode for s in self.body)
        return c.Module(self.header + (self._wrapper(body),) + self.footer)

    @property
    def children(self):
        return (self.body,)

class List(Block):

    """Class representing a sequence of one or more statements."""

    is_List = True

    _wrapper = c.Collection

class Element(Node):

    """A generic node that is worth identifying in an Iteration/Expression tree.

    It corresponds to a single :class:`cgen.Statement`.
    """

    is_Element = True

    def __init__(self, element):
        assert isinstance(element, (c.Comment, c.Statement, c.Value))
        self.element = element

    def __repr__(self):
        return "Element::\n\t%s" % (self.element)

    @property
    def ccode(self):
        return self.element

class Expression(Node):

    """Class encpasulating a single stencil expression"""

    is_Expression = True

    def __init__(self, stencil):
        assert isinstance(stencil, Eq)
        self.stencil = stencil

        self.dimensions = []
        self.functions = []
        # Traverse stencil to determine meta information
        for e in preorder_traversal(self.stencil):
            if isinstance(e, SymbolicData):
                self.dimensions += list(e.indices)
                self.functions += [e]
            if isinstance(e, IndexedBase):
                self.dimensions += list(e.function.indices)
                self.functions += [e.function]
        # Filter collected dimensions and functions
        self.dimensions = filter_ordered(self.dimensions)
        self.functions = filter_ordered(self.functions)

    def __repr__(self):
        return "<Expression::%s>" % filter_ordered([f.func for f in self.functions])

    def substitute(self, substitutions):
        """Apply substitutions to the expression stencil

        :param substitutions: Dict containing the substitutions to apply to
                              the stored loop stencils.
        """
        self.stencil = self.stencil.xreplace(substitutions)

    @property
    def ccode(self):
        return c.Assign(ccode(self.stencil.lhs), ccode(self.stencil.rhs))

    @property
    def output(self):
        """
        Return the symbol written by this Expression.
        """
        lhs = self.stencil.lhs
        if lhs.is_Symbol:
            return lhs
        else:
            # An Indexed
            return lhs.base.label

    @property
    def index_offsets(self):
        """Mapping of :class:`Dimension` symbols to each integer
        stencil offset used with it in this expression.

        This also include zero offsets, so that the keys of this map
        may be used as the set of :class:`Dimension` symbols used in
        this expression.

        Note: This assumes we have indexified the stencil expression.
        """
        offsets = DefaultOrderedDict(set)

        # Enforce left-first traversal order
        indexed = list(retrieve_indexed(self.stencil.lhs))
        indexed += list(retrieve_indexed(self.stencil.rhs))
        for e in indexed:
            for a in e.indices:
                if isinstance(a, Dimension):
                    offsets[a].update([0])
                d = None
                off = []
                for idx in a.args:
                    if isinstance(idx, Dimension):
                        d = idx
                    elif idx.is_integer:
                        off += [idx]
                if d is not None:
                    offsets[d].update(off)
        return offsets

class TypedExpression(Expression):

    """Class encpasulating a single stencil expression with known data type,
    represented as a NumPy data type."""

    def __init__(self, stencil, dtype):
        super(TypedExpression, self).__init__(stencil)
        self.dtype = dtype

    @property
    def ccode(self):
        ctype = c.dtype_to_ctype(self.dtype)
        return c.Initializer(c.Value(ctype, ccode(self.stencil.lhs)),
                             ccode(self.stencil.rhs))

class Iteration(Node):
    """Iteration object that encapsualtes a single loop over nodes, possibly
    just SymPy expressions.

    :param nodes: Single or list of :class:`Node` objects that
                        define the loop body.
    :param dimension: :class:`Dimension` object over which to iterate.
    :param limits: Limits for the iteration space, either the loop size or a
                   tuple of the form (start, finish, stepping).
    :param index: Symbol to be used as iteration variable.
    :param offsets: Optional map list of offsets to honour in the loop.
    :param properties: A bag of strings indicating properties of this Iteration.
                       For example, the string 'parallel' may be used to identify
                       a parallelizable Iteration.
    """

    is_Iteration = True

    _traversable = ['nodes']

    def __init__(self, nodes, dimension, limits, index=None, offsets=None,
                 properties=None):
        # Ensure we deal with a list of Expression objects internally
        nodes = as_tuple(nodes)
        self.nodes = as_tuple([n if isinstance(n, Node) else Expression(n)
                               for n in nodes])
        assert all(isinstance(i, Node) for i in self.nodes)

        self.dim = dimension
        self.index = index or self.dim.name

        # Generate loop limits
        if isinstance(limits, Iterable):
            assert(len(limits) == 3)
            self.limits = list(limits)
        else:
            self.limits = list((0, limits, 1))

        # Replace open limits with variables names
        if self.limits[1] is None:
            # FIXME: Add dimension size as variable bound.
            # Needs further generalisation to support loop blocking.
            self.limits[1] = IterationBound("%s_size" % self.dim.name, self.dim)

        # Record offsets to later adjust loop limits accordingly
        self.offsets = [0, 0]
        for off in (offsets or {}):
            self.offsets[0] = min(self.offsets[0], int(off))
            self.offsets[1] = max(self.offsets[1], int(off))

        # Track this Iteration's properties
        self.properties = as_tuple(properties)

    def __repr__(self):
        properties = ""
        if self.properties:
            properties = "WithProperties[%s]::" % ",".join(self.properties)
        return "<%sIteration %s; %s>" % (properties, self.index, self.limits)

    @property
    def ccode(self):
        """Generate C code for the represented stencil loop

        :returns: :class:`cgen.For` object representing the loop
        """
        loop_body = [s.ccode for s in self.nodes]

        # Start
        if self.offsets[0] != 0:
            val = "%s + %s" % (self.limits[0], -self.offsets[0])
            try:
                val = eval(val)
            except (NameError, TypeError):
                pass
        else:
            val = self.limits[0]
        loop_init = c.InlineInitializer(c.Value("int", self.index), ccode(val))

        # Bound
        if self.offsets[1] != 0:
            val = "%s - %s" % (self.limits[1], self.offsets[1])
            try:
                val = eval(val)
            except (NameError, TypeError):
                pass
        else:
            val = self.limits[1]
        loop_cond = '%s < %s' % (self.index, ccode(val))

        # Increment
        loop_inc = '%s += %s' % (self.index, self.limits[2])

        return c.For(loop_init, loop_cond, loop_inc, c.Block(loop_body))

    @property
    def is_Open(self):
        return self.dim.size is not None

    @property
    def is_Closed(self):
        return not self.is_Open

    @property
    def children(self):
        """Return the traversable children."""
        return (self.nodes,)

class Function(Node):

    """Represent a C function.

    :param name: The name of the function.
    :param body: A :class:`Node` or an iterable of :class:`Node` objects representing
                 the body of the function.
    :param retval: The type of the value returned by the function.
    :param parameters: An iterable of :class:`SymbolicData` objects in input to the
                       function, or ``None`` if the function takes no parameter.
    :param prefix: An iterable of qualifiers to prepend to the function declaration.
                   The default value is ('static', 'inline').
    """

    is_Function = True

    _traversable = ['body']

    def __init__(self, name, body, retval, parameters=None, prefix=('static', 'inline')):
        self.name = name
        self.body = as_tuple(body)
        self.retval = retval
        self.parameters = as_tuple(parameters)
        self.prefix = prefix

    def __repr__(self):
        parameters = ",".join([c.dtype_to_ctype(i.dtype) for i in self.parameters])
        body = "\n\t".join([str(s) for s in self.body])
        return "Function[%s]<%s; %s>::\n\t%s" % (self.name, self.retval, parameters, body)

    @property
    def _cparameters(self):
        """Generate arguments signature."""
        cparameters = []
        for v in self.parameters:
            if isinstance(v, Dimension):
                cparameters.append(v.decl)
            elif v.is_ScalarData:
                cparameters.append(c.Value('const int', v.name))
            else:
                cparameters.append(c.Pointer(c.POD(v.dtype, '%s_vec' % v.name)))
        return cparameters

    @property
    def _ccasts(self):
        """Generate data casts."""
        handle = [f for f in self.parameters if isinstance(f, TensorData)]
        shapes = [(f, ''.join(["[%s]" % i.ccode for i in f.indices[1:]])) for f in handle]
        casts = [c.Initializer(c.POD(v.dtype, '(*%s)%s' % (v.name, shape)),
                               '(%s (*)%s) %s' % (c.dtype_to_ctype(v.dtype),
                                                  shape, '%s_vec' % v.name))
                 for v, shape in shapes]
        return casts

    @property
    def _ctop(self):
        """Generate the function declaration."""
        return c.FunctionDeclaration(c.Value(self.retval, self.name), self._cparameters)

    @property
    def ccode(self):
        """Generate C code for the represented C routine.

        :returns: :class:`cgen.FunctionDeclaration` object representing the function.
        """
        body = [e.ccode for e in self.body]
        return c.FunctionBody(self._ctop, c.Block(self._ccasts + body))

    @property
    def children(self):
        return (self.body,)

# Utilities

class TimedList(List):

    """Wrap a Node with C-level timers."""

    def __init__(self, lname, gname, body):
        """
        Initialize a TimedList object.

        :param lname: Timer name in the local scope.
        :param gname: Name of the global struct tracking all timers.
        :param body: Timed block of code.
        """
        self._name = lname
        # TODO: need omp master pragma to be thread safe
        header = [c.Statement("struct timeval start_%s, end_%s" % (lname, lname)),
                  c.Statement("gettimeofday(&start_%s, NULL)" % lname)]
        footer = [c.Statement("gettimeofday(&end_%s, NULL)" % lname),
                  c.Statement(("%(gn)s->%(ln)s += " +
                               "(double)(end_%(ln)s.tv_sec-start_%(ln)s.tv_sec)+" +
                               "(double)(end_%(ln)s.tv_usec-start_%(ln)s.tv_usec)" +
                               "/1000000") % {'gn': gname, 'ln': lname})]
        super(TimedList, self).__init__(header, body, footer)

    def __repr__(self):
        body = "\n\t".join([str(s) for s in self.body])
        return "%s::\n\t%s" % (self.__class__.__name__, body)

    @property
    def name(self):
        return self._name

class Denormals(Block):

    """Macros to make sure denormal numbers are flushed in hardware."""

    def __init__(self, header=None, body=None, footer=None):
        b = [Element(c.Comment('Flush denormal numbers to zero in hardware')),
             Element(c.Statement('_MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON)')),
             Element(c.Statement('_MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON)'))]
        super(Denormals, self).__init__(header, b, footer)

    def __repr__(self):
        return "<DenormalsMacro>"

class IterationBound(object):
    """Utility class to encapsulate variable loop bounds and link them
    back to the respective :class:`Dimension` object.

    :param name: Variable name for the open loop bound variable.
    :param dim: The corresponding :class:`Dimension` object.
    :param expr: An expression to calculate the loop limit, in case this does
                 not coincide with the open loop bound variable itself.
    """

    def __init__(self, name, dim, expr=None):
        self.name = name
        self.dim = dim
        self.expr = expr

    def __repr__(self):
        return repr(self.expr).replace(' ', '') if self.expr else self.name

    def __eq__(self, bound):
        return self.name == bound.name and self.dim == bound.dim

    @property
    def ccode(self):
        """C code for the variable declaration within a kernel signature"""
        return c.Value('const int', self.name)
<commit_msg>nodes: Minor additions<commit_after>"""The Iteration/Expression hierarchy."""

from __future__ import absolute_import

import inspect
from collections import Iterable, OrderedDict

import cgen as c
from sympy import Eq, IndexedBase, preorder_traversal

from devito.codeprinter import ccode
from devito.dimension import Dimension
from devito.dse.inspection import retrieve_indexed
from devito.interfaces import SymbolicData, TensorData
from devito.tools import DefaultOrderedDict, as_tuple, filter_ordered

__all__ = ['Node', 'Block', 'Expression', 'Function', 'Iteration', 'List',
           'TimedList']

class Node(object):

    is_Node = True
    is_Block = False
    is_Iteration = False
    is_Expression = False
    is_Function = False
    is_List = False
    is_Element = False

    """
    :attr:`_traversable`. A list of traversable objects (ie, traversed by
    :class:`Visitor` objects). A traversable object is intended as an argument
    of a Node constructor and is represented as a string.
    """
    _traversable = []

    def __new__(cls, *args, **kwargs):
        obj = super(Node, cls).__new__(cls)
        argnames = inspect.getargspec(cls.__init__).args
        obj._args = {k: v for k, v in zip(argnames[1:], args)}
        obj._args.update(kwargs.items())
        obj._args.update({k: None for k in argnames[1:] if k not in obj._args})
        return obj

    def _rebuild(self, *args, **kwargs):
        """Reconstruct self. None of the embedded Sympy expressions are rebuilt."""
        handle = self._args.copy()  # Original constructor arguments
        argnames = [i for i in self._traversable if i not in kwargs]
        handle.update(OrderedDict([(k, v) for k, v in zip(argnames, args)]))
        handle.update(kwargs)
        return type(self)(**handle)

    @property
    def ccode(self):
        """Generate C code."""
        raise NotImplementedError()

    @property
    def children(self):
        """Return the traversable children."""
        return ()

    @property
    def args(self):
        """Arguments used to construct the Node."""
        return self._args.copy()

    @property
    def args_frozen(self):
        """Arguments used to construct the Node that cannot be traversed."""
        return {k: v for k, v in self.args.items() if k not in self._traversable}

class Block(Node):

    is_Block = True

    _wrapper = c.Block

    _traversable = ['body']

    def __init__(self, header=None, body=None, footer=None):
        self.header = as_tuple(header)
        self.body = as_tuple(body)
        self.footer = as_tuple(footer)

    def __repr__(self):
        return "<%s (%d, %d, %d)>" % (self.__class__.__name__, len(self.header),
                                      len(self.body), len(self.footer))

    @property
    def ccode(self):
        body = tuple(s.ccode for s in self.body)
        return c.Module(self.header + (self._wrapper(body),) + self.footer)

    @property
    def children(self):
        return (self.body,)

class List(Block):

    """Class representing a sequence of one or more statements."""

    is_List = True

    _wrapper = c.Collection

class Element(Node):

    """A generic node that is worth identifying in an Iteration/Expression tree.

    It corresponds to a single :class:`cgen.Statement`.
    """

    is_Element = True

    def __init__(self, element):
        assert isinstance(element, (c.Comment, c.Statement, c.Value,
                                    c.Pragma, c.Line))
        self.element = element

    def __repr__(self):
        return "Element::\n\t%s" % (self.element)

    @property
    def ccode(self):
        return self.element

class Expression(Node):

    """Class encpasulating a single stencil expression"""

    is_Expression = True

    def __init__(self, stencil):
        assert isinstance(stencil, Eq)
        self.stencil = stencil

        self.dimensions = []
        self.functions = []
        # Traverse stencil to determine meta information
        for e in preorder_traversal(self.stencil):
            if isinstance(e, SymbolicData):
                self.dimensions += list(e.indices)
                self.functions += [e]
            if isinstance(e, IndexedBase):
                self.dimensions += list(e.function.indices)
                self.functions += [e.function]
        # Filter collected dimensions and functions
        self.dimensions = filter_ordered(self.dimensions)
        self.functions = filter_ordered(self.functions)

    def __repr__(self):
        return "<Expression::%s>" % filter_ordered([f.func for f in self.functions])

    def substitute(self, substitutions):
        """Apply substitutions to the expression stencil

        :param substitutions: Dict containing the substitutions to apply to
                              the stored loop stencils.
        """
        self.stencil = self.stencil.xreplace(substitutions)

    @property
    def ccode(self):
        return c.Assign(ccode(self.stencil.lhs), ccode(self.stencil.rhs))

    @property
    def output(self):
        """
        Return the symbol written by this Expression.
        """
        lhs = self.stencil.lhs
        if lhs.is_Symbol:
            return lhs
        else:
            # An Indexed
            return lhs.base.label

    @property
    def index_offsets(self):
        """Mapping of :class:`Dimension` symbols to each integer
        stencil offset used with it in this expression.

        This also include zero offsets, so that the keys of this map
        may be used as the set of :class:`Dimension` symbols used in
        this expression.

        Note: This assumes we have indexified the stencil expression.
        """
        offsets = DefaultOrderedDict(set)

        # Enforce left-first traversal order
        indexed = list(retrieve_indexed(self.stencil.lhs))
        indexed += list(retrieve_indexed(self.stencil.rhs))
        for e in indexed:
            for a in e.indices:
                if isinstance(a, Dimension):
                    offsets[a].update([0])
                d = None
                off = []
                for idx in a.args:
                    if isinstance(idx, Dimension):
                        d = idx
                    elif idx.is_integer:
                        off += [idx]
                if d is not None:
                    offsets[d].update(off)
        return offsets

class TypedExpression(Expression):

    """Class encpasulating a single stencil expression with known data type,
    represented as a NumPy data type."""

    def __init__(self, stencil, dtype):
        super(TypedExpression, self).__init__(stencil)
        self.dtype = dtype

    @property
    def ccode(self):
        ctype = c.dtype_to_ctype(self.dtype)
        return c.Initializer(c.Value(ctype, ccode(self.stencil.lhs)),
                             ccode(self.stencil.rhs))

class Iteration(Node):
    """Iteration object that encapsualtes a single loop over nodes, possibly
    just SymPy expressions.

    :param nodes: Single or list of :class:`Node` objects that
                        define the loop body.
    :param dimension: :class:`Dimension` object over which to iterate.
    :param limits: Limits for the iteration space, either the loop size or a
                   tuple of the form (start, finish, stepping).
    :param index: Symbol to be used as iteration variable.
    :param offsets: Optional map list of offsets to honour in the loop.
    :param properties: A bag of strings indicating properties of this Iteration.
                       For example, the string 'parallel' may be used to identify
                       a parallelizable Iteration.
    """

    is_Iteration = True

    _traversable = ['nodes']

    def __init__(self, nodes, dimension, limits, index=None, offsets=None,
                 properties=None):
        # Ensure we deal with a list of Expression objects internally
        nodes = as_tuple(nodes)
        self.nodes = as_tuple([n if isinstance(n, Node) else Expression(n)
                               for n in nodes])
        assert all(isinstance(i, Node) for i in self.nodes)

        self.dim = dimension
        self.index = index or self.dim.name

        # Generate loop limits
        if isinstance(limits, Iterable):
            assert(len(limits) == 3)
            self.limits = list(limits)
        else:
            self.limits = list((0, limits, 1))

        # Replace open limits with variables names
        if self.limits[1] is None:
            # FIXME: Add dimension size as variable bound.
            # Needs further generalisation to support loop blocking.
            self.limits[1] = IterationBound("%s_size" % self.dim.name, self.dim)

        # Record offsets to later adjust loop limits accordingly
        self.offsets = [0, 0]
        for off in (offsets or {}):
            self.offsets[0] = min(self.offsets[0], int(off))
            self.offsets[1] = max(self.offsets[1], int(off))

        # Track this Iteration's properties
        self.properties = as_tuple(properties)

    def __repr__(self):
        properties = ""
        if self.properties:
            properties = "WithProperties[%s]::" % ",".join(self.properties)
        return "<%sIteration %s; %s>" % (properties, self.index, self.limits)

    @property
    def ccode(self):
        """Generate C code for the represented stencil loop

        :returns: :class:`cgen.For` object representing the loop
        """
        loop_body = [s.ccode for s in self.nodes]

        # Start
        if self.offsets[0] != 0:
            val = "%s + %s" % (self.limits[0], -self.offsets[0])
            try:
                val = eval(val)
            except (NameError, TypeError):
                pass
        else:
            val = self.limits[0]
        loop_init = c.InlineInitializer(c.Value("int", self.index), ccode(val))

        # Bound
        if self.offsets[1] != 0:
            val = "%s - %s" % (self.limits[1], self.offsets[1])
            try:
                val = eval(val)
            except (NameError, TypeError):
                pass
        else:
            val = self.limits[1]
        loop_cond = '%s < %s' % (self.index, ccode(val))

        # Increment
        loop_inc = '%s += %s' % (self.index, self.limits[2])

        return c.For(loop_init, loop_cond, loop_inc, c.Block(loop_body))

    @property
    def is_Open(self):
        return self.dim.size is not None

    @property
    def is_Closed(self):
        return not self.is_Open

    @property
    def extent(self):
        return (self.limits[1] - self.offsets[1]) - (self.limits[0] - self.offsets[0])

    @property
    def children(self):
        """Return the traversable children."""
        return (self.nodes,)

class Function(Node):

    """Represent a C function.

    :param name: The name of the function.
    :param body: A :class:`Node` or an iterable of :class:`Node` objects representing
                 the body of the function.
    :param retval: The type of the value returned by the function.
    :param parameters: An iterable of :class:`SymbolicData` objects in input to the
                       function, or ``None`` if the function takes no parameter.
    :param prefix: An iterable of qualifiers to prepend to the function declaration.
                   The default value is ('static', 'inline').
    """

    is_Function = True

    _traversable = ['body']

    def __init__(self, name, body, retval, parameters=None, prefix=('static', 'inline')):
        self.name = name
        self.body = as_tuple(body)
        self.retval = retval
        self.parameters = as_tuple(parameters)
        self.prefix = prefix

    def __repr__(self):
        parameters = ",".join([c.dtype_to_ctype(i.dtype) for i in self.parameters])
        body = "\n\t".join([str(s) for s in self.body])
        return "Function[%s]<%s; %s>::\n\t%s" % (self.name, self.retval, parameters, body)

    @property
    def _cparameters(self):
        """Generate arguments signature."""
        cparameters = []
        for v in self.parameters:
            if isinstance(v, Dimension):
                cparameters.append(v.decl)
            elif v.is_ScalarData:
                cparameters.append(c.Value('const int', v.name))
            else:
                cparameters.append(c.Pointer(c.POD(v.dtype, '%s_vec' % v.name)))
        return cparameters

    @property
    def _ccasts(self):
        """Generate data casts."""
        handle = [f for f in self.parameters if isinstance(f, TensorData)]
        shapes = [(f, ''.join(["[%s]" % i.ccode for i in f.indices[1:]])) for f in handle]
        casts = [c.Initializer(c.POD(v.dtype, '(*%s)%s' % (v.name, shape)),
                               '(%s (*)%s) %s' % (c.dtype_to_ctype(v.dtype),
                                                  shape, '%s_vec' % v.name))
                 for v, shape in shapes]
        return casts

    @property
    def _ctop(self):
        """Generate the function declaration."""
        return c.FunctionDeclaration(c.Value(self.retval, self.name), self._cparameters)

    @property
    def ccode(self):
        """Generate C code for the represented C routine.

        :returns: :class:`cgen.FunctionDeclaration` object representing the function.
        """
        body = [e.ccode for e in self.body]
        return c.FunctionBody(self._ctop, c.Block(self._ccasts + body))

    @property
    def children(self):
        return (self.body,)

# Utilities

class TimedList(List):

    """Wrap a Node with C-level timers."""

    def __init__(self, lname, gname, body):
        """
        Initialize a TimedList object.

        :param lname: Timer name in the local scope.
        :param gname: Name of the global struct tracking all timers.
        :param body: Timed block of code.
        """
        self._name = lname
        # TODO: need omp master pragma to be thread safe
        header = [c.Statement("struct timeval start_%s, end_%s" % (lname, lname)),
                  c.Statement("gettimeofday(&start_%s, NULL)" % lname)]
        footer = [c.Statement("gettimeofday(&end_%s, NULL)" % lname),
                  c.Statement(("%(gn)s->%(ln)s += " +
                               "(double)(end_%(ln)s.tv_sec-start_%(ln)s.tv_sec)+" +
                               "(double)(end_%(ln)s.tv_usec-start_%(ln)s.tv_usec)" +
                               "/1000000") % {'gn': gname, 'ln': lname})]
        super(TimedList, self).__init__(header, body, footer)

    def __repr__(self):
        body = "\n\t".join([str(s) for s in self.body])
        return "%s::\n\t%s" % (self.__class__.__name__, body)

    @property
    def name(self):
        return self._name

class Denormals(List):

    """Macros to make sure denormal numbers are flushed in hardware."""

    def __init__(self, header=None, body=None, footer=None):
        b = [Element(c.Comment('Flush denormal numbers to zero in hardware')),
             Element(c.Statement('_MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON)')),
             Element(c.Statement('_MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON)'))]
        super(Denormals, self).__init__(header, b, footer)

    def __repr__(self):
        return "<DenormalsMacro>"

class IterationBound(object):
    """Utility class to encapsulate variable loop bounds and link them
    back to the respective :class:`Dimension` object.

    :param name: Variable name for the open loop bound variable.
    :param dim: The corresponding :class:`Dimension` object.
    :param expr: An expression to calculate the loop limit, in case this does
                 not coincide with the open loop bound variable itself.
    """

    def __init__(self, name, dim, expr=None):
        self.name = name
        self.dim = dim
        self.expr = expr

    def __repr__(self):
        return repr(self.expr).replace(' ', '') if self.expr else self.name

    def __eq__(self, bound):
        return self.name == bound.name and self.dim == bound.dim

    @property
    def ccode(self):
        """C code for the variable declaration within a kernel signature"""
        return c.Value('const int', self.name)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Cross-correlograms."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import numpy as np

from ..ext import six

#------------------------------------------------------------------------------
# Cross-correlograms
#------------------------------------------------------------------------------

"""

Variables
---------

winsize_samples : int
    Number of time samples in the window. Needs to be an odd number.

binsize : int
    Number of time samples in one bin.

winsize_bins : int
    Number of bins in the window.

    winsize_bins = 2 * ((winsize_samples // 2) // binsize) + 1
    assert winsize_bins % 2 == 1

"""

def _increment(arr, indices):
    """Increment some indices in a 1D vector of non-negative integers.
    Repeated indices are taken into account."""
    arr = np.asarray(arr)
    indices = np.asarray(indices)
    bbins = np.bincount(indices)
    arr[:len(bbins)] += bbins
    return arr

def _diff_shifted(arr, steps=1):
    arr = np.asarray(arr)
    return arr[steps:] - arr[:len(arr)-steps]

def _index_of(arr, lookup):
    """Replace scalars in an array by their indices in a lookup table.

    Implicitely assume that:

    * All elements of arr and lookup are non-negative integers.
    * All elements or arr belong to lookup.

    This is not checked for performance reasons.

    """
    # Equivalent of np.digitize(arr, lookup) - 1, but much faster.
    # TODO: assertions to disable in production for performance reasons.
    m = lookup.max() + 1
    tmp = np.zeros(m, dtype=np.int)
    tmp[lookup] = np.arange(len(lookup))
    return tmp[arr]

def _create_correlograms_array(n_clusters, winsize_bins):
    return np.zeros((n_clusters, n_clusters, winsize_bins // 2 + 1),
                    dtype=np.int32)

def correlograms(spike_times, spike_clusters,
                 binsize=None, winsize_bins=None):

    spike_clusters = np.asarray(spike_clusters)
    spike_times = np.asarray(spike_times)

    assert spike_times.ndim == 1
    assert spike_times.shape == spike_clusters.shape

    assert winsize_bins % 2 == 1

    # TODO: use _unique()
    clusters = np.unique(spike_clusters)
    n_clusters = len(clusters)

    # Like spike_clusters, but with 0..n_clusters-1 indices.
    spike_clusters_i = _index_of(spike_clusters, clusters)

    # Shift between the two copies of the spike trains.
    shift = 1

    # At a given shift, the mask precises which spikes have matching spikes
    # within the correlogram time window.
    mask = np.ones_like(spike_times, dtype=np.bool)

    correlograms = _create_correlograms_array(n_clusters, winsize_bins)

    # The loop continues as long as there is at least one spike with
    # a matching spike.
    while mask[:-shift].any():
        # Number of time samples between spike i and spike i+shift.
        spike_diff = _diff_shifted(spike_times, shift)

        # Binarize the delays between spike i and spike i+shift.
        spike_diff_b = spike_diff // binsize

        # Spikes with no matching spikes are masked.
        mask[:-shift][spike_diff_b > (winsize_bins//2)] = False

        # Cache the masked spike delays.
        m = mask[:-shift].copy()
        d = spike_diff_b[m]

        # # Update the masks given the clusters to update.
        # m0 = np.in1d(spike_clusters[:-shift], clusters)
        # m = m & m0
        # d = spike_diff_b[m]
        d = spike_diff_b[m]

        # Find the indices in the raveled correlograms array that need
        # to be incremented, taking into account the spike clusters.
        indices = np.ravel_multi_index((spike_clusters_i[:-shift][m],
                                        spike_clusters_i[shift:][m], d),
                                       correlograms.shape)

        # Increment the matching spikes in the correlograms array.
        _increment(correlograms.ravel(), indices)

        shift += 1

    return correlograms
<commit_msg>Minor refactoring after rebasing.<commit_after># -*- coding: utf-8 -*-

"""Cross-correlograms."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

import numpy as np

from ..ext import six
from ..utils.array import _index_of, _unique

#------------------------------------------------------------------------------
# Cross-correlograms
#------------------------------------------------------------------------------

"""

Variables
---------

winsize_samples : int
    Number of time samples in the window. Needs to be an odd number.

binsize : int
    Number of time samples in one bin.

winsize_bins : int
    Number of bins in the window.

    winsize_bins = 2 * ((winsize_samples // 2) // binsize) + 1
    assert winsize_bins % 2 == 1

"""

def _increment(arr, indices):
    """Increment some indices in a 1D vector of non-negative integers.
    Repeated indices are taken into account."""
    arr = np.asarray(arr)
    indices = np.asarray(indices)
    bbins = np.bincount(indices)
    arr[:len(bbins)] += bbins
    return arr

def _diff_shifted(arr, steps=1):
    arr = np.asarray(arr)
    return arr[steps:] - arr[:len(arr)-steps]

def _create_correlograms_array(n_clusters, winsize_bins):
    return np.zeros((n_clusters, n_clusters, winsize_bins // 2 + 1),
                    dtype=np.int32)

def correlograms(spike_times, spike_clusters,
                 binsize=None, winsize_bins=None):

    spike_clusters = np.asarray(spike_clusters)
    spike_times = np.asarray(spike_times)

    assert spike_times.ndim == 1
    assert spike_times.shape == spike_clusters.shape

    assert winsize_bins % 2 == 1

    # TODO: use _unique()
    clusters = _unique(spike_clusters)
    n_clusters = len(clusters)

    # Like spike_clusters, but with 0..n_clusters-1 indices.
    spike_clusters_i = _index_of(spike_clusters, clusters)

    # Shift between the two copies of the spike trains.
    shift = 1

    # At a given shift, the mask precises which spikes have matching spikes
    # within the correlogram time window.
    mask = np.ones_like(spike_times, dtype=np.bool)

    correlograms = _create_correlograms_array(n_clusters, winsize_bins)

    # The loop continues as long as there is at least one spike with
    # a matching spike.
    while mask[:-shift].any():
        # Number of time samples between spike i and spike i+shift.
        spike_diff = _diff_shifted(spike_times, shift)

        # Binarize the delays between spike i and spike i+shift.
        spike_diff_b = spike_diff // binsize

        # Spikes with no matching spikes are masked.
        mask[:-shift][spike_diff_b > (winsize_bins//2)] = False

        # Cache the masked spike delays.
        m = mask[:-shift].copy()
        d = spike_diff_b[m]

        # # Update the masks given the clusters to update.
        # m0 = np.in1d(spike_clusters[:-shift], clusters)
        # m = m & m0
        # d = spike_diff_b[m]
        d = spike_diff_b[m]

        # Find the indices in the raveled correlograms array that need
        # to be incremented, taking into account the spike clusters.
        indices = np.ravel_multi_index((spike_clusters_i[:-shift][m],
                                        spike_clusters_i[shift:][m], d),
                                       correlograms.shape)

        # Increment the matching spikes in the correlograms array.
        _increment(correlograms.ravel(), indices)

        shift += 1

    return correlograms
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import sys
import configparser, os
from pathlib import Path
import sched
import time
import twitter

MEDIA_PATH = '.'
SCHED = sched.scheduler(time.time, time.sleep)
SECS = 3

def _main(argv):
    cfg_file = Path("twitter.cfg")
    if not cfg_file.is_file():
        ask_config()
    api = read_config()
    check_for_media(api)

def check_for_media(api):
    print('Checking for media in %s... (every %d secs)' % (MEDIA_PATH, SECS))
    for file in os.listdir(MEDIA_PATH):
        fname, fext = os.path.splitext(file)
        if fext in ('.jpg', '.gif', '.png'):
            print('%s found, posting on twitter...' % file)
            status = post_media(os.path.join(MEDIA_PATH, file), api)
            print(status.text)
            media_archive(file)
    SCHED.enter(SECS, 1, check_for_media, (api, ))
    SCHED.run()

def media_archive(file):
        os.makedirs(os.path.join(MEDIA_PATH, 'hist\\'))
    os.rename(os.path.join(MEDIA_PATH, file), os.path.join(MEDIA_PATH, 'hist\\' + file))

def post_media(media, api):

def ask_config():
    consumer_key = input('Consumer Key:')
    consumer_secret = input('Consumer Secret: ')
    access_token_key = input('Access Token Key: ')
    access_token_secret = input('Access Token Secret: ')
    global MEDIA_PATH
    global SECS
    MEDIA_PATH = input('Media Path: ')
    SECS = input('Check media path every N seconds: ')
    save_config(consumer_key, consumer_secret, access_token_key, access_token_secret)

def save_config(consumer_key, consumer_secret, access_token_key, access_token_secret):
    config = configparser.RawConfigParser()
    config.add_section('Main')
    config.set('Main', 'consumer_key', consumer_key)
    config.set('Main', 'consumer_secret', consumer_secret)
    config.set('Main', 'access_token_key', access_token_key)
    config.set('Main', 'access_token_secret', access_token_secret)
    config.set('Main', 'media_path', MEDIA_PATH)
    config.set('Main', 'secs', SECS)
    with open('twitter.cfg', 'w') as configfile:
        config.write(configfile)

def read_config():
    config = configparser.ConfigParser()
    _consumer_key = ''
    _consumer_secret = ''
    _access_token_key = ''
    _access_token_secret = ''

    if config.read('twitter.cfg'):
        _consumer_key = config.get('Main', 'consumer_key')
        _consumer_secret = config.get('Main', 'consumer_secret')
        _access_token_key = config.get('Main', 'access_token_key')
        _access_token_secret = config.get('Main', 'access_token_secret')
        global MEDIA_PATH
        global SECS
        MEDIA_PATH = config.get('Main', 'media_path')
        SECS = float(config.get('Main', 'secs'))

    return twitter.Api(consumer_key = _consumer_key,
                      consumer_secret = _consumer_secret,
                      access_token_key = _access_token_key,
                      access_token_secret = _access_token_secret)

if __name__ == '__main__':
    _main(sys.argv)
<commit_msg>initial commit<commit_after># -*- coding: utf-8 -*-
import sys
import configparser, os
from pathlib import Path
import sched
import time
import twitter

MEDIA_PATH = '.'
SCHED = sched.scheduler(time.time, time.sleep)
SECS = 3

def _main(argv):
    cfg_file = Path("twitter.cfg")
    if not cfg_file.is_file():
        ask_config()
    api = read_config()
    check_for_media(api)

def check_for_media(api):
    print('Checking for media in %s... (every %d secs)' % (MEDIA_PATH, SECS))
    for file in os.listdir(MEDIA_PATH):
        fname, fext = os.path.splitext(file)
        if fext in ('.jpg', '.gif', '.png'):
            print('%s found, posting on twitter...' % file)
            status = post_media(os.path.join(MEDIA_PATH, file), api)
            print(status.text)
            media_archive(file)
    SCHED.enter(SECS, 1, check_for_media, (api, ))
    SCHED.run()

def media_archive(file):
    if not os.path.exists(os.path.join(MEDIA_PATH, 'hist\\')): #TODO: history folder must be configurable?
        os.makedirs(os.path.join(MEDIA_PATH, 'hist\\'))
    os.rename(os.path.join(MEDIA_PATH, file), os.path.join(MEDIA_PATH, 'hist\\' + file))

def post_media(media, api):
    return api.PostUpdate('Ignore this tweet', media) #TODO: add logic for the tweet text

def ask_config():
    consumer_key = input('Consumer Key:')
    consumer_secret = input('Consumer Secret: ')
    access_token_key = input('Access Token Key: ')
    access_token_secret = input('Access Token Secret: ')
    global MEDIA_PATH
    global SECS
    MEDIA_PATH = input('Media Path: ')
    SECS = input('Check media path every N seconds: ')
    save_config(consumer_key, consumer_secret, access_token_key, access_token_secret)

def save_config(consumer_key, consumer_secret, access_token_key, access_token_secret):
    config = configparser.RawConfigParser()
    config.add_section('Main')
    config.set('Main', 'consumer_key', consumer_key)
    config.set('Main', 'consumer_secret', consumer_secret)
    config.set('Main', 'access_token_key', access_token_key)
    config.set('Main', 'access_token_secret', access_token_secret)
    config.set('Main', 'media_path', MEDIA_PATH)
    config.set('Main', 'secs', SECS)
    with open('twitter.cfg', 'w') as configfile:
        config.write(configfile)

def read_config():
    config = configparser.ConfigParser()
    _consumer_key = ''
    _consumer_secret = ''
    _access_token_key = ''
    _access_token_secret = ''

    if config.read('twitter.cfg'):
        _consumer_key = config.get('Main', 'consumer_key')
        _consumer_secret = config.get('Main', 'consumer_secret')
        _access_token_key = config.get('Main', 'access_token_key')
        _access_token_secret = config.get('Main', 'access_token_secret')
        global MEDIA_PATH
        global SECS
        MEDIA_PATH = config.get('Main', 'media_path')
        SECS = float(config.get('Main', 'secs'))

    return twitter.Api(consumer_key = _consumer_key,
                      consumer_secret = _consumer_secret,
                      access_token_key = _access_token_key,
                      access_token_secret = _access_token_secret)

if __name__ == '__main__':
    _main(sys.argv)
<|endoftext|>


<commit_before>from __future__ import absolute_import

import logging
from datetime import date, timedelta
from dateutil.parser import parse

from dexter.app import celery_app as app
from dexter.processing import DocumentProcessor, DocumentProcessorNT

# force configs for API keys to be set
import dexter.core

# This is a collection of periodic tasks for Dexter, using
# Celery to drive task completion.

log = logging.getLogger(__name__)

@app.task
def back_process_feeds():
    """ Enqueue a task to fetch yesterday's feeds. """

    if date.today() == date(2021, 2, 12):
        d1 = date(2021, 2, 9)
        # d2 = date(2020, 12, 31)
        days = [d1]

        for d in days:
            filter_parm = ''
            fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    elif date.today() == date(2021, 2, 11):
        d1 = date(2021, 1, 1)
        d2 = date(2021, 2, 8)
        days = [d1 + timedelta(days=x) for x in range((d2 - d1).days + 1)]

        for d in days:
            filter_parm = 'media=nationafrica'
            fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    # elif date.today() == date(2021, 2, 2):
    #     d1 = date(2021, 1, 21)
    #     days = [d1]
    #
    #     filter_parm = ''
    #
    #     for d in days:
    #         fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)
    #
    # elif date.today() == date(2021, 2, 3):
    #     d1 = date(2021, 1, 22)
    #     days = [d1]
    #
    #     filter_parm = ''
    #
    #     for d in days:
    #         fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)
    #
    # elif date.today() == date(2021, 2, 4):
    #     d1 = date(2021, 1, 23)
    #     days = [d1]
    #
    #     filter_parm = ''
    #
    #     for d in days:
    #         fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)
    #
    # elif date.today() == date(2021, 2, 5):
    #     d1 = date(2021, 1, 26)
    #     days = [d1]
    #
    #     filter_parm = ''
    #
    #     for d in days:
    #         fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)
    #
    # elif date.today() == date(2021, 2, 6):
    #     d1 = date(2021, 1, 27)
    #     days = [d1]
    #
    #     filter_parm = ''
    #
    #     for d in days:
    #         fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    else:
        print 'Already Done!'

@app.task
def fetch_yesterdays_feeds():
    """ Enqueue a task to fetch yesterday's feeds. """
    yesterday = date.today() - timedelta(days=1)
    fetch_daily_feeds.delay(yesterday.isoformat())

# retry after 60 minutes, retry for up to 7 days
@app.task(bind=True)
def fetch_filtered_daily_feeds(self, day, filter_parm):
    """ Fetch feed of URLs to crawl and queue up a task to grab and process
    each url. """
    try:
        day = parse(day)

        dp = DocumentProcessorNT()
        count = 0
        for item in dp.fetch_filtered_daily_feed_items(day, filter_parm):
            get_feed_item.delay(item)
            count += 1
    except Exception as e:
        log.error("Error processing daily feeds for %s" % day, exc_info=e)
        self.retry(exc=e)

# retry after 60 minutes, retry for up to 7 days
@app.task(bind=True, default_retry_delay=6*60*60, max_retries=7*4)
def fetch_daily_feeds(self, day):
    """ Fetch feed of URLs to crawl and queue up a task to grab and process
    each url. """
    try:
        day = parse(day)

        dp = DocumentProcessorNT()
        count = 0
        for item in dp.fetch_daily_feed_items(day):
            get_feed_item.delay(item)
            count += 1
    except Exception as e:
        log.error("Error processing daily feeds for %s" % day, exc_info=e)
        self.retry(exc=e)

    if count == 0:
        # nothing to do, retry later
        self.retry()

# retry every minute, for up to 24 hours.
@app.task(bind=True, rate_limit="10/m", default_retry_delay=30, max_retries=2)
def get_feed_item(self, item):
    """ Fetch and process a document feed item. """
    try:
        dp = DocumentProcessorNT()
        dp.process_feed_item(item)
    except Exception as e:
        log.error("Error processing feed item: %s" % item, exc_info=e)
        self.retry()

@app.task
def backfill_taxonomies():
    """ Enqueue a task to backfill taxonomies """
    try:
        dp = DocumentProcessorNT()
        dp.backfill_taxonomies()
    except Exception as e:
        log.error("Error backfilling taxonomies: %s" % e.message, exc_info=e)
<commit_msg>Backlog<commit_after>from __future__ import absolute_import

import logging
from datetime import date, timedelta
from dateutil.parser import parse

from dexter.app import celery_app as app
from dexter.processing import DocumentProcessor, DocumentProcessorNT

# force configs for API keys to be set
import dexter.core

# This is a collection of periodic tasks for Dexter, using
# Celery to drive task completion.

log = logging.getLogger(__name__)

@app.task
def back_process_feeds():
    """ Enqueue a task to fetch yesterday's feeds. """

    if date.today() == date(2021, 2, 16):
        d1 = date(2021, 1, 26)
        # d2 = date(2020, 12, 31)
        days = [d1]

        for d in days:
            filter_parm = ''
            fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    if date.today() == date(2021, 2, 17):
        d1 = date(2021, 2, 13)
        # d2 = date(2020, 12, 31)
        days = [d1]

        for d in days:
            filter_parm = ''
            fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    # elif date.today() == date(2021, 2, 11):
    #     d1 = date(2021, 1, 1)
    #     d2 = date(2021, 2, 8)
    #     days = [d1 + timedelta(days=x) for x in range((d2 - d1).days + 1)]
    #
    #     for d in days:
    #         filter_parm = 'media=nationafrica'
    #         fetch_filtered_daily_feeds.delay(d.isoformat(), filter_parm)

    else:
        print 'Already Done!'

@app.task
def fetch_yesterdays_feeds():
    """ Enqueue a task to fetch yesterday's feeds. """
    yesterday = date.today() - timedelta(days=1)
    fetch_daily_feeds.delay(yesterday.isoformat())

# retry after 60 minutes, retry for up to 7 days
@app.task(bind=True)
def fetch_filtered_daily_feeds(self, day, filter_parm):
    """ Fetch feed of URLs to crawl and queue up a task to grab and process
    each url. """
    try:
        day = parse(day)

        dp = DocumentProcessorNT()
        count = 0
        for item in dp.fetch_filtered_daily_feed_items(day, filter_parm):
            get_feed_item.delay(item)
            count += 1
    except Exception as e:
        log.error("Error processing daily feeds for %s" % day, exc_info=e)
        self.retry(exc=e)

# retry after 60 minutes, retry for up to 7 days
@app.task(bind=True, default_retry_delay=6*60*60, max_retries=7*4)
def fetch_daily_feeds(self, day):
    """ Fetch feed of URLs to crawl and queue up a task to grab and process
    each url. """
    try:
        day = parse(day)

        dp = DocumentProcessorNT()
        count = 0
        for item in dp.fetch_daily_feed_items(day):
            get_feed_item.delay(item)
            count += 1
    except Exception as e:
        log.error("Error processing daily feeds for %s" % day, exc_info=e)
        self.retry(exc=e)

    if count == 0:
        # nothing to do, retry later
        self.retry()

# retry every minute, for up to 24 hours.
@app.task(bind=True, rate_limit="10/m", default_retry_delay=30, max_retries=2)
def get_feed_item(self, item):
    """ Fetch and process a document feed item. """
    try:
        dp = DocumentProcessorNT()
        dp.process_feed_item(item)
    except Exception as e:
        log.error("Error processing feed item: %s" % item, exc_info=e)
        self.retry()

@app.task
def backfill_taxonomies():
    """ Enqueue a task to backfill taxonomies """
    try:
        dp = DocumentProcessorNT()
        dp.backfill_taxonomies()
    except Exception as e:
        log.error("Error backfilling taxonomies: %s" % e.message, exc_info=e)
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function

from flask import request, jsonify, send_file
from app import db
from flask.ext.login import current_user, login_required
from . import posts
from .models import Post, PostSerializer
from datetime import datetime
import json

@posts.route('/api/posts/all')
def all_posts():

    all_posts = Post.query.order_by('-pub_date')
    posts = PostSerializer(all_posts, many=True).json

    return jsonify({
        'status': 200,
        'error': None,
        'result':{
            'data':posts,
            'message':'Success returning all posts! whats gucci!'
            }
        }),200

@posts.route('/manage/my_posts')
@login_required
def my_posts():

    my_posts = Post.query.filter_by(author=current_user)
    posts = PostSerializer(my_posts, many=True).json

    return jsonify({
        'status': 200,
        'error': None,
        'result':{
            'data':posts,
            'message':'Success returning your posts!'
            }
        }),200

@posts.route('/manage/create_post', methods=['POST'])
@login_required
def create_post():

    print (request.data)
    data = json.loads(request.data)

    #data checking
    if not data.get('title'):
        return jsonify({
            'status': 400,
            'error': 'No title was provided',
            'result': {
                'message': 'No title was provided.'
            }
        }), 400

    if not data.get('body'):
        return jsonify({
            'status': 400,
            'error': 'No body was provided',
            'result': {
                'message': 'No body was provided.'
            }
        }), 400

    #valid data add to db
    post = Post(title=data['title'],body=data['body'],author=current_user)
    db.session.add(post)
    db.session.commit()

    return jsonify({
        'status': 200,
        'error': None,
        'result':{
            'message':'Post Successful!'
            }
        }),200

@posts.route('/manage/delete_post/<int:post_id>', methods=['GET'])
@login_required
def delete_post(post_id):

    try:
        post=Post.query.get(id=post_id, author=current_user)
        db.session.delete(post)
        db.session.commit()

    except:
        return jsonify({
            'status': 400,
            'error': 'No post found',
            'result': {
                'message': 'No post found.'
            }
        }), 400

    return jsonify({
        'status': 200,
        'error': None,
        'result':{
            'message':'Post deleted!'
            }
        }),200

@posts.route('/manage/edit_post/<int:post_id>', methods=['POST'])
@login_required
def edit_post():

    data=json_loads(request.data)
    post_id=request.args.get('post_id')
    title=data.get('title')
    body=data.get('body')

    try:
        post=Post.query.get(id=post_id, author=current_user)
    except:
        return jsonify({
            'status': 400,
            'error': 'No post found',
            'result': {
                'message': 'No post found.'
            }
        }), 400

    post.body=body
    db.session.commit()

    return jsonify({
            'status': 200,
            'error': None,
            'result':{
                'message':'Post edited!'
                }
            }),200<commit_msg>view changes<commit_after>from __future__ import absolute_import, print_function

from flask import request, jsonify, send_file
from app import db
from flask.ext.login import current_user, login_required
from . import posts
from .models import Post, PostSerializer
from datetime import datetime
import json

@posts.route('/api/posts/all')
def all_posts():

    all_posts = Post.query.order_by('-pub_date')
    posts = PostSerializer(all_posts, many=True).json

    return jsonify({
        'status': 200,
        'error': None,
        'result':{
            'data':posts,
            'message':'Success returning all posts! whats gucci!'
            }
        }),200

@posts.route('/manage/my_posts')
@login_required
def my_posts():

    my_posts = Post.query.filter_by(author=current_user)
    posts = PostSerializer(my_posts, many=True).json

    return jsonify({
        'status': 200,
        'error': None,
        'result':{
            'data':posts,
            'message':'Success returning your posts!'
            }
        }),200

@posts.route('/manage/create_post', methods=['POST'])
@login_required
def create_post():

    print (request.data)
    data = json.loads(request.data)

    #data checking
    if not data.get('title'):
        return jsonify({
            'status': 400,
            'error': 'No title was provided',
            'result': {
                'message': 'No title was provided.'
            }
        }), 400

    if not data.get('body'):
        return jsonify({
            'status': 400,
            'error': 'No body was provided',
            'result': {
                'message': 'No body was provided.'
            }
        }), 400

    #valid data add to db
    post = Post(title=data['title'],body=data['body'],author=current_user)
    db.session.add(post)
    db.session.commit()

    return jsonify({
        'status': 200,
        'error': None,
        'result':{
            'message':'Post Successful!'
            }
        }),200

@posts.route('/manage/delete_post/<int:post_id>', methods=['GET'])
@login_required
def delete_post(post_id):
    print(post_id);
    try:
        post=Post.query.get(id=post_id, author=current_user)
        db.session.delete(post)
        db.session.commit()

    except:
        return jsonify({
            'status': 400,
            'error': 'No post found',
            'result': {
                'message': 'No post found.'
            }
        }), 400

    return jsonify({
        'status': 200,
        'error': None,
        'result':{
            'message':'Post deleted!'
            }
        }),200

@posts.route('/manage/edit_post/<int:post_id>', methods=['POST'])
@login_required
def edit_post():

    data=json_loads(request.data)
    post_id=request.args.get('post_id')
    title=data.get('title')
    body=data.get('body')

    try:
        post=Post.query.get(id=post_id, author=current_user)
    except:
        return jsonify({
            'status': 400,
            'error': 'No post found',
            'result': {
                'message': 'No post found.'
            }
        }), 400

    post.body=body
    db.session.commit()

    return jsonify({
            'status': 200,
            'error': None,
            'result':{
                'message':'Post edited!'
                }
            }),200<|endoftext|>


<commit_before>import lxml.etree

WORD_BLACKLIST = ['quoted', 'Variant:', 'Retrieved', 'Notes:']
MIN_QUOTE_LEN = 6
MIN_QUOTE_WORDS = 3
MAIN_PAGE = "Main Page"

def is_cast_credit(txt_split):
    # Checks to see if the text is a cast credit:
    #   <actor name> as <character name>
    #   <actor name> - <character name>
    if not 2 < len(txt_split) < 7:
        return False

    separators = ['as', '-', '–']
    return all(w[0].isupper() or w in separators or w[0] == '"'
               for w in txt_split)

def is_quote(txt):
    txt_split = txt.split()
    invalid_conditions = [
        not txt or not txt[0].isupper() or len(txt) < MIN_QUOTE_LEN,
        len(txt_split) < MIN_QUOTE_WORDS,
        any(True for word in txt_split if word in WORD_BLACKLIST),
        txt.endswith(('(', ':', ']')),
        is_cast_credit(txt_split)
    ]

    # Returns false if any invalid conditions are true, otherwise returns True.
    return not any(invalid_conditions)

def is_quote_node(node):
    # Discard nodes with the <small> tag
    if node.find('small') is not None:
        return False

    # Discard nodes that are just a link
    # (using xpath so lxml will show text nodes)
    # The link may be inside <i> or <b> tags, so keep peeling layers
    suspect_node = node
    while True:
        node_children = suspect_node.xpath('child::node()')
        if len(node_children) != 1:
            break

        suspect_node = node_children[0]
        if not isinstance(suspect_node, lxml.etree._Element):
            break

        if suspect_node.tag == 'a':
            return False

    return True

def extract_quotes(tree, max_quotes):
    quotes_list = []

    # Remove table of contents
    toc_list = tree.xpath('//div[@id="toc"]')
    for toc in toc_list:
        toc.getparent().remove(toc)

    # List items inside unordered lists
    node_list = tree.xpath('//div/ul/li')

    # Description tags inside description lists,
    # first one is generally not a quote
    dd_list = tree.xpath('//div/dl/dd')[1:]
    if len(dd_list) > len(node_list):
        node_list += dd_list

    for node in node_list:
        uls = node.xpath('ul')
        for ul in uls:
            ul.getparent().remove(ul)

        if not is_quote_node(node):
            continue

        txt = node.text_content().strip()
        if is_quote(txt) and max_quotes > len(quotes_list):
            txt_normal = ' '.join(txt.split())
            quotes_list.append(txt_normal)

            if max_quotes == len(quotes_list):
                break

    return quotes_list

def qotd(html_tree):
    tree = html_tree.get_element_by_id('mf-qotd')

    raw_quote = tree.xpath('div/div/table/tr')[0].text_content().split('~')
    quote = raw_quote[0].strip()
    author = raw_quote[1].strip()
    return quote, author
<commit_msg>Skip common headlines that don't contain quotes<commit_after>import lxml.etree

WORD_BLACKLIST = ['quoted', 'Variant:', 'Retrieved', 'Notes:']
MIN_QUOTE_LEN = 6
MIN_QUOTE_WORDS = 3
MAIN_PAGE = "Main Page"

def is_quote(txt):
    txt_split = txt.split()
    invalid_conditions = [
        not txt or not txt[0].isupper() or len(txt) < MIN_QUOTE_LEN,
        len(txt_split) < MIN_QUOTE_WORDS,
        any(True for word in txt_split if word in WORD_BLACKLIST),
        txt.endswith(('(', ':', ']')),
    ]

    # Returns false if any invalid conditions are true, otherwise returns True.
    return not any(invalid_conditions)

def is_quote_node(node):
    # Discard nodes with the <small> tag
    if node.find('small') is not None:
        return False

    # Discard nodes that are just a link
    # (using xpath so lxml will show text nodes)
    # The link may be inside <i> or <b> tags, so keep peeling layers
    suspect_node = node
    while True:
        node_children = suspect_node.xpath('child::node()')
        if len(node_children) != 1:
            break

        suspect_node = node_children[0]
        if not isinstance(suspect_node, lxml.etree._Element):
            break

        if suspect_node.tag == 'a':
            return False

    return True

def extract_quotes(tree, max_quotes):
    quotes_list = []

    # Remove table of contents
    toc_list = tree.xpath('//div[@id="toc"]')
    for toc in toc_list:
        toc.getparent().remove(toc)

    # Scan list items description tags inside description lists.
    # Also grab headlines to skip some sections.
    node_list = tree.xpath('//div/ul/li|//div/dl/dd|//h2')

    # Skip all quotes above the first heading
    skip_to_next_heading = True

    for node in node_list:
        if node.tag != 'h2' and skip_to_next_heading:
            continue

        if node.tag == 'h2':
            skip_to_next_heading = False
            heading_text = node.text_content().lower()

            # Commence skipping
            if heading_text in ('cast', 'see also', 'external links'):
                skip_to_next_heading = True

            continue

        # Handle li/dd

        uls = node.xpath('ul')
        for ul in uls:
            ul.getparent().remove(ul)

        if not is_quote_node(node):
            continue

        txt = node.text_content().strip()
        if is_quote(txt) and max_quotes > len(quotes_list):
            txt_normal = ' '.join(txt.split())
            quotes_list.append(txt_normal)

            if max_quotes == len(quotes_list):
                break

    return quotes_list

def qotd(html_tree):
    tree = html_tree.get_element_by_id('mf-qotd')

    raw_quote = tree.xpath('div/div/table/tr')[0].text_content().split('~')
    quote = raw_quote[0].strip()
    author = raw_quote[1].strip()
    return quote, author
<|endoftext|>