<commit_before># -*- coding: utf-8 -*-

"""Lark parser to parse the expression."""

__author__ = "Yuan Chang"
__copyright__ = "Copyright (C) 2016-2020"
__license__ = "AGPL"
__email__ = "pyslvs@gmail.com"

from abc import abstractmethod
from typing import (
    cast, Tuple, List, Dict, Iterator, Optional, Union, TypeVar, Generic,
)
from dataclasses import dataclass
from lark import Lark, Transformer, LexError
from .expression import get_vlinks, VJoint, VPoint, VLink
from .graph import Graph

_T1 = TypeVar('_T1')
_T2 = TypeVar('_T2')
_Coord = Tuple[float, float]
_JointArgs = List[Union[str, VJoint, float, _Coord, Tuple[str, ...]]]

# Color dictionary
_color_list: Dict[str, Tuple[int, int, int]] = {
    'red': (172, 68, 68),
    'green': (110, 190, 30),
    'blue': (68, 120, 172),
    'cyan': (0, 255, 255),
    'magenta': (255, 0, 255),
    'brick-red': (255, 130, 130),
    'yellow': (255, 255, 0),
    'gray': (160, 160, 160),
    'orange': (225, 165, 0),
    'pink': (225, 192, 230),
    'black': (0, 0, 0),
    'white': (255, 255, 255),
    'dark-red': (128, 0, 0),
    'dark-green': (0, 128, 0),
    'dark-blue': (0, 0, 128),
    'dark-cyan': (128, 0, 128),
    'dark-magenta': (255, 0, 255),
    'dark-yellow': (128, 128, 0),
    'dark-gray': (128, 128, 128),
    'dark-orange': (225, 140, 0),
    'dark-pink': (225, 20, 147),
}
color_names = tuple(sorted(_color_list.keys()))

def color_rgb(name: str) -> Tuple[int, int, int]:
    """Get color by name.

    Get RGB color data by name, return `(0, 0, 0)` if it is invalid.
    Also support `"(R, G, B)"` string format.
    """
    name = name.lower()
    if name in _color_list:
        return _color_list[name]
    else:
        try:
            # Input RGB as a "(255, 255, 255)" string
            rgb = (
                name.replace('(', '')
                .replace(')', '')
                .replace(" ", '')
                .split(',', maxsplit=3)
            )
            color_text = (int(rgb[0]), int(rgb[1]), int(rgb[2]))
        except ValueError:
            return 0, 0, 0
        else:
            return color_text

@dataclass(repr=False, eq=False)
class PointArgs:
    """Point table argument."""
    links: str
    type: str
    color: str
    x: float
    y: float

@dataclass(repr=False, eq=False)
class LinkArgs:
    """Link table argument."""
    name: str
    color: str
    points: str

_GRAMMAR = Lark(r"""
// Number
DIGIT: "0".."9"
INT: DIGIT+
SIGNED_INT: ["+" | "-"] INT
DECIMAL: INT "." INT? | "." INT
_EXP: ("e" | "E") SIGNED_INT
FLOAT: INT _EXP | DECIMAL _EXP?
NUMBER: FLOAT | INT
SIGNED_NUMBER: ["+" | "-"] NUMBER

// Letters
LCASE_LETTER: "a".."z"
UCASE_LETTER: "A".."Z"
LETTER: UCASE_LETTER | LCASE_LETTER | "_"
CNAME: LETTER (LETTER | DIGIT)*

// White space and new line
WS: /\s+/
CR: /\r/
LF: /\n/
NEWLINE: (CR? LF)+
%ignore WS
%ignore NEWLINE

// Comment
LINE_COMMENT: /#[^\n]*/
MULTILINE_COMMENT: /#\[[\s\S]*#\][^\n]*/
%ignore LINE_COMMENT
%ignore MULTILINE_COMMENT

// Custom data type
JOINT_TYPE: "RP" | "R" | "P"
COLOR: """ + "|".join(f'"{color}"i' for color in color_names) + r"""
type: JOINT_TYPE
name: CNAME
number: SIGNED_NUMBER
color_value: INT

// Main grammar
joint: "J[" type ["," angle] ["," color] "," point "," link "]"
link: "L[" [name ("," name)* ","?] "]"
point: "P[" number "," number "]"
angle: "A[" number "]"
color: "color[" (("(" color_value ("," color_value) ~ 2 ")") | COLOR) "]"
mechanism: "M[" [joint ("," joint)* ","?] "]"
?start: mechanism
""", parser='lalr')

class _Transformer(Transformer, Generic[_T1, _T2]):
    """Base transformer implementation."""

    @staticmethod
    @abstractmethod
    def type(n: List[str]) -> _T1:
        ...

    @staticmethod
    def name(n: List[str]) -> str:
        return str(n[0])

    @staticmethod
    def color(n: List[str]) -> str:
        return str(n[0]) if len(n) == 1 else str(tuple(n))

    @staticmethod
    def color_value(n: List[str]) -> int:
        return int(n[0])

    @staticmethod
    def number(n: List[str]) -> float:
        return float(n[0])

    angle = number

    @staticmethod
    def point(c: List[float]) -> _Coord:
        return c[0], c[1]

    @staticmethod
    def link(a: List[str]) -> Tuple[str, ...]:
        return tuple(a)

    @staticmethod
    @abstractmethod
    def joint(args: _JointArgs) -> _T2:
        ...

    @staticmethod
    def mechanism(joints: List[_T2]) -> List[_T2]:
        return joints

class _ParamsTrans(_Transformer[str, PointArgs]):
    """Transformer will parse into a list of VPoint data."""

    @staticmethod
    def type(n: List[str]) -> str:
        return str(n[0])

    @staticmethod
    def joint(args: _JointArgs) -> PointArgs:
        """Sort the argument list.

        [0]: type
        ([1]: angle)
        ([2]: color)
        [-2]: point (coordinate)
        [-1]: link
        """
        type_str = cast(str, args[0])
        x, y = cast(_Coord, args[-2])
        links = ','.join(cast(Tuple[str, ...], args[-1]))
        if type_str == 'R':
            if len(args) == 3:
                return PointArgs(links, 'R', 'Green', x, y)
            elif len(args) == 4:
                color = cast(str, args[-3])
                return PointArgs(links, 'R', color, x, y)
        else:
            angle = cast(float, args[1])
            type_angle = f'{type_str}:{angle}'
            if len(args) == 4:
                return PointArgs(links, type_angle, 'Green', x, y)
            elif len(args) == 5:
                color = cast(str, args[-3])
                return PointArgs(links, type_angle, color, x, y)

        raise LexError(f"invalid options: {args}")

class _PositionTrans(_Transformer[str, _Coord]):
    """Transformer will parse into a list of position data."""

    @staticmethod
    def type(n: List[str]) -> str:
        return str(n[0])

    @staticmethod
    def joint(args: _JointArgs) -> _Coord:
        x, y = cast(_Coord, args[-2])
        return x, y

class _VPointsTrans(_Transformer[VJoint, VPoint]):
    """Using same grammar return as VPoints."""

    @staticmethod
    def type(n: List[str]) -> VJoint:
        """Return as int type."""
        type_str = str(n[0])
        if type_str == 'R':
            return VJoint.R
        elif type_str == 'P':
            return VJoint.P
        elif type_str == 'RP':
            return VJoint.RP
        else:
            raise ValueError(f"invalid joint type: {type_str}")

    @staticmethod
    def joint(args: _JointArgs) -> VPoint:
        """Sort the argument list.

        [0]: type
        ([1]: angle)
        ([2]: color)
        [-2]: point (coordinate)
        [-1]: link
        """
        type_int = cast(VJoint, args[0])
        x, y = cast(_Coord, args[-2])
        links = cast(Tuple[str, ...], args[-1])
        if type_int == VJoint.R:
            if len(args) == 3:
                return VPoint.r_joint(links, x, y)
            elif len(args) == 4:
                color = cast(str, args[-3])
                return VPoint(links, VJoint.R, 0., color, x, y, color_rgb)
        else:
            angle = cast(float, args[1])
            if len(args) == 4:
                return VPoint.slider_joint(links, type_int, angle, x, y)
            elif len(args) == 5:
                color = cast(str, args[-3])
                return VPoint(links, type_int, angle, color, x, y, color_rgb)

        raise LexError(f"invalid options: {args}")

_params_translator = _ParamsTrans()
_pos_translator = _PositionTrans()
_vpoint_translator = _VPointsTrans()

def parse_params(expr: str) -> List[PointArgs]:
    """Parse mechanism expression into VPoint constructor arguments."""
    return _params_translator.transform(_GRAMMAR.parse(expr))

def parse_pos(expr: str) -> List[_Coord]:
    """Parse mechanism expression into coordinates."""
    return _pos_translator.transform(_GRAMMAR.parse(expr))

def parse_vpoints(expr: str) -> List[VPoint]:
    """Parse mechanism expression into VPoint objects."""
    return _vpoint_translator.transform(_GRAMMAR.parse(expr))

def parse_vlinks(expr: str) -> List[VLink]:
    """Parse mechanism expression into VLink objects."""
    return get_vlinks(parse_vpoints(expr))

def _sorted_pair(a: int, b: int) -> Tuple[int, int]:
    return (a, b) if a < b else (b, a)

def edges_view(graph: Graph) -> Iterator[Tuple[int, Tuple[int, int]]]:
    """The iterator will yield the sorted edges from `graph`."""
    yield from enumerate(sorted(_sorted_pair(n1, n2) for n1, n2 in graph.edges))

def graph2vpoints(
    graph: Graph,
    pos: Dict[int, _Coord],
    cus: Optional[Dict[int, int]] = None,
    same: Optional[Dict[int, int]] = None,
    grounded: Optional[int] = None
) -> List[VPoint]:
    """Transform `graph` into [VPoint] objects. The vertices are mapped to links.

    + `pos`: Position for each vertices.
    + `cus`: Extra points on the specific links.
    + `same`: Multiple joint setting. The joints are according to [`edges_view`](#edges_view).
    + `grounded`: The ground link of vertices.
    """
    if cus is None:
        cus = {}
    if same is None:
        same = {}
    same_r: Dict[int, List[int]] = {}
    for k, v in same.items():
        if v in same_r:
            same_r[v].append(k)
        else:
            same_r[v] = [k]
    tmp_list = []
    ev = dict(edges_view(graph))
    for i, edge in ev.items():
        if i in same:
            # Do not connect to anyone!
            continue
        edges = set(edge)
        if i in same_r:
            for j in same_r[i]:
                edges.update(set(ev[j]))
        x, y = pos[i]
        links = [
            f"L{link}" if link != grounded else VLink.FRAME for link in edges
        ]
        tmp_list.append(VPoint.r_joint(links, x, y))
    for name in sorted(cus):
        link = f"L{cus[name]}" if cus[name] != grounded else VLink.FRAME
        x, y = pos[name]
        tmp_list.append(VPoint.r_joint((link,), x, y))
    return tmp_list
<commit_msg>Raise NotImplementedError instead of ellipse on abstract methods.<commit_after># -*- coding: utf-8 -*-

"""Lark parser to parse the expression."""

__author__ = "Yuan Chang"
__copyright__ = "Copyright (C) 2016-2020"
__license__ = "AGPL"
__email__ = "pyslvs@gmail.com"

from abc import abstractmethod
from typing import (
    cast, Tuple, List, Dict, Iterator, Optional, Union, TypeVar, Generic,
)
from dataclasses import dataclass
from lark import Lark, Transformer, LexError
from .expression import get_vlinks, VJoint, VPoint, VLink
from .graph import Graph

_T1 = TypeVar('_T1')
_T2 = TypeVar('_T2')
_Coord = Tuple[float, float]
_JointArgs = List[Union[str, VJoint, float, _Coord, Tuple[str, ...]]]

# Color dictionary
_color_list: Dict[str, Tuple[int, int, int]] = {
    'red': (172, 68, 68),
    'green': (110, 190, 30),
    'blue': (68, 120, 172),
    'cyan': (0, 255, 255),
    'magenta': (255, 0, 255),
    'brick-red': (255, 130, 130),
    'yellow': (255, 255, 0),
    'gray': (160, 160, 160),
    'orange': (225, 165, 0),
    'pink': (225, 192, 230),
    'black': (0, 0, 0),
    'white': (255, 255, 255),
    'dark-red': (128, 0, 0),
    'dark-green': (0, 128, 0),
    'dark-blue': (0, 0, 128),
    'dark-cyan': (128, 0, 128),
    'dark-magenta': (255, 0, 255),
    'dark-yellow': (128, 128, 0),
    'dark-gray': (128, 128, 128),
    'dark-orange': (225, 140, 0),
    'dark-pink': (225, 20, 147),
}
color_names = tuple(sorted(_color_list.keys()))

def color_rgb(name: str) -> Tuple[int, int, int]:
    """Get color by name.

    Get RGB color data by name, return `(0, 0, 0)` if it is invalid.
    Also support `"(R, G, B)"` string format.
    """
    name = name.lower()
    if name in _color_list:
        return _color_list[name]
    else:
        try:
            # Input RGB as a "(255, 255, 255)" string
            rgb = (
                name.replace('(', '')
                .replace(')', '')
                .replace(" ", '')
                .split(',', maxsplit=3)
            )
            color_text = (int(rgb[0]), int(rgb[1]), int(rgb[2]))
        except ValueError:
            return 0, 0, 0
        else:
            return color_text

@dataclass(repr=False, eq=False)
class PointArgs:
    """Point table argument."""
    links: str
    type: str
    color: str
    x: float
    y: float

@dataclass(repr=False, eq=False)
class LinkArgs:
    """Link table argument."""
    name: str
    color: str
    points: str

_GRAMMAR = Lark(r"""
// Number
DIGIT: "0".."9"
INT: DIGIT+
SIGNED_INT: ["+" | "-"] INT
DECIMAL: INT "." INT? | "." INT
_EXP: ("e" | "E") SIGNED_INT
FLOAT: INT _EXP | DECIMAL _EXP?
NUMBER: FLOAT | INT
SIGNED_NUMBER: ["+" | "-"] NUMBER

// Letters
LCASE_LETTER: "a".."z"
UCASE_LETTER: "A".."Z"
LETTER: UCASE_LETTER | LCASE_LETTER | "_"
CNAME: LETTER (LETTER | DIGIT)*

// White space and new line
WS: /\s+/
CR: /\r/
LF: /\n/
NEWLINE: (CR? LF)+
%ignore WS
%ignore NEWLINE

// Comment
LINE_COMMENT: /#[^\n]*/
MULTILINE_COMMENT: /#\[[\s\S]*#\][^\n]*/
%ignore LINE_COMMENT
%ignore MULTILINE_COMMENT

// Custom data type
JOINT_TYPE: "RP" | "R" | "P"
COLOR: """ + "|".join(f'"{color}"i' for color in color_names) + r"""
type: JOINT_TYPE
name: CNAME
number: SIGNED_NUMBER
color_value: INT

// Main grammar
joint: "J[" type ["," angle] ["," color] "," point "," link "]"
link: "L[" [name ("," name)* ","?] "]"
point: "P[" number "," number "]"
angle: "A[" number "]"
color: "color[" (("(" color_value ("," color_value) ~ 2 ")") | COLOR) "]"
mechanism: "M[" [joint ("," joint)* ","?] "]"
?start: mechanism
""", parser='lalr')

class _Transformer(Transformer, Generic[_T1, _T2]):
    """Base transformer implementation."""

    @staticmethod
    @abstractmethod
    def type(n: List[str]) -> _T1:
        raise NotImplementedError

    @staticmethod
    def name(n: List[str]) -> str:
        return str(n[0])

    @staticmethod
    def color(n: List[str]) -> str:
        return str(n[0]) if len(n) == 1 else str(tuple(n))

    @staticmethod
    def color_value(n: List[str]) -> int:
        return int(n[0])

    @staticmethod
    def number(n: List[str]) -> float:
        return float(n[0])

    angle = number

    @staticmethod
    def point(c: List[float]) -> _Coord:
        return c[0], c[1]

    @staticmethod
    def link(a: List[str]) -> Tuple[str, ...]:
        return tuple(a)

    @staticmethod
    @abstractmethod
    def joint(args: _JointArgs) -> _T2:
        raise NotImplementedError

    @staticmethod
    def mechanism(joints: List[_T2]) -> List[_T2]:
        return joints

class _ParamsTrans(_Transformer[str, PointArgs]):
    """Transformer will parse into a list of VPoint data."""

    @staticmethod
    def type(n: List[str]) -> str:
        return str(n[0])

    @staticmethod
    def joint(args: _JointArgs) -> PointArgs:
        """Sort the argument list.

        [0]: type
        ([1]: angle)
        ([2]: color)
        [-2]: point (coordinate)
        [-1]: link
        """
        type_str = cast(str, args[0])
        x, y = cast(_Coord, args[-2])
        links = ','.join(cast(Tuple[str, ...], args[-1]))
        if type_str == 'R':
            if len(args) == 3:
                return PointArgs(links, 'R', 'Green', x, y)
            elif len(args) == 4:
                color = cast(str, args[-3])
                return PointArgs(links, 'R', color, x, y)
        else:
            angle = cast(float, args[1])
            type_angle = f'{type_str}:{angle}'
            if len(args) == 4:
                return PointArgs(links, type_angle, 'Green', x, y)
            elif len(args) == 5:
                color = cast(str, args[-3])
                return PointArgs(links, type_angle, color, x, y)

        raise LexError(f"invalid options: {args}")

class _PositionTrans(_Transformer[str, _Coord]):
    """Transformer will parse into a list of position data."""

    @staticmethod
    def type(n: List[str]) -> str:
        return str(n[0])

    @staticmethod
    def joint(args: _JointArgs) -> _Coord:
        x, y = cast(_Coord, args[-2])
        return x, y

class _VPointsTrans(_Transformer[VJoint, VPoint]):
    """Using same grammar return as VPoints."""

    @staticmethod
    def type(n: List[str]) -> VJoint:
        """Return as int type."""
        type_str = str(n[0])
        if type_str == 'R':
            return VJoint.R
        elif type_str == 'P':
            return VJoint.P
        elif type_str == 'RP':
            return VJoint.RP
        else:
            raise ValueError(f"invalid joint type: {type_str}")

    @staticmethod
    def joint(args: _JointArgs) -> VPoint:
        """Sort the argument list.

        [0]: type
        ([1]: angle)
        ([2]: color)
        [-2]: point (coordinate)
        [-1]: link
        """
        type_int = cast(VJoint, args[0])
        x, y = cast(_Coord, args[-2])
        links = cast(Tuple[str, ...], args[-1])
        if type_int == VJoint.R:
            if len(args) == 3:
                return VPoint.r_joint(links, x, y)
            elif len(args) == 4:
                color = cast(str, args[-3])
                return VPoint(links, VJoint.R, 0., color, x, y, color_rgb)
        else:
            angle = cast(float, args[1])
            if len(args) == 4:
                return VPoint.slider_joint(links, type_int, angle, x, y)
            elif len(args) == 5:
                color = cast(str, args[-3])
                return VPoint(links, type_int, angle, color, x, y, color_rgb)

        raise LexError(f"invalid options: {args}")

_params_translator = _ParamsTrans()
_pos_translator = _PositionTrans()
_vpoint_translator = _VPointsTrans()

def parse_params(expr: str) -> List[PointArgs]:
    """Parse mechanism expression into VPoint constructor arguments."""
    return _params_translator.transform(_GRAMMAR.parse(expr))

def parse_pos(expr: str) -> List[_Coord]:
    """Parse mechanism expression into coordinates."""
    return _pos_translator.transform(_GRAMMAR.parse(expr))

def parse_vpoints(expr: str) -> List[VPoint]:
    """Parse mechanism expression into VPoint objects."""
    return _vpoint_translator.transform(_GRAMMAR.parse(expr))

def parse_vlinks(expr: str) -> List[VLink]:
    """Parse mechanism expression into VLink objects."""
    return get_vlinks(parse_vpoints(expr))

def _sorted_pair(a: int, b: int) -> Tuple[int, int]:
    return (a, b) if a < b else (b, a)

def edges_view(graph: Graph) -> Iterator[Tuple[int, Tuple[int, int]]]:
    """The iterator will yield the sorted edges from `graph`."""
    yield from enumerate(sorted(_sorted_pair(n1, n2) for n1, n2 in graph.edges))

def graph2vpoints(
    graph: Graph,
    pos: Dict[int, _Coord],
    cus: Optional[Dict[int, int]] = None,
    same: Optional[Dict[int, int]] = None,
    grounded: Optional[int] = None
) -> List[VPoint]:
    """Transform `graph` into [VPoint] objects. The vertices are mapped to links.

    + `pos`: Position for each vertices.
    + `cus`: Extra points on the specific links.
    + `same`: Multiple joint setting. The joints are according to [`edges_view`](#edges_view).
    + `grounded`: The ground link of vertices.
    """
    if cus is None:
        cus = {}
    if same is None:
        same = {}
    same_r: Dict[int, List[int]] = {}
    for k, v in same.items():
        if v in same_r:
            same_r[v].append(k)
        else:
            same_r[v] = [k]
    tmp_list = []
    ev = dict(edges_view(graph))
    for i, edge in ev.items():
        if i in same:
            # Do not connect to anyone!
            continue
        edges = set(edge)
        if i in same_r:
            for j in same_r[i]:
                edges.update(set(ev[j]))
        x, y = pos[i]
        links = [
            f"L{link}" if link != grounded else VLink.FRAME for link in edges
        ]
        tmp_list.append(VPoint.r_joint(links, x, y))
    for name in sorted(cus):
        link = f"L{cus[name]}" if cus[name] != grounded else VLink.FRAME
        x, y = pos[name]
        tmp_list.append(VPoint.r_joint((link,), x, y))
    return tmp_list
<|endoftext|>


<commit_before>from django.contrib.contenttypes.models import ContentType
from django.utils.encoding import force_unicode
from django.utils.functional import wraps
from django.template.defaultfilters import lower, slugify
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist

from core.taggit.models import Tag, TaggedItem, TagCategory

def stopwords(words):
    # Shacker fork - remove any stopwords defined in settings
    try:
        stoplist = settings.TAGGIT_STOPWORDS
        words = list(set(words) - set(stoplist))
    except:
        pass

    return words

def add_tags(obj, tag_str, tag_category_slug, creator, content_type_name):
    tag_name = tag_str.strip()
    # don't recreate the tag if it already exists
    try:
        t = Tag.objects.get(slug=slugify(tag_name))
    except ObjectDoesNotExist:
        t = Tag()
        t.name = tag_name[:99]
        t.slug = slugify(tag_name)
        t.save()
    ti = TaggedItem()
    ti.tag = t
    ti.object_id = obj.id
    ti.tag_category = TagCategory.objects.filter(slug=tag_category_slug)[0]
    ti.tag_creator = creator
    ti.content_type = ContentType.objects.filter(name=content_type_name)[0]
    ti.save()
    return ti

def parse_tags(tagstring):
    """
    Parses tag input, with multiple word input being activated and
    delineated by commas and double quotes. Quotes take precedence, so
    they may contain commas.

    Returns a sorted list of unique tag names.

    Ported from Jonathan Buchanan's `django-tagging
    <http://django-tagging.googlecode.com/>`_
    """
    if not tagstring:
        return []

    # Should all tags be handled as lowercase?
    try:
        settings.TAGGIT_FORCE_LOWERCASE
        tagstring = lower(force_unicode(tagstring))
    except:
        tagstring = force_unicode(tagstring)

    # Special case - if there are no commas or double quotes in the
    # input, we don't *do* a recall... I mean, we know we only need to
    # split on spaces.
    if u',' not in tagstring and u'"' not in tagstring:
        words = list(set(split_strip(tagstring, u' ')))
        words.sort()

        # shacker fork - remove any defined stopwords
        words = stopwords(words)

        return words

    words = []
    buffer = []
    # Defer splitting of non-quoted sections until we know if there are
    # any unquoted commas.
    to_be_split = []
    saw_loose_comma = False
    open_quote = False
    i = iter(tagstring)
    try:
        while True:
            c = i.next()
            if c == u'"':
                if buffer:
                    to_be_split.append(u''.join(buffer))
                    buffer = []
                # Find the matching quote
                open_quote = True
                c = i.next()
                while c != u'"':
                    buffer.append(c)
                    c = i.next()
                if buffer:
                    word = u''.join(buffer).strip()
                    if word:
                        words.append(word)
                    buffer = []
                open_quote = False
            else:
                if not saw_loose_comma and c == u',':
                    saw_loose_comma = True
                buffer.append(c)
    except StopIteration:
        # If we were parsing an open quote which was never closed treat
        # the buffer as unquoted.
        if buffer:
            if open_quote and u',' in buffer:
                saw_loose_comma = True
            to_be_split.append(u''.join(buffer))
    if to_be_split:
        if saw_loose_comma:
            delimiter = u','
        else:
            delimiter = u' '
        for chunk in to_be_split:
            words.extend(split_strip(chunk, delimiter))
    words = list(set(words))
    words.sort()

    # shacker fork - remove any defined stopwords
    words = stopwords(words)

    return words

def split_strip(string, delimiter=u','):
    """
    Splits ``string`` on ``delimiter``, stripping each resulting string
    and returning a list of non-empty strings.

    Ported from Jonathan Buchanan's `django-tagging
    <http://django-tagging.googlecode.com/>`_
    """
    if not string:
        return []

    words = [w.strip() for w in string.split(delimiter)]
    return [w for w in words if w]

def edit_string_for_tags(tags):
    """
    Given list of ``Tag`` instances, creates a string representation of
    the list suitable for editing by the user, such that submitting the
    given string representation back without changing it will give the
    same list of tags.

    Tag names which contain commas will be double quoted.

    If any tag name which isn't being quoted contains whitespace, the
    resulting string of tag names will be comma-delimited, otherwise
    it will be space-delimited.

    Ported from Jonathan Buchanan's `django-tagging
    <http://django-tagging.googlecode.com/>`_
    """
    names = []
    for tag in tags:
        name = tag.name
        if u',' in name or u' ' in name:
            names.append('"%s"' % name)
        else:
            names.append(name)
    return u', '.join(sorted(names))

def require_instance_manager(func):
    @wraps(func)
    def inner(self, *args, **kwargs):
        if self.instance is None:
            raise TypeError(
                "Can't call %s with a non-instance manager" % func.__name__)
        return func(self, *args, **kwargs)
    return inner
<commit_msg>Prevent error tag_category is 'None', which is a legal value for the field<commit_after>from django.contrib.contenttypes.models import ContentType
from django.utils.encoding import force_unicode
from django.utils.functional import wraps
from django.template.defaultfilters import lower, slugify
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist

from core.taggit.models import Tag, TaggedItem, TagCategory

def stopwords(words):
    # Shacker fork - remove any stopwords defined in settings
    try:
        stoplist = settings.TAGGIT_STOPWORDS
        words = list(set(words) - set(stoplist))
    except:
        pass

    return words

def add_tags(obj, tag_str, tag_category_slug, creator, content_type_name):
    tag_name = tag_str.strip()
    # don't recreate the tag if it already exists
    try:
        t = Tag.objects.get(slug=slugify(tag_name))
    except ObjectDoesNotExist:
        t = Tag()
        t.name = tag_name[:99]
        t.slug = slugify(tag_name)
        t.save()
    ti = TaggedItem()
    ti.tag = t
    ti.object_id = obj.id
    ti.tag_category = None if tag_category_slug is None else \
                      TagCategory.objects.filter(slug=tag_category_slug)[0]
    ti.tag_creator = creator
    ti.content_type = ContentType.objects.filter(name=content_type_name)[0]
    ti.save()
    return ti

def parse_tags(tagstring):
    """
    Parses tag input, with multiple word input being activated and
    delineated by commas and double quotes. Quotes take precedence, so
    they may contain commas.

    Returns a sorted list of unique tag names.

    Ported from Jonathan Buchanan's `django-tagging
    <http://django-tagging.googlecode.com/>`_
    """
    if not tagstring:
        return []

    # Should all tags be handled as lowercase?
    try:
        settings.TAGGIT_FORCE_LOWERCASE
        tagstring = lower(force_unicode(tagstring))
    except:
        tagstring = force_unicode(tagstring)

    # Special case - if there are no commas or double quotes in the
    # input, we don't *do* a recall... I mean, we know we only need to
    # split on spaces.
    if u',' not in tagstring and u'"' not in tagstring:
        words = list(set(split_strip(tagstring, u' ')))
        words.sort()

        # shacker fork - remove any defined stopwords
        words = stopwords(words)

        return words

    words = []
    buffer = []
    # Defer splitting of non-quoted sections until we know if there are
    # any unquoted commas.
    to_be_split = []
    saw_loose_comma = False
    open_quote = False
    i = iter(tagstring)
    try:
        while True:
            c = i.next()
            if c == u'"':
                if buffer:
                    to_be_split.append(u''.join(buffer))
                    buffer = []
                # Find the matching quote
                open_quote = True
                c = i.next()
                while c != u'"':
                    buffer.append(c)
                    c = i.next()
                if buffer:
                    word = u''.join(buffer).strip()
                    if word:
                        words.append(word)
                    buffer = []
                open_quote = False
            else:
                if not saw_loose_comma and c == u',':
                    saw_loose_comma = True
                buffer.append(c)
    except StopIteration:
        # If we were parsing an open quote which was never closed treat
        # the buffer as unquoted.
        if buffer:
            if open_quote and u',' in buffer:
                saw_loose_comma = True
            to_be_split.append(u''.join(buffer))
    if to_be_split:
        if saw_loose_comma:
            delimiter = u','
        else:
            delimiter = u' '
        for chunk in to_be_split:
            words.extend(split_strip(chunk, delimiter))
    words = list(set(words))
    words.sort()

    # shacker fork - remove any defined stopwords
    words = stopwords(words)

    return words

def split_strip(string, delimiter=u','):
    """
    Splits ``string`` on ``delimiter``, stripping each resulting string
    and returning a list of non-empty strings.

    Ported from Jonathan Buchanan's `django-tagging
    <http://django-tagging.googlecode.com/>`_
    """
    if not string:
        return []

    words = [w.strip() for w in string.split(delimiter)]
    return [w for w in words if w]

def edit_string_for_tags(tags):
    """
    Given list of ``Tag`` instances, creates a string representation of
    the list suitable for editing by the user, such that submitting the
    given string representation back without changing it will give the
    same list of tags.

    Tag names which contain commas will be double quoted.

    If any tag name which isn't being quoted contains whitespace, the
    resulting string of tag names will be comma-delimited, otherwise
    it will be space-delimited.

    Ported from Jonathan Buchanan's `django-tagging
    <http://django-tagging.googlecode.com/>`_
    """
    names = []
    for tag in tags:
        name = tag.name
        if u',' in name or u' ' in name:
            names.append('"%s"' % name)
        else:
            names.append(name)
    return u', '.join(sorted(names))

def require_instance_manager(func):
    @wraps(func)
    def inner(self, *args, **kwargs):
        if self.instance is None:
            raise TypeError(
                "Can't call %s with a non-instance manager" % func.__name__)
        return func(self, *args, **kwargs)
    return inner
<|endoftext|>


<commit_before>from __future__ import absolute_import
import os
import weakref
from contextlib import contextmanager
from io import UnsupportedOperation

from corehq.blobs import BlobInfo, DEFAULT_BUCKET
from corehq.blobs.exceptions import BadName, NotFound
from corehq.blobs.interface import AbstractBlobDB, SAFENAME
from corehq.blobs.util import ClosingContextProxy, set_blob_expire_object
from corehq.util.datadog.gauges import datadog_counter

from dimagi.utils.chunked import chunked

import boto3
from botocore.client import Config
from botocore.handlers import calculate_md5
from botocore.exceptions import ClientError
from botocore.utils import fix_s3_host

DEFAULT_S3_BUCKET = "blobdb"

class S3BlobDB(AbstractBlobDB):

    def __init__(self, config):
        kwargs = {}
        if "config" in config:
            kwargs["config"] = Config(**config["config"])
        self.db = boto3.resource(
            's3',
            endpoint_url=config.get("url"),
            aws_access_key_id=config.get("access_key", ""),
            aws_secret_access_key=config.get("secret_key", ""),
            **kwargs
        )
        self.s3_bucket_name = config.get("s3_bucket", DEFAULT_S3_BUCKET)
        self._s3_bucket_exists = False
        # https://github.com/boto/boto3/issues/259
        self.db.meta.client.meta.events.unregister('before-sign.s3', fix_s3_host)

    def put(self, content, identifier, bucket=DEFAULT_BUCKET, timeout=None):
        path = self.get_path(identifier, bucket)
        s3_bucket = self._s3_bucket(create=True)
        if isinstance(content, BlobStream) and content.blob_db is self:
            source = {"Bucket": self.s3_bucket_name, "Key": content.blob_path}
            s3_bucket.copy(source, path)
            obj = s3_bucket.Object(path)
            # unfortunately cannot get content-md5 here
            return BlobInfo(identifier, obj.content_length, None)
        content.seek(0)
        content_md5 = get_content_md5(content)
        content_length = get_file_size(content)
        s3_bucket.upload_fileobj(content, path)
        if timeout is not None:
            set_blob_expire_object(bucket, identifier, content_length, timeout)
        datadog_counter('commcare.blobs.added.count')
        datadog_counter('commcare.blobs.added.bytes', value=content_length)
        return BlobInfo(identifier, content_length, "md5-" + content_md5)

    def get(self, identifier, bucket=DEFAULT_BUCKET):
        path = self.get_path(identifier, bucket)
        with maybe_not_found(throw=NotFound(identifier, bucket)):
            resp = self._s3_bucket().Object(path).get()
        return BlobStream(resp["Body"], self, path)

    def size(self, identifier, bucket=DEFAULT_BUCKET):
        path = self.get_path(identifier, bucket)
        with maybe_not_found(throw=NotFound(identifier, bucket)):
            return self._s3_bucket().Object(path).content_length

    def exists(self, identifier, bucket=DEFAULT_BUCKET):
        path = self.get_path(identifier, bucket)
        try:
            with maybe_not_found(throw=NotFound(identifier, bucket)):
                self._s3_bucket().Object(path).load()
            return True
        except NotFound:
            return False

    def delete(self, *args, **kw):
        identifier, bucket = self.get_args_for_delete(*args, **kw)
        path = self.get_path(identifier, bucket)
        s3_bucket = self._s3_bucket()
        with maybe_not_found():
            if identifier is None:
                summaries = s3_bucket.objects.filter(Prefix=path + "/")
                pages = ([{"Key": o.key} for o in page]
                         for page in summaries.pages())
                deleted_bytes = sum(o.size for page in summaries.pages()
                                    for o in page)
            else:
                pages = [[{"Key": path}]]
                # may raise a not found error -> return False
                deleted_bytes = s3_bucket.Object(path).content_length
            success = True
            deleted_count = 0
            for objects in pages:
                resp = s3_bucket.delete_objects(Delete={"Objects": objects})
                if success:
                    deleted = set(d["Key"] for d in resp.get("Deleted", []))
                    success = all(o["Key"] in deleted for o in objects)
                    deleted_count += len(deleted)
            datadog_counter('commcare.blobs.deleted.count', value=deleted_count)
            datadog_counter('commcare.blobs.deleted.bytes', value=deleted_bytes)
            return success
        return False

    def bulk_delete(self, paths):
        success = True
        for chunk in chunked(paths, 1000):
            objects = [{"Key": path} for path in chunk]
            s3_bucket = self._s3_bucket()
            deleted_bytes = 0
            for path in chunk:
                with maybe_not_found():
                    deleted_bytes += s3_bucket.Object(path).content_length
            resp = s3_bucket.delete_objects(Delete={"Objects": objects})
            deleted = set(d["Key"] for d in resp.get("Deleted", []))
            success = success and all(o["Key"] in deleted for o in objects)
            datadog_counter('commcare.blobs.deleted.count', value=len(deleted))
            datadog_counter('commcare.blobs.deleted.bytes', value=deleted_bytes)
        return success

    def copy_blob(self, content, info, bucket):
        self._s3_bucket(create=True)
        path = self.get_path(info.identifier, bucket)
        self._s3_bucket().upload_fileobj(content, path)

    def _s3_bucket(self, create=False):
        if create and not self._s3_bucket_exists:
            try:
                self.db.meta.client.head_bucket(Bucket=self.s3_bucket_name)
            except ClientError as err:
                if not is_not_found(err):
                    datadog_counter('commcare.blobdb.notfound')
                    raise
                self.db.create_bucket(Bucket=self.s3_bucket_name)
            self._s3_bucket_exists = True
        return self.db.Bucket(self.s3_bucket_name)

    def get_path(self, identifier=None, bucket=DEFAULT_BUCKET):
        if identifier is None:
            return safepath(bucket)
        return safejoin(bucket, identifier)

class BlobStream(ClosingContextProxy):

    def __init__(self, stream, blob_db, blob_path):
        super(BlobStream, self).__init__(stream)
        self._blob_db = weakref.ref(blob_db)
        self.blob_path = blob_path

    @property
    def blob_db(self):
        return self._blob_db()

def safepath(path):
    if (path.startswith(("/", ".")) or
            "/../" in path or
            path.endswith("/..") or
            not SAFENAME.match(path)):
        raise BadName(u"unsafe path name: %r" % path)
    return path

def safejoin(root, subpath):
    """Join root to subpath ensuring that the result is actually inside root
    """
    return safepath(root) + "/" + safepath(subpath)

def is_not_found(err, not_found_codes=["NoSuchKey", "NoSuchBucket", "404"]):
    return (err.response["Error"]["Code"] in not_found_codes or
        err.response.get("Errors", {}).get("Error", {}).get("Code") in not_found_codes)

def get_content_md5(content):
    params = {"body": content, "headers": {}}
    calculate_md5(params)
    return params["headers"]["Content-MD5"]

def get_file_size(fileobj):

    def tell_end(fileobj_):
        pos = fileobj_.tell()
        try:
            fileobj_.seek(0, os.SEEK_END)
            return fileobj_.tell()
        finally:
            fileobj_.seek(pos)

    if not hasattr(fileobj, 'fileno'):
        return tell_end(fileobj)
    try:
        fileno = fileobj.fileno()
    except UnsupportedOperation:
        return tell_end(fileobj)
    return os.fstat(fileno).st_size

@contextmanager
def maybe_not_found(throw=None):
    try:
        yield
    except ClientError as err:
        if not is_not_found(err):
            datadog_counter('commcare.blobdb.notfound')
            raise
        if throw is not None:
            raise throw
<commit_msg>Use DELETE method when deleting single object<commit_after>from __future__ import absolute_import
import os
import weakref
from contextlib import contextmanager
from io import UnsupportedOperation

from corehq.blobs import BlobInfo, DEFAULT_BUCKET
from corehq.blobs.exceptions import BadName, NotFound
from corehq.blobs.interface import AbstractBlobDB, SAFENAME
from corehq.blobs.util import ClosingContextProxy, set_blob_expire_object
from corehq.util.datadog.gauges import datadog_counter

from dimagi.utils.chunked import chunked

import boto3
from botocore.client import Config
from botocore.handlers import calculate_md5
from botocore.exceptions import ClientError
from botocore.utils import fix_s3_host

DEFAULT_S3_BUCKET = "blobdb"

class S3BlobDB(AbstractBlobDB):

    def __init__(self, config):
        kwargs = {}
        if "config" in config:
            kwargs["config"] = Config(**config["config"])
        self.db = boto3.resource(
            's3',
            endpoint_url=config.get("url"),
            aws_access_key_id=config.get("access_key", ""),
            aws_secret_access_key=config.get("secret_key", ""),
            **kwargs
        )
        self.s3_bucket_name = config.get("s3_bucket", DEFAULT_S3_BUCKET)
        self._s3_bucket_exists = False
        # https://github.com/boto/boto3/issues/259
        self.db.meta.client.meta.events.unregister('before-sign.s3', fix_s3_host)

    def put(self, content, identifier, bucket=DEFAULT_BUCKET, timeout=None):
        path = self.get_path(identifier, bucket)
        s3_bucket = self._s3_bucket(create=True)
        if isinstance(content, BlobStream) and content.blob_db is self:
            source = {"Bucket": self.s3_bucket_name, "Key": content.blob_path}
            s3_bucket.copy(source, path)
            obj = s3_bucket.Object(path)
            # unfortunately cannot get content-md5 here
            return BlobInfo(identifier, obj.content_length, None)
        content.seek(0)
        content_md5 = get_content_md5(content)
        content_length = get_file_size(content)
        s3_bucket.upload_fileobj(content, path)
        if timeout is not None:
            set_blob_expire_object(bucket, identifier, content_length, timeout)
        datadog_counter('commcare.blobs.added.count')
        datadog_counter('commcare.blobs.added.bytes', value=content_length)
        return BlobInfo(identifier, content_length, "md5-" + content_md5)

    def get(self, identifier, bucket=DEFAULT_BUCKET):
        path = self.get_path(identifier, bucket)
        with maybe_not_found(throw=NotFound(identifier, bucket)):
            resp = self._s3_bucket().Object(path).get()
        return BlobStream(resp["Body"], self, path)

    def size(self, identifier, bucket=DEFAULT_BUCKET):
        path = self.get_path(identifier, bucket)
        with maybe_not_found(throw=NotFound(identifier, bucket)):
            return self._s3_bucket().Object(path).content_length

    def exists(self, identifier, bucket=DEFAULT_BUCKET):
        path = self.get_path(identifier, bucket)
        try:
            with maybe_not_found(throw=NotFound(identifier, bucket)):
                self._s3_bucket().Object(path).load()
            return True
        except NotFound:
            return False

    def delete(self, *args, **kw):
        identifier, bucket = self.get_args_for_delete(*args, **kw)
        path = self.get_path(identifier, bucket)
        s3_bucket = self._s3_bucket()
        with maybe_not_found():
            success = True
            if identifier is None:
                summaries = s3_bucket.objects.filter(Prefix=path + "/")
                pages = ([{"Key": o.key} for o in page]
                         for page in summaries.pages())
                deleted_bytes = sum(o.size for page in summaries.pages()
                                    for o in page)
                deleted_count = 0
                for objects in pages:
                    resp = s3_bucket.delete_objects(Delete={"Objects": objects})
                    if success:
                        deleted = set(d["Key"] for d in resp.get("Deleted", []))
                        success = all(o["Key"] in deleted for o in objects)
                        deleted_count += len(deleted)
            else:
                obj = s3_bucket.Object(path)
                deleted_count = 1
                # may raise a not found error -> return False
                deleted_bytes = obj.content_length
                obj.delete()
            datadog_counter('commcare.blobs.deleted.count', value=deleted_count)
            datadog_counter('commcare.blobs.deleted.bytes', value=deleted_bytes)
            return success
        return False

    def bulk_delete(self, paths):
        success = True
        for chunk in chunked(paths, 1000):
            objects = [{"Key": path} for path in chunk]
            s3_bucket = self._s3_bucket()
            deleted_bytes = 0
            for path in chunk:
                with maybe_not_found():
                    deleted_bytes += s3_bucket.Object(path).content_length
            resp = s3_bucket.delete_objects(Delete={"Objects": objects})
            deleted = set(d["Key"] for d in resp.get("Deleted", []))
            success = success and all(o["Key"] in deleted for o in objects)
            datadog_counter('commcare.blobs.deleted.count', value=len(deleted))
            datadog_counter('commcare.blobs.deleted.bytes', value=deleted_bytes)
        return success

    def copy_blob(self, content, info, bucket):
        self._s3_bucket(create=True)
        path = self.get_path(info.identifier, bucket)
        self._s3_bucket().upload_fileobj(content, path)

    def _s3_bucket(self, create=False):
        if create and not self._s3_bucket_exists:
            try:
                self.db.meta.client.head_bucket(Bucket=self.s3_bucket_name)
            except ClientError as err:
                if not is_not_found(err):
                    datadog_counter('commcare.blobdb.notfound')
                    raise
                self.db.create_bucket(Bucket=self.s3_bucket_name)
            self._s3_bucket_exists = True
        return self.db.Bucket(self.s3_bucket_name)

    def get_path(self, identifier=None, bucket=DEFAULT_BUCKET):
        if identifier is None:
            return safepath(bucket)
        return safejoin(bucket, identifier)

class BlobStream(ClosingContextProxy):

    def __init__(self, stream, blob_db, blob_path):
        super(BlobStream, self).__init__(stream)
        self._blob_db = weakref.ref(blob_db)
        self.blob_path = blob_path

    @property
    def blob_db(self):
        return self._blob_db()

def safepath(path):
    if (path.startswith(("/", ".")) or
            "/../" in path or
            path.endswith("/..") or
            not SAFENAME.match(path)):
        raise BadName(u"unsafe path name: %r" % path)
    return path

def safejoin(root, subpath):
    """Join root to subpath ensuring that the result is actually inside root
    """
    return safepath(root) + "/" + safepath(subpath)

def is_not_found(err, not_found_codes=["NoSuchKey", "NoSuchBucket", "404"]):
    return (err.response["Error"]["Code"] in not_found_codes or
        err.response.get("Errors", {}).get("Error", {}).get("Code") in not_found_codes)

def get_content_md5(content):
    params = {"body": content, "headers": {}}
    calculate_md5(params)
    return params["headers"]["Content-MD5"]

def get_file_size(fileobj):

    def tell_end(fileobj_):
        pos = fileobj_.tell()
        try:
            fileobj_.seek(0, os.SEEK_END)
            return fileobj_.tell()
        finally:
            fileobj_.seek(pos)

    if not hasattr(fileobj, 'fileno'):
        return tell_end(fileobj)
    try:
        fileno = fileobj.fileno()
    except UnsupportedOperation:
        return tell_end(fileobj)
    return os.fstat(fileno).st_size

@contextmanager
def maybe_not_found(throw=None):
    try:
        yield
    except ClientError as err:
        if not is_not_found(err):
            datadog_counter('commcare.blobdb.notfound')
            raise
        if throw is not None:
            raise throw
<|endoftext|>


<commit_before>from weakref import ref
from decimal import Decimal
import re
import datetime
import traceback, sys

from kivy.app import App
from kivy.cache import Cache
from kivy.clock import Clock
from kivy.compat import string_types
from kivy.properties import (ObjectProperty, DictProperty, NumericProperty,
                             ListProperty)
from kivy.lang import Builder
from kivy.factory import Factory

from electrum.i18n import _
from electrum.util import profiler
from electrum import bitcoin
from electrum.util import timestamp_to_datetime
from electrum.plugins import run_hook

class CScreen(Factory.Screen):

    __events__ = ('on_activate', 'on_deactivate', 'on_enter', 'on_leave')
    action_view = ObjectProperty(None)
    loaded = False
    kvname = None
    app = App.get_running_app()

    def _change_action_view(self):
        app = App.get_running_app()
        action_bar = app.root.manager.current_screen.ids.action_bar
        _action_view = self.action_view

        if (not _action_view) or _action_view.parent:
            return
        action_bar.clear_widgets()
        action_bar.add_widget(_action_view)

    def on_enter(self):
        # FIXME: use a proper event don't use animation time of screen
        Clock.schedule_once(lambda dt: self.dispatch('on_activate'), .25)
        pass

    def update(self):
        pass

    @profiler
    def load_screen(self):
        self.screen = Builder.load_file('gui/kivy/uix/ui_screens/' + self.kvname + '.kv')
        self.add_widget(self.screen)
        self.loaded = True
        self.update()
        setattr(self.app, self.kvname + '_screen', self)

    def on_activate(self):
        if self.kvname and not self.loaded:
            self.load_screen()
        #Clock.schedule_once(lambda dt: self._change_action_view())

    def on_leave(self):
        self.dispatch('on_deactivate')

    def on_deactivate(self):
        pass
        #Clock.schedule_once(lambda dt: self._change_action_view())

class HistoryScreen(CScreen):

    tab = ObjectProperty(None)
    kvname = 'history'

    def __init__(self, **kwargs):
        self.ra_dialog = None
        super(HistoryScreen, self).__init__(**kwargs)

    def get_history_rate(self, btc_balance, timestamp):
        date = timestamp_to_datetime(timestamp)
        return run_hook('historical_value_str', btc_balance, date)

    def parse_history(self, items):
        for item in items:
            tx_hash, conf, value, timestamp, balance = item
            time_str = _("unknown")
            if conf > 0:
                try:
                    time_str = datetime.datetime.fromtimestamp(timestamp).isoformat(' ')[:-3]
                except Exception:
                    time_str = _("error")
            if conf == -1:
                time_str = _('unverified')
                icon = "atlas://gui/kivy/theming/light/close"
            elif conf == 0:
                time_str = _('pending')
                icon = "atlas://gui/kivy/theming/light/unconfirmed"
            elif conf < 6:
                time_str = ''  # add new to fix error when conf < 0
                conf = max(1, conf)
                icon = "atlas://gui/kivy/theming/light/clock{}".format(conf)
            else:
                icon = "atlas://gui/kivy/theming/light/confirmed"

            if tx_hash:
                label, is_default_label = self.app.wallet.get_label(tx_hash)
            else:
                label = _('Pruned transaction outputs')
                is_default_label = False

            quote_currency = 'USD'
            rate = self.get_history_rate(value, timestamp)
            quote_text = "..." if rate is None else "{0:.3} {1}".format(rate, quote_currency)

            yield (conf, icon, time_str, label, value, tx_hash, quote_text)

    def update(self, see_all=False):
        if self.app.wallet is None:
            return

        history_card = self.screen.ids.recent_activity_card
        history = self.parse_history(reversed(
            self.app.wallet.get_history(self.app.current_account)))
        # repopulate History Card
        last_widget = history_card.ids.content.children[-1]
        history_card.ids.content.clear_widgets()
        history_add = history_card.ids.content.add_widget
        history_add(last_widget)
        RecentActivityItem = Factory.RecentActivityItem
        count = 0
        for item in history:
            count += 1
            conf, icon, date_time, address, value, tx, quote_text = item
            ri = RecentActivityItem()
            ri.icon = icon
            ri.date = date_time
            ri.address = address
            ri.value = value
            ri.quote_text = quote_text
            ri.confirmations = conf
            ri.tx_hash = tx
            history_add(ri)
            if count == 8 and not see_all:
                break

        history_card.ids.btn_see_all.opacity = (0 if count < 8 else 1)

class ScreenAddress(CScreen):
    '''This is the dialog that shows a carousel of the currently available
    addresses.
    '''

    labels  = DictProperty({})
    ''' A precached list of address labels.
    '''

    tab =  ObjectProperty(None)
    ''' The tab associated With this Carousel
    '''

class ScreenPassword(Factory.Screen):

    __events__ = ('on_release', 'on_deactivate', 'on_activate')

    def on_activate(self):
        app = App.get_running_app()
        action_bar = app.root.main_screen.ids.action_bar
        action_bar.add_widget(self._action_view)

    def on_deactivate(self):
        self.ids.password.text = ''

    def on_release(self, *args):
        pass

class SendScreen(CScreen):
    kvname = 'send'
    def set_qr_data(self, uri):
        self.ids.payto_e.text = uri.get('address', '')
        self.ids.message_e.text = uri.get('message', '')
        amount = uri.get('amount')
        if amount:
            amount_str = str( Decimal(amount) / pow(10, self.app.decimal_point()))
            self.ids.amount_e.text = amount_str + ' ' + self.app.base_unit

    def do_clear(self):
        self.ids.payto_e.text = ''
        self.ids.message_e.text = ''
        self.ids.amount_e.text = 'Amount'
        #self.set_frozen(content, False)
        #self.update_status()

    def do_send(self):
        scrn = self.ids
        label = unicode(scrn.message_e.text)
        r = unicode(scrn.payto_e.text).strip()
        # label or alias, with address in brackets
        m = re.match('(.*?)\s*\<([1-9A-HJ-NP-Za-km-z]{26,})\>', r)
        to_address = m.group(2) if m else r

        if not bitcoin.is_address(to_address):
            self.app.show_error(_('Invalid Bitcoin Address') + ':\n' + to_address)
            return

        amount = self.app.get_amount(scrn.amount_e.text)
        #fee = scrn.fee_e.amt
        #if not fee:
        #    app.show_error(_('Invalid Fee'))
        #    return
        fee = None
        message = 'sending {} {} to {}'.format(self.app.base_unit, scrn.amount_e.text, r)
        outputs = [('address', to_address, amount)]
        self.app.password_dialog(self.send_tx, (outputs, fee, label))

    def send_tx(self, outputs, fee, label, password):
        # make unsigned transaction
        coins = self.app.wallet.get_spendable_coins()
        try:
            tx = self.app.wallet.make_unsigned_transaction(coins, outputs, self.app.electrum_config, fee)
        except Exception as e:
            traceback.print_exc(file=sys.stdout)
            self.app.show_error(str(e))
            return
        # sign transaction
        try:
            self.app.wallet.sign_transaction(tx, password)
        except Exception as e:
            traceback.print_exc(file=sys.stdout)
            self.app.show_error(str(e))
            return
        # broadcast
        ok, txid = self.app.wallet.sendtx(tx)
        self.app.show_info(txid)

class ReceiveScreen(CScreen):
    kvname = 'receive'

    def update(self):
        addr = self.app.wallet.get_unused_address(None)
        address_label = self.screen.ids.get('address')
        address_label.text = addr
        self.update_qr()

    def amount_callback(self, popup):
        amount_label = self.screen.ids.get('amount')
        amount_label.text = popup.ids.amount_label.text
        self.update_qr()

    @profiler
    def update_qrtt(self):
        raise

    def update_qr(self):
        from electrum.util import create_URI
        address = self.screen.ids.get('address').text
        amount = self.screen.ids.get('amount').text
        default_text = self.screen.ids.get('amount').default_text
        if amount == default_text:
            amount = None
        else:
            a, u = amount.split()
            assert u == self.app.base_unit
            amount = Decimal(a) * pow(10, self.app.decimal_point())
        msg = self.screen.ids.get('message').text
        uri = create_URI(address, amount, msg)
        qr = self.screen.ids.get('qr')
        qr.set_data(uri)

    def do_share(self):
        pass

    def do_clear(self):
        a = self.screen.ids.get('amount')
        a.text = a.default_text
        self.screen.ids.get('message').text = ''

class ContactsScreen(CScreen):
    kvname = 'contacts'

    def add_new_contact(self):
        dlg = Cache.get('electrum_widgets', 'NewContactDialog')
        if not dlg:
            dlg = NewContactDialog()
            Cache.append('electrum_widgets', 'NewContactDialog', dlg)
        dlg.open()

    def update(self):
        contact_list = self.screen.ids.contact_container
        contact_list.clear_widgets()
        child = -1
        children = contact_list.children
        for key in sorted(self.app.contacts.keys()):
            _type, value = self.app.contacts[key]
            child += 1
            try:
                if children[child].label == value:
                    continue
            except IndexError:
                pass
            ci = Factory.ContactItem()
            ci.address = key
            ci.label = value
            contact_list.add_widget(ci)

class CSpinner(Factory.Spinner):
    '''CustomDropDown that allows fading out the dropdown
    '''

    def _update_dropdown(self, *largs):
        dp = self._dropdown
        cls = self.option_cls
        if isinstance(cls, string_types):
            cls = Factory.get(cls)
        dp.clear_widgets()
        def do_release(option):
            Clock.schedule_once(lambda dt: dp.select(option.text), .25)
        for value in self.values:
            item = cls(text=value)
            item.bind(on_release=do_release)
            dp.add_widget(item)

class TabbedCarousel(Factory.TabbedPanel):
    '''Custom TabbedPanel using a carousel used in the Main Screen
    '''

    carousel = ObjectProperty(None)

    def animate_tab_to_center(self, value):
        scrlv = self._tab_strip.parent
        if not scrlv:
            return

        idx = self.tab_list.index(value)
        if  idx == 0:
            scroll_x = 1
        elif idx == len(self.tab_list) - 1:
            scroll_x = 0
        else:
            self_center_x = scrlv.center_x
            vcenter_x = value.center_x
            diff_x = (self_center_x - vcenter_x)
            try:
                scroll_x = scrlv.scroll_x - (diff_x / scrlv.width)
            except ZeroDivisionError:
                pass
        mation = Factory.Animation(scroll_x=scroll_x, d=.25)
        mation.cancel_all(scrlv)
        mation.start(scrlv)

    def on_current_tab(self, instance, value):
        if value.text == 'default_tab':
            return
        self.animate_tab_to_center(value)

    def on_index(self, instance, value):
        current_slide = instance.current_slide
        if not hasattr(current_slide, 'tab'):
            return
        tab = current_slide.tab
        ct = self.current_tab
        try:
            if ct.text != tab.text:
                carousel = self.carousel
                carousel.slides[ct.slide].dispatch('on_leave')
                self.switch_to(tab)
                carousel.slides[tab.slide].dispatch('on_enter')
        except AttributeError:
            current_slide.dispatch('on_enter')

    def switch_to(self, header):
        # we have to replace the functionality of the original switch_to
        if not header:
            return
        if not hasattr(header, 'slide'):
            header.content = self.carousel
            super(TabbedCarousel, self).switch_to(header)
            try:
                tab = self.tab_list[-1]
            except IndexError:
                return
            self._current_tab = tab
            tab.state = 'down'
            return

        carousel = self.carousel
        self.current_tab.state = "normal"
        header.state = 'down'
        self._current_tab = header
        # set the carousel to load  the appropriate slide
        # saved in the screen attribute of the tab head
        slide = carousel.slides[header.slide]
        if carousel.current_slide != slide:
            carousel.current_slide.dispatch('on_leave')
            carousel.load_slide(slide)
            slide.dispatch('on_enter')

    def add_widget(self, widget, index=0):
        if isinstance(widget, Factory.CScreen):
            self.carousel.add_widget(widget)
            return
        super(TabbedCarousel, self).add_widget(widget, index=index)

class ELTextInput(Factory.TextInput):
    '''Custom TextInput used in main screens for numeric entry
    '''

    def insert_text(self, substring, from_undo=False):
        if not from_undo:
            if self.input_type == 'numbers':
                numeric_list = map(str, range(10))
                if '.' not in self.text:
                    numeric_list.append('.')
                if substring not in numeric_list:
                    return
        super(ELTextInput, self).insert_text(substring, from_undo=from_undo)
<commit_msg>kivy: sign and send in a thread<commit_after>from weakref import ref
from decimal import Decimal
import re
import datetime
import traceback, sys
import threading

from kivy.app import App
from kivy.cache import Cache
from kivy.clock import Clock
from kivy.compat import string_types
from kivy.properties import (ObjectProperty, DictProperty, NumericProperty,
                             ListProperty)
from kivy.lang import Builder
from kivy.factory import Factory

from electrum.i18n import _
from electrum.util import profiler
from electrum import bitcoin
from electrum.util import timestamp_to_datetime
from electrum.plugins import run_hook

class CScreen(Factory.Screen):

    __events__ = ('on_activate', 'on_deactivate', 'on_enter', 'on_leave')
    action_view = ObjectProperty(None)
    loaded = False
    kvname = None
    app = App.get_running_app()

    def _change_action_view(self):
        app = App.get_running_app()
        action_bar = app.root.manager.current_screen.ids.action_bar
        _action_view = self.action_view

        if (not _action_view) or _action_view.parent:
            return
        action_bar.clear_widgets()
        action_bar.add_widget(_action_view)

    def on_enter(self):
        # FIXME: use a proper event don't use animation time of screen
        Clock.schedule_once(lambda dt: self.dispatch('on_activate'), .25)
        pass

    def update(self):
        pass

    @profiler
    def load_screen(self):
        self.screen = Builder.load_file('gui/kivy/uix/ui_screens/' + self.kvname + '.kv')
        self.add_widget(self.screen)
        self.loaded = True
        self.update()
        setattr(self.app, self.kvname + '_screen', self)

    def on_activate(self):
        if self.kvname and not self.loaded:
            self.load_screen()
        #Clock.schedule_once(lambda dt: self._change_action_view())

    def on_leave(self):
        self.dispatch('on_deactivate')

    def on_deactivate(self):
        pass
        #Clock.schedule_once(lambda dt: self._change_action_view())

class HistoryScreen(CScreen):

    tab = ObjectProperty(None)
    kvname = 'history'

    def __init__(self, **kwargs):
        self.ra_dialog = None
        super(HistoryScreen, self).__init__(**kwargs)

    def get_history_rate(self, btc_balance, timestamp):
        date = timestamp_to_datetime(timestamp)
        return run_hook('historical_value_str', btc_balance, date)

    def parse_history(self, items):
        for item in items:
            tx_hash, conf, value, timestamp, balance = item
            time_str = _("unknown")
            if conf > 0:
                try:
                    time_str = datetime.datetime.fromtimestamp(timestamp).isoformat(' ')[:-3]
                except Exception:
                    time_str = _("error")
            if conf == -1:
                time_str = _('unverified')
                icon = "atlas://gui/kivy/theming/light/close"
            elif conf == 0:
                time_str = _('pending')
                icon = "atlas://gui/kivy/theming/light/unconfirmed"
            elif conf < 6:
                time_str = ''  # add new to fix error when conf < 0
                conf = max(1, conf)
                icon = "atlas://gui/kivy/theming/light/clock{}".format(conf)
            else:
                icon = "atlas://gui/kivy/theming/light/confirmed"

            if tx_hash:
                label, is_default_label = self.app.wallet.get_label(tx_hash)
            else:
                label = _('Pruned transaction outputs')
                is_default_label = False

            quote_currency = 'USD'
            rate = self.get_history_rate(value, timestamp)
            quote_text = "..." if rate is None else "{0:.3} {1}".format(rate, quote_currency)

            yield (conf, icon, time_str, label, value, tx_hash, quote_text)

    def update(self, see_all=False):
        if self.app.wallet is None:
            return

        history_card = self.screen.ids.recent_activity_card
        history = self.parse_history(reversed(
            self.app.wallet.get_history(self.app.current_account)))
        # repopulate History Card
        last_widget = history_card.ids.content.children[-1]
        history_card.ids.content.clear_widgets()
        history_add = history_card.ids.content.add_widget
        history_add(last_widget)
        RecentActivityItem = Factory.RecentActivityItem
        count = 0
        for item in history:
            count += 1
            conf, icon, date_time, address, value, tx, quote_text = item
            ri = RecentActivityItem()
            ri.icon = icon
            ri.date = date_time
            ri.address = address
            ri.value = value
            ri.quote_text = quote_text
            ri.confirmations = conf
            ri.tx_hash = tx
            history_add(ri)
            if count == 8 and not see_all:
                break

        history_card.ids.btn_see_all.opacity = (0 if count < 8 else 1)

class ScreenAddress(CScreen):
    '''This is the dialog that shows a carousel of the currently available
    addresses.
    '''

    labels  = DictProperty({})
    ''' A precached list of address labels.
    '''

    tab =  ObjectProperty(None)
    ''' The tab associated With this Carousel
    '''

class ScreenPassword(Factory.Screen):

    __events__ = ('on_release', 'on_deactivate', 'on_activate')

    def on_activate(self):
        app = App.get_running_app()
        action_bar = app.root.main_screen.ids.action_bar
        action_bar.add_widget(self._action_view)

    def on_deactivate(self):
        self.ids.password.text = ''

    def on_release(self, *args):
        pass

class SendScreen(CScreen):
    kvname = 'send'
    def set_qr_data(self, uri):
        self.ids.payto_e.text = uri.get('address', '')
        self.ids.message_e.text = uri.get('message', '')
        amount = uri.get('amount')
        if amount:
            amount_str = str( Decimal(amount) / pow(10, self.app.decimal_point()))
            self.ids.amount_e.text = amount_str + ' ' + self.app.base_unit

    def do_clear(self):
        self.ids.payto_e.text = ''
        self.ids.message_e.text = ''
        self.ids.amount_e.text = 'Amount'
        #self.set_frozen(content, False)
        #self.update_status()

    def do_send(self):
        scrn = self.ids
        label = unicode(scrn.message_e.text)
        r = unicode(scrn.payto_e.text).strip()
        # label or alias, with address in brackets
        m = re.match('(.*?)\s*\<([1-9A-HJ-NP-Za-km-z]{26,})\>', r)
        to_address = m.group(2) if m else r

        if not bitcoin.is_address(to_address):
            self.app.show_error(_('Invalid Bitcoin Address') + ':\n' + to_address)
            return

        amount = self.app.get_amount(scrn.amount_e.text)
        #fee = scrn.fee_e.amt
        #if not fee:
        #    app.show_error(_('Invalid Fee'))
        #    return
        fee = None
        message = 'sending {} {} to {}'.format(self.app.base_unit, scrn.amount_e.text, r)
        outputs = [('address', to_address, amount)]
        self.app.password_dialog(self.send_tx, (outputs, fee, label))

    def send_tx(self, *args):
        self.app.show_info("Sending...")
        threading.Thread(target=self.send_tx_thread, args=args).start()

    def send_tx_thread(self, outputs, fee, label, password):
        # make unsigned transaction
        coins = self.app.wallet.get_spendable_coins()
        try:
            tx = self.app.wallet.make_unsigned_transaction(coins, outputs, self.app.electrum_config, fee)
        except Exception as e:
            traceback.print_exc(file=sys.stdout)
            self.app.show_error(str(e))
            return
        # sign transaction
        try:
            self.app.wallet.sign_transaction(tx, password)
        except Exception as e:
            traceback.print_exc(file=sys.stdout)
            self.app.show_error(str(e))
            return
        # broadcast
        ok, txid = self.app.wallet.sendtx(tx)
        self.app.show_info(txid)

class ReceiveScreen(CScreen):
    kvname = 'receive'

    def update(self):
        addr = self.app.wallet.get_unused_address(None)
        address_label = self.screen.ids.get('address')
        address_label.text = addr
        self.update_qr()

    def amount_callback(self, popup):
        amount_label = self.screen.ids.get('amount')
        amount_label.text = popup.ids.amount_label.text
        self.update_qr()

    @profiler
    def update_qrtt(self):
        raise

    def update_qr(self):
        from electrum.util import create_URI
        address = self.screen.ids.get('address').text
        amount = self.screen.ids.get('amount').text
        default_text = self.screen.ids.get('amount').default_text
        if amount == default_text:
            amount = None
        else:
            a, u = amount.split()
            assert u == self.app.base_unit
            amount = Decimal(a) * pow(10, self.app.decimal_point())
        msg = self.screen.ids.get('message').text
        uri = create_URI(address, amount, msg)
        qr = self.screen.ids.get('qr')
        qr.set_data(uri)

    def do_share(self):
        pass

    def do_clear(self):
        a = self.screen.ids.get('amount')
        a.text = a.default_text
        self.screen.ids.get('message').text = ''

class ContactsScreen(CScreen):
    kvname = 'contacts'

    def add_new_contact(self):
        dlg = Cache.get('electrum_widgets', 'NewContactDialog')
        if not dlg:
            dlg = NewContactDialog()
            Cache.append('electrum_widgets', 'NewContactDialog', dlg)
        dlg.open()

    def update(self):
        contact_list = self.screen.ids.contact_container
        contact_list.clear_widgets()
        child = -1
        children = contact_list.children
        for key in sorted(self.app.contacts.keys()):
            _type, value = self.app.contacts[key]
            child += 1
            try:
                if children[child].label == value:
                    continue
            except IndexError:
                pass
            ci = Factory.ContactItem()
            ci.address = key
            ci.label = value
            contact_list.add_widget(ci)

class CSpinner(Factory.Spinner):
    '''CustomDropDown that allows fading out the dropdown
    '''

    def _update_dropdown(self, *largs):
        dp = self._dropdown
        cls = self.option_cls
        if isinstance(cls, string_types):
            cls = Factory.get(cls)
        dp.clear_widgets()
        def do_release(option):
            Clock.schedule_once(lambda dt: dp.select(option.text), .25)
        for value in self.values:
            item = cls(text=value)
            item.bind(on_release=do_release)
            dp.add_widget(item)

class TabbedCarousel(Factory.TabbedPanel):
    '''Custom TabbedPanel using a carousel used in the Main Screen
    '''

    carousel = ObjectProperty(None)

    def animate_tab_to_center(self, value):
        scrlv = self._tab_strip.parent
        if not scrlv:
            return

        idx = self.tab_list.index(value)
        if  idx == 0:
            scroll_x = 1
        elif idx == len(self.tab_list) - 1:
            scroll_x = 0
        else:
            self_center_x = scrlv.center_x
            vcenter_x = value.center_x
            diff_x = (self_center_x - vcenter_x)
            try:
                scroll_x = scrlv.scroll_x - (diff_x / scrlv.width)
            except ZeroDivisionError:
                pass
        mation = Factory.Animation(scroll_x=scroll_x, d=.25)
        mation.cancel_all(scrlv)
        mation.start(scrlv)

    def on_current_tab(self, instance, value):
        if value.text == 'default_tab':
            return
        self.animate_tab_to_center(value)

    def on_index(self, instance, value):
        current_slide = instance.current_slide
        if not hasattr(current_slide, 'tab'):
            return
        tab = current_slide.tab
        ct = self.current_tab
        try:
            if ct.text != tab.text:
                carousel = self.carousel
                carousel.slides[ct.slide].dispatch('on_leave')
                self.switch_to(tab)
                carousel.slides[tab.slide].dispatch('on_enter')
        except AttributeError:
            current_slide.dispatch('on_enter')

    def switch_to(self, header):
        # we have to replace the functionality of the original switch_to
        if not header:
            return
        if not hasattr(header, 'slide'):
            header.content = self.carousel
            super(TabbedCarousel, self).switch_to(header)
            try:
                tab = self.tab_list[-1]
            except IndexError:
                return
            self._current_tab = tab
            tab.state = 'down'
            return

        carousel = self.carousel
        self.current_tab.state = "normal"
        header.state = 'down'
        self._current_tab = header
        # set the carousel to load  the appropriate slide
        # saved in the screen attribute of the tab head
        slide = carousel.slides[header.slide]
        if carousel.current_slide != slide:
            carousel.current_slide.dispatch('on_leave')
            carousel.load_slide(slide)
            slide.dispatch('on_enter')

    def add_widget(self, widget, index=0):
        if isinstance(widget, Factory.CScreen):
            self.carousel.add_widget(widget)
            return
        super(TabbedCarousel, self).add_widget(widget, index=index)

class ELTextInput(Factory.TextInput):
    '''Custom TextInput used in main screens for numeric entry
    '''

    def insert_text(self, substring, from_undo=False):
        if not from_undo:
            if self.input_type == 'numbers':
                numeric_list = map(str, range(10))
                if '.' not in self.text:
                    numeric_list.append('.')
                if substring not in numeric_list:
                    return
        super(ELTextInput, self).insert_text(substring, from_undo=from_undo)
<|endoftext|>


<commit_before># Scrapy settings for pystock-crawler project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#

BOT_NAME = 'pystock-crawler'

SPIDER_MODULES = ['pystock_crawler.spiders']
NEWSPIDER_MODULE = 'pystock_crawler.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'stockcrawler (+http://www.yourdomain.com)'

LOG_LEVEL = 'DEBUG'

HTTPCACHE_ENABLED = True
<commit_msg>Adjust log and cache settings.<commit_after># Scrapy settings for pystock-crawler project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#

BOT_NAME = 'pystock-crawler'

SPIDER_MODULES = ['pystock_crawler.spiders']
NEWSPIDER_MODULE = 'pystock_crawler.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'stockcrawler (+http://www.yourdomain.com)'

LOG_LEVEL = 'INFO'

HTTPCACHE_ENABLED = True

HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.RFC2616Policy'
<|endoftext|>


<commit_before>import functools
import logging
import mimetypes
import os
import datetime
import re
from django.conf import settings
from django.contrib.sessions.middleware import SessionMiddleware
from django.core.exceptions import MiddlewareNotUsed
from django.http import HttpResponseRedirect
from django.urls import reverse
from django.contrib.auth.views import LogoutView
from django.utils.deprecation import MiddlewareMixin

from corehq.apps.domain.models import Domain
from corehq.apps.domain.utils import get_domain_from_url, legacy_domain_re
from corehq.apps.linked_domain.dbaccessors import get_domain_master_link
from corehq.const import OPENROSA_DEFAULT_VERSION
from corehq.toggles import LOGOUT_SENSITIVE_DOMAIN
from corehq.util.soft_assert import soft_assert
from dimagi.utils.logging import notify_exception
from dimagi.utils.web import get_ip

from dimagi.utils.parsing import json_format_datetime, string_to_utc_datetime

try:
    import psutil
except ImportError:
    psutil = None

# this isn't OR specific, but we like it to be included
OPENROSA_ACCEPT_LANGUAGE = "HTTP_ACCEPT_LANGUAGE"
OPENROSA_VERSION_HEADER = "HTTP_X_OPENROSA_VERSION"
OPENROSA_DATE_HEADER = "HTTP_DATE"
OPENROSA_HEADERS = [OPENROSA_VERSION_HEADER, OPENROSA_DATE_HEADER, OPENROSA_ACCEPT_LANGUAGE]

class OpenRosaMiddleware(MiddlewareMixin):
    """
    Middleware to support OpenRosa request/response standards compliance
    https://bitbucket.org/javarosa/javarosa/wiki/OpenRosaRequest
    """

    def process_request(self, request):
        # if there's a date header specified add that to the request
        # as a first class property
        or_headers = {}
        for header in OPENROSA_HEADERS:
            if header in request.META:
                or_headers[header] = request.META[header]
        request.openrosa_headers = or_headers

    def process_response(self, request, response):
        response[OPENROSA_VERSION_HEADER] = OPENROSA_DEFAULT_VERSION
        return response

profile_logger = logging.getLogger('profile_middleware')

class MemoryUsageMiddleware(object):
    """
    Stolen and modified from http://stackoverflow.com/a/12254394/8207

    This is a pretty poor, blunt tool and is not recommended to be treated as definitive truth.
    """
    _psutil_installed = None

    def _check_psutil(self):
        if self._psutil_installed is None:
            if psutil is None:
                profile_logger.warning('Install dev-requirements (psutil) in order to use MemoryUsageMiddleware')
                self._psutil_installed = False
            else:
                self._psutil_installed = True
        return self._psutil_installed

    def process_request(self, request):
        if self._check_psutil():
            request._profile_memory = psutil.Process(os.getpid()).get_memory_info()

    def process_response(self, request, response):
        if self._check_psutil() and hasattr(request, '_profile_memory'):
            mem = psutil.Process(os.getpid()).get_memory_info()
            diff = (mem.rss - request._profile_memory.rss) // 1024
            profile_logger.info('{} memory usage {} KB'.format(request.path, diff))
        return response

class TimingMiddleware(object):

    def process_request(self, request):
        request._profile_starttime = datetime.datetime.utcnow()

    def process_response(self, request, response):
        if hasattr(request, '_profile_starttime'):
            duration = datetime.datetime.utcnow() - request._profile_starttime
            profile_logger.info('{} time {}'.format(request.path, duration), extra={'duration': duration})
        return response

class LogLongRequestMiddleware(MiddlewareMixin):

    def process_request(self, request):
        request._profile_starttime = datetime.datetime.utcnow()

    def process_response(self, request, response):
        if hasattr(request, '_profile_starttime'):
            duration = datetime.datetime.utcnow() - request._profile_starttime
            if duration > datetime.timedelta(minutes=10):
                notify_exception(request, "Request took a very long time.", details={
                    'duration': duration.total_seconds(),
                })
        return response

class TimeoutMiddleware(MiddlewareMixin):

    @staticmethod
    def _session_expired(timeout, activity, time):
        if activity is None:
            return False

        time_since_activity = time - string_to_utc_datetime(activity)
        return time_since_activity > datetime.timedelta(minutes=timeout)

    @staticmethod
    def _user_requires_secure_session(couch_user):
        if not couch_user:
            return False

        domains = couch_user.get_domains()
        if any(Domain.is_secure_session_required(domain) for domain in domains):
            return True

        from corehq.apps.users.models import DomainPermissionsMirror
        for domain in domains:
            mirrors = DomainPermissionsMirror.mirror_domains(domain)
            if any(Domain.is_secure_session_required(m) for m in mirrors):
                return True

        return False

    def process_view(self, request, view_func, view_args, view_kwargs):
        if not request.user.is_authenticated:
            return

        secure_session = request.session.get('secure_session')
        domain = getattr(request, "domain", None)
        domain_obj = Domain.get_by_name(domain) if domain else None
        now = datetime.datetime.utcnow()

        # figure out if we want to switch to secure_sessions
        change_to_secure_session = (
            not secure_session
            and (
                (domain_obj and domain_obj.secure_sessions)
                or self._user_requires_secure_session(request.couch_user)))

        timeout = None
        if secure_session or change_to_secure_session:
            if domain_obj:
                timeout = domain_obj.secure_timeout
            if not timeout:
                timeout = settings.SECURE_TIMEOUT
        else:
            timeout = settings.INACTIVITY_TIMEOUT

        if change_to_secure_session:
            # force re-authentication if the user has been logged in longer than the secure timeout
            if self._session_expired(timeout, request.user.last_login, now):
                LogoutView.as_view(template_name=settings.BASE_TEMPLATE)(request)
                # this must be after logout so it is attached to the new session
                request.session['secure_session'] = True
                request.session.set_expiry(timeout * 60)
                return HttpResponseRedirect(reverse('login') + '?next=' + request.path)

            request.session['secure_session'] = True

        if not getattr(request, '_bypass_sessions', False):
            request.session.set_expiry(timeout * 60)
            request.session['last_request'] = json_format_datetime(now)

def always_allow_browser_caching(fn):
    @functools.wraps(fn)
    def inner(*args, **kwargs):
        response = fn(*args, **kwargs)
        response._always_allow_browser_caching = True
        return response
    return inner

class NoCacheMiddleware(MiddlewareMixin):

    def process_response(self, request, response):
        if not self._explicitly_marked_safe(response):
            response['Cache-Control'] = "private, no-cache, no-store, must-revalidate, proxy-revalidate"
            response['Expires'] = "Thu, 01 Dec 1994 16:00:00 GMT"
            response['Pragma'] = "no-cache"
        else:
            max_age = getattr(response, '_cache_max_age', "31536000")
            content_type, _ = mimetypes.guess_type(request.path)
            response['Cache-Control'] = "max-age={}".format(max_age)
            del response['Vary']
            del response['Set-Cookie']
            response['Content-Type'] = content_type
            del response['Content-Language']
            response['Content-Length'] = len(response.content)
            del response['HTTP_X_OPENROSA_VERSION']
        return response

    @staticmethod
    def _explicitly_marked_safe(response):
        return getattr(response, '_always_allow_browser_caching', False)

class SentryContextMiddleware(MiddlewareMixin):
    """Add details to Sentry context.
    Should be placed after 'corehq.apps.users.middleware.UsersMiddleware'
    """
    def __init__(self, get_response=None):
        super(SentryContextMiddleware, self).__init__(get_response)
        try:
            from sentry_sdk import configure_scope
        except ImportError:
            raise MiddlewareNotUsed

        if not getattr(settings, 'SENTRY_CONFIGURED', None):
            raise MiddlewareNotUsed

    def process_view(self, request, view_func, view_args, view_kwargs):
        from sentry_sdk import configure_scope

        with configure_scope() as scope:
            if getattr(request, 'couch_user', None):
                scope.set_extra('couch_user_id', request.couch_user.get_id)
                scope.set_tag('user.username', request.couch_user.username)

            if getattr(request, 'domain', None):
                scope.set_tag('domain', request.domain)

class SelectiveSessionMiddleware(SessionMiddleware):

    def __init__(self, get_response=None):
        super().__init__(get_response)
        regexes = [
            '/favicon.ico$',
            '/ping_login/$',
            '/downloads/temp/ajax/',  # soil polling
            '/downloads/temp/heartbeat/',  # soil status
            '/a/{domain}/apps/view/[A-Za-z0-9-]+/current_version/$'  # app manager new changes polling
            '/hq/notifications/service/$',  # background request for notification (bell menu in top nav)
        ]
        if settings.BYPASS_SESSIONS_FOR_MOBILE:
            regexes.extend(getattr(settings, 'SESSION_BYPASS_URLS', []))
        self.bypass_re = [
            re.compile(regex.format(domain=legacy_domain_re)) for regex in regexes
        ]

    def _bypass_sessions(self, request):
        return any(rx.match(request.path_info) for rx in self.bypass_re)

    def process_request(self, request):
        super().process_request(request)
        if self._bypass_sessions(request):
            request.session.save = lambda *x: None
            request._bypass_sessions = True

class LoggedOutDomainUserMiddleware(MiddlewareMixin):
    """Soft assert with domain and client IP details if user is logged out

    Used to get a lead on what was happening when someone tried to
    access a page and was not logged in.
    """

    def process_request(self, request):
        if request.user.is_authenticated:
            return
        domain_name = get_domain_from_url(request.path)
        if self.is_logout_sensitive_domain(domain_name):
            msg = "Unauthenticated client"
            data = {
                "domain": domain_name,
                "ip_address": get_ip(request),
                "path_info": request.path_info,
            }
            _soft_assert_is_authenticated(False, msg, data)
            notify_exception(request, msg, details=data)

    def is_logout_sensitive_domain(self, domain_name):
        if domain_name:
            if LOGOUT_SENSITIVE_DOMAIN.enabled(domain_name):
                return True
            link = get_domain_master_link(domain_name)
            if link and LOGOUT_SENSITIVE_DOMAIN.enabled(link.master_domain):
                return True
        return False

_soft_assert_is_authenticated = soft_assert(
    to=['{}@{}'.format(email, 'dimagi.com') for email in [
        'jschweers',
        'cellowitz',
        'kcowger',
        'btalbot',
    ]],
    exponential_backoff=False,
)
<commit_msg>Simplified TimeoutMiddleware<commit_after>import functools
import logging
import mimetypes
import os
import datetime
import re
from django.conf import settings
from django.contrib.sessions.middleware import SessionMiddleware
from django.core.exceptions import MiddlewareNotUsed
from django.http import HttpResponseRedirect
from django.urls import reverse
from django.contrib.auth.views import LogoutView
from django.utils.deprecation import MiddlewareMixin

from corehq.apps.domain.models import Domain
from corehq.apps.domain.utils import get_domain_from_url, legacy_domain_re
from corehq.apps.linked_domain.dbaccessors import get_domain_master_link
from corehq.const import OPENROSA_DEFAULT_VERSION
from corehq.toggles import LOGOUT_SENSITIVE_DOMAIN
from corehq.util.soft_assert import soft_assert
from dimagi.utils.logging import notify_exception
from dimagi.utils.web import get_ip

from dimagi.utils.parsing import json_format_datetime, string_to_utc_datetime

try:
    import psutil
except ImportError:
    psutil = None

# this isn't OR specific, but we like it to be included
OPENROSA_ACCEPT_LANGUAGE = "HTTP_ACCEPT_LANGUAGE"
OPENROSA_VERSION_HEADER = "HTTP_X_OPENROSA_VERSION"
OPENROSA_DATE_HEADER = "HTTP_DATE"
OPENROSA_HEADERS = [OPENROSA_VERSION_HEADER, OPENROSA_DATE_HEADER, OPENROSA_ACCEPT_LANGUAGE]

class OpenRosaMiddleware(MiddlewareMixin):
    """
    Middleware to support OpenRosa request/response standards compliance
    https://bitbucket.org/javarosa/javarosa/wiki/OpenRosaRequest
    """

    def process_request(self, request):
        # if there's a date header specified add that to the request
        # as a first class property
        or_headers = {}
        for header in OPENROSA_HEADERS:
            if header in request.META:
                or_headers[header] = request.META[header]
        request.openrosa_headers = or_headers

    def process_response(self, request, response):
        response[OPENROSA_VERSION_HEADER] = OPENROSA_DEFAULT_VERSION
        return response

profile_logger = logging.getLogger('profile_middleware')

class MemoryUsageMiddleware(object):
    """
    Stolen and modified from http://stackoverflow.com/a/12254394/8207

    This is a pretty poor, blunt tool and is not recommended to be treated as definitive truth.
    """
    _psutil_installed = None

    def _check_psutil(self):
        if self._psutil_installed is None:
            if psutil is None:
                profile_logger.warning('Install dev-requirements (psutil) in order to use MemoryUsageMiddleware')
                self._psutil_installed = False
            else:
                self._psutil_installed = True
        return self._psutil_installed

    def process_request(self, request):
        if self._check_psutil():
            request._profile_memory = psutil.Process(os.getpid()).get_memory_info()

    def process_response(self, request, response):
        if self._check_psutil() and hasattr(request, '_profile_memory'):
            mem = psutil.Process(os.getpid()).get_memory_info()
            diff = (mem.rss - request._profile_memory.rss) // 1024
            profile_logger.info('{} memory usage {} KB'.format(request.path, diff))
        return response

class TimingMiddleware(object):

    def process_request(self, request):
        request._profile_starttime = datetime.datetime.utcnow()

    def process_response(self, request, response):
        if hasattr(request, '_profile_starttime'):
            duration = datetime.datetime.utcnow() - request._profile_starttime
            profile_logger.info('{} time {}'.format(request.path, duration), extra={'duration': duration})
        return response

class LogLongRequestMiddleware(MiddlewareMixin):

    def process_request(self, request):
        request._profile_starttime = datetime.datetime.utcnow()

    def process_response(self, request, response):
        if hasattr(request, '_profile_starttime'):
            duration = datetime.datetime.utcnow() - request._profile_starttime
            if duration > datetime.timedelta(minutes=10):
                notify_exception(request, "Request took a very long time.", details={
                    'duration': duration.total_seconds(),
                })
        return response

class TimeoutMiddleware(MiddlewareMixin):

    @staticmethod
    def _session_expired(timeout, activity, time):
        if activity is None:
            return False

        time_since_activity = time - string_to_utc_datetime(activity)
        return time_since_activity > datetime.timedelta(minutes=timeout)

    @staticmethod
    def _user_requires_secure_session(couch_user):
        if not couch_user:
            return False

        domains = couch_user.get_domains()
        if any(Domain.is_secure_session_required(domain) for domain in domains):
            return True

        from corehq.apps.users.models import DomainPermissionsMirror
        for domain in domains:
            mirrors = DomainPermissionsMirror.mirror_domains(domain)
            if any(Domain.is_secure_session_required(m) for m in mirrors):
                return True

        return False

    def process_view(self, request, view_func, view_args, view_kwargs):
        if not request.user.is_authenticated:
            return

        secure_session = request.session.get('secure_session')
        domain = getattr(request, "domain", None)
        domain_obj = Domain.get_by_name(domain) if domain else None
        now = datetime.datetime.utcnow()

        # figure out if we want to switch to secure_sessions
        change_to_secure_session = (
            not secure_session
            and (
                (domain_obj and domain_obj.secure_sessions)
                or self._user_requires_secure_session(request.couch_user)))

        use_secure_timeout = secure_session or change_to_secure_session
        timeout = settings.SECURE_TIMEOUT if use_secure_timeout else settings.INACTIVITY_TIMEOUT
        if change_to_secure_session:
            # force re-authentication if the user has been logged in longer than the secure timeout
            if self._session_expired(timeout, request.user.last_login, now):
                LogoutView.as_view(template_name=settings.BASE_TEMPLATE)(request)
                # this must be after logout so it is attached to the new session
                request.session['secure_session'] = True
                request.session.set_expiry(timeout * 60)
                return HttpResponseRedirect(reverse('login') + '?next=' + request.path)

            request.session['secure_session'] = True

        if not getattr(request, '_bypass_sessions', False):
            request.session.set_expiry(timeout * 60)
            request.session['last_request'] = json_format_datetime(now)

def always_allow_browser_caching(fn):
    @functools.wraps(fn)
    def inner(*args, **kwargs):
        response = fn(*args, **kwargs)
        response._always_allow_browser_caching = True
        return response
    return inner

class NoCacheMiddleware(MiddlewareMixin):

    def process_response(self, request, response):
        if not self._explicitly_marked_safe(response):
            response['Cache-Control'] = "private, no-cache, no-store, must-revalidate, proxy-revalidate"
            response['Expires'] = "Thu, 01 Dec 1994 16:00:00 GMT"
            response['Pragma'] = "no-cache"
        else:
            max_age = getattr(response, '_cache_max_age', "31536000")
            content_type, _ = mimetypes.guess_type(request.path)
            response['Cache-Control'] = "max-age={}".format(max_age)
            del response['Vary']
            del response['Set-Cookie']
            response['Content-Type'] = content_type
            del response['Content-Language']
            response['Content-Length'] = len(response.content)
            del response['HTTP_X_OPENROSA_VERSION']
        return response

    @staticmethod
    def _explicitly_marked_safe(response):
        return getattr(response, '_always_allow_browser_caching', False)

class SentryContextMiddleware(MiddlewareMixin):
    """Add details to Sentry context.
    Should be placed after 'corehq.apps.users.middleware.UsersMiddleware'
    """
    def __init__(self, get_response=None):
        super(SentryContextMiddleware, self).__init__(get_response)
        try:
            from sentry_sdk import configure_scope
        except ImportError:
            raise MiddlewareNotUsed

        if not getattr(settings, 'SENTRY_CONFIGURED', None):
            raise MiddlewareNotUsed

    def process_view(self, request, view_func, view_args, view_kwargs):
        from sentry_sdk import configure_scope

        with configure_scope() as scope:
            if getattr(request, 'couch_user', None):
                scope.set_extra('couch_user_id', request.couch_user.get_id)
                scope.set_tag('user.username', request.couch_user.username)

            if getattr(request, 'domain', None):
                scope.set_tag('domain', request.domain)

class SelectiveSessionMiddleware(SessionMiddleware):

    def __init__(self, get_response=None):
        super().__init__(get_response)
        regexes = [
            '/favicon.ico$',
            '/ping_login/$',
            '/downloads/temp/ajax/',  # soil polling
            '/downloads/temp/heartbeat/',  # soil status
            '/a/{domain}/apps/view/[A-Za-z0-9-]+/current_version/$'  # app manager new changes polling
            '/hq/notifications/service/$',  # background request for notification (bell menu in top nav)
        ]
        if settings.BYPASS_SESSIONS_FOR_MOBILE:
            regexes.extend(getattr(settings, 'SESSION_BYPASS_URLS', []))
        self.bypass_re = [
            re.compile(regex.format(domain=legacy_domain_re)) for regex in regexes
        ]

    def _bypass_sessions(self, request):
        return any(rx.match(request.path_info) for rx in self.bypass_re)

    def process_request(self, request):
        super().process_request(request)
        if self._bypass_sessions(request):
            request.session.save = lambda *x: None
            request._bypass_sessions = True

class LoggedOutDomainUserMiddleware(MiddlewareMixin):
    """Soft assert with domain and client IP details if user is logged out

    Used to get a lead on what was happening when someone tried to
    access a page and was not logged in.
    """

    def process_request(self, request):
        if request.user.is_authenticated:
            return
        domain_name = get_domain_from_url(request.path)
        if self.is_logout_sensitive_domain(domain_name):
            msg = "Unauthenticated client"
            data = {
                "domain": domain_name,
                "ip_address": get_ip(request),
                "path_info": request.path_info,
            }
            _soft_assert_is_authenticated(False, msg, data)
            notify_exception(request, msg, details=data)

    def is_logout_sensitive_domain(self, domain_name):
        if domain_name:
            if LOGOUT_SENSITIVE_DOMAIN.enabled(domain_name):
                return True
            link = get_domain_master_link(domain_name)
            if link and LOGOUT_SENSITIVE_DOMAIN.enabled(link.master_domain):
                return True
        return False

_soft_assert_is_authenticated = soft_assert(
    to=['{}@{}'.format(email, 'dimagi.com') for email in [
        'jschweers',
        'cellowitz',
        'kcowger',
        'btalbot',
    ]],
    exponential_backoff=False,
)
<|endoftext|>