<commit_before>from vint.ast.traversing import traverse
from vint.ast.node_type import NodeType

IDENTIFIER_ATTRIBUTE = 'VINT:identifier_attribute'
IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG = 'is_definition'
IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG = 'is_dynamic'
IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG = 'is_member_of_subscript'
IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG = 'is_function'
REFERENCING_IDENTIFIERS = 'VINT:referencing_identifiers'
DECLARING_IDENTIFIERS = 'VINT:declaring_identifiers'

DeclarativeNodeTypes = {
    NodeType.LET: True,
    NodeType.FUNCTION: True,
    NodeType.FOR: True,
}

IdentifierTerminateNodeTypes = {
    NodeType.IDENTIFIER: True,
    NodeType.ENV: True,
    NodeType.REG: True,
    NodeType.OPTION: True,
    NodeType.CURLYNAME: True,
}

AccessorLikeNodeTypes = {
    NodeType.DOT: True,
    NodeType.SUBSCRIPT: True,
    NodeType.SLICE: True,
}

# These node types is identifier-like when:
#   let object['name'] = 0
AnalyzableSubScriptChildNodeTypes = {
    NodeType.NUMBER: True,
    NodeType.STRING: True,
}

class IdentifierClassifier(object):
    """ A class for identifier classifiers.
    This class classify nodes by 3 flags:

    - is dynamic: True if the identifier name can be determined by static analysis.
    - is member: True if the identifier is a member of a subscription/dot/slice node.
    - is declaring: True if the identifier is used to declare.
    """

    class IdentifierCollector(object):
        """ A class for identifier node collectors.
        Only static and not member nodes will be collected and the nodes will
        be grouped by 2 types; declaring or referencing.
        """

        def collect_identifiers(self, ast):
            self.static_referencing_identifiers = []
            self.static_declaring_identifiers = []

            # TODO: Make more performance efficiency.
            traverse(ast, on_enter=self._enter_handler)

            return {
                'static_declaring_identifiers': self.static_declaring_identifiers,
                'static_referencing_identifiers': self.static_referencing_identifiers,
            }

        def _enter_handler(self, node):
            is_identifier_like_node = IDENTIFIER_ATTRIBUTE in node

            if not is_identifier_like_node:
                return

            id_attr = node[IDENTIFIER_ATTRIBUTE]
            if id_attr[IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG] or \
                    id_attr[IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG]:
                return

            if id_attr[IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG]:
                self.static_declaring_identifiers.append(node)
            else:
                self.static_referencing_identifiers.append(node)

    def attach_identifier_attributes(self, ast):
        """ Attach 3 flags to the AST.

        - is dynamic: True if the identifier name can be determined by static analysis.
        - is member: True if the identifier is a member of a subscription/dot/slice node.
        - is declaring: True if the identifier is used to declare.
        - is function: True if the identifier is a function. Vim distinguish
                       between function identifiers and variable identifiers.
        """
        traverse(ast, on_enter=self._enter_handler)
        return ast

    def _set_identifier_attribute(self, node, is_definition=None, is_dynamic=None,
                                  is_member_of_subscript=None, is_function=None):

        id_attr = node.setdefault(IDENTIFIER_ATTRIBUTE, {
            IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG: False,
            IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG: False,
            IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG: False,
            IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG: False,
        })

        if is_definition is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG] = is_definition

        if is_dynamic is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG] = is_dynamic

        if is_member_of_subscript is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG] = is_member_of_subscript

        if is_function is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG] = is_function

    def _enter_handler(self, node):
        node_type = NodeType(node['type'])

        if node_type in IdentifierTerminateNodeTypes:
            # Attach identifier attributes to all IdentifierTerminateNodeTypes.
            self._set_identifier_attribute(node)

        if node_type in AccessorLikeNodeTypes:
            self._pre_mark_accessor_children(node)

        if node_type in DeclarativeNodeTypes:
            self._enter_declarative_node(node)

        if node_type is NodeType.CALL:
            self._enter_call_node(node)

    def _pre_mark_accessor_children(self, node):
        node_type = NodeType(node['type'])
        dict_node = node['left']

        if NodeType(dict_node['type']) in AccessorLikeNodeTypes:
            self._pre_mark_accessor_children(dict_node)

        if node_type is NodeType.SLICE:
            for member_node in node['rlist']:
                # In VimLParser spec, an empty array means null.
                #   list[1:] => {rlist: [node, []]}
                if type(member_node) is list:
                    continue

                self._pre_mark_member_node(member_node)
        else:
            member_node = node['right']
            self._pre_mark_member_node(member_node)

    def _pre_mark_member_node(self, member_node):
        member_node_type = NodeType(member_node['type'])

        if member_node_type in IdentifierTerminateNodeTypes or \
                member_node_type in AnalyzableSubScriptChildNodeTypes:
            self._set_identifier_attribute(member_node,
                                           is_member_of_subscript=True)

    def _enter_identifier_like_node(self, id_like_node, is_definition=None,
                                    is_function=None):
        id_like_node_type = NodeType(id_like_node['type'])

        if id_like_node_type in AccessorLikeNodeTypes:
            self._enter_accessor_node(id_like_node,
                                      is_definition=is_definition,
                                      is_function=is_function)
            return

        if id_like_node_type in IdentifierTerminateNodeTypes:
            self._enter_identifier_node(id_like_node,
                                        is_definition=is_definition,
                                        is_function=is_function)
            return

        # Curlyname node can have a dynamic name. For example:
        #   let s:var = 'VAR'
        #   let my_{s:var} = 0
        if id_like_node_type is NodeType.CURLYNAME:
            self._enter_curlyname_node(id_like_node,
                                       is_definition=is_definition,
                                       is_function=is_function)
            return

    def _enter_function_node(self, func_node):
        # Function node has declarative identifiers as the function name and
        # the paramerter names.

        # Function name is in the left.
        func_name_node = func_node['left']
        self._enter_identifier_like_node(func_name_node,
                                         is_definition=True,
                                         is_function=True)

        # Function parameter names is in the r_list.
        func_param_nodes = func_node['rlist']
        for func_param_node in func_param_nodes:
            self._enter_identifier_like_node(func_param_node,
                                             is_definition=True)

    def _enter_curlyname_node(self, curlyname_node, is_definition=None, is_function=None):
        self._set_identifier_attribute(curlyname_node,
                                       is_dynamic=True,
                                       is_definition=is_definition,
                                       is_function=is_function)

    def _enter_identifier_node(self, def_id_node, is_definition=None, is_function=None):
        self._set_identifier_attribute(def_id_node,
                                       is_definition=is_definition,
                                       is_function=is_function)

    def _enter_accessor_node(self, accessor_node, is_definition=None, is_function=None):
        accessor_node_type = NodeType(accessor_node['type'])

        if accessor_node_type is NodeType.DOT:
            self._set_identifier_attribute(accessor_node['right'],
                                           is_definition=is_definition,
                                           is_dynamic=False,
                                           is_function=is_function)
            return

        if accessor_node_type is NodeType.SUBSCRIPT:
            subscript_right_type = NodeType(accessor_node['right']['type'])

            # We can do static analysis NodeType.SUBSCRIPT such as:
            #   let object['name'] = 0
            #
            # but we cannot do it in other cases such as:
            #   let object[var] = 0
            is_dynamic = subscript_right_type not in AnalyzableSubScriptChildNodeTypes

            if not is_dynamic:
                self._set_identifier_attribute(accessor_node['right'],
                                               is_definition=is_definition,
                                               is_dynamic=False,
                                               is_function=is_function)
            return

        if accessor_node_type is NodeType.SLICE:
            for elem_node in accessor_node['rlist']:
                elem_node_type = NodeType(elem_node['type'])

                # We can do static analysis NodeType.SLICE such as:
                #   let object[0:1] = 0
                #
                # but we cannot do it in other cases such as:
                #   let object[0:var] = 0
                is_dynamic = elem_node_type not in AnalyzableSubScriptChildNodeTypes

                # In the following case, 0 is a definition but var is not definition.
                # It is more like a reference.
                #   let object[0:var] = 0
                is_definition = elem_node_type in AnalyzableSubScriptChildNodeTypes

                self._set_identifier_attribute(elem_node,
                                               is_definition=is_definition,
                                               is_dynamic=is_dynamic,
                                               is_function=is_function)
            return

        raise Exception()

    def _enter_let_node(self, let_node):
        self._enter_destructuring_assignment_node(let_node)

    def _enter_for_node(self, for_node):
        self._enter_destructuring_assignment_node(for_node)

    def _enter_destructuring_assignment_node(self, node):
        # In VimLParser spec, an empty array means null.
        #
        # | Normal assignment    | Destructuring assignment    |
        # |:--------------------:|:---------------------------:|
        # | node['left'] == Node | node['left'] == []          |
        # | node['list'] == []   | node['list'] == [Node, ...] |

        left_node = node['left']
        is_destructuring_assignment = type(left_node) is list

        if is_destructuring_assignment:
            for elem_node in node['list']:
                self._enter_identifier_like_node(elem_node, is_definition=True)
        else:
            self._enter_identifier_like_node(left_node, is_definition=True)

    def _enter_declarative_node(self, node):
        node_type = NodeType(node['type'])

        if node_type is NodeType.FUNCTION:
            self._enter_function_node(node)
            return
        if node_type is NodeType.LET:
            self._enter_let_node(node)
            return
        if node_type is NodeType.FOR:
            self._enter_for_node(node)
            return

    def _enter_call_node(self, call_node):
        left_node = call_node['left']
        self._enter_identifier_like_node(left_node, is_function=True)
<commit_msg>Can hide flag attachment algorithm<commit_after>from vint.ast.traversing import traverse
from vint.ast.node_type import NodeType

IDENTIFIER_ATTRIBUTE = 'VINT:identifier_attribute'
IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG = 'is_definition'
IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG = 'is_dynamic'
IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG = 'is_member_of_subscript'
IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG = 'is_function'
REFERENCING_IDENTIFIERS = 'VINT:referencing_identifiers'
DECLARING_IDENTIFIERS = 'VINT:declaring_identifiers'

DeclarativeNodeTypes = {
    NodeType.LET: True,
    NodeType.FUNCTION: True,
    NodeType.FOR: True,
}

IdentifierTerminateNodeTypes = {
    NodeType.IDENTIFIER: True,
    NodeType.ENV: True,
    NodeType.REG: True,
    NodeType.OPTION: True,
    NodeType.CURLYNAME: True,
}

AccessorLikeNodeTypes = {
    NodeType.DOT: True,
    NodeType.SUBSCRIPT: True,
    NodeType.SLICE: True,
}

# These node types is identifier-like when:
#   let object['name'] = 0
AnalyzableSubScriptChildNodeTypes = {
    NodeType.NUMBER: True,
    NodeType.STRING: True,
}

class IdentifierClassifier(object):
    """ A class for identifier classifiers.
    This class classify nodes by 3 flags:

    - is dynamic: True if the identifier name can be determined by static analysis.
    - is member: True if the identifier is a member of a subscription/dot/slice node.
    - is declaring: True if the identifier is used to declare.
    """

    class IdentifierCollector(object):
        """ A class for identifier node collectors.
        Only static and not member nodes will be collected and the nodes will
        be grouped by 2 types; declaring or referencing.
        """

        def collect_identifiers(self, ast):
            self.static_referencing_identifiers = []
            self.static_declaring_identifiers = []

            # TODO: Make more performance efficiency.
            traverse(ast, on_enter=self._enter_handler)

            return {
                'static_declaring_identifiers': self.static_declaring_identifiers,
                'static_referencing_identifiers': self.static_referencing_identifiers,
            }

        def _enter_handler(self, node):
            is_identifier_like_node = IDENTIFIER_ATTRIBUTE in node

            if not is_identifier_like_node:
                return

            id_attr = node[IDENTIFIER_ATTRIBUTE]
            if id_attr[IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG] or \
                    id_attr[IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG]:
                return

            if id_attr[IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG]:
                self.static_declaring_identifiers.append(node)
            else:
                self.static_referencing_identifiers.append(node)

    def attach_identifier_attributes(self, ast):
        """ Attach 3 flags to the AST.

        - is dynamic: True if the identifier name can be determined by static analysis.
        - is member: True if the identifier is a member of a subscription/dot/slice node.
        - is declaring: True if the identifier is used to declare.
        - is function: True if the identifier is a function. Vim distinguish
                       between function identifiers and variable identifiers.
        """
        traverse(ast, on_enter=self._enter_handler)
        return ast

    def _set_identifier_attribute(self, node, is_definition=None, is_dynamic=None,
                                  is_member_of_subscript=None, is_function=None):

        id_attr = node.setdefault(IDENTIFIER_ATTRIBUTE, {
            IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG: False,
            IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG: False,
            IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG: False,
            IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG: False,
        })

        if is_definition is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG] = is_definition

        if is_dynamic is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG] = is_dynamic

        if is_member_of_subscript is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG] = is_member_of_subscript

        if is_function is not None:
            id_attr[IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG] = is_function

    def _enter_handler(self, node):
        node_type = NodeType(node['type'])

        if node_type in IdentifierTerminateNodeTypes:
            # Attach identifier attributes to all IdentifierTerminateNodeTypes.
            self._set_identifier_attribute(node)

        if node_type in AccessorLikeNodeTypes:
            self._pre_mark_accessor_children(node)

        if node_type in DeclarativeNodeTypes:
            self._enter_declarative_node(node)

        if node_type is NodeType.CALL:
            self._enter_call_node(node)

    def _pre_mark_accessor_children(self, node):
        node_type = NodeType(node['type'])
        dict_node = node['left']

        if NodeType(dict_node['type']) in AccessorLikeNodeTypes:
            self._pre_mark_accessor_children(dict_node)

        if node_type is NodeType.SLICE:
            for member_node in node['rlist']:
                # In VimLParser spec, an empty array means null.
                #   list[1:] => {rlist: [node, []]}
                if type(member_node) is list:
                    continue

                self._pre_mark_member_node(member_node)
        else:
            member_node = node['right']
            self._pre_mark_member_node(member_node)

    def _pre_mark_member_node(self, member_node):
        member_node_type = NodeType(member_node['type'])

        if member_node_type in IdentifierTerminateNodeTypes or \
                member_node_type in AnalyzableSubScriptChildNodeTypes:
            self._set_identifier_attribute(member_node,
                                           is_member_of_subscript=True)

    def _enter_identifier_like_node(self, id_like_node, is_definition=None,
                                    is_function=None):
        id_like_node_type = NodeType(id_like_node['type'])

        if id_like_node_type in AccessorLikeNodeTypes:
            self._enter_accessor_node(id_like_node,
                                      is_definition=is_definition,
                                      is_function=is_function)
            return

        if id_like_node_type in IdentifierTerminateNodeTypes:
            self._enter_identifier_node(id_like_node,
                                        is_definition=is_definition,
                                        is_function=is_function)
            return

        # Curlyname node can have a dynamic name. For example:
        #   let s:var = 'VAR'
        #   let my_{s:var} = 0
        if id_like_node_type is NodeType.CURLYNAME:
            self._enter_curlyname_node(id_like_node,
                                       is_definition=is_definition,
                                       is_function=is_function)
            return

    def _enter_function_node(self, func_node):
        # Function node has declarative identifiers as the function name and
        # the paramerter names.

        # Function name is in the left.
        func_name_node = func_node['left']
        self._enter_identifier_like_node(func_name_node,
                                         is_definition=True,
                                         is_function=True)

        # Function parameter names is in the r_list.
        func_param_nodes = func_node['rlist']
        for func_param_node in func_param_nodes:
            self._enter_identifier_like_node(func_param_node,
                                             is_definition=True)

    def _enter_curlyname_node(self, curlyname_node, is_definition=None, is_function=None):
        self._set_identifier_attribute(curlyname_node,
                                       is_dynamic=True,
                                       is_definition=is_definition,
                                       is_function=is_function)

    def _enter_identifier_node(self, def_id_node, is_definition=None, is_function=None):
        self._set_identifier_attribute(def_id_node,
                                       is_definition=is_definition,
                                       is_function=is_function)

    def _enter_accessor_node(self, accessor_node, is_definition=None, is_function=None):
        accessor_node_type = NodeType(accessor_node['type'])

        if accessor_node_type is NodeType.DOT:
            self._set_identifier_attribute(accessor_node['right'],
                                           is_definition=is_definition,
                                           is_dynamic=False,
                                           is_function=is_function)
            return

        if accessor_node_type is NodeType.SUBSCRIPT:
            subscript_right_type = NodeType(accessor_node['right']['type'])

            # We can do static analysis NodeType.SUBSCRIPT such as:
            #   let object['name'] = 0
            #
            # but we cannot do it in other cases such as:
            #   let object[var] = 0
            is_dynamic = subscript_right_type not in AnalyzableSubScriptChildNodeTypes

            if not is_dynamic:
                self._set_identifier_attribute(accessor_node['right'],
                                               is_definition=is_definition,
                                               is_dynamic=False,
                                               is_function=is_function)
            return

        if accessor_node_type is NodeType.SLICE:
            for elem_node in accessor_node['rlist']:
                elem_node_type = NodeType(elem_node['type'])

                # We can do static analysis NodeType.SLICE such as:
                #   let object[0:1] = 0
                #
                # but we cannot do it in other cases such as:
                #   let object[0:var] = 0
                is_dynamic = elem_node_type not in AnalyzableSubScriptChildNodeTypes

                # In the following case, 0 is a definition but var is not definition.
                # It is more like a reference.
                #   let object[0:var] = 0
                is_definition = elem_node_type in AnalyzableSubScriptChildNodeTypes

                self._set_identifier_attribute(elem_node,
                                               is_definition=is_definition,
                                               is_dynamic=is_dynamic,
                                               is_function=is_function)
            return

        raise Exception()

    def _enter_let_node(self, let_node):
        self._enter_destructuring_assignment_node(let_node)

    def _enter_for_node(self, for_node):
        self._enter_destructuring_assignment_node(for_node)

    def _enter_destructuring_assignment_node(self, node):
        # In VimLParser spec, an empty array means null.
        #
        # | Normal assignment    | Destructuring assignment    |
        # |:--------------------:|:---------------------------:|
        # | node['left'] == Node | node['left'] == []          |
        # | node['list'] == []   | node['list'] == [Node, ...] |

        left_node = node['left']
        is_destructuring_assignment = type(left_node) is list

        if is_destructuring_assignment:
            for elem_node in node['list']:
                self._enter_identifier_like_node(elem_node, is_definition=True)
        else:
            self._enter_identifier_like_node(left_node, is_definition=True)

    def _enter_declarative_node(self, node):
        node_type = NodeType(node['type'])

        if node_type is NodeType.FUNCTION:
            self._enter_function_node(node)
            return
        if node_type is NodeType.LET:
            self._enter_let_node(node)
            return
        if node_type is NodeType.FOR:
            self._enter_for_node(node)
            return

    def _enter_call_node(self, call_node):
        left_node = call_node['left']
        self._enter_identifier_like_node(left_node, is_function=True)

def is_identifier_like_node(node):
    return IDENTIFIER_ATTRIBUTE in node

def is_function_identifier(node):
    if not is_identifier_like_node(node):
        return False

    return node[IDENTIFIER_ATTRIBUTE][IDENTIFIER_ATTRIBUTE_FUNCTION_FLAG]

def is_dynamic_identifier(node):
    if not is_identifier_like_node(node):
        return False

    return node[IDENTIFIER_ATTRIBUTE][IDENTIFIER_ATTRIBUTE_DYNAMIC_FLAG]

def is_declarative_identifier(node):
    if not is_identifier_like_node(node):
        return False

    return node[IDENTIFIER_ATTRIBUTE][IDENTIFIER_ATTRIBUTE_DEFINITION_FLAG]

def is_member_identifier(node):
    if not is_identifier_like_node(node):
        return False

    return node[IDENTIFIER_ATTRIBUTE][IDENTIFIER_ATTRIBUTE_SUBSCRIPT_MEMBER_FLAG]
<|endoftext|>


<commit_before>"""
XXX
"""

import io
import random

import pytest
from _pytest.fixtures import SubRequest
from PIL import Image

def _image_file(
    file_format: str,
    color_space: str,
    width: int,
    height: int,
) -> io.BytesIO:
    """
    Return an image file in the given format and color space.

    The image file is filled with randomly colored pixels.

    Args:
        file_format: See
            http://pillow.readthedocs.io/en/3.1.x/handbook/image-file-formats.html
        color_space: One of "L", "RGB", or "CMYK". "L" means greyscale.
        width: The width, in pixels of the image.
        height: The width, in pixels of the image.

    Returns:
        An image file in the given format and color space.
    """
    image_buffer = io.BytesIO()
    image = Image.new(color_space, (width, height))
    pixels = image.load()
    for i in range(height):
        for j in range(width):
            red = random.randint(0, 255)
            green = random.randint(0, 255)
            blue = random.randint(0, 255)
            if color_space != 'L':
                pixels[i, j] = (red, green, blue)
    image.save(image_buffer, file_format)
    image_buffer.seek(0)
    return image_buffer

@pytest.fixture
def png_rgb_success() -> io.BytesIO:
    """
    Return a PNG file in the RGB color space which is expected to have a
    'success' status when added to a target.
    """
    return _image_file(file_format='PNG', color_space='RGB', width=5, height=5)

@pytest.fixture
def png_rgb() -> io.BytesIO:
    """
    Return a 1x1 PNG file in the RGB color space.
    """
    return _image_file(file_format='PNG', color_space='RGB', width=1, height=1)

@pytest.fixture
def png_greyscale() -> io.BytesIO:
    """
    Return a 1x1 PNG file in the greyscale color space.
    """
    return _image_file(file_format='PNG', color_space='L', width=1, height=1)

@pytest.fixture()
def png_large(
    png_rgb: io.BytesIO,  # pylint: disable=redefined-outer-name
) -> io.BytesIO:
    """
    Return a PNG file of the maximum allowed file size.

    https://library.vuforia.com/articles/Training/Cloud-Recognition-Guide
    describes that the maximum allowed file size of an image is 2 MB.
    However, tests using this fixture demonstrate that the maximum allowed
    size is actually slightly greater than that.
    """
    png_size = len(png_rgb.getbuffer())
    max_size = 2359293
    filler_length = max_size - png_size
    filler_data = b'\x00' * int(filler_length)
    original_data = png_rgb.getvalue()
    longer_data = original_data.replace(b'IEND', filler_data + b'IEND')
    png = io.BytesIO(longer_data)
    return png

@pytest.fixture
def jpeg_cmyk() -> io.BytesIO:
    """
    Return a 1x1 JPEG file in the CMYK color space.
    """
    return _image_file(
        file_format='JPEG',
        color_space='CMYK',
        width=1,
        height=1,
    )

@pytest.fixture
def jpeg_rgb() -> io.BytesIO:
    """
    Return a 1x1 JPEG file in the RGB color space.
    """
    return _image_file(
        file_format='JPEG',
        color_space='RGB',
        width=1,
        height=1,
    )

@pytest.fixture
def tiff_rgb() -> io.BytesIO:
    """
    Return a 1x1 TIFF file in the RGB color space.

    This is given as an option which is not supported by Vuforia as Vuforia
    supports only JPEG and PNG files.
    """
    return _image_file(
        file_format='TIFF',
        color_space='RGB',
        width=1,
        height=1,
    )

@pytest.fixture(params=['png_rgb', 'jpeg_rgb', 'png_greyscale', 'png_large'])
def image_file(request: SubRequest) -> io.BytesIO:
    """
    Return an image file which is expected to work on Vuforia.

    "work" means that this will be added as a target. However, this may or may
    not result in target with a 'success' status.
    """
    file_bytes_io: io.BytesIO = request.getfixturevalue(request.param)
    return file_bytes_io

@pytest.fixture(params=['tiff_rgb', 'jpeg_cmyk'])
def bad_image_file(request: SubRequest) -> io.BytesIO:
    """
    Return an image file which is expected to work on Vuforia which is
    expected to cause a `BadImage` result when an attempt is made to add it to
    the target database.
    """
    file_bytes_io: io.BytesIO = request.getfixturevalue(request.param)
    return file_bytes_io

@pytest.fixture()
def high_quality_image() -> io.BytesIO:
    """
    Return an image file which is expected to have a 'success' status when
    added to a target and a high tracking rating.

    At the time of writing, this image gains a tracking rating of 5.
    """
    path = 'tests/mock_vws/data/high_quality_image.jpg'
    with open(path, 'rb') as high_quality_image_file:
        return io.BytesIO(high_quality_image_file.read())
<commit_msg>Add docstring<commit_after>"""
Fixtures for images.
"""

import io
import random

import pytest
from _pytest.fixtures import SubRequest
from PIL import Image

def _image_file(
    file_format: str,
    color_space: str,
    width: int,
    height: int,
) -> io.BytesIO:
    """
    Return an image file in the given format and color space.

    The image file is filled with randomly colored pixels.

    Args:
        file_format: See
            http://pillow.readthedocs.io/en/3.1.x/handbook/image-file-formats.html
        color_space: One of "L", "RGB", or "CMYK". "L" means greyscale.
        width: The width, in pixels of the image.
        height: The width, in pixels of the image.

    Returns:
        An image file in the given format and color space.
    """
    image_buffer = io.BytesIO()
    image = Image.new(color_space, (width, height))
    pixels = image.load()
    for i in range(height):
        for j in range(width):
            red = random.randint(0, 255)
            green = random.randint(0, 255)
            blue = random.randint(0, 255)
            if color_space != 'L':
                pixels[i, j] = (red, green, blue)
    image.save(image_buffer, file_format)
    image_buffer.seek(0)
    return image_buffer

@pytest.fixture
def png_rgb_success() -> io.BytesIO:
    """
    Return a PNG file in the RGB color space which is expected to have a
    'success' status when added to a target.
    """
    return _image_file(file_format='PNG', color_space='RGB', width=5, height=5)

@pytest.fixture
def png_rgb() -> io.BytesIO:
    """
    Return a 1x1 PNG file in the RGB color space.
    """
    return _image_file(file_format='PNG', color_space='RGB', width=1, height=1)

@pytest.fixture
def png_greyscale() -> io.BytesIO:
    """
    Return a 1x1 PNG file in the greyscale color space.
    """
    return _image_file(file_format='PNG', color_space='L', width=1, height=1)

@pytest.fixture()
def png_large(
    png_rgb: io.BytesIO,  # pylint: disable=redefined-outer-name
) -> io.BytesIO:
    """
    Return a PNG file of the maximum allowed file size.

    https://library.vuforia.com/articles/Training/Cloud-Recognition-Guide
    describes that the maximum allowed file size of an image is 2 MB.
    However, tests using this fixture demonstrate that the maximum allowed
    size is actually slightly greater than that.
    """
    png_size = len(png_rgb.getbuffer())
    max_size = 2359293
    filler_length = max_size - png_size
    filler_data = b'\x00' * int(filler_length)
    original_data = png_rgb.getvalue()
    longer_data = original_data.replace(b'IEND', filler_data + b'IEND')
    png = io.BytesIO(longer_data)
    return png

@pytest.fixture
def jpeg_cmyk() -> io.BytesIO:
    """
    Return a 1x1 JPEG file in the CMYK color space.
    """
    return _image_file(
        file_format='JPEG',
        color_space='CMYK',
        width=1,
        height=1,
    )

@pytest.fixture
def jpeg_rgb() -> io.BytesIO:
    """
    Return a 1x1 JPEG file in the RGB color space.
    """
    return _image_file(
        file_format='JPEG',
        color_space='RGB',
        width=1,
        height=1,
    )

@pytest.fixture
def tiff_rgb() -> io.BytesIO:
    """
    Return a 1x1 TIFF file in the RGB color space.

    This is given as an option which is not supported by Vuforia as Vuforia
    supports only JPEG and PNG files.
    """
    return _image_file(
        file_format='TIFF',
        color_space='RGB',
        width=1,
        height=1,
    )

@pytest.fixture(params=['png_rgb', 'jpeg_rgb', 'png_greyscale', 'png_large'])
def image_file(request: SubRequest) -> io.BytesIO:
    """
    Return an image file which is expected to work on Vuforia.

    "work" means that this will be added as a target. However, this may or may
    not result in target with a 'success' status.
    """
    file_bytes_io: io.BytesIO = request.getfixturevalue(request.param)
    return file_bytes_io

@pytest.fixture(params=['tiff_rgb', 'jpeg_cmyk'])
def bad_image_file(request: SubRequest) -> io.BytesIO:
    """
    Return an image file which is expected to work on Vuforia which is
    expected to cause a `BadImage` result when an attempt is made to add it to
    the target database.
    """
    file_bytes_io: io.BytesIO = request.getfixturevalue(request.param)
    return file_bytes_io

@pytest.fixture()
def high_quality_image() -> io.BytesIO:
    """
    Return an image file which is expected to have a 'success' status when
    added to a target and a high tracking rating.

    At the time of writing, this image gains a tracking rating of 5.
    """
    path = 'tests/mock_vws/data/high_quality_image.jpg'
    with open(path, 'rb') as high_quality_image_file:
        return io.BytesIO(high_quality_image_file.read())
<|endoftext|>


<commit_before>import json
import os
import pprint
import subprocess
from copy import copy
from datetime import datetime, timezone
from pathlib import Path

import git
from github import Github

# The owner and repository name. For example, octocat/Hello-World.
GITHUB_REPOSITORY = os.getenv('GITHUB_REPOSITORY')

GITHUB_TOKEN = os.getenv('GITHUB_TOKEN')
PRODUCTION = os.getenv('PRODUCTION', False)

CURRENT_FILE = Path(__file__)
ROOT = CURRENT_FILE.parents[1]
CHANGELOG_PATH = ROOT / 'CHANGELOG.md'
LIBRARY_FILE_PATH = ROOT / 'src/Curl/Curl.php'

# TODO: Adjust number of recent pull requests to include likely number of
# pull requests since the last release.
RECENT_PULL_REQUEST_LIMIT = 10

def main():
    # Find most recent tag and timestamp.
    #   git for-each-ref --format="%(refname:short) | %(creatordate)" "refs/tags/*"
    local_repo = git.Repo(ROOT)

    # Fetch tags since `git fetch' is run with --no-tags during actions/checkout.
    #   git fetch --tags
    for remote in local_repo.remotes:
        remote.fetch('--tags')

    # Sort the tags by version.
    #   git tag --list | sort --reverse --version-sort
    tags = sorted(
        local_repo.tags,
        key=lambda tag: list(map(int, tag.name.split('.'))), reverse=True)

    most_recent_tag = tags[0]
    print('most_recent_tag: {}'.format(most_recent_tag))
    most_recent_tag_datetime = most_recent_tag.commit.committed_datetime
    print('most_recent_tag_datetime: {}'.format(most_recent_tag_datetime))

    # Find merged pull requests since the most recent tag.
    github_repo = Github(login_or_token=GITHUB_TOKEN).get_repo(GITHUB_REPOSITORY)
    recent_pulls = github_repo.get_pulls(
        state='closed',
        sort='updated',
        direction='desc',
    )[:RECENT_PULL_REQUEST_LIMIT]

    pull_request_changes = []

    # Group pull requests by semantic version change type.
    pull_request_by_type = {
        'major': [],
        'minor': [],
        'patch': [],
        'unspecified': [],
    }

    # Track if any pull request is missing a semantic version change type.
    pulls_missing_semver_label = []

    for pull in recent_pulls:
        # print('-' * 10)

        if not pull.merged:
            # print('skipping since not merged: {}'.format(pull.title))
            # print(pull.html_url)
            continue

        # Make merged_at timestamp offset-aware. Without this, the following
        # error will appear:
        #   TypeError: can't compare offset-naive and offset-aware datetimes
        pull_merged_at = copy(pull.merged_at).replace(tzinfo=timezone.utc)

        if pull_merged_at < most_recent_tag_datetime:
            # print('skipping since merged prior to last release: {}'.format(pull.title))
            # print(pull.html_url)
            continue

        pull_labels = {
            label.name for label in pull.labels
        }
        if 'major-incompatible-changes' in pull_labels:
            group_name = 'major'
        elif 'minor-backwards-compatible-added-functionality' in pull_labels:
            group_name = 'minor'
        elif 'patch-backwards-compatible-bug-fixes' in pull_labels:
            group_name = 'patch'
        else:
            group_name = 'unspecified'
            pulls_missing_semver_label.append(pull)
        pull_request_by_type[group_name].append(pull)

        # pprint.pprint(pull.title)
        # pprint.pprint('most recent: {}'.format(most_recent_tag_datetime))
        # pprint.pprint('merged at:   {}'.format(pull.merged_at))
        # print(pull.html_url)

        pull_request_changes.append(
            '- {} ([#{}]({}))'.format(pull.title, pull.number, pull.html_url)
        )

        # print('-' * 10)

    # pprint.pprint(pull_request_changes)

    # pprint.pprint(pull_request_by_type)
    highest_semantic_version = None
    php_file_path = ''
    if pull_request_by_type.get('major'):
        highest_semantic_version = 'major'
        php_file_path = 'scripts/bump_major_version.php'
    elif pull_request_by_type.get('minor'):
        highest_semantic_version = 'minor'
        php_file_path = 'scripts/bump_minor_version.php'
    elif pull_request_by_type.get('patch'):
        highest_semantic_version = 'patch'
        php_file_path = 'scripts/bump_patch_version.php'
    print('highest_semantic_version: {}'.format(highest_semantic_version))

    # Bump version and get next semantic version.
    command = ['php', php_file_path]
    print('running command: {}'.format(command))
    proc = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
    stdout, stderr = proc.communicate()
    print('stdout: {}'.format(stdout))
    print('stderr: {}'.format(stderr))
    result = json.loads(stdout)
    pprint.pprint(result)

    release_version = result['new_version']
    release_date = datetime.today().strftime('%Y-%m-%d')
    release_title = '{} - {}'.format(release_version, release_date)
    print('release_title: {}'.format(release_title))

    release_content = (
        '## {}\n'
        '\n'
        '{}'
    ).format(release_title, '\n'.join(pull_request_changes))

    old_content = CHANGELOG_PATH.read_text()
    new_content = old_content.replace(
        '<!-- CHANGELOG_PLACEHOLDER -->',
        '<!-- CHANGELOG_PLACEHOLDER -->\n\n{}'.format(release_content),
    )
    # print(new_content[:800])
    CHANGELOG_PATH.write_text(new_content)

    # Raise error if any pull request is missing a semantic version change type.
    if pulls_missing_semver_label:
        error_message = (
            'Merged pull request(s) found without semantic version label:\n'
            '{}'.format('\n'.join(
                '  {}'.format(pull.html_url)
                for pull in pulls_missing_semver_label)))
        raise Exception(error_message)

    # print('before:')
    # print(local_repo.git.status())

    local_repo.git.add(CHANGELOG_PATH)
    local_repo.git.add(LIBRARY_FILE_PATH)

    # print('after:')
    # print(local_repo.git.status())

    print('diff:')
    print(local_repo.git.diff(cached=True, color='always'))

if __name__ == '__main__':
    main()
<commit_msg>Commit changes<commit_after>import json
import os
import pprint
import subprocess
from copy import copy
from datetime import datetime, timezone
from pathlib import Path

import git
from github import Github

# The owner and repository name. For example, octocat/Hello-World.
GITHUB_REPOSITORY = os.getenv('GITHUB_REPOSITORY')

GITHUB_TOKEN = os.getenv('GITHUB_TOKEN')
PRODUCTION = os.getenv('PRODUCTION', False)

CURRENT_FILE = Path(__file__)
ROOT = CURRENT_FILE.parents[1]
CHANGELOG_PATH = ROOT / 'CHANGELOG.md'
LIBRARY_FILE_PATH = ROOT / 'src/Curl/Curl.php'

# TODO: Adjust number of recent pull requests to include likely number of
# pull requests since the last release.
RECENT_PULL_REQUEST_LIMIT = 10

def main():
    # Find most recent tag and timestamp.
    #   git for-each-ref --format="%(refname:short) | %(creatordate)" "refs/tags/*"
    local_repo = git.Repo(ROOT)

    # Fetch tags since `git fetch' is run with --no-tags during actions/checkout.
    #   git fetch --tags
    for remote in local_repo.remotes:
        remote.fetch('--tags')

    # Sort the tags by version.
    #   git tag --list | sort --reverse --version-sort
    tags = sorted(
        local_repo.tags,
        key=lambda tag: list(map(int, tag.name.split('.'))), reverse=True)

    most_recent_tag = tags[0]
    print('most_recent_tag: {}'.format(most_recent_tag))
    most_recent_tag_datetime = most_recent_tag.commit.committed_datetime
    print('most_recent_tag_datetime: {}'.format(most_recent_tag_datetime))

    # Find merged pull requests since the most recent tag.
    github_repo = Github(login_or_token=GITHUB_TOKEN).get_repo(GITHUB_REPOSITORY)
    recent_pulls = github_repo.get_pulls(
        state='closed',
        sort='updated',
        direction='desc',
    )[:RECENT_PULL_REQUEST_LIMIT]

    pull_request_changes = []

    # Group pull requests by semantic version change type.
    pull_request_by_type = {
        'major': [],
        'minor': [],
        'patch': [],
        'unspecified': [],
    }

    # Track if any pull request is missing a semantic version change type.
    pulls_missing_semver_label = []

    for pull in recent_pulls:
        # print('-' * 10)

        if not pull.merged:
            # print('skipping since not merged: {}'.format(pull.title))
            # print(pull.html_url)
            continue

        # Make merged_at timestamp offset-aware. Without this, the following
        # error will appear:
        #   TypeError: can't compare offset-naive and offset-aware datetimes
        pull_merged_at = copy(pull.merged_at).replace(tzinfo=timezone.utc)

        if pull_merged_at < most_recent_tag_datetime:
            # print('skipping since merged prior to last release: {}'.format(pull.title))
            # print(pull.html_url)
            continue

        pull_labels = {
            label.name for label in pull.labels
        }
        if 'major-incompatible-changes' in pull_labels:
            group_name = 'major'
        elif 'minor-backwards-compatible-added-functionality' in pull_labels:
            group_name = 'minor'
        elif 'patch-backwards-compatible-bug-fixes' in pull_labels:
            group_name = 'patch'
        else:
            group_name = 'unspecified'
            pulls_missing_semver_label.append(pull)
        pull_request_by_type[group_name].append(pull)

        # pprint.pprint(pull.title)
        # pprint.pprint('most recent: {}'.format(most_recent_tag_datetime))
        # pprint.pprint('merged at:   {}'.format(pull.merged_at))
        # print(pull.html_url)

        pull_request_changes.append(
            '- {} ([#{}]({}))'.format(pull.title, pull.number, pull.html_url)
        )

        # print('-' * 10)

    # pprint.pprint(pull_request_changes)

    # pprint.pprint(pull_request_by_type)
    highest_semantic_version = None
    php_file_path = ''
    if pull_request_by_type.get('major'):
        highest_semantic_version = 'major'
        php_file_path = 'scripts/bump_major_version.php'
    elif pull_request_by_type.get('minor'):
        highest_semantic_version = 'minor'
        php_file_path = 'scripts/bump_minor_version.php'
    elif pull_request_by_type.get('patch'):
        highest_semantic_version = 'patch'
        php_file_path = 'scripts/bump_patch_version.php'
    print('highest_semantic_version: {}'.format(highest_semantic_version))

    # Bump version and get next semantic version.
    command = ['php', php_file_path]
    print('running command: {}'.format(command))
    proc = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
    stdout, stderr = proc.communicate()
    print('stdout: {}'.format(stdout))
    print('stderr: {}'.format(stderr))
    result = json.loads(stdout)
    pprint.pprint(result)

    release_version = result['new_version']
    release_date = datetime.today().strftime('%Y-%m-%d')
    release_title = '{} - {}'.format(release_version, release_date)
    print('release_title: {}'.format(release_title))

    release_content = (
        '## {}\n'
        '\n'
        '{}'
    ).format(release_title, '\n'.join(pull_request_changes))

    old_content = CHANGELOG_PATH.read_text()
    new_content = old_content.replace(
        '<!-- CHANGELOG_PLACEHOLDER -->',
        '<!-- CHANGELOG_PLACEHOLDER -->\n\n{}'.format(release_content),
    )
    # print(new_content[:800])
    CHANGELOG_PATH.write_text(new_content)

    # Raise error if any pull request is missing a semantic version change type.
    if pulls_missing_semver_label:
        error_message = (
            'Merged pull request(s) found without semantic version label:\n'
            '{}'.format('\n'.join(
                '  {}'.format(pull.html_url)
                for pull in pulls_missing_semver_label)))
        raise Exception(error_message)

    # print('before:')
    # print(local_repo.git.status())

    local_repo.git.add(CHANGELOG_PATH)
    local_repo.git.add(LIBRARY_FILE_PATH)

    # print('after:')
    # print(local_repo.git.status())

    print('diff:')
    print(local_repo.git.diff(cached=True, color='always'))

    local_repo.git.commit(
        message=result['message'],
        author='{} <{}>'.format(
            local_repo.git.config('--get', 'user.name'),
            local_repo.git.config('--get', 'user.email')))

    print('diff after commit:')
    print(local_repo.git.diff(color='always'))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

import re
import logging
import datetime
import dateutil.rrule

# Django
from django.db import models
from django.db.models.query import QuerySet
from django.utils.timezone import now, make_aware, get_default_timezone
from django.utils.translation import ugettext_lazy as _

# AWX
from awx.api.versioning import reverse
from awx.main.models.base import * # noqa
from awx.main.models.jobs import LaunchTimeConfig
from awx.main.utils import ignore_inventory_computed_fields
from awx.main.consumers import emit_channel_notification

logger = logging.getLogger('awx.main.models.schedule')

__all__ = ['Schedule']

class ScheduleFilterMethods(object):

    def enabled(self, enabled=True):
        return self.filter(enabled=enabled)

    def before(self, dt):
        return self.filter(next_run__lt=dt)

    def after(self, dt):
        return self.filter(next_run__gt=dt)

    def between(self, begin, end):
        return self.after(begin).before(end)

class ScheduleQuerySet(ScheduleFilterMethods, QuerySet):
    pass

class ScheduleManager(ScheduleFilterMethods, models.Manager):

    use_for_related_objects = True

    def get_queryset(self):
        return ScheduleQuerySet(self.model, using=self._db)

class Schedule(CommonModel, LaunchTimeConfig):

    class Meta:
        app_label = 'main'
        ordering = ['-next_run']

    objects = ScheduleManager()

    unified_job_template = models.ForeignKey(
        'UnifiedJobTemplate',
        related_name='schedules',
        on_delete=models.CASCADE,
    )
    enabled = models.BooleanField(
        default=True,
        help_text=_("Enables processing of this schedule.")
    )
    dtstart = models.DateTimeField(
        null=True,
        default=None,
        editable=False,
        help_text=_("The first occurrence of the schedule occurs on or after this time.")
    )
    dtend = models.DateTimeField(
        null=True,
        default=None,
        editable=False,
        help_text=_("The last occurrence of the schedule occurs before this time, aftewards the schedule expires.")
    )
    rrule = models.CharField(
        max_length=255,
        help_text=_("A value representing the schedules iCal recurrence rule.")
    )
    next_run = models.DateTimeField(
        null=True,
        default=None,
        editable=False,
        help_text=_("The next time that the scheduled action will run.")
    )

    # extra_data is actually a string with a JSON payload in it. This
    # is technically OK because a string is a valid JSON. One day we will
    # enforce non-string JSON.
    def _clean_extra_data_system_jobs(self):
        extra_data = self.extra_data
        if not isinstance(extra_data, dict):
            try:
                extra_data = json.loads(self.extra_data)
            except Exception:
                raise ValidationError(_("Expected JSON"))

        if extra_data and 'days' in extra_data:
            try:
                if type(extra_data['days']) is bool:
                    raise ValueError
                if float(extra_data['days']) != int(extra_data['days']):
                    raise ValueError
                days = int(extra_data['days'])
                if days < 0:
                    raise ValueError
            except ValueError:
                raise ValidationError(_("days must be a positive integer."))
        return self.extra_data

    def clean_extra_data(self):
        if not self.unified_job_template:
            return self.extra_data

        # Compare class by string name because it's hard to import SystemJobTemplate
        if type(self.unified_job_template).__name__ is not 'SystemJobTemplate':
            return self.extra_data

        return self._clean_extra_data_system_jobs()

    def __unicode__(self):
        return u'%s_t%s_%s_%s' % (self.name, self.unified_job_template.id, self.id, self.next_run)

    def get_absolute_url(self, request=None):
        return reverse('api:schedule_detail', kwargs={'pk': self.pk}, request=request)

    def get_job_kwargs(self):
        config_data = self.prompts_dict()
        job_kwargs, rejected, errors = self.unified_job_template._accept_or_ignore_job_kwargs(**config_data)
        if errors:
            logger.info('Errors creating scheduled job: {}'.format(errors))
        job_kwargs['_eager_fields'] = {'launch_type': 'scheduled', 'schedule': self}
        return job_kwargs

    def update_computed_fields(self):
        future_rs = dateutil.rrule.rrulestr(self.rrule, forceset=True)
        next_run_actual = future_rs.after(now())

        self.next_run = next_run_actual
        try:
            self.dtstart = future_rs[0]
        except IndexError:
            self.dtstart = None
        self.dtend = None
        if 'until' in self.rrule.lower():
            match_until = re.match(".*?(UNTIL\=[0-9]+T[0-9]+Z)", self.rrule)
            until_date = match_until.groups()[0].split("=")[1]
            self.dtend = make_aware(datetime.datetime.strptime(until_date, "%Y%m%dT%H%M%SZ"), get_default_timezone())
        if 'count' in self.rrule.lower():
            self.dtend = future_rs[-1]
        emit_channel_notification('schedules-changed', dict(id=self.id, group_name='schedules'))
        with ignore_inventory_computed_fields():
            self.unified_job_template.update_computed_fields()

    def save(self, *args, **kwargs):
        self.update_computed_fields()
        super(Schedule, self).save(*args, **kwargs)
<commit_msg>fix bug when creating system job schedule<commit_after># Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

import re
import logging
import datetime
import dateutil.rrule

# Django
from django.db import models
from django.db.models.query import QuerySet
from django.utils.timezone import now, make_aware, get_default_timezone
from django.utils.translation import ugettext_lazy as _

# AWX
from awx.api.versioning import reverse
from awx.main.models.base import * # noqa
from awx.main.models.jobs import LaunchTimeConfig
from awx.main.utils import ignore_inventory_computed_fields
from awx.main.consumers import emit_channel_notification

logger = logging.getLogger('awx.main.models.schedule')

__all__ = ['Schedule']

class ScheduleFilterMethods(object):

    def enabled(self, enabled=True):
        return self.filter(enabled=enabled)

    def before(self, dt):
        return self.filter(next_run__lt=dt)

    def after(self, dt):
        return self.filter(next_run__gt=dt)

    def between(self, begin, end):
        return self.after(begin).before(end)

class ScheduleQuerySet(ScheduleFilterMethods, QuerySet):
    pass

class ScheduleManager(ScheduleFilterMethods, models.Manager):

    use_for_related_objects = True

    def get_queryset(self):
        return ScheduleQuerySet(self.model, using=self._db)

class Schedule(CommonModel, LaunchTimeConfig):

    class Meta:
        app_label = 'main'
        ordering = ['-next_run']

    objects = ScheduleManager()

    unified_job_template = models.ForeignKey(
        'UnifiedJobTemplate',
        related_name='schedules',
        on_delete=models.CASCADE,
    )
    enabled = models.BooleanField(
        default=True,
        help_text=_("Enables processing of this schedule.")
    )
    dtstart = models.DateTimeField(
        null=True,
        default=None,
        editable=False,
        help_text=_("The first occurrence of the schedule occurs on or after this time.")
    )
    dtend = models.DateTimeField(
        null=True,
        default=None,
        editable=False,
        help_text=_("The last occurrence of the schedule occurs before this time, aftewards the schedule expires.")
    )
    rrule = models.CharField(
        max_length=255,
        help_text=_("A value representing the schedules iCal recurrence rule.")
    )
    next_run = models.DateTimeField(
        null=True,
        default=None,
        editable=False,
        help_text=_("The next time that the scheduled action will run.")
    )

    def __unicode__(self):
        return u'%s_t%s_%s_%s' % (self.name, self.unified_job_template.id, self.id, self.next_run)

    def get_absolute_url(self, request=None):
        return reverse('api:schedule_detail', kwargs={'pk': self.pk}, request=request)

    def get_job_kwargs(self):
        config_data = self.prompts_dict()
        job_kwargs, rejected, errors = self.unified_job_template._accept_or_ignore_job_kwargs(**config_data)
        if errors:
            logger.info('Errors creating scheduled job: {}'.format(errors))
        job_kwargs['_eager_fields'] = {'launch_type': 'scheduled', 'schedule': self}
        return job_kwargs

    def update_computed_fields(self):
        future_rs = dateutil.rrule.rrulestr(self.rrule, forceset=True)
        next_run_actual = future_rs.after(now())

        self.next_run = next_run_actual
        try:
            self.dtstart = future_rs[0]
        except IndexError:
            self.dtstart = None
        self.dtend = None
        if 'until' in self.rrule.lower():
            match_until = re.match(".*?(UNTIL\=[0-9]+T[0-9]+Z)", self.rrule)
            until_date = match_until.groups()[0].split("=")[1]
            self.dtend = make_aware(datetime.datetime.strptime(until_date, "%Y%m%dT%H%M%SZ"), get_default_timezone())
        if 'count' in self.rrule.lower():
            self.dtend = future_rs[-1]
        emit_channel_notification('schedules-changed', dict(id=self.id, group_name='schedules'))
        with ignore_inventory_computed_fields():
            self.unified_job_template.update_computed_fields()

    def save(self, *args, **kwargs):
        self.update_computed_fields()
        super(Schedule, self).save(*args, **kwargs)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: UTF-8 -*-
from __future__ import print_function
from niprov.exceptions import UnknownFileError
from niprov.dependencies import Dependencies

class Commandline(object):

    vlevels = ['debug','info','warning','error']

    def __init__(self, dependencies=Dependencies()):
        self.config = dependencies.config
        self.verbosity = dependencies.config.verbosity
        assert self.verbosity in self.vlevels, "Unknown verbosity value"

    def fileAdded(self, image):
        if image.status == 'new':
            self.log('info', 'New file: {0}'.format(image.path))
        if image.status == 'series-new-file':
            template = 'Added {0} file to series: {1}'
            nfiles = len(image.provenance['filesInSeries'])
            self.log('info', template.format(ordinal(nfiles), image.getSeriesId()))
        if image.status == 'new-version':
            template = 'Added {0} file to series: {1}'
            nfiles = len(image.provenance['filesInSeries'])
            self.log('info', 'Added new version for: {}'.format(image.path))

    def missingDependencyForImage(self, lib, fpath):
        template = 'Missing python package "{0}" to read file: {1}'
        self.log('warning', template.format(lib, fpath))

    def fileError(self, fpath):
        import traceback
        traceback.print_exc()
        self.log('warning', 'Error inspecting file: {0}'.format(fpath))

    def interpretedRecording(self, new, transform, parents):
        template = ('[provenance] Recorded the command [{1}] to create [{0}] '+
            'based on [{2}]')
        self.log('info', template.format(', '.join(new), transform,
            ', '.join(parents)))

    def renamedDicom(self, fpath):
        self.log('info', 'Renamed dicom file: '+fpath)

    def discoveryFinished(self, nnew, nadded, nfailed, ntotal):
        self.log('info', 'Discovered {0} new, added {1} to series, failed to read {2}, '
           'processed {3} total files.'.format(nnew, nadded, nfailed, ntotal))

    def mnefunEventReceived(self, operationName):
        self.log('info', 'Mnefun operation: '+operationName)

    def receivedBashCommand(self, command):
        self.log('info', 'Recording command: \n'+(' '.join(command)))

    def filesMarkedForApproval(self, images):
        paths = '\n'.join([img.path for img in images])
        self.log('info', 'Files marked for approval: \n{0}'.format(paths))

    def exportedToFile(self, fname):
        self.log('info', 'Exported to file: {0}'.format(fname))

    def log(self, level, message, exceptionClass=None):
        if self.vlevels.index(level) >= self.vlevels.index(self.verbosity):
            if level == 'error':
                if exceptionClass is None:
                    raise NiprovError(message)
                else:
                    raise exceptionClass(message)
            else:
                print('[provenance:{0}] {1}'.format(level, message))

    def addUnknownParent(self, fpath):
        self.log('warning', '{0} unknown. Adding to provenance'.format(fpath))

SUFFIXES = {1: 'st', 2: 'nd', 3: 'rd'}
def ordinal(num):
    if 10 <= num % 100 <= 20:
        suffix = 'th'
    else:
        # the second parameter is a default.
        suffix = SUFFIXES.get(num % 10, 'th')
    return str(num) + suffix

<commit_msg>Fixed bug in Commandline.fileAdded()<commit_after>#!/usr/bin/python
# -*- coding: UTF-8 -*-
from __future__ import print_function
from niprov.exceptions import UnknownFileError
from niprov.dependencies import Dependencies

class Commandline(object):

    vlevels = ['debug','info','warning','error']

    def __init__(self, dependencies=Dependencies()):
        self.config = dependencies.config
        self.verbosity = dependencies.config.verbosity
        assert self.verbosity in self.vlevels, "Unknown verbosity value"

    def fileAdded(self, image):
        if image.status == 'new':
            self.log('info', 'New file: {0}'.format(image.path))
        if image.status == 'series-new-file':
            template = 'Added {0} file to series: {1}'
            nfiles = len(image.provenance['filesInSeries'])
            self.log('info', template.format(ordinal(nfiles), image.getSeriesId()))
        if image.status == 'new-version':
            self.log('info', 'Added new version for: {}'.format(image.path))

    def missingDependencyForImage(self, lib, fpath):
        template = 'Missing python package "{0}" to read file: {1}'
        self.log('warning', template.format(lib, fpath))

    def fileError(self, fpath):
        import traceback
        traceback.print_exc()
        self.log('warning', 'Error inspecting file: {0}'.format(fpath))

    def interpretedRecording(self, new, transform, parents):
        template = ('[provenance] Recorded the command [{1}] to create [{0}] '+
            'based on [{2}]')
        self.log('info', template.format(', '.join(new), transform,
            ', '.join(parents)))

    def renamedDicom(self, fpath):
        self.log('info', 'Renamed dicom file: '+fpath)

    def discoveryFinished(self, nnew, nadded, nfailed, ntotal):
        self.log('info', 'Discovered {0} new, added {1} to series, failed to read {2}, '
           'processed {3} total files.'.format(nnew, nadded, nfailed, ntotal))

    def mnefunEventReceived(self, operationName):
        self.log('info', 'Mnefun operation: '+operationName)

    def receivedBashCommand(self, command):
        self.log('info', 'Recording command: \n'+(' '.join(command)))

    def filesMarkedForApproval(self, images):
        paths = '\n'.join([img.path for img in images])
        self.log('info', 'Files marked for approval: \n{0}'.format(paths))

    def exportedToFile(self, fname):
        self.log('info', 'Exported to file: {0}'.format(fname))

    def log(self, level, message, exceptionClass=None):
        if self.vlevels.index(level) >= self.vlevels.index(self.verbosity):
            if level == 'error':
                if exceptionClass is None:
                    raise NiprovError(message)
                else:
                    raise exceptionClass(message)
            else:
                print('[provenance:{0}] {1}'.format(level, message))

    def addUnknownParent(self, fpath):
        self.log('warning', '{0} unknown. Adding to provenance'.format(fpath))

SUFFIXES = {1: 'st', 2: 'nd', 3: 'rd'}
def ordinal(num):
    if 10 <= num % 100 <= 20:
        suffix = 'th'
    else:
        # the second parameter is a default.
        suffix = SUFFIXES.get(num % 10, 'th')
    return str(num) + suffix

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Wrapper to run myapp from the command line.

:copyright: Copyright (c) 2017 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkio
from pykern.pkdebug import pkdp, pkdc
from sirepo import simulation_db
from sirepo.template import template_common
import sirepo.template.myapp as template

_SCHEMA = simulation_db.get_schema(template.SIM_TYPE)

def run(cfg_dir):
    with pkio.save_chdir(cfg_dir):
        exec(pkio.read_text(template_common.PARAMETERS_PYTHON_FILE), locals(), locals())
    data = simulation_db.read_json(template_common.INPUT_BASE_NAME)
    if data['report'] == 'dogReport':
        dog = data.models.dog
        res = {
            'title': 'Dog Height/Weight',
            'x_range': [0, 1],
            'y_label': 'y label',
            'x_label': 'x label',
            'x_points': [0, 1],
            'points': [
                [dog.height, dog.height],
                [dog.weight, dog.weight],
            ],
            'y_range': [0, max(dog.height, dog.weight)],
            'y1_title': _SCHEMA['model']['dog']['height'][0],
            'y2_title': _SCHEMA['model']['dog']['weight'][0],
        }
    else:
        raise RuntimeError('unknown report: {}'.format(data['report']))
    simulation_db.write_result(res)
<commit_msg>fix #1291 previously merged hello-world branch. Made height/weight graph more interesting. Updated wiki<commit_after># -*- coding: utf-8 -*-
"""Wrapper to run myapp from the command line.

:copyright: Copyright (c) 2017 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkio
from pykern.pkdebug import pkdp, pkdc
from sirepo import simulation_db
from sirepo.template import template_common
import numpy as np
import random
import sirepo.template.myapp as template

_SCHEMA = simulation_db.get_schema(template.SIM_TYPE)

_MAX_AGE_BY_WEIGHT = [
    [0, 16],
    [20, 13],
    [50, 11],
    [90, 10],
]

def run(cfg_dir):
    with pkio.save_chdir(cfg_dir):
        exec(pkio.read_text(template_common.PARAMETERS_PYTHON_FILE), locals(), locals())
    data = simulation_db.read_json(template_common.INPUT_BASE_NAME)
    if data['report'] == 'dogReport':
        dog = data.models.dog
        max_age = _max_age(dog.weight)
        x = np.linspace(0, max_age, int(max_age) + 1).tolist()
        plots = [
            _plot(dog, 'height', x),
            _plot(dog, 'weight', x),
        ]
        res = {
            'title': 'Dog Height and Weight Over Time',
            'x_range': [0, max_age],
            'y_label': '',
            'x_label': 'Age (years)',
            'x_points': x,
            'plots': plots,
            'y_range': template_common.compute_plot_color_and_range(plots),
        }
    else:
        raise RuntimeError('unknown report: {}'.format(data['report']))
    simulation_db.write_result(res)

def _max_age(weight):
    prev = None
    for bracket in _MAX_AGE_BY_WEIGHT:
        if weight <= bracket[0]:
            break;
        prev = bracket[1]
    return prev

def _plot(dog, field, x):
    return {
        'name': field,
        'label': _SCHEMA.model.dog[field][0],
        'points': _points(dog[field], x),
    }

def _points(max_value, x):
    return map(lambda v: (random.random() * 0.5) * max_value / 20.0 + max_value * 19.0 / 20.0 * (1.0 - 1.0 / (1.0 + v ** 2)), x)
<|endoftext|>


<commit_before>import csv
import sys
import time

from django.contrib.auth.models import User
from django.db.models import CharField, F
from django.db.models.functions import Cast

USER_COLS = [
    'id',
    'username',
    'is_superuser',
    'is_staff',
    'date_joined_str',
    'last_login_str',
    'first_name',
    'last_name',
    'is_active',
    'email',
    'mfa_is_active',
]
METADATA_COL = ['metadata']
EXTRA_DETAILS_COLS = [
    'bio',
    'city',
    'name',
    'gender',
    'sector',
    'country',
    'twitter',
    'linkedin',
    'metadata',
    'instagram',
    'organization',
    'require_auth',
    'organization_website',
    'phone_number',
    'primarySector',
    'address',
    'default_language',
]

def get_user_data():
    values = USER_COLS + METADATA_COL
    return (
        User.objects.all()
        .annotate(
            mfa_is_active=F('mfa_methods__is_active'),
            metadata=F('extra_details__data'),
            date_joined_str=Cast('date_joined', CharField()),
            last_login_str=Cast('last_login', CharField()),
        )
        .values(*values)
        .order_by('id')
    )

def flatten_metadata_inplace(metadata):
    for k, v in metadata.items():
        if isinstance(v, list):
            v = v[0] if v else ''
        if isinstance(v, dict) and 'label' in v:
            metadata[k] = v['label']

def write_csv(filename, data):
    with open(filename, 'w') as f:
        columns = USER_COLS + EXTRA_DETAILS_COLS
        writer = csv.DictWriter(f, fieldnames=columns)
        writer.writeheader()
        for item in data:
            metadata = item.pop('metadata', {}) or dict()
            flatten_metadata_inplace(metadata)
            item.update(metadata)
            writer.writerow(item)

def run(*args):
    if not args:
        print('Inlcude filename and path in `--script-args`')
        sys.exit()
    filename = args[0]
    t0 = time.time()
    data = get_user_data()
    write_csv(filename, data)
    t1 = time.time()
    print(f'Completed in {t1-t0:.4f}s.')
<commit_msg>add asset_count to columns<commit_after>import csv
import sys
import time

from django.contrib.auth.models import User
from django.db.models import CharField, F, Count
from django.db.models.functions import Cast

USER_COLS = [
    'id',
    'username',
    'is_superuser',
    'is_staff',
    'date_joined_str',
    'last_login_str',
    'first_name',
    'last_name',
    'is_active',
    'email',
    'mfa_is_active',
    'asset_count',
]
METADATA_COL = ['metadata']
EXTRA_DETAILS_COLS = [
    'bio',
    'city',
    'name',
    'gender',
    'sector',
    'country',
    'twitter',
    'linkedin',
    'metadata',
    'instagram',
    'organization',
    'require_auth',
    'organization_website',
    'phone_number',
    'primarySector',
    'address',
    'default_language',
]

def get_user_data():
    values = USER_COLS + METADATA_COL
    return (
        User.objects.all()
        .annotate(
            mfa_is_active=F('mfa_methods__is_active'),
            metadata=F('extra_details__data'),
            date_joined_str=Cast('date_joined', CharField()),
            last_login_str=Cast('last_login', CharField()),
            asset_count=Count('assets'),
        )
        .values(*values)
        .order_by('id')
    )

def flatten_metadata_inplace(metadata):
    for k, v in metadata.items():
        if isinstance(v, list):
            v = v[0] if v else ''
        if isinstance(v, dict) and 'label' in v:
            metadata[k] = v['label']

def write_csv(filename, data):
    with open(filename, 'w') as f:
        columns = USER_COLS + EXTRA_DETAILS_COLS
        writer = csv.DictWriter(f, fieldnames=columns)
        writer.writeheader()
        for item in data:
            metadata = item.pop('metadata', {}) or dict()
            flatten_metadata_inplace(metadata)
            item.update(metadata)
            writer.writerow(item)

def run(*args):
    if not args:
        print('Inlcude filename and path in `--script-args`')
        sys.exit()
    filename = args[0]
    t0 = time.time()
    data = get_user_data()
    write_csv(filename, data)
    t1 = time.time()
    print(f'Completed in {t1-t0:.4f}s.')
<|endoftext|>


<commit_before>"""Module for dealing with WSGI errors."""

from cgi import escape
import sys
import traceback
import logging

try:
    from paste.httpexceptions import HTTPException as PasteException
except ImportError:
    class PasteException(object):
        pass

from werkzeug.datastructures import Headers
from mako.exceptions import RichTraceback as MakoTraceback

from . import status

log = logging.getLogger(__name__)

DEFAULT_ERROR_HTTP_STATUS = '500 Internal Server Error'
DEFAULT_ERROR_HTTP_HEADERS = [('Content-Type','text/plain')]
DEFAULT_ERROR_BODY = """Uh oh... we dropped the ball on this one.

But don't worry! The site builders have been notified.

Please make your request again. If this error continues to occur please wait a
little while and try again.

Thank you for you patience as we resolve this matter.
"""

def format_report_iter(environ, html=False):
    """Returns an iterator of consecutive parts of an error report with
    traceback, environment, and buffered output, if supplied.

    Must be called while handling an exception.

    Params:
        environ -- The WSGI environ that the error occoured in.
        output -- Any WSGI output captured.

    Examples:

        >>> try:
        ...     raise ValueError('Testing')
        ... except:
        ...     print format_report({'HTTP_HOST': 'example.com'}, ['one', 'two'])
        ... # doctest: +ELLIPSIS
        Traceback (most recent call last):
          File "<doctest ....format_report_iter[0]>", line 2, in <module>
            raise ValueError('Testing')
        ValueError: Testing
        Environment:
          HTTP_HOST: 'example.com'
        Output 2 chunks (6 bytes):
        ==============================================================================
        onetwo
        ==============================================================================
        <BLANKLINE>

        >>> try:
        ...     raise ValueError('Testing')
        ... except:
        ...     print format_report(environ={'HTTP_HOST': 'example.com'}, output=['one', 'two'], html=True).replace('\\t', '    ')
        ... # doctest: +ELLIPSIS

Traceback (most recent call last):

                File "<doctest ....format_report_iter[1]>", line 2, in <module>

                raise ValueError('Testing')

ValueError: Testing

Environment:

HTTP_HOST: 'example.com'

Output 2 chunks (6 bytes):

onetwo

    """

    type, value, tb = sys.exc_info()

    if html:
        yield '
Traceback (most recent call last):
\n'
        yield '
\n'
        for filename, lineno, function, line in get_cleaned_traceback():
            yield '\t
\n'
            yield '\t\tFile "%s", line %d, in %s
' % (escape(filename), lineno, escape(function))
            if line:
                yield '\n\t\t%s' % escape(line)
            yield '
\n'
        yield '
\n'
        yield '
'
        yield ''.join(traceback.format_exception_only(type, value)).strip()
        yield '
\n'

    else:
        yield 'Traceback (most recent call last):\n'
        for x in traceback.format_list(get_cleaned_traceback()):
            yield x
        for x in traceback.format_exception_only(type, value):
            yield x

    if html:
        yield '
Environment:
\n'
        yield '
\n'
        for k, v in sorted(environ.items()):
            yield '\t
%s: %s
\n' % (
                escape(str(k)), escape(repr(v)))
        yield '
\n'

    else:
        yield 'Environment:\n'
        yield '\n'.join('  %s: %r' % x for x in sorted(environ.items()))
        yield '\n'

def format_report(environ, html=False):
    """Returns a stringified error report as built by format_report_iter."""
    return ''.join(format_report_iter(environ, html))

def get_cleaned_traceback():
    """Returns a traceback cleaned of mako jibberish.

    Must be called from within an exception handler.

    """

    type, value, tb = sys.exc_info()
    try:
        return list(MakoTraceback().traceback)
    except:
        log.exception('Error while Mako was cleaning traceback')
        return traceback.extract_tb(tb)

def exception_logger(app, level=logging.ERROR, ignore=None):
    """WSGI middleware which logs errors, environments, and caught output.

    Reraises the exception after logging it. Ignored exceptions are still
    reraised.

    """

    ignore = ignore or ()
    def _exception_logger(environ, start):
        try:
            for x in app(environ, start):
                yield x
        except ignore:
            raise
        except Exception as e:
            report = format_report(environ).strip()
            log.log(level, 'Unexpected %r\n' % e + report)
            raise

    return _exception_logger

error_logger = exception_logger
logger = exception_logger

def exception_handler(app, get_template=None, debug=False):
    def _exception_handler(environ, start):

        try:
            app_iter = iter(app(environ, start))
            try:
                yield next(app_iter)
            except StopIteration:
                pass
        except status.HTTPException as e:
            e.original = None
        except PasteException as original:
            e = status.exceptions.get(original.code) or status.InternalServerError
            e = e()
            e.original = original
        except Exception as original:
            e = status.InternalServerError()
            e.original = original
        else:
            for x in app_iter:
                yield x
            return

        if isinstance(e, status.HTTPRedirection):
            log.info('caught %d %s; redirects to %r' % (e.code, e.title, e.location))
            for x in e(environ, start):
                yield x
            return

        log.info('caught %d %s: %r' % (e.code, e.title, e.description))

        try:
            text_report = format_report(environ, False) if debug else None
            html_report = format_report(environ, True) if debug else None
        except:
            text_report = html_report = None
            log.exception('Exception while formating error report.')

        template = get_template and (get_template('/status/%d.html' % e.code) or get_template('/status/generic.html'))

        if template:
            try:
                output = template.render_unicode(
                    exception=e,
                    environ=environ,
                    text_report=text_report,
                    html_report=html_report,
                ).encode('utf8')
            except:
                log.exception('Exception while building error page.')
            else:
                try:
                    start('%d %s' % (e.code, e.title), [('Content-Type', 'text/html; charset=utf-8')])
                except:
                    pass
                yield output
                return

        for x in e(environ, start):
            yield x

        # Need to break these out like this incase there was an issue while
        # building them.
        if html_report:
            yield html_report
        if text_report:
            yield ''

    return _exception_handler

handler = exception_handler

class ExceptionAppMixin(object):

    def __init__(self, *args, **kwargs):
        super(ExceptionAppMixin, self).__init__(*args, **kwargs)
        self.register_middleware((self.FRAMEWORK_LAYER, 99), logger, None, dict(ignore=(status.HTTPException, )))
        self.register_middleware((self.FRAMEWORK_LAYER, 100), handler, None, dict(
            debug=self.config.debug,
            get_template=getattr(self, 'get_template', None),
        ))

if __name__ == '__main__':
    import nose; nose.run(defaultTest=__name__)<commit_msg>Exception page renderer uses App render method.<commit_after>"""Module for dealing with WSGI errors."""

from cgi import escape
import sys
import traceback
import logging

try:
    from paste.httpexceptions import HTTPException as PasteException
except ImportError:
    class PasteException(object):
        pass

from werkzeug.datastructures import Headers
from mako.exceptions import RichTraceback as MakoTraceback, TopLevelLookupException as MakoLookupError

from . import status

log = logging.getLogger(__name__)

DEFAULT_ERROR_HTTP_STATUS = '500 Internal Server Error'
DEFAULT_ERROR_HTTP_HEADERS = [('Content-Type','text/plain')]
DEFAULT_ERROR_BODY = """Uh oh... we dropped the ball on this one.

But don't worry! The site builders have been notified.

Please make your request again. If this error continues to occur please wait a
little while and try again.

Thank you for you patience as we resolve this matter.
"""

def format_report_iter(environ, html=False):
    """Returns an iterator of consecutive parts of an error report with
    traceback, environment, and buffered output, if supplied.

    Must be called while handling an exception.

    Params:
        environ -- The WSGI environ that the error occoured in.
        output -- Any WSGI output captured.

    Examples:

        >>> try:
        ...     raise ValueError('Testing')
        ... except:
        ...     print format_report({'HTTP_HOST': 'example.com'}, ['one', 'two'])
        ... # doctest: +ELLIPSIS
        Traceback (most recent call last):
          File "<doctest ....format_report_iter[0]>", line 2, in <module>
            raise ValueError('Testing')
        ValueError: Testing
        Environment:
          HTTP_HOST: 'example.com'
        Output 2 chunks (6 bytes):
        ==============================================================================
        onetwo
        ==============================================================================
        <BLANKLINE>

        >>> try:
        ...     raise ValueError('Testing')
        ... except:
        ...     print format_report(environ={'HTTP_HOST': 'example.com'}, output=['one', 'two'], html=True).replace('\\t', '    ')
        ... # doctest: +ELLIPSIS

Traceback (most recent call last):

                File "<doctest ....format_report_iter[1]>", line 2, in <module>

                raise ValueError('Testing')

ValueError: Testing

Environment:

HTTP_HOST: 'example.com'

Output 2 chunks (6 bytes):

onetwo

    """

    type, value, tb = sys.exc_info()

    if html:
        yield '
Traceback (most recent call last):
\n'
        yield '
\n'
        for filename, lineno, function, line in get_cleaned_traceback():
            yield '\t
\n'
            yield '\t\tFile "%s", line %d, in %s
' % (escape(filename), lineno, escape(function))
            if line:
                yield '\n\t\t%s' % escape(line)
            yield '
\n'
        yield '
\n'
        yield '
'
        yield ''.join(traceback.format_exception_only(type, value)).strip()
        yield '
\n'

    else:
        yield 'Traceback (most recent call last):\n'
        for x in traceback.format_list(get_cleaned_traceback()):
            yield x
        for x in traceback.format_exception_only(type, value):
            yield x

    if html:
        yield '
Environment:
\n'
        yield '
\n'
        for k, v in sorted(environ.items()):
            yield '\t
%s: %s
\n' % (
                escape(str(k)), escape(repr(v)))
        yield '
\n'

    else:
        yield 'Environment:\n'
        yield '\n'.join('  %s: %r' % x for x in sorted(environ.items()))
        yield '\n'

def format_report(environ, html=False):
    """Returns a stringified error report as built by format_report_iter."""
    return ''.join(format_report_iter(environ, html))

def get_cleaned_traceback():
    """Returns a traceback cleaned of mako jibberish.

    Must be called from within an exception handler.

    """

    type, value, tb = sys.exc_info()
    try:
        return list(MakoTraceback().traceback)
    except:
        log.exception('Error while Mako was cleaning traceback')
        return traceback.extract_tb(tb)

def exception_logger(app, level=logging.ERROR, ignore=None):
    """WSGI middleware which logs errors, environments, and caught output.

    Reraises the exception after logging it. Ignored exceptions are still
    reraised.

    """

    ignore = ignore or ()
    def _exception_logger(environ, start):
        try:
            for x in app(environ, start):
                yield x
        except ignore:
            raise
        except Exception as e:
            report = format_report(environ).strip()
            log.log(level, 'Unexpected %r\n' % e + report)
            raise

    return _exception_logger

error_logger = exception_logger
logger = exception_logger

def exception_handler(app, render=None, debug=False):
    def _exception_handler(environ, start):

        try:
            app_iter = iter(app(environ, start))
            try:
                yield next(app_iter)
            except StopIteration:
                pass
        except status.HTTPException as e:
            e.original = None
        except PasteException as original:
            e = status.exceptions.get(original.code) or status.InternalServerError
            e = e()
            e.original = original
        except Exception as original:
            e = status.InternalServerError()
            e.original = original
        else:
            for x in app_iter:
                yield x
            return

        if isinstance(e, status.HTTPRedirection):
            log.info('caught %d %s; redirects to %r' % (e.code, e.title, e.location))
            for x in e(environ, start):
                yield x
            return

        log.info('caught %d %s: %r' % (e.code, e.title, e.description))

        try:
            text_report = format_report(environ, False) if debug else None
            html_report = format_report(environ, True) if debug else None
        except:
            text_report = html_report = None
            log.exception('Exception while formating error report.')

        if render:
            output = None
            for template in ('/status/%d.html' % e.code, '/status/generic.html'):
                try:
                    output = render(template, exception=e,
                        environ=environ,
                        text_report=text_report,
                        html_report=html_report,
                    ).encode('utf8')
                except MakoLookupError:
                    continue
                except:
                    log.exception('Exception while building error page.')
                break
            if output:
                try:
                    start('%d %s' % (e.code, e.title), [('Content-Type', 'text/html; charset=utf-8')])
                except:
                    pass
                yield output
                return

        for x in e(environ, start):
            yield x

        # Need to break these out like this incase there was an issue while
        # building them.
        if html_report:
            yield html_report
        if text_report:
            yield ''

    return _exception_handler

handler = exception_handler

class ExceptionAppMixin(object):

    def __init__(self, *args, **kwargs):
        super(ExceptionAppMixin, self).__init__(*args, **kwargs)
        self.register_middleware((self.FRAMEWORK_LAYER, 99), logger, None, dict(ignore=(status.HTTPException, )))
        self.register_middleware((self.FRAMEWORK_LAYER, 100), handler, None, dict(
            debug=self.config.debug,
            render=getattr(self, 'render', None),
        ))

if __name__ == '__main__':
    import nose; nose.run(defaultTest=__name__)<|endoftext|>