<commit_before># coding=utf8
"""
wikipedia.py - Willie Wikipedia Module
Copyright 2013 Edward Powell - embolalia.net
Licensed under the Eiffel Forum License 2.

http://willie.dftba.net
"""
from __future__ import unicode_literals
from willie import web, tools
from willie.config import StaticSection, ValidatedAttribute
from willie.module import NOLIMIT, commands, example, rule
import json
import re

import sys
if sys.version_info.major < 3:
    from urlparse import unquote
else:
    from urllib.parse import unquote

REDIRECT = re.compile(r'^REDIRECT (.*)')

class WikipediaSection(StaticSection):
    default_lang = ValidatedAttribute('default_lang')
    """The default language to find articles from."""
    lang_per_channel = ValidatedAttribute('lang_per_channel')

def setup(bot):
    bot.config.define_section('wikipedia', WikipediaSection, default="en")

    regex = re.compile('([a-z]+).(wikipedia.org/wiki/)([^ ]+)')
    if not bot.memory.contains('url_callbacks'):
        bot.memory['url_callbacks'] = tools.WillieMemory()
    bot.memory['url_callbacks'][regex] = mw_info

def configure(config):
    config.define_section('wikipedia', WikipediaSection)
    config.wikipedia.configure_setting(
        'default_lang',
        "Enter the default language to find articles from."
    )

def mw_search(server, query, num):
    """
    Searches the specified MediaWiki server for the given query, and returns
    the specified number of results.
    """
    search_url = ('http://%s/w/api.php?format=json&action=query'
                  '&list=search&srlimit=%d&srprop=timestamp&srwhat=text'
                  '&srsearch=') % (server, num)
    search_url += query
    query = json.loads(web.get(search_url))
    if 'query' in query:
        query = query['query']['search']
        return [r['title'] for r in query]
    else:
        return None

def say_snippet(bot, server, query, show_url=True):
    page_name = query.replace('_', ' ')
    query = query.replace(' ', '_')
    snippet = mw_snippet(server, query)
    msg = '[WIKIPEDIA] {} | "{}"'.format(page_name, snippet)
    if show_url:
        msg = msg + ' | https://{}/wiki/{}'.format(server, query)
    bot.say(msg)

def mw_snippet(server, query):
    """
    Retrives a snippet of the specified length from the given page on the given
    server.
    """
    snippet_url = ('https://' + server + '/w/api.php?format=json'
                   '&action=query&prop=extracts&exintro&explaintext'
                   '&exchars=300&redirects&titles=')
    snippet_url += query
    snippet = json.loads(web.get(snippet_url))
    snippet = snippet['query']['pages']

    # For some reason, the API gives the page *number* as the key, so we just
    # grab the first page number in the results.
    snippet = snippet[list(snippet.keys())[0]]

    return snippet['extract']

@rule('.*/([a-z]+\.wikipedia.org)/wiki/([^ ]+).*')
def mw_info(bot, trigger, found_match=None):
    """
    Retrives a snippet of the specified length from the given page on the given
    server.
    """
    match = found_match or trigger
    say_snippet(bot, match.group(1), unquote(match.group(2)), show_url=False)

@commands('w', 'wiki', 'wik')
@example('.w San Francisco')
def wikipedia(bot, trigger):
    lang = bot.config.wikipedia.default_lang

    #change lang if channel has custom language set
    if (trigger.sender and not trigger.sender.is_nick() and
            bot.config.wikipedia.lang_per_channel):
        customlang = re.search('(' + trigger.sender + '):(\w+)',
                               bot.config.wikipedia.lang_per_channel)
        if customlang is not None:
            lang = customlang.group(2)

    if trigger.group(2) is None:
        bot.reply("What do you want me to look up?")
        return NOLIMIT

    query = trigger.group(2)
    args = re.search(r'^-([a-z]{2,12})\s(.*)', query)
    if args is not None:
        lang = args.group(1)
        query = args.group(2)

    if not query:
        bot.reply('What do you want me to look up?')
        return NOLIMIT
    server = lang + '.wikipedia.org'
    query = mw_search(server, query, 1)
    if not query:
        bot.reply("I can't find any results for that.")
        return NOLIMIT
    else:
        query = query[0]
    say_snippet(bot, server, query)
<commit_msg>wikipedia: Fix bad config import<commit_after># coding=utf8
"""
wikipedia.py - Willie Wikipedia Module
Copyright 2013 Edward Powell - embolalia.net
Licensed under the Eiffel Forum License 2.

http://willie.dftba.net
"""
from __future__ import unicode_literals
from willie import web, tools
from willie.config.types import StaticSection, ValidatedAttribute
from willie.module import NOLIMIT, commands, example, rule
import json
import re

import sys
if sys.version_info.major < 3:
    from urlparse import unquote
else:
    from urllib.parse import unquote

REDIRECT = re.compile(r'^REDIRECT (.*)')

class WikipediaSection(StaticSection):
    default_lang = ValidatedAttribute('default_lang')
    """The default language to find articles from."""
    lang_per_channel = ValidatedAttribute('lang_per_channel')

def setup(bot):
    bot.config.define_section('wikipedia', WikipediaSection, default="en")

    regex = re.compile('([a-z]+).(wikipedia.org/wiki/)([^ ]+)')
    if not bot.memory.contains('url_callbacks'):
        bot.memory['url_callbacks'] = tools.WillieMemory()
    bot.memory['url_callbacks'][regex] = mw_info

def configure(config):
    config.define_section('wikipedia', WikipediaSection)
    config.wikipedia.configure_setting(
        'default_lang',
        "Enter the default language to find articles from."
    )

def mw_search(server, query, num):
    """
    Searches the specified MediaWiki server for the given query, and returns
    the specified number of results.
    """
    search_url = ('http://%s/w/api.php?format=json&action=query'
                  '&list=search&srlimit=%d&srprop=timestamp&srwhat=text'
                  '&srsearch=') % (server, num)
    search_url += query
    query = json.loads(web.get(search_url))
    if 'query' in query:
        query = query['query']['search']
        return [r['title'] for r in query]
    else:
        return None

def say_snippet(bot, server, query, show_url=True):
    page_name = query.replace('_', ' ')
    query = query.replace(' ', '_')
    snippet = mw_snippet(server, query)
    msg = '[WIKIPEDIA] {} | "{}"'.format(page_name, snippet)
    if show_url:
        msg = msg + ' | https://{}/wiki/{}'.format(server, query)
    bot.say(msg)

def mw_snippet(server, query):
    """
    Retrives a snippet of the specified length from the given page on the given
    server.
    """
    snippet_url = ('https://' + server + '/w/api.php?format=json'
                   '&action=query&prop=extracts&exintro&explaintext'
                   '&exchars=300&redirects&titles=')
    snippet_url += query
    snippet = json.loads(web.get(snippet_url))
    snippet = snippet['query']['pages']

    # For some reason, the API gives the page *number* as the key, so we just
    # grab the first page number in the results.
    snippet = snippet[list(snippet.keys())[0]]

    return snippet['extract']

@rule('.*/([a-z]+\.wikipedia.org)/wiki/([^ ]+).*')
def mw_info(bot, trigger, found_match=None):
    """
    Retrives a snippet of the specified length from the given page on the given
    server.
    """
    match = found_match or trigger
    say_snippet(bot, match.group(1), unquote(match.group(2)), show_url=False)

@commands('w', 'wiki', 'wik')
@example('.w San Francisco')
def wikipedia(bot, trigger):
    lang = bot.config.wikipedia.default_lang

    #change lang if channel has custom language set
    if (trigger.sender and not trigger.sender.is_nick() and
            bot.config.wikipedia.lang_per_channel):
        customlang = re.search('(' + trigger.sender + '):(\w+)',
                               bot.config.wikipedia.lang_per_channel)
        if customlang is not None:
            lang = customlang.group(2)

    if trigger.group(2) is None:
        bot.reply("What do you want me to look up?")
        return NOLIMIT

    query = trigger.group(2)
    args = re.search(r'^-([a-z]{2,12})\s(.*)', query)
    if args is not None:
        lang = args.group(1)
        query = args.group(2)

    if not query:
        bot.reply('What do you want me to look up?')
        return NOLIMIT
    server = lang + '.wikipedia.org'
    query = mw_search(server, query, 1)
    if not query:
        bot.reply("I can't find any results for that.")
        return NOLIMIT
    else:
        query = query[0]
    say_snippet(bot, server, query)
<|endoftext|>


<commit_before>import hail as hl

from .utils import benchmark

@benchmark()
def block_matrix_nested_multiply():
    bm = hl.linalg.BlockMatrix.random(8 * 1024, 8 * 1024).checkpoint(hl.utils.new_temp_file(suffix='bm'))
    path = hl.utils.new_temp_file(suffix='bm')
    ((bm @ bm) @ bm @ bm @ (bm @ bm)).write(path, overwrite=True)

@benchmark()
def make_ndarray_bench():
    ht = hl.utils.range_table(200_000)
    ht = ht.annotate(x=hl._nd.array(hl.range(200_000)))
    ht._force_count()
<commit_msg>Benchmark NDArray Addition and Multiplication (#7724)<commit_after>import hail as hl

from .utils import benchmark

@benchmark()
def block_matrix_nested_multiply():
    bm = hl.linalg.BlockMatrix.random(8 * 1024, 8 * 1024).checkpoint(hl.utils.new_temp_file(suffix='bm'))
    path = hl.utils.new_temp_file(suffix='bm')
    ((bm @ bm) @ bm @ bm @ (bm @ bm)).write(path, overwrite=True)

@benchmark()
def make_ndarray_bench():
    ht = hl.utils.range_table(200_000)
    ht = ht.annotate(x=hl._nd.array(hl.range(200_000)))
    ht._force_count()

@benchmark()
def ndarray_addition_benchmark():
    arr = hl._nd.ones((1024, 1024))
    hl.eval(arr + arr)

@benchmark()
def ndarray_matmul_int64_benchmark():
    arr = hl._nd.arange(1024 * 1024).map(hl.int64).reshape((1024, 1024))
    hl.eval(arr @ arr)

@benchmark()
def ndarray_matmul_float64_benchmark():
    arr = hl._nd.arange(1024 * 1024).map(hl.float64).reshape((1024, 1024))
    hl.eval(arr @ arr)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) 2015, Carnegie Mellon University
# All rights reserved.
# Authors: Jennifer King <jeking@cs.cmu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# - Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# - Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# - Neither the name of Carnegie Mellon University nor the names of its
#   contributors may be used to endorse or promote products derived from this
#   software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import logging
from base import PerceptionModule, PerceptionMethod

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

class ApriltagsModule(PerceptionModule):

    def __init__(self, marker_topic, marker_data_path, kinbody_path,
                 detection_frame, destination_frame):
        """
        This initializes an April Tags detector.

        @param marker_topic The ROS topic to read markers from. Typically the output topic for April Tags
        @param marker_data_path The json file where the association between tag and object is stored
        @param kinbody_path The path to the folder where kinbodies are stored
        @param detection_frame The TF frame of the camera
        @param destination_frame The desired world TF frame
        """

        super(ApriltagsModule, self).__init__()

        self.marker_topic = marker_topic
        self.marker_data_path = marker_data_path
        self.kinbody_path = kinbody_path
        self.detection_frame = detection_frame
        self.destination_frame = destination_frame

    def __str__(self):
        return self.__class__.__name__

    def DetectObjects(self, env, object_names, **kw_args):
        """
        This hack detects only the objects in object_names. Updates existing
        objects, but only adds objects in the object_names list.

        @param env: The current OpenRAVE environment
        @param object_names: The list of names of objects to detect
        """
        added_kinbodies, updated_kinbodies = self._DetectObjects(env, **kw_args);
        detected = [];
        for body in added_kinbodies:
            if not (body.GetName() in object_names):
                env.RemoveKinbody(body);
            else:
                detected.append(body);
        return detected;

    def DetectObject(self, env, object_name, **kw_args):
        """
        Detects a single named object.
        """
        return (self._DetectObjects( env, object_names=[object_name], **kw_args))[0][0];

    def _DetectObjects(self, env, marker_topic, marker_data_path, kinbody_path,
                 detection_frame, destination_frame,**kwargs):
        """
        Use the apriltags service to detect objects and add them to the
        environment. Params are as in __init__.

        @param env: The current OpenRAVE environment
        @param marker_topic The ROS topic to read markers from. Typically the output topic for April Tags
        @param marker_data_path The json file where the association between tag and object is stored
        @param kinbody_path The path to the folder where kinbodies are stored
        @param detection_frame The TF frame of the camera
        @param destination_frame The desired world TF frame

        @return The list of kinbodies associated with the detected apriltags
        """
        try:
            # Allow caller to override any of the initial parameters
            # loaded into the module
            if marker_topic is None:
                marker_topic = self.marker_topic

            if marker_data_path is None:
                marker_data_path = self.marker_data_path

            if kinbody_path is None:
                kinbody_path = self.kinbody_path

            if detection_frame is None:
                detection_frame = self.detection_frame

            if destination_frame is None:
                destination_frame = self.destination_frame

            # TODO: Creating detector is not instant...might want
            #  to just do this once in the constructor
            import kinbody_detector.kinbody_detector as kd
            detector = kd.KinBodyDetector(env,
                                          marker_data_path,
                                          kinbody_path,
                                          marker_topic,
                                          detection_frame,
                                          destination_frame)

            logger.warn('Waiting to detect objects...')
            return detector.Update()
        except Exception, e:
            logger.error('Detecton failed update: %s' % str(e))
            raise

    @PerceptionMethod
    def DetectObjects(self, robot, **kw_args):
        """
        Overriden method for detection_frame
        """
        return self._DetectObjects(robot.GetEnv(),**kwargs)

<commit_msg>Last minor changes for merge<commit_after>#!/usr/bin/env python

# Copyright (c) 2015, Carnegie Mellon University
# All rights reserved.
# Authors: Jennifer King <jeking@cs.cmu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# - Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# - Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# - Neither the name of Carnegie Mellon University nor the names of its
#   contributors may be used to endorse or promote products derived from this
#   software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import logging
from base import PerceptionModule, PerceptionMethod

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

class ApriltagsModule(PerceptionModule):

    def __init__(self, marker_topic, marker_data_path, kinbody_path,
                 detection_frame, destination_frame):
        """
        This initializes an April Tags detector.

        @param marker_topic The ROS topic to read markers from. Typically the output topic for April Tags
        @param marker_data_path The json file where the association between tag and object is stored
        @param kinbody_path The path to the folder where kinbodies are stored
        @param detection_frame The TF frame of the camera
        @param destination_frame The desired world TF frame
        """

        super(ApriltagsModule, self).__init__()

        self.marker_topic = marker_topic
        self.marker_data_path = marker_data_path
        self.kinbody_path = kinbody_path
        self.detection_frame = detection_frame
        self.destination_frame = destination_frame

    def __str__(self):
        return self.__class__.__name__

    def DetectObjects(self, env, object_names, **kw_args):
        """
        This hack detects only the objects in object_names. Updates existing
        objects, but only adds objects in the object_names list.

        @param env: The current OpenRAVE environment
        @param object_names: The list of names of objects to detect
        """
        added_kinbodies, updated_kinbodies = self._DetectObjects(env, **kw_args);
        detected = [];
        for body in added_kinbodies:
            if not (body.GetName() in object_names):
                env.RemoveKinbody(body);
            else:
                detected.append(body);
        return detected;

    def DetectObject(self, env, object_name, **kw_args):
        """
        Detects a single named object.
        """
        return (self._DetectObjects( env, object_names=[object_name], **kw_args))[0][0];

    def _DetectObjects(self, env, marker_topic, marker_data_path, kinbody_path,
                 detection_frame, destination_frame,**kw_args):
        """
        Use the apriltags service to detect objects and add them to the
        environment. Params are as in __init__.

        @param env: The current OpenRAVE environment
        @param marker_topic The ROS topic to read markers from. Typically the output topic for April Tags
        @param marker_data_path The json file where the association between tag and object is stored
        @param kinbody_path The path to the folder where kinbodies are stored
        @param detection_frame The TF frame of the camera
        @param destination_frame The desired world TF frame

        @return The list of kinbodies associated with the detected apriltags
        """
        try:
            # Allow caller to override any of the initial parameters
            # loaded into the module
            if marker_topic is None:
                marker_topic = self.marker_topic

            if marker_data_path is None:
                marker_data_path = self.marker_data_path

            if kinbody_path is None:
                kinbody_path = self.kinbody_path

            if detection_frame is None:
                detection_frame = self.detection_frame

            if destination_frame is None:
                destination_frame = self.destination_frame

            # TODO: Creating detector is not instant...might want
            #  to just do this once in the constructor
            import kinbody_detector.kinbody_detector as kd
            detector = kd.KinBodyDetector(env,
                                          marker_data_path,
                                          kinbody_path,
                                          marker_topic,
                                          detection_frame,
                                          destination_frame)

            logger.warn('Waiting to detect objects...')
            return detector.Update()
        except Exception, e:
            logger.error('Detecton failed update: %s' % str(e))
            raise

    @PerceptionMethod
    def DetectObjects(self, robot, **kw_args):
        """
        Overriden method for detection_frame
        """
        return self._DetectObjects(robot.GetEnv(),**kwargs)

<|endoftext|>


<commit_before>"""
Vector module.
"""
from __future__ import absolute_import, unicode_literals
import logging

from psd_tools.api.pil_io import convert_pattern_to_pil
from psd_tools.composer.blend import blend
from psd_tools.terminology import Enum, Key

logger = logging.getLogger(__name__)

def draw_vector_mask(layer):
    from PIL import Image, ImageChops
    width = layer._psd.width
    height = layer._psd.height
    color = 255 * layer.vector_mask.initial_fill_rule

    mask = Image.new('L', (width, height), color)
    first = True
    for subpath in layer.vector_mask.paths:
        plane = _draw_subpath(subpath, width, height)
        if subpath.operation == 0:
            mask = ImageChops.difference(mask, plane)
        elif subpath.operation == 1:
            mask = ImageChops.lighter(mask, plane)
        elif subpath.operation == 2:
            if first:
                mask = ImageChops.invert(mask)
            mask = ImageChops.subtract(mask, plane)
        elif subpath.operation == 3:
            if first:
                mask = ImageChops.invert(mask)
            mask = ImageChops.darker(mask, plane)
        first = False
    return mask.crop(layer.bbox)

def _draw_subpath(subpath, width, height):
    from PIL import Image
    import aggdraw
    mask = Image.new('L', (width, height), 0)
    path = ' '.join(map(str, _generate_symbol(subpath, width, height)))
    draw = aggdraw.Draw(mask)
    brush = aggdraw.Brush(255)
    symbol = aggdraw.Symbol(path)
    draw.symbol((0, 0), symbol, None, brush)
    draw.flush()
    del draw
    return mask

def _generate_symbol(path, width, height, command='C'):
    """Sequence generator for SVG path."""
    if len(path) == 0:
        return

    # Initial point.
    yield 'M'
    yield path[0].anchor[1] * width
    yield path[0].anchor[0] * height
    yield command

    # Closed path or open path
    points = (
        zip(path, path[1:] +
            path[0:1]) if path.is_closed() else zip(path, path[1:])
    )

    # Rest of the points.
    for p1, p2 in points:
        yield p1.leaving[1] * width
        yield p1.leaving[0] * height
        yield p2.preceding[1] * width
        yield p2.preceding[0] * height
        yield p2.anchor[1] * width
        yield p2.anchor[0] * height

    if path.is_closed():
        yield 'Z'

def draw_solid_color_fill(image, setting, mode=None):
    from PIL import Image, ImageDraw, ImageChops
    color = tuple(int(x) for x in setting.get(b'Clr ').values())
    canvas = Image.new(image.mode, image.size)
    draw = ImageDraw.Draw(canvas)
    draw.rectangle((0, 0, canvas.width, canvas.height), fill=color)
    del draw
    if mode:
        if image.mode.endswith('A'):
            canvas.putalpha(image.getchannel('A'))
        blend(image, canvas, (0, 0), mode=mode)
    else:
        image.paste(canvas)

def draw_pattern_fill(image, psd, setting, mode=None):
    """
    Draw pattern fill on the image.

    :param image: Image to be filled.
    :param psd: :py:class:`PSDImage`.
    :param setting: Descriptor containing pattern fill.
    :param mode: Blend the fill or ignore if None.
    """
    from PIL import Image
    pattern_id = setting[b'Ptrn'][b'Idnt'].value.rstrip('\x00')
    pattern = psd._get_pattern(pattern_id)
    if not pattern:
        logger.error('Pattern not found: %s' % (pattern_id))
        return None
    panel = convert_pattern_to_pil(pattern, psd._record.header.version)

    scale = setting.get(b'Scl ', 100) / 100.
    if scale != 1.:
        panel = panel.resize(
            (int(panel.width * scale), int(panel.height * scale))
        )

    opacity = int(setting.get(b'Opct', 100) / 100. * 255)
    if opacity != 255:
        panel.putalpha(opacity)

    pattern_image = Image.new(image.mode, image.size)
    if mode and image.mode.endswith('A'):
        mask = image.getchannel('A')
    else:
        mask = Image.new('L', image.size, 255)

    for left in range(0, pattern_image.width, panel.width):
        for top in range(0, pattern_image.height, panel.height):
            panel_mask = mask.crop(
                (left, top, left + panel.width, top + panel.height)
            )
            pattern_image.paste(panel, (left, top), panel_mask)

    if mode:
        image.paste(blend(image, pattern_image, (0, 0), mode=mode))
    else:
        image.paste(pattern_image)

def draw_gradient_fill(image, setting, mode=None):
    try:
        import numpy as np
    except ImportError:
        logger.error('Gradient fill requires numpy and scipy.')
        return None

    angle = float(setting.get(Key.Angle, 0))
    scale = float(setting.get(Key.Scale, 100.)) / 100.
    scale *= min(image.height, image.width)
    X, Y = np.meshgrid(
        np.linspace(-image.width / scale, image.width / scale, image.width),
        np.linspace(-image.height / scale, image.height / scale, image.height),
    )

    gradient_kind = setting.get(Key.Type).enum
    if gradient_kind == Enum.Linear:
        Z = _make_linear_gradient(X, Y, angle)
    elif gradient_kind == Enum.Radial:
        Z = _make_radial_gradient(X, Y)
    elif gradient_kind == Enum.Angle:
        Z = _make_angle_gradient(X, Y, angle)
    elif gradient_kind == Enum.Reflected:
        Z = _make_reflected_gradient(X, Y, angle)
    elif gradient_kind == Enum.Diamond:
        Z = _make_diamond_gradient(X, Y, angle)
    else:
        logger.warning('Unknown gradient style: %s.' % (gradient_kind))
        Z = np.ones((image.height, image.width)) * 0.5

    Z = np.maximum(0, np.minimum(1, Z))
    if bool(setting.get(Key.Reverse, False)):
        Z = 1 - Z

    gradient_image = _apply_color_map(image.mode, setting.get(Key.Gradient), Z)
    if gradient_image:
        if mode:
            if image.mode.endswith('A'):
                gradient_image.putalpha(image.getchannel('A'))
            blend(image, gradient_image, (0, 0), mode=mode)
        else:
            image.paste(gradient_image)

def _make_linear_gradient(X, Y, angle):
    """Generates index map for linear gradients."""
    import numpy as np
    theta = np.radians(angle % 360)
    Z = .5 * (np.cos(theta) * X - np.sin(theta) * Y + 1)
    return Z

def _make_radial_gradient(X, Y):
    """Generates index map for radial gradients."""
    import numpy as np
    Z = np.sqrt(np.power(X, 2) + np.power(Y, 2))
    return Z

def _make_angle_gradient(X, Y, angle):
    """Generates index map for angle gradients."""
    import numpy as np
    Z = (((180 * np.arctan2(Y, X) / np.pi) + angle) % 360) / 360
    return Z

def _make_reflected_gradient(X, Y, angle):
    """Generates index map for reflected gradients."""
    import numpy as np
    theta = np.radians(angle % 360)
    Z = np.abs((np.cos(theta) * X - np.sin(theta) * Y))
    return Z

def _make_diamond_gradient(X, Y, angle):
    """Generates index map for diamond gradients."""
    import numpy as np
    theta = np.radians(angle % 360)
    Z = np.abs(np.cos(theta) * X - np.sin(theta) *
               Y) + np.abs(np.sin(theta) * X + np.cos(theta) * Y)
    return Z

def _apply_color_map(mode, grad, Z):
    """"""
    import numpy as np
    from scipy import interpolate
    from PIL import Image

    if grad.get(b'GrdF').enum == Enum.ColorNoise:
        """
        TODO: Improve noise gradient quality.

        Example:

            Descriptor(b'Grdn'){
                'Nm  ': 'Custom\x00',
                'GrdF': (b'GrdF', b'ClNs'),
                'ShTr': False,
                'VctC': False,
                'ClrS': (b'ClrS', b'RGBC'),
                'RndS': 3650322,
                'Smth': 2048,
                'Mnm ': [0, 0, 0, 0],
                'Mxm ': [0, 100, 100, 100]
            }
        """
        logger.debug('Noise gradient is not accurate.')
        from scipy.ndimage.filters import maximum_filter1d, uniform_filter1d
        roughness = grad.get(
            Key.Smoothness
        ).value / 4096.  # Larger is sharper.
        maximum = np.array([x.value for x in grad.get(Key.Maximum)])
        minimum = np.array([x.value for x in grad.get(Key.Minimum)])
        seed = grad.get(Key.RandomSeed).value

        rng = np.random.RandomState(seed)
        G = rng.binomial(1, .5, (256, len(maximum))).astype(np.float)
        size = max(1, int(roughness * 4))
        G = maximum_filter1d(G, size, axis=0)
        G = uniform_filter1d(G, size * 64, axis=0)
        G = (2.55 * ((maximum - minimum) * G + minimum)).astype(np.uint8)
        Z = (255 * Z).astype(np.uint8)
        pixels = G[Z]
        if pixels.shape[-1] == 1:
            pixels = pixels[:, :, 0]
        image = Image.fromarray(pixels, mode)
    elif grad.get(b'GrdF').enum == Enum.CustomStops:
        scalar = {
            'RGB': 1.0,
            'L': 2.55,
            'CMYK': 2.55,
        }.get(mode, 1.0)
        stops = grad.get(b'Clrs')
        X = [stop.get(b'Lctn').value / 4096. for stop in stops]
        Y = [
            tuple(int(scalar * x.value) for x in stop.get(b'Clr ').values())
            for stop in stops
        ]
        if len(stops) == 1:
            X = [0., 1.]
            Y = [Y[0], Y[0]]
        # TODO: Workaround zero-division.
        G = interpolate.interp1d(X, Y, axis=0, fill_value='extrapolate')
        pixels = G(Z).astype(np.uint8)
        if pixels.shape[-1] == 1:
            pixels = pixels[:, :, 0]

        image = Image.fromarray(pixels, mode.rstrip('A'))
        if b'Trns' in grad and mode.endswith('A'):
            stops = grad.get(b'Trns')
            X = [stop.get(b'Lctn').value / 4096 for stop in stops]
            Y = [stop.get(b'Opct').value * 2.55 for stop in stops]
            if len(stops) == 1:
                X = [0., 1.]
                Y = [Y[0], Y[0]]
            G_opacity = interpolate.interp1d(
                X, Y, axis=0, fill_value='extrapolate'
            )
            alpha = G_opacity(Z).astype(np.uint8)
            image.putalpha(Image.fromarray(alpha, 'L'))
    return image
<commit_msg>workaround zero-division for duplicated gradient stops<commit_after>"""
Vector module.
"""
from __future__ import absolute_import, unicode_literals
import logging

from psd_tools.api.pil_io import convert_pattern_to_pil
from psd_tools.composer.blend import blend
from psd_tools.terminology import Enum, Key

logger = logging.getLogger(__name__)

def draw_vector_mask(layer):
    from PIL import Image, ImageChops
    width = layer._psd.width
    height = layer._psd.height
    color = 255 * layer.vector_mask.initial_fill_rule

    mask = Image.new('L', (width, height), color)
    first = True
    for subpath in layer.vector_mask.paths:
        plane = _draw_subpath(subpath, width, height)
        if subpath.operation == 0:
            mask = ImageChops.difference(mask, plane)
        elif subpath.operation == 1:
            mask = ImageChops.lighter(mask, plane)
        elif subpath.operation == 2:
            if first:
                mask = ImageChops.invert(mask)
            mask = ImageChops.subtract(mask, plane)
        elif subpath.operation == 3:
            if first:
                mask = ImageChops.invert(mask)
            mask = ImageChops.darker(mask, plane)
        first = False
    return mask.crop(layer.bbox)

def _draw_subpath(subpath, width, height):
    from PIL import Image
    import aggdraw
    mask = Image.new('L', (width, height), 0)
    path = ' '.join(map(str, _generate_symbol(subpath, width, height)))
    draw = aggdraw.Draw(mask)
    brush = aggdraw.Brush(255)
    symbol = aggdraw.Symbol(path)
    draw.symbol((0, 0), symbol, None, brush)
    draw.flush()
    del draw
    return mask

def _generate_symbol(path, width, height, command='C'):
    """Sequence generator for SVG path."""
    if len(path) == 0:
        return

    # Initial point.
    yield 'M'
    yield path[0].anchor[1] * width
    yield path[0].anchor[0] * height
    yield command

    # Closed path or open path
    points = (
        zip(path, path[1:] +
            path[0:1]) if path.is_closed() else zip(path, path[1:])
    )

    # Rest of the points.
    for p1, p2 in points:
        yield p1.leaving[1] * width
        yield p1.leaving[0] * height
        yield p2.preceding[1] * width
        yield p2.preceding[0] * height
        yield p2.anchor[1] * width
        yield p2.anchor[0] * height

    if path.is_closed():
        yield 'Z'

def draw_solid_color_fill(image, setting, mode=None):
    from PIL import Image, ImageDraw, ImageChops
    color = tuple(int(x) for x in setting.get(b'Clr ').values())
    canvas = Image.new(image.mode, image.size)
    draw = ImageDraw.Draw(canvas)
    draw.rectangle((0, 0, canvas.width, canvas.height), fill=color)
    del draw
    if mode:
        if image.mode.endswith('A'):
            canvas.putalpha(image.getchannel('A'))
        blend(image, canvas, (0, 0), mode=mode)
    else:
        image.paste(canvas)

def draw_pattern_fill(image, psd, setting, mode=None):
    """
    Draw pattern fill on the image.

    :param image: Image to be filled.
    :param psd: :py:class:`PSDImage`.
    :param setting: Descriptor containing pattern fill.
    :param mode: Blend the fill or ignore if None.
    """
    from PIL import Image
    pattern_id = setting[b'Ptrn'][b'Idnt'].value.rstrip('\x00')
    pattern = psd._get_pattern(pattern_id)
    if not pattern:
        logger.error('Pattern not found: %s' % (pattern_id))
        return None
    panel = convert_pattern_to_pil(pattern, psd._record.header.version)

    scale = setting.get(b'Scl ', 100) / 100.
    if scale != 1.:
        panel = panel.resize(
            (int(panel.width * scale), int(panel.height * scale))
        )

    opacity = int(setting.get(b'Opct', 100) / 100. * 255)
    if opacity != 255:
        panel.putalpha(opacity)

    pattern_image = Image.new(image.mode, image.size)
    if mode and image.mode.endswith('A'):
        mask = image.getchannel('A')
    else:
        mask = Image.new('L', image.size, 255)

    for left in range(0, pattern_image.width, panel.width):
        for top in range(0, pattern_image.height, panel.height):
            panel_mask = mask.crop(
                (left, top, left + panel.width, top + panel.height)
            )
            pattern_image.paste(panel, (left, top), panel_mask)

    if mode:
        image.paste(blend(image, pattern_image, (0, 0), mode=mode))
    else:
        image.paste(pattern_image)

def draw_gradient_fill(image, setting, mode=None):
    try:
        import numpy as np
    except ImportError:
        logger.error('Gradient fill requires numpy and scipy.')
        return None

    angle = float(setting.get(Key.Angle, 0))
    scale = float(setting.get(Key.Scale, 100.)) / 100.
    scale *= min(image.height, image.width)
    X, Y = np.meshgrid(
        np.linspace(-image.width / scale, image.width / scale, image.width),
        np.linspace(-image.height / scale, image.height / scale, image.height),
    )

    gradient_kind = setting.get(Key.Type).enum
    if gradient_kind == Enum.Linear:
        Z = _make_linear_gradient(X, Y, angle)
    elif gradient_kind == Enum.Radial:
        Z = _make_radial_gradient(X, Y)
    elif gradient_kind == Enum.Angle:
        Z = _make_angle_gradient(X, Y, angle)
    elif gradient_kind == Enum.Reflected:
        Z = _make_reflected_gradient(X, Y, angle)
    elif gradient_kind == Enum.Diamond:
        Z = _make_diamond_gradient(X, Y, angle)
    else:
        logger.warning('Unknown gradient style: %s.' % (gradient_kind))
        Z = np.ones((image.height, image.width)) * 0.5

    Z = np.maximum(0, np.minimum(1, Z))
    if bool(setting.get(Key.Reverse, False)):
        Z = 1 - Z

    gradient_image = _apply_color_map(image.mode, setting.get(Key.Gradient), Z)
    if gradient_image:
        if mode:
            if image.mode.endswith('A'):
                gradient_image.putalpha(image.getchannel('A'))
            blend(image, gradient_image, (0, 0), mode=mode)
        else:
            image.paste(gradient_image)

def _make_linear_gradient(X, Y, angle):
    """Generates index map for linear gradients."""
    import numpy as np
    theta = np.radians(angle % 360)
    Z = .5 * (np.cos(theta) * X - np.sin(theta) * Y + 1)
    return Z

def _make_radial_gradient(X, Y):
    """Generates index map for radial gradients."""
    import numpy as np
    Z = np.sqrt(np.power(X, 2) + np.power(Y, 2))
    return Z

def _make_angle_gradient(X, Y, angle):
    """Generates index map for angle gradients."""
    import numpy as np
    Z = (((180 * np.arctan2(Y, X) / np.pi) + angle) % 360) / 360
    return Z

def _make_reflected_gradient(X, Y, angle):
    """Generates index map for reflected gradients."""
    import numpy as np
    theta = np.radians(angle % 360)
    Z = np.abs((np.cos(theta) * X - np.sin(theta) * Y))
    return Z

def _make_diamond_gradient(X, Y, angle):
    """Generates index map for diamond gradients."""
    import numpy as np
    theta = np.radians(angle % 360)
    Z = np.abs(np.cos(theta) * X - np.sin(theta) *
               Y) + np.abs(np.sin(theta) * X + np.cos(theta) * Y)
    return Z

def _apply_color_map(mode, grad, Z):
    """"""
    import numpy as np
    from scipy import interpolate
    from PIL import Image

    if grad.get(b'GrdF').enum == Enum.ColorNoise:
        """
        TODO: Improve noise gradient quality.

        Example:

            Descriptor(b'Grdn'){
                'Nm  ': 'Custom\x00',
                'GrdF': (b'GrdF', b'ClNs'),
                'ShTr': False,
                'VctC': False,
                'ClrS': (b'ClrS', b'RGBC'),
                'RndS': 3650322,
                'Smth': 2048,
                'Mnm ': [0, 0, 0, 0],
                'Mxm ': [0, 100, 100, 100]
            }
        """
        logger.debug('Noise gradient is not accurate.')
        from scipy.ndimage.filters import maximum_filter1d, uniform_filter1d
        roughness = grad.get(
            Key.Smoothness
        ).value / 4096.  # Larger is sharper.
        maximum = np.array([x.value for x in grad.get(Key.Maximum)])
        minimum = np.array([x.value for x in grad.get(Key.Minimum)])
        seed = grad.get(Key.RandomSeed).value

        rng = np.random.RandomState(seed)
        G = rng.binomial(1, .5, (256, len(maximum))).astype(np.float)
        size = max(1, int(roughness * 4))
        G = maximum_filter1d(G, size, axis=0)
        G = uniform_filter1d(G, size * 64, axis=0)
        G = (2.55 * ((maximum - minimum) * G + minimum)).astype(np.uint8)
        Z = (255 * Z).astype(np.uint8)
        pixels = G[Z]
        if pixels.shape[-1] == 1:
            pixels = pixels[:, :, 0]
        image = Image.fromarray(pixels, mode)
    elif grad.get(b'GrdF').enum == Enum.CustomStops:
        scalar = {
            'RGB': 1.0,
            'L': 2.55,
            'CMYK': 2.55,
        }.get(mode, 1.0)
        X, Y = [], []
        for stop in grad.get(Key.Colors, []):
            location = int(stop.get(Key.Location)) / 4096.
            color = tuple(scalar * x for x in stop.get(Key.Color).values())
            if len(X) and X[-1] == location:
                logger.debug('Duplicate stop at %d' % location)
                X.pop(), Y.pop()
            X.append(location), Y.append(color)
        assert len(X) > 0
        if len(X) == 1:
            X = [0., 1.]
            Y = [Y[0], Y[0]]
        G = interpolate.interp1d(X, Y, axis=0, fill_value='extrapolate')
        pixels = G(Z).astype(np.uint8)
        if pixels.shape[-1] == 1:
            pixels = pixels[:, :, 0]

        image = Image.fromarray(pixels, mode.rstrip('A'))
        if Key.Transparency in grad and mode.endswith('A'):
            X, Y = [], []
            for stop in grad.get(Key.Transparency):
                location = int(stop.get(Key.Location)) / 4096.
                opacity = float(stop.get(Key.Opacity)) * 2.55
                if len(X) and X[-1] == location:
                    logger.debug('Duplicate stop at %d' % location)
                    X.pop(), Y.pop()
                X.append(location), Y.append(opacity)
            assert len(X) > 0
            if len(X) == 1:
                X = [0., 1.]
                Y = [Y[0], Y[0]]
            G = interpolate.interp1d(X, Y, axis=0, fill_value='extrapolate')
            alpha = G(Z).astype(np.uint8)
            image.putalpha(Image.fromarray(alpha, 'L'))
    return image
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Provides an ``ActivityEvent`` model to store arbitrary event data and a
  ``WorkStatusMixin`` with generic association table that provides a
  ``work_status`` relationship when mixed into an ORM class.
"""

__all__ = [
    'ActivityEvent',
    'Notification',
    'NotificationDispatch',
    'WorkStatus',
    'WorkStatusMixin',
]

import os

from datetime import datetime

from sqlalchemy import event
from sqlalchemy import orm
from sqlalchemy import schema
from sqlalchemy import types
from sqlalchemy.dialects import postgresql
from sqlalchemy.ext import associationproxy as proxy
from sqlalchemy.ext import declarative
from sqlalchemy.ext import hybrid

import pyramid_basemodel as bm
from pyramid_simpleauth import model as simpleauth_model

import zope.interface as zi
from . import interfaces

# XXX It may be better to require the code that creates a work status to
# explicitly set the value rather than relying on this abitrary default.
DEFAULT_STATE = os.environ.get('ENGINE_DEFAULT_STATE', u'state:CREATED')

class ActivityEventAssociation(bm.Base, bm.BaseMixin):
    """Polymorphic base that's used to associate a collection of
      ``ActivityEvent``s with a parent.
    """

    __tablename__ = 'activity_event_associations'
    discriminator = schema.Column(types.Unicode(64))
    __mapper_args__ = {'polymorphic_on': discriminator}

class ActivityEvent(bm.Base, bm.BaseMixin):
    """Something that happened. Has an event type made out of target
      and action (e.g.: `job:confirmed`) and an arbitrary JSON payload.
    """

    # Store all events in a single table...
    __tablename__ = 'activity_events'

    # ... whilst allowing sub classes to add fields by specifying a discriminator.
    discriminator = schema.Column(types.Unicode(64))
    __mapper_args__ = {'polymorphic_on': discriminator}

    # An event is normally -- but not always -- performed by a user.
    user_id = schema.Column(
        types.Integer,
        schema.ForeignKey('auth_users.id'),
    )
    user = orm.relationship(
        simpleauth_model.User,
        backref='activity_events',
    )

    # Can belong to a ``parent`` via a ``ActivityEventAssociation``.
    association_id = schema.Column(
        types.Integer,
        schema.ForeignKey('activity_event_associations.id'),
    )
    assocation = orm.relationship(ActivityEventAssociation, backref='activity_events')

    @property
    def parent(self):
        return self.assocation.parent

    @parent.setter
    def parent(self, value):
        self.assocation.parent = value

    # Has an arbitrary data payload.
    # data = schema.Column(postgresql.JSONB, default={}, nullable=False)
    data = schema.Column(postgresql.JSON, default={}, nullable=False)

    # Has a `target:action` as string identifiers of the event type, e.g.:
    # `message:created`, `job:confirmed`, etc.
    target = schema.Column(types.Unicode, nullable=False)
    action = schema.Column(types.Unicode, nullable=False)

    # These are exposed and can be managed using `type_`.
    @hybrid.hybrid_property
    def type_(self):
        return u'{0}:{1}'.format(self.target, self.action)

    @type_.setter
    def type_(self, value):
        self.target, self.action = value.split(u':')

    def __json__(self, request=None):
        """Represent the event as a JSON serialisable dict."""

        data = {
            'type': self.class_slug,
            'id': self.id,
            'created_at': self.created.isoformat(),
            'modified_at': self.modified.isoformat(),
            'type': self.type_,
            'data': self.data,
        }
        if self.parent:
            data['parent'] = {
                'type': self.parent.class_slug,
                'id': self.parent.id,
            }
        if self.user:
            data['user'] = {
                'type': self.user.class_slug,
                'id': self.user_id,
            }
        return data

class WorkStatusAssociation(bm.Base, bm.BaseMixin):
    """Polymorphic base that's used to associate a collection of
      ``WorkStatus``s with a parent.
    """

    __tablename__ = 'work_status_associations'
    discriminator = schema.Column(types.Unicode(64))
    __mapper_args__ = {'polymorphic_on': discriminator}

class WorkStatus(bm.Base, bm.BaseMixin):
    """Define the properties provided by a work status entry."""

    # Store in `work_statuses`.
    __tablename__ = 'work_statuses'

    # Must have a string status value
    value = schema.Column(
        types.Unicode(64),
        default=DEFAULT_STATE,
        nullable=False,
    )

    # Can belong to a ``parent`` via a ``WorkStatusAssociation``.
    association_id = schema.Column(
        types.Integer,
        schema.ForeignKey('work_status_associations.id'),
    )
    assocation = orm.relationship(WorkStatusAssociation, backref='work_statuses')

    @property
    def parent(self):
        return self.assocation.parent

    # Can have an event (i.e.: the change to this state was triggered by).
    event_id = schema.Column(
        types.Integer,
        schema.ForeignKey('activity_events.id'),
    )
    event = orm.relationship(
        ActivityEvent,
        backref=orm.backref(
            'work_status',
            lazy='joined',
            single_parent=True,
            uselist=False,
        ),
        lazy='joined',
        uselist=False,
    )

    def __json__(self, request=None):
        data = {
            'type': self.class_slug,
            'id': self.id,
            'value': self.value,
        }
        return data

@zi.implementer(interfaces.IWorkStatus)
class WorkStatusMixin(object):
    """Mixin a collection of work_statuses and activity_events to each target
      ORM class.

      - `activity_events` is a collection of `ActivityEvent` instances
      - `work_statuses` is a collection of `WorkStatusEntry` instances
      - use the `parent.work_status` property to get the most recent entry
      - use `set_work_status(value, event)` to update the work status
    """

    @declarative.declared_attr
    def activity_event_association_id(cls):
        return schema.Column(
            types.Integer,
            schema.ForeignKey('activity_event_associations.id'),
        )

    @declarative.declared_attr
    def activity_event_association(cls):
        """Dynamically defined association table relationship."""

        class_name = '{0}ActivityEventAssociation'.format(cls.__name__)
        bases = (ActivityEventAssociation,)
        mapping = {
            '__mapper_args__': {
                'polymorphic_identity': cls.singular_class_slug,
            }
        }
        association_cls = type(class_name, bases, mapping)
        cls.ActivityEventAssociation = association_cls
        cls.activity_events = proxy.association_proxy(
            'activity_event_association',
            'activity_events',
            creator=lambda x: association_cls(activity_events=x)
        )
        return orm.relationship(
            association_cls,
            backref=orm.backref('parent', uselist=False),
        )

    @declarative.declared_attr
    def work_status_association_id(cls):
        return schema.Column(
            types.Integer,
            schema.ForeignKey('work_status_associations.id'),
        )

    @declarative.declared_attr
    def work_status_association(cls):
        """Dynamically defined association table relationship."""

        class_name = '{0}WorkStatusAssociation'.format(cls.__name__)
        bases = (WorkStatusAssociation,)
        mapping = {
            '__mapper_args__': {
                'polymorphic_identity': cls.singular_class_slug,
            }
        }
        association_cls = type(class_name, bases, mapping)
        cls.WorkStatus = WorkStatus # <!-- just for backwards compatibility
        cls.WorkStatusAssociation = association_cls
        cls.work_statuses = proxy.association_proxy(
            'work_status_association',
            'work_statuses',
            creator=lambda work_statuses: association_cls(work_statuses=work_statuses)
        )
        return orm.relationship(
            association_cls,
            backref=orm.backref('parent', uselist=False),
        )

    def set_work_status(self, value, event=None, model_cls=WorkStatus):
        """Append a new work status to the entry list."""

        # Make sure we're not detatched o_O.
        bm.Session.add(self)

        # Add a new entry to the status collection.
        status = model_cls(value=value, event=event)
        if self.work_statuses:
            self.work_statuses.append(status)
        else:
            self.work_statuses = [status]

        # Update timestamps.
        self.modified = datetime.utcnow()

        # Make sure everything gets saved.
        bm.Session.add_all([self, status])
        bm.Session.flush()

        # Return the new status instance.
        return status

    def get_work_status(self, value=None, model_cls=WorkStatus):
        """Return the most recent work status, optionally filtered by value."""

        query = model_cls.query
        query = query.filter_by(association_id=self.work_status_association_id)
        if value is not None:
            query = query.filter_by(value=value)
        query = query.order_by(model_cls.created.desc())
        return query.first()

    @property
    def work_status(self):
        return self.get_work_status()

class NotificationDispatch(bm.Base, bm.BaseMixin):
    """A notification dispatch to an user, holds information about how to deliver
    and when."""

    __tablename__ = 'notifications_dispatch'

    # Has a due date.
    due = schema.Column(types.DateTime)

    # Has a sent date.
    sent = schema.Column(types.DateTime)

    # has a Notification.
    notification_id = schema.Column(
        types.Integer,
        schema.ForeignKey('notifications.id'),
    )

    # view  -> function to decode things
    view = schema.Column(types.Unicode(96))
    # simple for the moment, either single or batch text. XXX use ENUM.
    type_ = schema.Column(types.Unicode(96))
    # dotted path for the asset spec.
    single_spec = schema.Column(types.Unicode(96))
    batch_spec = schema.Column(types.Unicode(96))
    # simple for the moment, either email or sms. XXX use ENUM.
    category = schema.Column(types.Unicode(96))
    # email or telephone number
    address = schema.Column(types.Unicode(96))

class Notification(bm.Base, bm.BaseMixin):
    """A notification about an event that should be sent to an user."""

    __tablename__ = 'notifications'

    # has an user.
    user_id = schema.Column(
        types.Integer,
        schema.ForeignKey('auth_users.id'),
    )

    user = orm.relationship(
        simpleauth_model.User,
        backref='notification',
    )

    # Has a read date.
    read = schema.Column(types.DateTime)

    notification_dispatch = orm.relationship(
        NotificationDispatch,
        backref='notification')

    # has an Activity event.
    # One to many
    event_id = schema.Column(
        types.Integer,
        schema.ForeignKey('activity_events.id'),
    )
    event = orm.relationship(
        ActivityEvent,
        backref=orm.backref(
            'notification',
        ),
    )

    def __json__(self, request=None):
        """Represent the event as a JSON serialisable dict."""

        data = {
            'id': self.id,
            'user_id': self.user_id,
            'created_at': self.created.isoformat(),
            'read_at': self.read.isoformat(),
            'event_id': self.event_id,
        }
        return data
<commit_msg>workstatus: removed by mistake.<commit_after># -*- coding: utf-8 -*-

"""Provides an ``ActivityEvent`` model to store arbitrary event data and a
  ``WorkStatusMixin`` with generic association table that provides a
  ``work_status`` relationship when mixed into an ORM class.
"""

__all__ = [
    'ActivityEvent',
    'Notification',
    'NotificationDispatch',
    'WorkStatus',
    'WorkStatusMixin',
]

import os

from datetime import datetime

from sqlalchemy import event
from sqlalchemy import orm
from sqlalchemy import schema
from sqlalchemy import types
from sqlalchemy.dialects import postgresql
from sqlalchemy.ext import associationproxy as proxy
from sqlalchemy.ext import declarative
from sqlalchemy.ext import hybrid

import pyramid_basemodel as bm
from pyramid_simpleauth import model as simpleauth_model

import zope.interface as zi
from . import interfaces

# XXX It may be better to require the code that creates a work status to
# explicitly set the value rather than relying on this abitrary default.
DEFAULT_STATE = os.environ.get('ENGINE_DEFAULT_STATE', u'state:CREATED')

class ActivityEventAssociation(bm.Base, bm.BaseMixin):
    """Polymorphic base that's used to associate a collection of
      ``ActivityEvent``s with a parent.
    """

    __tablename__ = 'activity_event_associations'
    discriminator = schema.Column(types.Unicode(64))
    __mapper_args__ = {'polymorphic_on': discriminator}

class ActivityEvent(bm.Base, bm.BaseMixin):
    """Something that happened. Has an event type made out of target
      and action (e.g.: `job:confirmed`) and an arbitrary JSON payload.
    """

    # Store all events in a single table...
    __tablename__ = 'activity_events'

    # ... whilst allowing sub classes to add fields by specifying a discriminator.
    discriminator = schema.Column(types.Unicode(64))
    __mapper_args__ = {'polymorphic_on': discriminator}

    # An event is normally -- but not always -- performed by a user.
    user_id = schema.Column(
        types.Integer,
        schema.ForeignKey('auth_users.id'),
    )
    user = orm.relationship(
        simpleauth_model.User,
        backref='activity_events',
    )

    # Can belong to a ``parent`` via a ``ActivityEventAssociation``.
    association_id = schema.Column(
        types.Integer,
        schema.ForeignKey('activity_event_associations.id'),
    )
    assocation = orm.relationship(ActivityEventAssociation, backref='activity_events')

    @property
    def parent(self):
        return self.assocation.parent

    @parent.setter
    def parent(self, value):
        self.assocation.parent = value

    # Has an arbitrary data payload.
    # data = schema.Column(postgresql.JSONB, default={}, nullable=False)
    data = schema.Column(postgresql.JSON, default={}, nullable=False)

    # Has a `target:action` as string identifiers of the event type, e.g.:
    # `message:created`, `job:confirmed`, etc.
    target = schema.Column(types.Unicode, nullable=False)
    action = schema.Column(types.Unicode, nullable=False)

    # These are exposed and can be managed using `type_`.
    @hybrid.hybrid_property
    def type_(self):
        return u'{0}:{1}'.format(self.target, self.action)

    @type_.setter
    def type_(self, value):
        self.target, self.action = value.split(u':')

    def __json__(self, request=None):
        """Represent the event as a JSON serialisable dict."""

        data = {
            'type': self.class_slug,
            'id': self.id,
            'created_at': self.created.isoformat(),
            'modified_at': self.modified.isoformat(),
            'type': self.type_,
            'data': self.data,
        }
        if self.parent:
            data['parent'] = {
                'type': self.parent.class_slug,
                'id': self.parent.id,
            }
        if self.user:
            data['user'] = {
                'type': self.user.class_slug,
                'id': self.user_id,
            }
        return data

class WorkStatusAssociation(bm.Base, bm.BaseMixin):
    """Polymorphic base that's used to associate a collection of
      ``WorkStatus``s with a parent.
    """

    __tablename__ = 'work_status_associations'
    discriminator = schema.Column(types.Unicode(64))
    __mapper_args__ = {'polymorphic_on': discriminator}

class WorkStatus(bm.Base, bm.BaseMixin):
    """Define the properties provided by a work status entry."""

    # Store in `work_statuses`.
    __tablename__ = 'work_statuses'

    # Must have a string status value
    value = schema.Column(
        types.Unicode(64),
        default=DEFAULT_STATE,
        nullable=False,
    )

    # Can belong to a ``parent`` via a ``WorkStatusAssociation``.
    association_id = schema.Column(
        types.Integer,
        schema.ForeignKey('work_status_associations.id'),
    )
    assocation = orm.relationship(WorkStatusAssociation, backref='work_statuses')

    @property
    def parent(self):
        return self.assocation.parent

    # Can have an event (i.e.: the change to this state was triggered by).
    event_id = schema.Column(
        types.Integer,
        schema.ForeignKey('activity_events.id'),
    )
    event = orm.relationship(
        ActivityEvent,
        backref=orm.backref(
            'work_status',
            lazy='joined',
            single_parent=True,
            uselist=False,
        ),
        lazy='joined',
        uselist=False,
    )

    def __json__(self, request=None):
        data = {
            'type': self.class_slug,
            'id': self.id,
            'value': self.value,
        }
        return data

@zi.implementer(interfaces.IWorkStatus)
class WorkStatusMixin(object):
    """Mixin a collection of work_statuses and activity_events to each target
      ORM class.

      - `activity_events` is a collection of `ActivityEvent` instances
      - `work_statuses` is a collection of `WorkStatusEntry` instances
      - use the `parent.work_status` property to get the most recent entry
      - use `set_work_status(value, event)` to update the work status
    """

    @declarative.declared_attr
    def activity_event_association_id(cls):
        return schema.Column(
            types.Integer,
            schema.ForeignKey('activity_event_associations.id'),
        )

    @declarative.declared_attr
    def activity_event_association(cls):
        """Dynamically defined association table relationship."""

        class_name = '{0}ActivityEventAssociation'.format(cls.__name__)
        bases = (ActivityEventAssociation,)
        mapping = {
            '__mapper_args__': {
                'polymorphic_identity': cls.singular_class_slug,
            }
        }
        association_cls = type(class_name, bases, mapping)
        cls.ActivityEventAssociation = association_cls
        cls.activity_events = proxy.association_proxy(
            'activity_event_association',
            'activity_events',
            creator=lambda x: association_cls(activity_events=x)
        )
        return orm.relationship(
            association_cls,
            backref=orm.backref('parent', uselist=False),
        )

    @declarative.declared_attr
    def work_status_association_id(cls):
        return schema.Column(
            types.Integer,
            schema.ForeignKey('work_status_associations.id'),
        )

    @declarative.declared_attr
    def work_status_association(cls):
        """Dynamically defined association table relationship."""

        class_name = '{0}WorkStatusAssociation'.format(cls.__name__)
        bases = (WorkStatusAssociation,)
        mapping = {
            '__mapper_args__': {
                'polymorphic_identity': cls.singular_class_slug,
            }
        }
        association_cls = type(class_name, bases, mapping)
        cls.WorkStatus = WorkStatus # <!-- just for backwards compatibility
        cls.WorkStatusAssociation = association_cls
        cls.work_statuses = proxy.association_proxy(
            'work_status_association',
            'work_statuses',
            creator=lambda work_statuses: association_cls(work_statuses=work_statuses)
        )
        return orm.relationship(
            association_cls,
            backref=orm.backref('parent', uselist=False),
        )

    def set_work_status(self, value, event=None, model_cls=WorkStatus):
        """Append a new work status to the entry list."""

        # Make sure we're not detatched o_O.
        bm.Session.add(self)

        # Add a new entry to the status collection.
        status = model_cls(value=value, event=event)
        if self.work_statuses:
            self.work_statuses.append(status)
        else:
            self.work_statuses = [status]

        # Update timestamps.
        self.modified = datetime.utcnow()

        # Make sure everything gets saved.
        bm.Session.add_all([self, status])
        bm.Session.flush()

        # Return the new status instance.
        return status

    def get_work_status(self, value=None, model_cls=WorkStatus):
        """Return the most recent work status, optionally filtered by value."""

        query = model_cls.query
        query = query.filter_by(association_id=self.work_status_association_id)
        if value is not None:
            query = query.filter_by(value=value)
        query = query.order_by(model_cls.created.desc())
        return query.first()

    @property
    def work_status(self):
        return self.get_work_status()

    @classmethod
    def status_query(cls, value_or_values, model_cls=WorkStatus):
        """Returns a query for ``cls`` instances whose current work_status
          value matches the ``value_or_values`` provided.

          As you can see from the implementation, this is non-trivial, so
          handy to have as a class method.

          The solution was ported from http://stackoverflow.com/a/2111420
        """

        # Prepare by aliasing the work status class twice.
        ws1 = orm.aliased(model_cls)
        ws2 = orm.aliased(model_cls)

        # And build a query for instances that have work statuses.
        query = cls.query
        query = query.join(ws1, ws1.association_id==cls.work_status_association_id)

        # Now for the magic: use a left outer join to rule out work statuses
        # that aren't current.
        query = query.join(
            ws2,
            sql.and_(
                ws2.association_id==cls.work_status_association_id,
                sql.or_(
                    # Created date can (in theory) be the same, so make sure
                    # there's a winner by also falling back on id.
                    ws1.created < ws2.created,
                    sql.and_(
                        ws1.created==ws2.created,
                        ws1.id < ws2.id
                    )
                )
            ),
            isouter=True
        )
        query = query.filter(ws2.id==None)

        # Before filtering for the status value or values.
        if hasattr(value_or_values, '__iter__'):
            values = value_or_values
            clause = ws1.value.in_(values)
        else:
            value = value_or_values
            clause = ws1.value==value
        query = query.filter(clause)
        return query

class NotificationDispatch(bm.Base, bm.BaseMixin):
    """A notification dispatch to an user, holds information about how to deliver
    and when."""

    __tablename__ = 'notifications_dispatch'

    # Has a due date.
    due = schema.Column(types.DateTime)

    # Has a sent date.
    sent = schema.Column(types.DateTime)

    # has a Notification.
    notification_id = schema.Column(
        types.Integer,
        schema.ForeignKey('notifications.id'),
    )

    # view  -> function to decode things
    view = schema.Column(types.Unicode(96))
    # simple for the moment, either single or batch text. XXX use ENUM.
    type_ = schema.Column(types.Unicode(96))
    # dotted path for the asset spec.
    single_spec = schema.Column(types.Unicode(96))
    batch_spec = schema.Column(types.Unicode(96))
    # simple for the moment, either email or sms. XXX use ENUM.
    category = schema.Column(types.Unicode(96))
    # email or telephone number
    address = schema.Column(types.Unicode(96))

class Notification(bm.Base, bm.BaseMixin):
    """A notification about an event that should be sent to an user."""

    __tablename__ = 'notifications'

    # has an user.
    user_id = schema.Column(
        types.Integer,
        schema.ForeignKey('auth_users.id'),
    )

    user = orm.relationship(
        simpleauth_model.User,
        backref='notification',
    )

    # Has a read date.
    read = schema.Column(types.DateTime)

    notification_dispatch = orm.relationship(
        NotificationDispatch,
        backref='notification')

    # has an Activity event.
    # One to many
    event_id = schema.Column(
        types.Integer,
        schema.ForeignKey('activity_events.id'),
    )
    event = orm.relationship(
        ActivityEvent,
        backref=orm.backref(
            'notification',
        ),
    )

    def __json__(self, request=None):
        """Represent the event as a JSON serialisable dict."""

        data = {
            'id': self.id,
            'user_id': self.user_id,
            'created_at': self.created.isoformat(),
            'read_at': self.read.isoformat(),
            'event_id': self.event_id,
        }
        return data
<|endoftext|>


<commit_before># Copyright (c) 2012-2013 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Andreas Hansson
#          Uri Wiener

#####################################################################
#
# System visualization using DOT
#
# While config.ini and config.json provide an almost complete listing
# of a system's components and connectivity, they lack a birds-eye
# view. The output generated by do_dot() is a DOT-based figure (as a
# pdf and an editable svg file) and its source dot code. Nodes are
# components, and edges represent the memory hierarchy: the edges are
# directed, from a master to slave. Initially all nodes are
# generated, and then all edges are added. do_dot should be called
# with the top-most SimObject (namely root but not necessarily), the
# output folder and the output dot source filename. From the given
# node, both processes (node and edge creation) is performed
# recursivly, traversing all children of the given root.
#
# pydot is required. When missing, no output will be generated.
#
#####################################################################

import m5, os, re
from m5.SimObject import isRoot, isSimObjectVector
from m5.util import warn
try:
    import pydot
except:
    pydot = False

# need to create all nodes (components) before creating edges (memory channels)
def dot_create_nodes(simNode, callgraph):
    if isRoot(simNode):
        label = "root"
    else:
        label = simNode._name
    full_path = re.sub('\.', '_', simNode.path())
    # add class name under the label
    label = "\"" + label + " \\n: " + simNode.__class__.__name__ + "\""

    # each component is a sub-graph (cluster)
    cluster = dot_create_cluster(simNode, full_path, label)

    # create nodes per port
    for port_name in simNode._ports.keys():
        port = simNode._port_refs.get(port_name, None)
        if port != None:
            full_port_name = full_path + "_" + port_name
            port_node = dot_create_node(simNode, full_port_name, port_name)
            cluster.add_node(port_node)

    # recurse to children
    if simNode._children:
        for c in simNode._children:
            child = simNode._children[c]
            if isSimObjectVector(child):
                for obj in child:
                    dot_create_nodes(obj, cluster)
            else:
                dot_create_nodes(child, cluster)

    callgraph.add_subgraph(cluster)

# create all edges according to memory hierarchy
def dot_create_edges(simNode, callgraph):
    for port_name in simNode._ports.keys():
        port = simNode._port_refs.get(port_name, None)
        if port != None:
            full_path = re.sub('\.', '_', simNode.path())
            full_port_name = full_path + "_" + port_name
            port_node = dot_create_node(simNode, full_port_name, port_name)
            # create edges
            if type(port) is m5.params.PortRef:
                dot_add_edge(simNode, callgraph, full_port_name, port)
            else:
                for p in port.elements:
                    dot_add_edge(simNode, callgraph, full_port_name, p)

    # recurse to children
    if simNode._children:
        for c in simNode._children:
            child = simNode._children[c]
            if isSimObjectVector(child):
                for obj in child:
                    dot_create_edges(obj, callgraph)
            else:
                dot_create_edges(child, callgraph)

def dot_add_edge(simNode, callgraph, full_port_name, peerPort):
    if peerPort.role == "MASTER":
        peer_port_name = re.sub('\.', '_', peerPort.peer.simobj.path() \
                + "." + peerPort.peer.name)
        callgraph.add_edge(pydot.Edge(full_port_name, peer_port_name))

def dot_create_cluster(simNode, full_path, label):
    # get the parameter values of the node and use them as a tooltip
    ini_strings = []
    for param in sorted(simNode._params.keys()):
        value = simNode._values.get(param)
        if value != None:
            # parameter name = value in HTML friendly format
            ini_strings.append(str(param) + "=" +
                               simNode._values[param].ini_str())
    # join all the parameters with an HTML newline
    tooltip = "
".join(ini_strings)

    return pydot.Cluster( \
                         full_path, \
                         shape = "Mrecord", \
                         label = label, \
                         tooltip = "\"" + tooltip + "\"", \
                         style = "\"rounded, filled\"", \
                         color = "#000000", \
                         fillcolor = dot_gen_colour(simNode), \
                         fontname = "Arial", \
                         fontsize = "14", \
                         fontcolor = "#000000" \
                         )

def dot_create_node(simNode, full_path, label):
    return pydot.Node( \
                         full_path, \
                         shape = "Mrecord", \
                         label = label, \
                         style = "\"rounded, filled\"", \
                         color = "#000000", \
                         fillcolor = dot_gen_colour(simNode, True), \
                         fontname = "Arial", \
                         fontsize = "14", \
                         fontcolor = "#000000" \
                         )

# an enumerator for different kinds of node types, at the moment we
# discern the majority of node types, with the caches being the
# notable exception
class NodeType:
    SYS = 0
    CPU = 1
    BUS = 2
    MEM = 3
    DEV = 4
    OTHER = 5

# based on the sim object, determine the node type
def get_node_type(simNode):
    if isinstance(simNode, m5.objects.System):
        return NodeType.SYS
    # NULL ISA has no BaseCPU or PioDevice, so check if these names
    # exists before using them
    elif 'BaseCPU' in dir(m5.objects) and \
            isinstance(simNode, m5.objects.BaseCPU):
        return NodeType.CPU
    elif 'PioDevice' in dir(m5.objects) and \
            isinstance(simNode, m5.objects.PioDevice):
        return NodeType.DEV
    elif isinstance(simNode, m5.objects.BaseBus):
        return NodeType.BUS
    elif isinstance(simNode, m5.objects.AbstractMemory):
        return NodeType.MEM
    else:
        return NodeType.OTHER

# based on the node type, determine the colour as an RGB tuple, the
# palette is rather arbitrary at this point (some coherent natural
# tones), and someone that feels artistic should probably have a look
def get_type_colour(nodeType):
    if nodeType == NodeType.SYS:
        return (228, 231, 235)
    elif nodeType == NodeType.CPU:
        return (187, 198, 217)
    elif nodeType == NodeType.BUS:
        return (111, 121, 140)
    elif nodeType == NodeType.MEM:
        return (94, 89, 88)
    elif nodeType == NodeType.DEV:
        return (199, 167, 147)
    elif nodeType == NodeType.OTHER:
        # use a relatively gray shade
        return (186, 182, 174)

# generate colour for a node, either corresponding to a sim object or a
# port
def dot_gen_colour(simNode, isPort = False):
    # determine the type of the current node, and also its parent, if
    # the node is not the same type as the parent then we use the base
    # colour for its type
    node_type = get_node_type(simNode)
    if simNode._parent:
        parent_type = get_node_type(simNode._parent)
    else:
        parent_type = NodeType.OTHER

    # if this node is the same type as the parent, then scale the
    # colour based on the depth such that the deeper levels in the
    # hierarchy get darker colours
    if node_type == parent_type:
        # start out with a depth of zero
        depth = 0
        parent = simNode._parent
        # find the closes parent that is not the same type
        while parent and get_node_type(parent) == parent_type:
            depth = depth + 1
            parent = parent._parent
        node_colour = get_type_colour(parent_type)
        # slightly arbitrary, but assume that the depth is less than
        # five levels
        r, g, b = map(lambda x: x * max(1 - depth / 7.0, 0.3), node_colour)
    else:
        node_colour = get_type_colour(node_type)
        r, g, b = node_colour

    # if we are colouring a port, then make it a slightly darker shade
    # than the node that encapsulates it, once again use a magic constant
    if isPort:
        r, g, b = map(lambda x: 0.8 * x, (r, g, b))

    return dot_rgb_to_html(r, g, b)

def dot_rgb_to_html(r, g, b):
    return "#%.2x%.2x%.2x" % (r, g, b)

def do_dot(root, outdir, dotFilename):
    if not pydot:
        return
    # * use ranksep > 1.0 for for vertical separation between nodes
    # especially useful if you need to annotate edges using e.g. visio
    # which accepts svg format
    # * no need for hoizontal separation as nothing moves horizonally
    callgraph = pydot.Dot(graph_type='digraph', ranksep='1.3')
    dot_create_nodes(root, callgraph)
    dot_create_edges(root, callgraph)
    dot_filename = os.path.join(outdir, dotFilename)
    callgraph.write(dot_filename)
    try:
        # dot crashes if the figure is extremely wide.
        # So avoid terminating simulation unnecessarily
        callgraph.write_svg(dot_filename + ".svg")
        callgraph.write_pdf(dot_filename + ".pdf")
    except:
        warn("failed to generate dot output from %s", dot_filename)
<commit_msg>misc: Proper type check and import for PortRef<commit_after># Copyright (c) 2012-2013 ARM Limited
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Andreas Hansson
#          Uri Wiener

#####################################################################
#
# System visualization using DOT
#
# While config.ini and config.json provide an almost complete listing
# of a system's components and connectivity, they lack a birds-eye
# view. The output generated by do_dot() is a DOT-based figure (as a
# pdf and an editable svg file) and its source dot code. Nodes are
# components, and edges represent the memory hierarchy: the edges are
# directed, from a master to slave. Initially all nodes are
# generated, and then all edges are added. do_dot should be called
# with the top-most SimObject (namely root but not necessarily), the
# output folder and the output dot source filename. From the given
# node, both processes (node and edge creation) is performed
# recursivly, traversing all children of the given root.
#
# pydot is required. When missing, no output will be generated.
#
#####################################################################

import m5, os, re
from m5.SimObject import isRoot, isSimObjectVector
from m5.params import PortRef
from m5.util import warn
try:
    import pydot
except:
    pydot = False

# need to create all nodes (components) before creating edges (memory channels)
def dot_create_nodes(simNode, callgraph):
    if isRoot(simNode):
        label = "root"
    else:
        label = simNode._name
    full_path = re.sub('\.', '_', simNode.path())
    # add class name under the label
    label = "\"" + label + " \\n: " + simNode.__class__.__name__ + "\""

    # each component is a sub-graph (cluster)
    cluster = dot_create_cluster(simNode, full_path, label)

    # create nodes per port
    for port_name in simNode._ports.keys():
        port = simNode._port_refs.get(port_name, None)
        if port != None:
            full_port_name = full_path + "_" + port_name
            port_node = dot_create_node(simNode, full_port_name, port_name)
            cluster.add_node(port_node)

    # recurse to children
    if simNode._children:
        for c in simNode._children:
            child = simNode._children[c]
            if isSimObjectVector(child):
                for obj in child:
                    dot_create_nodes(obj, cluster)
            else:
                dot_create_nodes(child, cluster)

    callgraph.add_subgraph(cluster)

# create all edges according to memory hierarchy
def dot_create_edges(simNode, callgraph):
    for port_name in simNode._ports.keys():
        port = simNode._port_refs.get(port_name, None)
        if port != None:
            full_path = re.sub('\.', '_', simNode.path())
            full_port_name = full_path + "_" + port_name
            port_node = dot_create_node(simNode, full_port_name, port_name)
            # create edges
            if isinstance(port, PortRef):
                dot_add_edge(simNode, callgraph, full_port_name, port)
            else:
                for p in port.elements:
                    dot_add_edge(simNode, callgraph, full_port_name, p)

    # recurse to children
    if simNode._children:
        for c in simNode._children:
            child = simNode._children[c]
            if isSimObjectVector(child):
                for obj in child:
                    dot_create_edges(obj, callgraph)
            else:
                dot_create_edges(child, callgraph)

def dot_add_edge(simNode, callgraph, full_port_name, peerPort):
    if peerPort.role == "MASTER":
        peer_port_name = re.sub('\.', '_', peerPort.peer.simobj.path() \
                + "." + peerPort.peer.name)
        callgraph.add_edge(pydot.Edge(full_port_name, peer_port_name))

def dot_create_cluster(simNode, full_path, label):
    # get the parameter values of the node and use them as a tooltip
    ini_strings = []
    for param in sorted(simNode._params.keys()):
        value = simNode._values.get(param)
        if value != None:
            # parameter name = value in HTML friendly format
            ini_strings.append(str(param) + "=" +
                               simNode._values[param].ini_str())
    # join all the parameters with an HTML newline
    tooltip = "
".join(ini_strings)

    return pydot.Cluster( \
                         full_path, \
                         shape = "Mrecord", \
                         label = label, \
                         tooltip = "\"" + tooltip + "\"", \
                         style = "\"rounded, filled\"", \
                         color = "#000000", \
                         fillcolor = dot_gen_colour(simNode), \
                         fontname = "Arial", \
                         fontsize = "14", \
                         fontcolor = "#000000" \
                         )

def dot_create_node(simNode, full_path, label):
    return pydot.Node( \
                         full_path, \
                         shape = "Mrecord", \
                         label = label, \
                         style = "\"rounded, filled\"", \
                         color = "#000000", \
                         fillcolor = dot_gen_colour(simNode, True), \
                         fontname = "Arial", \
                         fontsize = "14", \
                         fontcolor = "#000000" \
                         )

# an enumerator for different kinds of node types, at the moment we
# discern the majority of node types, with the caches being the
# notable exception
class NodeType:
    SYS = 0
    CPU = 1
    BUS = 2
    MEM = 3
    DEV = 4
    OTHER = 5

# based on the sim object, determine the node type
def get_node_type(simNode):
    if isinstance(simNode, m5.objects.System):
        return NodeType.SYS
    # NULL ISA has no BaseCPU or PioDevice, so check if these names
    # exists before using them
    elif 'BaseCPU' in dir(m5.objects) and \
            isinstance(simNode, m5.objects.BaseCPU):
        return NodeType.CPU
    elif 'PioDevice' in dir(m5.objects) and \
            isinstance(simNode, m5.objects.PioDevice):
        return NodeType.DEV
    elif isinstance(simNode, m5.objects.BaseBus):
        return NodeType.BUS
    elif isinstance(simNode, m5.objects.AbstractMemory):
        return NodeType.MEM
    else:
        return NodeType.OTHER

# based on the node type, determine the colour as an RGB tuple, the
# palette is rather arbitrary at this point (some coherent natural
# tones), and someone that feels artistic should probably have a look
def get_type_colour(nodeType):
    if nodeType == NodeType.SYS:
        return (228, 231, 235)
    elif nodeType == NodeType.CPU:
        return (187, 198, 217)
    elif nodeType == NodeType.BUS:
        return (111, 121, 140)
    elif nodeType == NodeType.MEM:
        return (94, 89, 88)
    elif nodeType == NodeType.DEV:
        return (199, 167, 147)
    elif nodeType == NodeType.OTHER:
        # use a relatively gray shade
        return (186, 182, 174)

# generate colour for a node, either corresponding to a sim object or a
# port
def dot_gen_colour(simNode, isPort = False):
    # determine the type of the current node, and also its parent, if
    # the node is not the same type as the parent then we use the base
    # colour for its type
    node_type = get_node_type(simNode)
    if simNode._parent:
        parent_type = get_node_type(simNode._parent)
    else:
        parent_type = NodeType.OTHER

    # if this node is the same type as the parent, then scale the
    # colour based on the depth such that the deeper levels in the
    # hierarchy get darker colours
    if node_type == parent_type:
        # start out with a depth of zero
        depth = 0
        parent = simNode._parent
        # find the closes parent that is not the same type
        while parent and get_node_type(parent) == parent_type:
            depth = depth + 1
            parent = parent._parent
        node_colour = get_type_colour(parent_type)
        # slightly arbitrary, but assume that the depth is less than
        # five levels
        r, g, b = map(lambda x: x * max(1 - depth / 7.0, 0.3), node_colour)
    else:
        node_colour = get_type_colour(node_type)
        r, g, b = node_colour

    # if we are colouring a port, then make it a slightly darker shade
    # than the node that encapsulates it, once again use a magic constant
    if isPort:
        r, g, b = map(lambda x: 0.8 * x, (r, g, b))

    return dot_rgb_to_html(r, g, b)

def dot_rgb_to_html(r, g, b):
    return "#%.2x%.2x%.2x" % (r, g, b)

def do_dot(root, outdir, dotFilename):
    if not pydot:
        return
    # * use ranksep > 1.0 for for vertical separation between nodes
    # especially useful if you need to annotate edges using e.g. visio
    # which accepts svg format
    # * no need for hoizontal separation as nothing moves horizonally
    callgraph = pydot.Dot(graph_type='digraph', ranksep='1.3')
    dot_create_nodes(root, callgraph)
    dot_create_edges(root, callgraph)
    dot_filename = os.path.join(outdir, dotFilename)
    callgraph.write(dot_filename)
    try:
        # dot crashes if the figure is extremely wide.
        # So avoid terminating simulation unnecessarily
        callgraph.write_svg(dot_filename + ".svg")
        callgraph.write_pdf(dot_filename + ".pdf")
    except:
        warn("failed to generate dot output from %s", dot_filename)
<|endoftext|>