<commit_before>#!/usr/bin/env python2
# -*- coding:utf-8 -*-

"""
Generate html pages for each directory of images
"""

import os
import codecs
import Image
from distutils.dir_util import copy_tree
from jinja2 import Environment, PackageLoader
import sigal.image

DEFAULT_THEME = "default"
INDEX_PAGE = "index.html"
IGNORED_DIR = ['css', 'js', 'img']
DESCRIPTION_FILE = "album_description"
SIGAL_LINK = "https://github.com/saimn/sigal"
PATH_SEP = u" » "

def do_link(link, title):
    "return html link"
    return '%s' % (link, title)

class Theme():
    """ Generate html pages for each directory of images """

    def __init__(self, params, path, theme=DEFAULT_THEME, tpl=INDEX_PAGE):
        self.data = {}
        self.path = os.path.normpath(path)
        self.bigimg = params.getint('sigal', 'big_img')
        self.bigimg_dir = params.get('sigal', 'bigimg_dir')
        self.thumb_dir = params.get('sigal', 'thumb_dir')
        self.thumb_prefix = params.get('sigal', 'thumb_prefix')
        self.fileExtList = params.get('sigal', 'fileExtList').split(',')

        if params.has_option('sigal', 'theme'):
            theme = params.get('sigal', 'theme')

        self.theme_dir = os.path.join('themes', theme)
        env = Environment(loader=PackageLoader('sigal',
                                               os.path.join('..', self.theme_dir)))
        self.template = env.get_template(tpl)

    def directory_list(self):
        "get the list of directories with files of particular extensions"
        ignored = [self.thumb_dir, self.bigimg_dir] + IGNORED_DIR

        for dirpath, dirnames, filenames in os.walk(self.path):
            # filelist = [os.path.normcase(f) for f in os.listdir(dir)]
            dirpath = os.path.normpath(dirpath)
            if os.path.split(dirpath)[1] not in ignored:
                # sort images and sub-albums by name
                filenames.sort(key=str.lower)
                dirnames.sort(key=str.lower)

                self.data[dirpath] = {}
                self.data[dirpath]['img'] = [f for f in filenames \
                                             if os.path.splitext(f)[1] in self.fileExtList]
                self.data[dirpath]['subdir'] = [d for d in dirnames if d not in ignored]

    def get_meta_value(self, data):
        """
        Return the value for a line like:
           key = "value"
        """
        data = data.split('=')[1].strip()

        # Strip quotes
        if data[0] == '"':
            data = data[1:]
        if data[-1] == '"':
            data = data[:-1]

        return data

    def get_metadata(self, path):
        """
        Get album metadata from DESCRIPTION_FILE:
          - album_name
          - album_description
          - album_representative
        """
        descfile = os.path.join(path, DESCRIPTION_FILE)
        if not os.path.isfile(descfile):
            return

        with codecs.open(descfile, "r", "utf-8") as f:
            for l in f:
                if "album_name" in l:
                    self.data[path]['title'] = self.get_meta_value(l)
                if "album_description" in l:
                    self.data[path]['description'] = self.get_meta_value(l)
                if "album_representative" in l:
                    self.data[path]['representative'] = self.get_meta_value(l)

    def find_representative(self, path):
        """
        find the representative image for a given album/path
        at the moment, this is the first image found.
        """

        files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) \
                 and os.path.splitext(f)[1] in self.fileExtList]

        for f in files:
            # find and return the first landscape image
            im = Image.open(os.path.join(path, f))
            if im.size[0] > im.size[1]:
                return f

        # else simply return the 1st image
        return files[0]

    def render(self):
        """
        Render the html page
        """

        # copy static files in the output dir
        copy_tree(os.path.abspath(self.theme_dir), os.path.abspath(self.path))

        sigal_link = do_link(SIGAL_LINK, "sigal")

        self.directory_list()

        for dirpath in self.data.keys():
            # default: get title from directory name
            self.data[dirpath]['title'] = os.path.basename(dirpath).replace('_',' ').\
                                          replace('-',' ').capitalize()

            self.get_metadata(dirpath)
            # print self.data[dirpath]

        # loop on directories
        for dirpath in self.data.keys():
            theme = { 'path': os.path.relpath(self.path, dirpath) }
            home_path = os.path.join(os.path.relpath(self.path, dirpath), INDEX_PAGE)

            # paths to upper directories (with titles and links)
            tmp_path = dirpath
            paths = do_link(INDEX_PAGE, self.data[tmp_path]['title'])

            while tmp_path != self.path:
                tmp_path = os.path.normpath(os.path.join(tmp_path, '..'))
                tmp_link = os.path.relpath(tmp_path, dirpath) + "/" + INDEX_PAGE
                paths = do_link(tmp_link, self.data[tmp_path]['title']) + \
                        PATH_SEP + paths

            images = []
            for i in self.data[dirpath]['img']:
                image = {
                    'file': i,
                    'thumb': os.path.join(self.thumb_dir, self.thumb_prefix+i)
                    }
                images.append(image)

            albums = []
            for d in self.data[dirpath]['subdir']:
                alb_thumb = ''
                if self.data[dirpath].has_key('representative'):
                    alb_thumb = self.data[dirpath]['representative']

                if not alb_thumb or not os.path.isfile(alb_thumb):
                    alb_thumb = self.find_representative(os.path.join(dirpath, d))

                album = {
                    'path': os.path.join(d, INDEX_PAGE),
                    'title': self.data[os.path.join(dirpath, d)]['title'],
                    'thumb': os.path.join(d, self.thumb_dir,
                                          self.thumb_prefix+alb_thumb),
                    }
                albums.append(album)

            page = self.template.render(self.data[dirpath], paths=paths,
                                        home_path=home_path, images=images,
                                        albums=albums, theme=theme,
                                        sigal_link=sigal_link).encode('utf-8')

            # save page
            f = open(os.path.join(dirpath, INDEX_PAGE),"w")
            f.write(page)
            f.close()
<commit_msg>bugfix for representative image<commit_after>#!/usr/bin/env python2
# -*- coding:utf-8 -*-

"""
Generate html pages for each directory of images
"""

import os
import codecs
import Image
from distutils.dir_util import copy_tree
from jinja2 import Environment, PackageLoader
import sigal.image

DEFAULT_THEME = "default"
INDEX_PAGE = "index.html"
IGNORED_DIR = ['css', 'js', 'img']
DESCRIPTION_FILE = "album_description"
SIGAL_LINK = "https://github.com/saimn/sigal"
PATH_SEP = u" » "

def do_link(link, title):
    "return html link"
    return '%s' % (link, title)

class Theme():
    """ Generate html pages for each directory of images """

    def __init__(self, params, path, theme=DEFAULT_THEME, tpl=INDEX_PAGE):
        self.data = {}
        self.path = os.path.normpath(path)
        self.bigimg = params.getint('sigal', 'big_img')
        self.bigimg_dir = params.get('sigal', 'bigimg_dir')
        self.thumb_dir = params.get('sigal', 'thumb_dir')
        self.thumb_prefix = params.get('sigal', 'thumb_prefix')
        self.fileExtList = params.get('sigal', 'fileExtList').split(',')

        if params.has_option('sigal', 'theme'):
            theme = params.get('sigal', 'theme')

        self.theme_dir = os.path.join('themes', theme)
        env = Environment(loader=PackageLoader('sigal',
                                               os.path.join('..', self.theme_dir)))
        self.template = env.get_template(tpl)

    def directory_list(self):
        "get the list of directories with files of particular extensions"
        ignored = [self.thumb_dir, self.bigimg_dir] + IGNORED_DIR

        for dirpath, dirnames, filenames in os.walk(self.path):
            # filelist = [os.path.normcase(f) for f in os.listdir(dir)]
            dirpath = os.path.normpath(dirpath)
            if os.path.split(dirpath)[1] not in ignored:
                # sort images and sub-albums by name
                filenames.sort(key=str.lower)
                dirnames.sort(key=str.lower)

                self.data[dirpath] = {}
                self.data[dirpath]['img'] = [f for f in filenames \
                                             if os.path.splitext(f)[1] in self.fileExtList]
                self.data[dirpath]['subdir'] = [d for d in dirnames if d not in ignored]

    def get_meta_value(self, data):
        """
        Return the value for a line like:
           key = "value"
        """
        data = data.split('=')[1].strip()

        # Strip quotes
        if data[0] == '"':
            data = data[1:]
        if data[-1] == '"':
            data = data[:-1]

        return data

    def get_metadata(self, path):
        """
        Get album metadata from DESCRIPTION_FILE:
          - album_name
          - album_description
          - album_representative
        """
        descfile = os.path.join(path, DESCRIPTION_FILE)
        if not os.path.isfile(descfile):
            return

        with codecs.open(descfile, "r", "utf-8") as f:
            for l in f:
                if "album_name" in l:
                    self.data[path]['title'] = self.get_meta_value(l)
                if "album_description" in l:
                    self.data[path]['description'] = self.get_meta_value(l)
                if "album_representative" in l:
                    self.data[path]['representative'] = self.get_meta_value(l)

    def find_representative(self, path):
        """
        find the representative image for a given album/path
        at the moment, this is the first image found.
        """

        files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) \
                 and os.path.splitext(f)[1] in self.fileExtList]

        for f in files:
            # find and return the first landscape image
            im = Image.open(os.path.join(path, f))
            if im.size[0] > im.size[1]:
                return f

        # else simply return the 1st image
        return files[0]

    def render(self):
        """
        Render the html page
        """

        # copy static files in the output dir
        copy_tree(os.path.abspath(self.theme_dir), os.path.abspath(self.path))

        sigal_link = do_link(SIGAL_LINK, "sigal")

        self.directory_list()

        for dirpath in self.data.keys():
            # default: get title from directory name
            self.data[dirpath]['title'] = os.path.basename(dirpath).replace('_',' ').\
                                          replace('-',' ').capitalize()

            self.get_metadata(dirpath)
            # print self.data[dirpath]

        # loop on directories
        for dirpath in self.data.keys():
            theme = { 'path': os.path.relpath(self.path, dirpath) }
            home_path = os.path.join(os.path.relpath(self.path, dirpath), INDEX_PAGE)

            # paths to upper directories (with titles and links)
            tmp_path = dirpath
            paths = do_link(INDEX_PAGE, self.data[tmp_path]['title'])

            while tmp_path != self.path:
                tmp_path = os.path.normpath(os.path.join(tmp_path, '..'))
                tmp_link = os.path.relpath(tmp_path, dirpath) + "/" + INDEX_PAGE
                paths = do_link(tmp_link, self.data[tmp_path]['title']) + \
                        PATH_SEP + paths

            images = []
            for i in self.data[dirpath]['img']:
                image = {
                    'file': i,
                    'thumb': os.path.join(self.thumb_dir, self.thumb_prefix+i)
                    }
                images.append(image)

            albums = []
            for d in self.data[dirpath]['subdir']:

                dpath = os.path.join(dirpath, d)

                alb_thumb = ''
                if self.data[dpath].has_key('representative'):
                    alb_thumb = self.data[dpath]['representative']

                if not alb_thumb or \
                   not os.path.isfile(os.path.join(dpath, alb_thumb)):
                    alb_thumb = self.find_representative(dpath)

                album = {
                    'path': os.path.join(d, INDEX_PAGE),
                    'title': self.data[dpath]['title'],
                    'thumb': os.path.join(d, self.thumb_dir,
                                          self.thumb_prefix+alb_thumb),
                    }
                albums.append(album)

            page = self.template.render(self.data[dirpath], paths=paths,
                                        home_path=home_path, images=images,
                                        albums=albums, theme=theme,
                                        sigal_link=sigal_link).encode('utf-8')

            # save page
            f = open(os.path.join(dirpath, INDEX_PAGE),"w")
            f.write(page)
            f.close()
<|endoftext|>


<commit_before>from __future__ import unicode_literals
import six

from django import forms
from django.db import models
from django import forms
from django.contrib.contenttypes.generic import GenericForeignKey
from django.contrib.contenttypes.models import ContentType

from .registry import registry as default_registry
from .widgets import ChoiceWidget, MultipleChoiceWidget, TextWidget

__all__ = ['FieldBase', 'ChoiceField', 'MultipleChoiceField',
    'ModelChoiceField', 'ModelMultipleChoiceField', 'GenericModelChoiceField',
    'GenericModelMultipleChoiceField']

class FieldBase(object):
    def __init__(self, autocomplete=None, registry=None, widget=None,
            widget_js_attributes=None, autocomplete_js_attributes=None,
            extra_context=None, *args, **kwargs):

        self.autocomplete = self.get_autocomplete(autocomplete, registry,
                widget)

        widget = widget or self.widget
        if isinstance(widget, type):
            widget = widget(autocomplete, widget_js_attributes,
                    autocomplete_js_attributes, extra_context)
        kwargs['widget'] = widget

        parents = super(FieldBase, self).__self_class__.__bases__
        if ((forms.ModelChoiceField in parents or
                forms.ModelMultipleChoiceField in parents)
                and isinstance(self.autocomplete.choices, models.QuerySet)):
            kwargs['queryset'] = self.autocomplete.choices

        super(FieldBase, self).__init__(*args, **kwargs)

    def get_autocomplete(self, autocomplete, registry, widget):
        if widget:
            # BC: maybe defining the autocomplete as a widget argument ?
            autocomplete = getattr(widget, 'autocomplete', None)

        registry = registry or default_registry
        return registry.get_autocomplete_from_arg(autocomplete)

    def validate(self, value):
        """
        Wrap around Autocomplete.validate_values().
        """
        super(FieldBase, self).validate(value)

        # FIXME: we might actually want to change the Autocomplete API to
        # support python values instead of raw values, that would probably be
        # more performant.
        values = self.prepare_value(value)

        if value and not self.autocomplete(values=values).validate_values():
            raise forms.ValidationError('%s cannot validate %s' % (
                self.autocomplete.__name__, value))

class ChoiceField(FieldBase, forms.ChoiceField):
    widget = ChoiceWidget

    def __init__(self, autocomplete=None, registry=None, widget=None,
            widget_js_attributes=None, autocomplete_js_attributes=None,
            extra_context=None, *args, **kwargs):

        kwargs.update({'choices':
            self.get_choices(autocomplete, registry, widget)})

        super(ChoiceField, self).__init__(autocomplete, registry, widget,
                widget_js_attributes, autocomplete_js_attributes,
                extra_context, *args, **kwargs)

    def get_choices(self, autocomplete, registry, widget):
        a = self.get_autocomplete(autocomplete, registry, widget)()
        return ((a.choice_value(c), a.choice_label(c)) for c in a.choices)

class MultipleChoiceField(ChoiceField, forms.MultipleChoiceField):
    widget = MultipleChoiceWidget

class ModelChoiceField(FieldBase, forms.ModelChoiceField):
    widget = ChoiceWidget

class ModelMultipleChoiceField(FieldBase,
        forms.ModelMultipleChoiceField):
    widget = MultipleChoiceWidget

class GenericModelChoiceField(FieldBase, forms.Field):
    """
    Simple form field that converts strings to models.
    """
    widget = ChoiceWidget

    def prepare_value(self, value):
        """
        Given a model instance as value, with content type id of 3 and pk of 5,
        return such a string '3-5'.
        """
        if isinstance(value, six.string_types):
            # Apparently there's a bug in django, that causes a python value to
            # be passed here. This ONLY happens when in an inline ....
            return value
        elif isinstance(value, models.Model):
            return '%s-%s' % (ContentType.objects.get_for_model(value).pk,
                              value.pk)

    def to_python(self, value):
        """
        Given a string like '3-5', return the model of content type id 3 and pk
        5.
        """
        if not value:
            return value

        content_type_id, object_id = value.split('-', 1)
        try:
            content_type = ContentType.objects.get_for_id(content_type_id)
        except ContentType.DoesNotExist:
            raise forms.ValidationError('Wrong content type')
        else:
            model = content_type.model_class()

        try:
            return model.objects.get(pk=object_id)
        except model.DoesNotExist:
            raise forms.ValidationError('Wrong object id')

class GenericModelMultipleChoiceField(GenericModelChoiceField):
    """
    Simple form field that converts strings to models.
    """
    widget = MultipleChoiceWidget

    def prepare_value(self, value):
        return [super(GenericModelMultipleChoiceField, self
            ).prepare_value(v) for v in value]

    def to_python(self, value):
        return [super(GenericModelMultipleChoiceField, self).to_python(v)
            for v in value]
<commit_msg>Fixed QuerySet import<commit_after>from __future__ import unicode_literals
import six

from django import forms
from django.db import models
from django.db.models.query import QuerySet
from django import forms
from django.contrib.contenttypes.generic import GenericForeignKey
from django.contrib.contenttypes.models import ContentType

from .registry import registry as default_registry
from .widgets import ChoiceWidget, MultipleChoiceWidget, TextWidget

__all__ = ['FieldBase', 'ChoiceField', 'MultipleChoiceField',
    'ModelChoiceField', 'ModelMultipleChoiceField', 'GenericModelChoiceField',
    'GenericModelMultipleChoiceField']

class FieldBase(object):
    def __init__(self, autocomplete=None, registry=None, widget=None,
            widget_js_attributes=None, autocomplete_js_attributes=None,
            extra_context=None, *args, **kwargs):

        self.autocomplete = self.get_autocomplete(autocomplete, registry,
                widget)

        widget = widget or self.widget
        if isinstance(widget, type):
            widget = widget(autocomplete, widget_js_attributes,
                    autocomplete_js_attributes, extra_context)
        kwargs['widget'] = widget

        parents = super(FieldBase, self).__self_class__.__bases__
        if ((forms.ModelChoiceField in parents or
                forms.ModelMultipleChoiceField in parents)
                and isinstance(self.autocomplete.choices, QuerySet)):
            kwargs['queryset'] = self.autocomplete.choices

        super(FieldBase, self).__init__(*args, **kwargs)

    def get_autocomplete(self, autocomplete, registry, widget):
        if widget:
            # BC: maybe defining the autocomplete as a widget argument ?
            autocomplete = getattr(widget, 'autocomplete', None)

        registry = registry or default_registry
        return registry.get_autocomplete_from_arg(autocomplete)

    def validate(self, value):
        """
        Wrap around Autocomplete.validate_values().
        """
        super(FieldBase, self).validate(value)

        # FIXME: we might actually want to change the Autocomplete API to
        # support python values instead of raw values, that would probably be
        # more performant.
        values = self.prepare_value(value)

        if value and not self.autocomplete(values=values).validate_values():
            raise forms.ValidationError('%s cannot validate %s' % (
                self.autocomplete.__name__, value))

class ChoiceField(FieldBase, forms.ChoiceField):
    widget = ChoiceWidget

    def __init__(self, autocomplete=None, registry=None, widget=None,
            widget_js_attributes=None, autocomplete_js_attributes=None,
            extra_context=None, *args, **kwargs):

        kwargs.update({'choices':
            self.get_choices(autocomplete, registry, widget)})

        super(ChoiceField, self).__init__(autocomplete, registry, widget,
                widget_js_attributes, autocomplete_js_attributes,
                extra_context, *args, **kwargs)

    def get_choices(self, autocomplete, registry, widget):
        a = self.get_autocomplete(autocomplete, registry, widget)()
        return ((a.choice_value(c), a.choice_label(c)) for c in a.choices)

class MultipleChoiceField(ChoiceField, forms.MultipleChoiceField):
    widget = MultipleChoiceWidget

class ModelChoiceField(FieldBase, forms.ModelChoiceField):
    widget = ChoiceWidget

class ModelMultipleChoiceField(FieldBase,
        forms.ModelMultipleChoiceField):
    widget = MultipleChoiceWidget

class GenericModelChoiceField(FieldBase, forms.Field):
    """
    Simple form field that converts strings to models.
    """
    widget = ChoiceWidget

    def prepare_value(self, value):
        """
        Given a model instance as value, with content type id of 3 and pk of 5,
        return such a string '3-5'.
        """
        if isinstance(value, six.string_types):
            # Apparently there's a bug in django, that causes a python value to
            # be passed here. This ONLY happens when in an inline ....
            return value
        elif isinstance(value, models.Model):
            return '%s-%s' % (ContentType.objects.get_for_model(value).pk,
                              value.pk)

    def to_python(self, value):
        """
        Given a string like '3-5', return the model of content type id 3 and pk
        5.
        """
        if not value:
            return value

        content_type_id, object_id = value.split('-', 1)
        try:
            content_type = ContentType.objects.get_for_id(content_type_id)
        except ContentType.DoesNotExist:
            raise forms.ValidationError('Wrong content type')
        else:
            model = content_type.model_class()

        try:
            return model.objects.get(pk=object_id)
        except model.DoesNotExist:
            raise forms.ValidationError('Wrong object id')

class GenericModelMultipleChoiceField(GenericModelChoiceField):
    """
    Simple form field that converts strings to models.
    """
    widget = MultipleChoiceWidget

    def prepare_value(self, value):
        return [super(GenericModelMultipleChoiceField, self
            ).prepare_value(v) for v in value]

    def to_python(self, value):
        return [super(GenericModelMultipleChoiceField, self).to_python(v)
            for v in value]
<|endoftext|>


<commit_before>"""
Tests for the PlotFileSaver classes. Also tests analysis commands.

$Id$
"""
__version__='$Revision$'

import unittest
import os
import tempfile
import shutil

from topo.base.simulation import Simulation
from topo.base.cf import CFSheet, CFProjection
import topo.misc.filepath

from topo.sheet import GeneratorSheet

from topo.plotting.plotfilesaver import PlotGroupSaver

from topo.plotting.plotgroup import PlotGroup

from topo.command.analysis import save_plotgroup

import __main__
exec "from topo.command.analysis import *" in __main__.__dict__

class TestPlotGroupSaverBase(unittest.TestCase):

    def exists(self,name):
        self.assert_(os.path.exists(os.path.join(topo.misc.filepath.output_path,name)))

    def setUp(self):
        self.original_output_path = topo.misc.filepath.output_path
        topo.misc.filepath.output_path = tempfile.mkdtemp()
        self.sim = Simulation(register=True,name="testplotfilesaver")
        self.sim['A'] = GeneratorSheet(nominal_density=2)
        self.sim['B'] = CFSheet(nominal_density=2)
        self.sim.connect('A','B',connection_type=CFProjection,name='Afferent')

    def tearDown(self):
        shutil.rmtree(topo.misc.filepath.output_path)
        topo.misc.filepath.output_path = self.original_output_path

class TestPlotGroupSaver(TestPlotGroupSaverBase):

    def test_activity_saving(self):
        save_plotgroup('Activity')
        self.exists("testplotfilesaver_000000.00_A_Activity.png")
        self.exists("testplotfilesaver_000000.00_B_Activity.png")

    def test_orientation_preference_saving(self):
        save_plotgroup('Orientation Preference')
        self.exists("testplotfilesaver_000000.00_B_Orientation_Preference.png")
        self.exists("testplotfilesaver_000000.00_B_Orientation_PreferenceAndSelectivity.png")
        self.exists("testplotfilesaver_000000.00_B_Orientation_Selectivity.png")
        self.exists("testplotfilesaver_000000.00__Color_Key.png")

    def test_cf_saving(self):
        save_plotgroup("Connection Fields",sheet=self.sim['B'])
        self.exists("testplotfilesaver_000000.00_Afferent_(from_A).png")

class TestCFProjectionPlotGroupSaver(TestPlotGroupSaverBase):

    def test_cfprojection_saving(self):
        save_plotgroup('Projection',
                       projection=self.sim['B'].projections('Afferent'))
        self.exists("testplotfilesaver_000000.00_B_Afferent.png")

###########################################################

cases = [TestPlotGroupSaver,TestCFProjectionPlotGroupSaver]

suite = unittest.TestSuite()
suite.addTests([unittest.makeSuite(case) for case in cases])

if __name__ == '__main__':
    unittest.TextTestRunner(verbosity=2).run(suite)

<commit_msg>Improved error message.<commit_after>"""
Tests for the PlotFileSaver classes. Also tests analysis commands.

$Id$
"""
__version__='$Revision$'

import unittest
import os
import tempfile
import shutil
import glob

from topo.base.simulation import Simulation
from topo.base.cf import CFSheet, CFProjection
import topo.misc.filepath

from topo.sheet import GeneratorSheet

from topo.plotting.plotfilesaver import PlotGroupSaver

from topo.plotting.plotgroup import PlotGroup

from topo.command.analysis import save_plotgroup

import __main__
exec "from topo.command.analysis import *" in __main__.__dict__

class TestPlotGroupSaverBase(unittest.TestCase):

    def exists(self,name):
        target = os.path.join(topo.misc.filepath.output_path,name)
        files = glob.glob(os.path.join(topo.misc.filepath.output_path,"*"))
        self.assert_(os.path.exists(target),
                     "'%s' not among '%s'"%(target,files))

    def setUp(self):
        self.original_output_path = topo.misc.filepath.output_path
        topo.misc.filepath.output_path = tempfile.mkdtemp()
        self.sim = Simulation(register=True,name="testplotfilesaver")
        self.sim['A'] = GeneratorSheet(nominal_density=2)
        self.sim['B'] = CFSheet(nominal_density=2)
        self.sim.connect('A','B',connection_type=CFProjection,name='Afferent')

    def tearDown(self):
        shutil.rmtree(topo.misc.filepath.output_path)
        topo.misc.filepath.output_path = self.original_output_path

class TestPlotGroupSaver(TestPlotGroupSaverBase):

    def test_activity_saving(self):
        save_plotgroup('Activity')
        self.exists("testplotfilesaver_000000.00_A_Activity.png")
        self.exists("testplotfilesaver_000000.00_B_Activity.png")

    def test_orientation_preference_saving(self):
        save_plotgroup('Orientation Preference')
        self.exists("testplotfilesaver_000000.00_B_Orientation_Preference.png")
        self.exists("testplotfilesaver_000000.00_B_Orientation_PreferenceAndSelectivity.png")
        self.exists("testplotfilesaver_000000.00_B_Orientation_Selectivity.png")
        self.exists("testplotfilesaver_000000.00__Color_Key.png")

    def test_cf_saving(self):
        save_plotgroup("Connection Fields",sheet=self.sim['B'])
        self.exists("testplotfilesaver_000000.00_Afferent_(from_A).png")

class TestCFProjectionPlotGroupSaver(TestPlotGroupSaverBase):

    def test_cfprojection_saving(self):
        save_plotgroup('Projection',
                       projection=self.sim['B'].projections('Afferent'))
        self.exists("testplotfilesaver_000000.00_B_Afferent.png")

###########################################################

cases = [TestPlotGroupSaver,TestCFProjectionPlotGroupSaver]

suite = unittest.TestSuite()
suite.addTests([unittest.makeSuite(case) for case in cases])

if __name__ == '__main__':
    unittest.TextTestRunner(verbosity=2).run(suite)

<|endoftext|>


<commit_before>from __future__ import unicode_literals, division, print_function, absolute_import
from builtins import range, object
import numpy as np
np.random.seed(28)
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data as tdata

class Dense(nn.Linear):
    """
    Shorthand for a nn.Linear layer with an activation function

    Args:
        in_dim (int): number of input feature
        out_dim (int): number of output features
        bias (bool): If set to False, the layer will not adapt the bias. (default: True)
        activation (callable): activation function or string (default: None)
    """
    def __init__(self, in_dim, out_dim, bias=True, activation=None):
        activation_map = {"tanh": F.tanh,
                          "sigmoid": F.sigmoid,
                          "relu": F.relu}
        if activation in activation_map:
            activation = activation_map[activation]
        self.activation = activation
        super(Dense, self).__init__(in_dim, out_dim, bias)

    def forward(self, inputs):
        y = super(Dense, self).forward(inputs)
        if self.activation:
            y = self.activation(y)
        return y

class FFNet(nn.Module):

    def __init__(self, in_dim, out_dim, hidden_layers=[]):
        """
        Neural network PyTorch model; shortcut for creating a feed forward NN that can be used as an in_net for a SimEcModel

        Input:
            - in_dim: input dimensionality
            - out_dim: output dimensionality
            - hidden_layers: list with tuples of (number of hidden units [int], activation function [string or function])
        """
        super(FFNet, self).__init__()
        # get a list of layer dimensions: in_dim --> hl --> out_dim
        dimensions = [in_dim]
        dimensions.extend([h[0] for h in hidden_layers])
        dimensions.append(out_dim)
        # get list of activation functions (output (i.e. embedding) layer has no activation)
        activations = [h[1] for h in hidden_layers]
        activations.append(None)
        # initialize dense layers
        layers = [Dense(dimensions[i], dimensions[i+1], activation=activations[i]) for i in range(len(activations))]
        # construct feed forward network
        self.net = nn.Sequential(*layers)

    def forward(self, inputs):
        return self.net(inputs)

class SimEcModel(nn.Module):

    def __init__(self, in_net, embedding_dim, out_dim):
        """
        Similarity Encoder (SimEc) neural network PyTorch model

        Input:
            - in_net: input network mapping from whatever original input to the embedding (e.g. a FFNet)
            - embedding_dim: dimensionality of the embedding layer
            - out_dim: dimensionality of the output / number of targets
        """
        super(SimEcModel, self).__init__()
        # the simec model is the in_net, which creates the embedding,
        self.embedding_net = in_net
        # plus a last layer to compute the similarity approximation
        self.W_ll = Dense(embedding_dim, out_dim, bias=False)

    def forward(self, inputs):
        x = self.embedding_net(inputs)
        x = self.W_ll(x)
        return x

class SimilarityEncoder(object):

    def __init__(self, in_net, embedding_dim, out_dim, **kwargs):
        """
        Similarity Encoder (SimEc) neural network model wrapper

        Input:
            - in_net: either a NN model that maps from the input to the embedding OR the dimensionality of
                      the input feature vector (int), in which case a FFNet will be created by supplying
                      the kwargs, which should probably contain a "hidden_layers" argument
            - embedding_dim: dimensionality of the embedding layer
            - out_dim: dimensionality of the output / number of targets
        """
        if isinstance(in_net, int):
            in_net = FFNet(in_net, embedding_dim, **kwargs)
        self.model = SimEcModel(in_net, embedding_dim, out_dim)
        self.device = "cpu"  # by default, before training, the model is on the cpu

    def fit(self, X, S, epochs=25, batch_size=32, lr=0.0005, weight_decay=0., s_ll_reg=0., S_ll=None, orth_reg=0.):
        """
        Train the SimEc model

        Input:
            - X: n x in_dim feature matrix
            - S: n x out_dim target similarity matrix
            - epochs: int, number of epochs to train (default: 25)
            - batch_size: int, number of samples per batch (default: 32)
            - lr: float used as the learning rate for the Adam optimizer (default: lr=0.0005)
            - s_ll_reg: float, regularization strength for (S - W_ll^T W_ll), i.e. how much the dot product of the
                        last layer weights should approximate the target similarities; useful when factoring a square symmetric
                        similarity matrix. (default: 0.; if > 0. need to give S_ll)
            - S_ll: matrix that the dot product of the last layer should approximate (see above), needs to be (out_dim x out_dim)
        """
        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        if not self.device == "cpu":
            self.model.to(self.device)

        if s_ll_reg > 0:
            if S_ll is None:
                print("Warning: need to give S_ll if s_ll_reg > 0.")
                s_ll_reg = 0.
            else:
                S_ll = torch.from_numpy(S_ll).float()
                if not self.device == "cpu":
                    S_ll = S_ll.to(self.device)
        if orth_reg > 0:
            edim = self.model.W_ll.weight.size()[1]
            Ones = torch.from_numpy((np.ones((edim, edim)) - np.eye(edim))).float()
            if not self.device == "cpu":
                Ones = Ones.to(self.device)

        criterion = nn.MSELoss()
        optimizer = optim.Adam(self.model.parameters(), lr=lr, weight_decay=weight_decay)

        trainloader = tdata.DataLoader(tdata.TensorDataset(torch.from_numpy(X).float(), torch.from_numpy(S).float()),
                                       batch_size=batch_size, shuffle=False)
        # loop over the dataset multiple times
        for epoch in range(epochs):

            running_loss = 0.0
            for i, data in enumerate(trainloader, 0):
                # get the inputs
                x_batch, s_batch = data
                if not self.device == "cpu":
                    x_batch, s_batch = x_batch.to(self.device), s_batch.to(self.device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward + backward + optimize
                outputs = self.model(x_batch)
                loss = criterion(outputs, s_batch)
                running_loss += loss.item()
                # possibly some regularization stuff based on the last layer
                # CAREFUL! linear layer weights are stored with dimensions (out_dim, in_dim) instead of
                # how you initialized them with (in_dim, out_dim), therefore the transpose is mixed up here!
                if s_ll_reg > 0:
                    loss += s_ll_reg*criterion(torch.mm(self.model.W_ll.weight, self.model.W_ll.weight.t()), S_ll)
                if orth_reg > 0:
                    loss += orth_reg * torch.mean((Ones * torch.mm(self.model.W_ll.weight.t(), self.model.W_ll.weight))**2)
                loss.backward()
                optimizer.step()

            print('[epoch %d] loss: %.7f' % (epoch + 1, running_loss / (i + 1)))

    def transform(self, X):
        """
        Project the input feature vectors to the embedding space

        Input:
            - X: m x in_dim feature matrix

        Returns:
            - Y: m x embedding_dim embedding matrix
        """
        X = torch.from_numpy(X).float()
        if not self.device == "cpu":
            X = X.to(self.device)
        with torch.no_grad():
            Y = self.model.embedding_net(X)
            if not self.device == "cpu":
                Y = Y.cpu()
        return Y.numpy()

    def predict(self, X):
        """
        Generate the output of the network, i.e. the predicted similarities

        Input:
            - X: m x in_dim feature matrix

        Returns:
            - S': m x out_dim output matrix with approximated similarities to the out_dim targets
        """
        X = torch.from_numpy(X).float()
        if not self.device == "cpu":
            X = X.to(self.device)
        with torch.no_grad():
            S = self.model(X)
            if not self.device == "cpu":
                S = S.cpu()
        return S.numpy()
<commit_msg>make torch simec version 4.1 compatible to remove deprecation warnings<commit_after>from __future__ import unicode_literals, division, print_function, absolute_import
from builtins import range, object
import numpy as np
np.random.seed(28)
import torch
import torch.nn as nn
import torch.optim as optim
import torch.utils.data as tdata

class Dense(nn.Linear):
    """
    Shorthand for a nn.Linear layer with an activation function

    Args:
        in_dim (int): number of input feature
        out_dim (int): number of output features
        bias (bool): If set to False, the layer will not adapt the bias. (default: True)
        activation (callable): activation function or string (default: None)
    """
    def __init__(self, in_dim, out_dim, bias=True, activation=None):
        activation_map = {"tanh": torch.tanh,
                          "sigmoid": torch.sigmoid,
                          "relu": torch.relu}
        if activation in activation_map:
            activation = activation_map[activation]
        self.activation = activation
        super(Dense, self).__init__(in_dim, out_dim, bias)

    def forward(self, inputs):
        y = super(Dense, self).forward(inputs)
        if self.activation:
            y = self.activation(y)
        return y

class FFNet(nn.Module):

    def __init__(self, in_dim, out_dim, hidden_layers=[]):
        """
        Neural network PyTorch model; shortcut for creating a feed forward NN that can be used as an in_net for a SimEcModel

        Input:
            - in_dim: input dimensionality
            - out_dim: output dimensionality
            - hidden_layers: list with tuples of (number of hidden units [int], activation function [string or function])
        """
        super(FFNet, self).__init__()
        # get a list of layer dimensions: in_dim --> hl --> out_dim
        dimensions = [in_dim]
        dimensions.extend([h[0] for h in hidden_layers])
        dimensions.append(out_dim)
        # get list of activation functions (output (i.e. embedding) layer has no activation)
        activations = [h[1] for h in hidden_layers]
        activations.append(None)
        # initialize dense layers
        layers = [Dense(dimensions[i], dimensions[i+1], activation=activations[i]) for i in range(len(activations))]
        # construct feed forward network
        self.net = nn.Sequential(*layers)

    def forward(self, inputs):
        return self.net(inputs)

class SimEcModel(nn.Module):

    def __init__(self, in_net, embedding_dim, out_dim):
        """
        Similarity Encoder (SimEc) neural network PyTorch model

        Input:
            - in_net: input network mapping from whatever original input to the embedding (e.g. a FFNet)
            - embedding_dim: dimensionality of the embedding layer
            - out_dim: dimensionality of the output / number of targets
        """
        super(SimEcModel, self).__init__()
        # the simec model is the in_net, which creates the embedding,
        self.embedding_net = in_net
        # plus a last layer to compute the similarity approximation
        self.W_ll = Dense(embedding_dim, out_dim, bias=False)

    def forward(self, inputs):
        x = self.embedding_net(inputs)
        x = self.W_ll(x)
        return x

class SimilarityEncoder(object):

    def __init__(self, in_net, embedding_dim, out_dim, **kwargs):
        """
        Similarity Encoder (SimEc) neural network model wrapper

        Input:
            - in_net: either a NN model that maps from the input to the embedding OR the dimensionality of
                      the input feature vector (int), in which case a FFNet will be created by supplying
                      the kwargs, which should probably contain a "hidden_layers" argument
            - embedding_dim: dimensionality of the embedding layer
            - out_dim: dimensionality of the output / number of targets
        """
        if isinstance(in_net, int):
            in_net = FFNet(in_net, embedding_dim, **kwargs)
        self.model = SimEcModel(in_net, embedding_dim, out_dim)
        self.device = "cpu"  # by default, before training, the model is on the cpu

    def fit(self, X, S, epochs=25, batch_size=32, lr=0.0005, weight_decay=0., s_ll_reg=0., S_ll=None, orth_reg=0.):
        """
        Train the SimEc model

        Input:
            - X: n x in_dim feature matrix
            - S: n x out_dim target similarity matrix
            - epochs: int, number of epochs to train (default: 25)
            - batch_size: int, number of samples per batch (default: 32)
            - lr: float used as the learning rate for the Adam optimizer (default: lr=0.0005)
            - s_ll_reg: float, regularization strength for (S - W_ll^T W_ll), i.e. how much the dot product of the
                        last layer weights should approximate the target similarities; useful when factoring a square symmetric
                        similarity matrix. (default: 0.; if > 0. need to give S_ll)
            - S_ll: matrix that the dot product of the last layer should approximate (see above), needs to be (out_dim x out_dim)
        """
        self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        if not self.device == "cpu":
            self.model.to(self.device)

        if s_ll_reg > 0:
            if S_ll is None:
                print("Warning: need to give S_ll if s_ll_reg > 0.")
                s_ll_reg = 0.
            else:
                S_ll = torch.from_numpy(S_ll).float()
                if not self.device == "cpu":
                    S_ll = S_ll.to(self.device)
        if orth_reg > 0:
            edim = self.model.W_ll.weight.size()[1]
            Ones = torch.from_numpy((np.ones((edim, edim)) - np.eye(edim))).float()
            if not self.device == "cpu":
                Ones = Ones.to(self.device)

        criterion = nn.MSELoss()
        optimizer = optim.Adam(self.model.parameters(), lr=lr, weight_decay=weight_decay)

        trainloader = tdata.DataLoader(tdata.TensorDataset(torch.from_numpy(X).float(), torch.from_numpy(S).float()),
                                       batch_size=batch_size, shuffle=False)
        # loop over the dataset multiple times
        for epoch in range(epochs):

            running_loss = 0.0
            for i, data in enumerate(trainloader, 0):
                # get the inputs
                x_batch, s_batch = data
                if not self.device == "cpu":
                    x_batch, s_batch = x_batch.to(self.device), s_batch.to(self.device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward + backward + optimize
                outputs = self.model(x_batch)
                loss = criterion(outputs, s_batch)
                running_loss += loss.item()
                # possibly some regularization stuff based on the last layer
                # CAREFUL! linear layer weights are stored with dimensions (out_dim, in_dim) instead of
                # how you initialized them with (in_dim, out_dim), therefore the transpose is mixed up here!
                if s_ll_reg > 0:
                    loss += s_ll_reg*criterion(torch.mm(self.model.W_ll.weight, self.model.W_ll.weight.t()), S_ll)
                if orth_reg > 0:
                    loss += orth_reg * torch.mean((Ones * torch.mm(self.model.W_ll.weight.t(), self.model.W_ll.weight))**2)
                loss.backward()
                optimizer.step()

            print('[epoch %d] loss: %.7f' % (epoch + 1, running_loss / (i + 1)))

    def transform(self, X):
        """
        Project the input feature vectors to the embedding space

        Input:
            - X: m x in_dim feature matrix

        Returns:
            - Y: m x embedding_dim embedding matrix
        """
        X = torch.from_numpy(X).float()
        if not self.device == "cpu":
            X = X.to(self.device)
        with torch.no_grad():
            Y = self.model.embedding_net(X)
            if not self.device == "cpu":
                Y = Y.cpu()
        return Y.numpy()

    def predict(self, X):
        """
        Generate the output of the network, i.e. the predicted similarities

        Input:
            - X: m x in_dim feature matrix

        Returns:
            - S': m x out_dim output matrix with approximated similarities to the out_dim targets
        """
        X = torch.from_numpy(X).float()
        if not self.device == "cpu":
            X = X.to(self.device)
        with torch.no_grad():
            S = self.model(X)
            if not self.device == "cpu":
                S = S.cpu()
        return S.numpy()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from functools import partial
from copy import copy
from numpy import *
import numpy as np_orig
import operator as op
from autograd.core import primitive, Node, log, getval

# ----- Objects in numpy.__dict__ not imported by * -----

int     = np_orig.int
unicode = np_orig.unicode
complex = np_orig.complex
long    = np_orig.long
abs     = np_orig.abs
bool    = np_orig.bool
float   = np_orig.float
max     = np_orig.max
object  = np_orig.object
min     = np_orig.min
str     = np_orig.str
round   = np_orig.round

# ----- Broadcasting logic -----

isarray = lambda x : isinstance(getval(x), ndarray)
isfloat = lambda x : isinstance(getval(x), float)

def unbroadcast(ans, x, y, funs):
    return [unbroadcast_fun(ans, x, funs[0]),
            unbroadcast_fun(ans, y, funs[1])]

def unbroadcast_fun(ans, x, fun):
    if isfloat(x) and isarray(ans):
        return lambda g : sum(fun(g))
    elif isarray(x):
        shape = x.shape
        def new_fun(g):
            result = fun(g)
            while result.ndim > len(shape):
                result = sum(result, axis=0)
            for axis, size in enumerate(shape):
                if size is 1:
                    result = sum(result, axis=axis, keepdims=True)
            assert result.shape == shape
            return result
        return new_fun
    else:
        return fun

def make_grad_np_sum(ans, x, axis=None, keepdims=False):
    if not isarray(x):
        return [I]
    shape = x.shape
    if axis is None:
        return [lambda g : full(shape, g)]
    else:
        if keepdims:
            return [lambda g : repeat(g, shape[axis], axis)]
        else:
            return [lambda g : repeat(expand_dims(g, axis),
                                      shape[axis], axis)]

# Numpy doesn't support keepdims for subclasses so this is the workaround
def keep_keepdims(fun, funname):
    def new_fun(*args, **kwargs):
        x = args[0]
        return getattr(x, funname)(*args[1:], **kwargs) if isinstance(x, np_orig.ndarray) else x
    return new_fun
sum = primitive(keep_keepdims(sum, 'sum'), make_grad_np_sum)

def make_grad_np_mean(ans, x, axis=None, keepdims=False):
    if not isarray(x):
        return [I]
    shape = x.shape
    if axis is None:
        return [lambda g : full(shape, g) / prod(shape)]
    else:
        if keepdims:
            return [lambda g : repeat(g, shape[axis], axis) / shape[axis]]
        else:
            return [lambda g : repeat(expand_dims(g, axis),
                                      shape[axis], axis) / shape[axis]]
mean = primitive(keep_keepdims(mean, 'mean'), make_grad_np_mean)

# ----- Slightly modified version of ndarray -----

P = primitive
I = lambda x : x # Identity operator

grad_neg = lambda ans, x    : [op.neg]
grad_add = lambda ans, x, y : unbroadcast(ans, x, y, [I, I])
grad_mul = lambda ans, x, y : unbroadcast(ans, x, y, [lambda g : y * g, lambda g : x * g])
grad_sub = lambda ans, x, y : unbroadcast(ans, x, y, [I, op.neg])
grad_div = lambda ans, x, y : unbroadcast(ans, x, y, [lambda g : g / y, lambda g : - g * x / y**2])
grad_pow = lambda ans, x, y : unbroadcast(ans, x, y, [lambda g : g * y * x ** (y - 1),
                                                      lambda g : g * log(x) * x ** y])
grad_log = lambda ans, x    : [lambda g : g / x]

def reverse_args(fun):
    def new_fun(ans, x, y):
        return fun(ans, y, x)[::-1]
    return new_fun

def make_grad_take(ans, A, idx):
    shape = A.shape
    return [lambda g : untake(g, idx, shape)]

class ndarray(np_orig.ndarray):
    # Wrap binary ops since the other operand could be a Node
    __add__  = P(np_orig.ndarray.__add__ , grad_add)
    __sub__  = P(np_orig.ndarray.__sub__,  grad_sub)
    __mul__  = P(np_orig.ndarray.__mul__,  grad_mul)
    __pow__  = P(np_orig.ndarray.__pow__,  grad_pow)
    __div__  = P(np_orig.ndarray.__div__,  grad_div)
    __radd__ = P(np_orig.ndarray.__radd__, reverse_args(grad_add))
    __rsub__ = P(np_orig.ndarray.__rsub__, reverse_args(grad_sub))
    __rmul__ = P(np_orig.ndarray.__rmul__, reverse_args(grad_mul))
    __rpow__ = P(np_orig.ndarray.__rpow__, reverse_args(grad_pow))
    __rdiv__ = P(np_orig.ndarray.__rdiv__, reverse_args(grad_div))

# ----- Sparse array -----

class SparseArray(object):
    __array_priority__ = 50.0
    def __init__(self, shape, idx, val):
        self.shape = shape
        self.idx = idx
        self.val = val

    def __add__(self, other):
        array = zeros(self.shape) if other is 0 else copy(other)
        array[self.idx] += self.val
        return array

    def __radd__(self, other):
        return self + other

class SparseArrayNode(Node):
    pass
Node.type_mappings[SparseArray] = SparseArrayNode

take = lambda A, idx : A[idx]
def make_grad_take(ans, A, idx):
    shape = A.shape
    return [lambda g : untake(g, idx, shape)]
take = primitive(take, make_grad_take)

untake = lambda x, idx, shape : SparseArray(shape, idx, x)
untake = primitive(untake, lambda ans, x, idx, shape : [lambda g : take(g, idx)])

def wrap_output(fun):
    def wrapped_fun(*args, **kwargs):
        ans = fun(*args, **kwargs)
        if isinstance(ans, np_orig.ndarray):
            ans = ans.view(ndarray)
        return ans
    return wrapped_fun
zeros = wrap_output(zeros)
ones = wrap_output(ones)

# ----- Numpy gradients -----

W = wrap_output
P = primitive
isarray = lambda x : isinstance(getval(x), ndarray)
I = lambda x : x

abs    = P(abs,    lambda ans, x : [lambda g : sign(x) * g])
exp    = P(exp,    lambda ans, x : [lambda g : ans * g])
sin    = P(sin,    lambda ans, x : [lambda g : g * cos(x)])
cos    = P(cos,    lambda ans, x : [lambda g : - g * sin(x)])
tan    = P(tan,    lambda ans, x : [lambda g : g / cos(x) **2])
sinh   = P(sinh,   lambda ans, x : [lambda g : g * cosh(x)])
cosh   = P(cosh,   lambda ans, x : [lambda g : g * sinh(x)])
tanh   = P(tanh,   lambda ans, x : [lambda g : g / cosh(x) **2])
square = P(square, lambda ans, x : [lambda g : g * 2 * x])
sqrt   = P(sqrt,   lambda ans, x : [lambda g : g * 0.5 * x**-0.5])
sign   = P(sign,   lambda ans, x : [lambda g : 0.0])
full   = P(W(full),   lambda ans, shape, fill_value : [None, lambda g :  sum(g)])
reshape  = P(reshape, lambda ans, x, shape, order=None : [lambda g : reshape(g, x.shape, order=order)])
ravel    = P(W(ravel), lambda ans, x, order=None    : [lambda g : reshape(g, x.shape, order=order)])
expand_dims = P(W(expand_dims), lambda ans, x, axis : [lambda g : squeeze(g, axis)])
squeeze     = P(squeeze,        lambda ans, x, axis : [lambda g : repeat(g, x.shape[axis], axis)])
repeat      = P(repeat,      lambda ans, x, shape, axis  : [lambda g : sum(g, axis, keepdims=True)])
transpose   = P(transpose,   lambda ans, x               : [lambda g : transpose(g)])
split       = P(split,       lambda ans, x, idxs, axis=0 : [lambda g : concatenate(g, axis=axis)])
diag        = P(W(diag),     lambda ans, x               : [lambda g : diag(g)])
trace       = P(trace,       lambda ans, x               : [lambda g : g * eye(x.shape[0])])

# ----- Subtler gradients -----

def make_grad_np_max(ans, x):
    def gradfun(g):
        idxs = argmax(getval(x))
        shape = x.shape
        return untake(g, unravel_index(idxs, shape), shape)
    return [gradfun]
max = P(max, make_grad_np_max)

def make_grad_np_dot(ans, A, B):
    def grad_np_dot_A(g):
        if B.ndim is 2:
            return dot(g, B.T)
        elif A.ndim is 2:
            return outer(g, B)
        else:
            return g * B
    def grad_np_dot_B(g):
        if A.ndim is 2:
            return dot(A.T, g)
        elif B.ndim is 2:
            return outer(A, g)
        else:
            return g * A
    return [grad_np_dot_A, grad_np_dot_B]
dot = P(dot, make_grad_np_dot)

concatenate_orig = concatenate
def concatenate_args(axis, *args):
    return concatenate_orig(args, axis)
def make_grad_concatenate_args(ans, axis, *args):
    high = 0
    gradfuns = [None]
    for a in args:
        low = high
        high += a.shape[axis]
        idxs = [slice(None)] * ans.ndim
        idxs[axis] = slice(low, high)
        gradfuns.append(partial(take, idx=idxs))
    return gradfuns
concatenate_args = P(W(concatenate_args), make_grad_concatenate_args)
concatenate = lambda arr_list, axis=0 : concatenate_args(axis, *arr_list)

# ----- Node version of ndarray -----

class ArrayNode(Node):
    # Constants w.r.t float data just pass though
    @property
    def shape(self): return self.value.shape
    @property
    def ndim(self): return self.value.ndim
    @property
    def size(self): return self.value.size

    # Differentiable unary methods just apply to self
    squeeze = squeeze
    ravel = ravel
    reshape = reshape
    sum = sum
    mean = mean

    @property
    def T(self): return transpose(self)
    __neg__ = P(op.neg,  grad_neg)

    # Binary ops already wrapped by autograd.numpy.ndarray
    __add__  = ndarray.__add__.__func__
    __sub__  = ndarray.__sub__.__func__
    __mul__  = ndarray.__mul__.__func__
    __pow__  = ndarray.__pow__.__func__
    __div__  = ndarray.__div__.__func__
    __radd__ = ndarray.__radd__.__func__
    __rsub__ = ndarray.__rsub__.__func__
    __rmul__ = ndarray.__rmul__.__func__
    __rpow__ = ndarray.__rpow__.__func__
    __rdiv__ = ndarray.__rdiv__.__func__
Node.type_mappings[ndarray] = ArrayNode
<commit_msg>All tests passing with new factoring!<commit_after>from __future__ import absolute_import
from functools import partial
from numpy import *
from copy import copy
import numpy as np_orig
import operator as op
from autograd.core import primitive, Node, log, getval

# ----- Objects in numpy.__dict__ not imported by * -----

int     = np_orig.int
unicode = np_orig.unicode
complex = np_orig.complex
long    = np_orig.long
abs     = np_orig.abs
bool    = np_orig.bool
float   = np_orig.float
max     = np_orig.max
object  = np_orig.object
min     = np_orig.min
str     = np_orig.str
round   = np_orig.round

# ----- Broadcasting logic -----

isarray = lambda x : isinstance(getval(x), ndarray)
isfloat = lambda x : isinstance(getval(x), float)

def unbroadcast(ans, x, y, funs):
    return [unbroadcast_fun(ans, x, funs[0]),
            unbroadcast_fun(ans, y, funs[1])]

def unbroadcast_fun(ans, x, fun):
    if isfloat(x) and isarray(ans):
        return lambda g : sum(fun(g))
    elif isarray(x):
        shape = x.shape
        def new_fun(g):
            result = fun(g)
            while result.ndim > len(shape):
                result = sum(result, axis=0)
            for axis, size in enumerate(shape):
                if size is 1:
                    result = sum(result, axis=axis, keepdims=True)
            assert result.shape == shape
            return result
        return new_fun
    else:
        return fun

def make_grad_np_sum(ans, x, axis=None, keepdims=False):
    if not isarray(x):
        return [I]
    shape = x.shape
    if axis is None:
        return [lambda g : full(shape, g)]
    else:
        if keepdims:
            return [lambda g : repeat(g, shape[axis], axis)]
        else:
            return [lambda g : repeat(expand_dims(g, axis),
                                      shape[axis], axis)]

# Numpy doesn't support keepdims for subclasses so this is the workaround
def keep_keepdims(fun, funname):
    def new_fun(*args, **kwargs):
        x = args[0]
        return getattr(x, funname)(*args[1:], **kwargs) if isinstance(x, np_orig.ndarray) else x
    return new_fun
sum = primitive(keep_keepdims(sum, 'sum'), make_grad_np_sum)

def make_grad_np_mean(ans, x, axis=None, keepdims=False):
    if not isarray(x):
        return [I]
    shape = x.shape
    if axis is None:
        return [lambda g : full(shape, g) / prod(shape)]
    else:
        if keepdims:
            return [lambda g : repeat(g, shape[axis], axis) / shape[axis]]
        else:
            return [lambda g : repeat(expand_dims(g, axis),
                                      shape[axis], axis) / shape[axis]]
mean = primitive(keep_keepdims(mean, 'mean'), make_grad_np_mean)

# ----- Slightly modified version of ndarray -----

P = primitive
I = lambda x : x # Identity operator

grad_neg = lambda ans, x    : [op.neg]
grad_add = lambda ans, x, y : unbroadcast(ans, x, y, [I, I])
grad_mul = lambda ans, x, y : unbroadcast(ans, x, y, [lambda g : y * g, lambda g : x * g])
grad_sub = lambda ans, x, y : unbroadcast(ans, x, y, [I, op.neg])
grad_div = lambda ans, x, y : unbroadcast(ans, x, y, [lambda g : g / y, lambda g : - g * x / y**2])
grad_pow = lambda ans, x, y : unbroadcast(ans, x, y, [lambda g : g * y * x ** (y - 1),
                                                      lambda g : g * log(x) * x ** y])
grad_log = lambda ans, x    : [lambda g : g / x]

def reverse_args(fun):
    def new_fun(ans, x, y):
        return fun(ans, y, x)[::-1]
    return new_fun

def make_grad_take(ans, A, idx):
    shape = A.shape
    return [lambda g : untake(g, idx, shape)]

class ndarray(np_orig.ndarray):
    # Wrap binary ops since the other operand could be a Node
    __add__  = P(np_orig.ndarray.__add__ , grad_add)
    __sub__  = P(np_orig.ndarray.__sub__,  grad_sub)
    __mul__  = P(np_orig.ndarray.__mul__,  grad_mul)
    __pow__  = P(np_orig.ndarray.__pow__,  grad_pow)
    __div__  = P(np_orig.ndarray.__div__,  grad_div)
    __radd__ = P(np_orig.ndarray.__radd__, reverse_args(grad_add))
    __rsub__ = P(np_orig.ndarray.__rsub__, reverse_args(grad_sub))
    __rmul__ = P(np_orig.ndarray.__rmul__, reverse_args(grad_mul))
    __rpow__ = P(np_orig.ndarray.__rpow__, reverse_args(grad_pow))
    __rdiv__ = P(np_orig.ndarray.__rdiv__, reverse_args(grad_div))

take = lambda A, idx : A[idx]
def make_grad_take(ans, A, idx):
    shape = A.shape
    return [lambda g : untake(g, idx, shape)]
take = primitive(take, make_grad_take)

untake = lambda x, idx, shape : SparseArray(shape, idx, x)
untake = primitive(untake, lambda ans, x, idx, shape : [lambda g : take(g, idx)])

def wrap_output(fun):
    def wrapped_fun(*args, **kwargs):
        ans = fun(*args, **kwargs)
        if isinstance(ans, np_orig.ndarray):
            ans = ans.view(ndarray)
        return ans
    return wrapped_fun
zeros = wrap_output(zeros)
ones = wrap_output(ones)
eye = wrap_output(eye)

# ----- Numpy gradients -----

W = wrap_output
P = primitive
isarray = lambda x : isinstance(getval(x), ndarray)
I = lambda x : x

abs    = P(abs,    lambda ans, x : [lambda g : sign(x) * g])
exp    = P(exp,    lambda ans, x : [lambda g : ans * g])
sin    = P(sin,    lambda ans, x : [lambda g : g * cos(x)])
cos    = P(cos,    lambda ans, x : [lambda g : - g * sin(x)])
tan    = P(tan,    lambda ans, x : [lambda g : g / cos(x) **2])
sinh   = P(sinh,   lambda ans, x : [lambda g : g * cosh(x)])
cosh   = P(cosh,   lambda ans, x : [lambda g : g * sinh(x)])
tanh   = P(tanh,   lambda ans, x : [lambda g : g / cosh(x) **2])
square = P(square, lambda ans, x : [lambda g : g * 2 * x])
sqrt   = P(sqrt,   lambda ans, x : [lambda g : g * 0.5 * x**-0.5])
sign   = P(sign,   lambda ans, x : [lambda g : 0.0])
full   = P(W(full),   lambda ans, shape, fill_value : [None, lambda g :  sum(g)])
reshape  = P(reshape, lambda ans, x, shape, order=None : [lambda g : reshape(g, x.shape, order=order)])
ravel    = P(W(ravel), lambda ans, x, order=None    : [lambda g : reshape(g, x.shape, order=order)])
expand_dims = P(W(expand_dims), lambda ans, x, axis : [lambda g : squeeze(g, axis)])
squeeze     = P(squeeze,        lambda ans, x, axis : [lambda g : repeat(g, x.shape[axis], axis)])
repeat      = P(repeat,      lambda ans, x, shape, axis  : [lambda g : sum(g, axis, keepdims=True)])
transpose   = P(transpose,   lambda ans, x               : [lambda g : transpose(g)])
split       = P(split,       lambda ans, x, idxs, axis=0 : [lambda g : concatenate(g, axis=axis)])
diag        = P(W(diag),     lambda ans, x               : [lambda g : diag(g)])
trace       = P(trace,       lambda ans, x               : [lambda g : g * eye(x.shape[0])])

# ----- Subtler gradients -----

def make_grad_np_max(ans, x):
    def gradfun(g):
        idxs = argmax(getval(x))
        shape = x.shape
        return untake(g, unravel_index(idxs, shape), shape)
    return [gradfun]
max = P(max, make_grad_np_max)

def make_grad_np_dot(ans, A, B):
    def grad_np_dot_A(g):
        if B.ndim is 2:
            return dot(g, B.T)
        elif A.ndim is 2:
            return outer(g, B).view(ndarray)
        else:
            return g * B
    def grad_np_dot_B(g):
        if A.ndim is 2:
            return dot(A.T, g)
        elif B.ndim is 2:
            return outer(A, g).view(ndarray)
        else:
            return g * A
    return [grad_np_dot_A, grad_np_dot_B]
dot = P(dot, make_grad_np_dot)

concatenate_orig = concatenate
def concatenate_args(axis, *args):
    return concatenate_orig(args, axis)
def make_grad_concatenate_args(ans, axis, *args):
    high = 0
    gradfuns = [None]
    for a in args:
        low = high
        high += a.shape[axis]
        idxs = [slice(None)] * ans.ndim
        idxs[axis] = slice(low, high)
        gradfuns.append(partial(take, idx=idxs))
    return gradfuns
concatenate_args = P(W(concatenate_args), make_grad_concatenate_args)
concatenate = lambda arr_list, axis=0 : concatenate_args(axis, *arr_list)

# ----- Node version of ndarray -----

class ArrayNode(Node):
    __getitem__ = take
    # Constants w.r.t float data just pass though
    @property
    def shape(self): return self.value.shape
    @property
    def ndim(self): return self.value.ndim
    @property
    def size(self): return self.value.size

    # Differentiable unary methods just apply to self
    squeeze = squeeze
    ravel = ravel
    reshape = reshape
    sum = sum
    mean = mean
    @property
    def T(self): return transpose(self)
    __neg__ = P(op.neg,  grad_neg)

    # Binary ops already wrapped by autograd.numpy.ndarray
    __add__  = ndarray.__add__.__func__
    __sub__  = ndarray.__sub__.__func__
    __mul__  = ndarray.__mul__.__func__
    __pow__  = ndarray.__pow__.__func__
    __div__  = ndarray.__div__.__func__
    __radd__ = ndarray.__radd__.__func__
    __rsub__ = ndarray.__rsub__.__func__
    __rmul__ = ndarray.__rmul__.__func__
    __rpow__ = ndarray.__rpow__.__func__
    __rdiv__ = ndarray.__rdiv__.__func__

Node.type_mappings[ndarray] = ArrayNode

# ----- Sparse array -----

class SparseArray(object):
    __array_priority__ = 150.0
    def __init__(self, shape, idx, val):
        self.shape = shape
        self.idx = idx
        self.val = val

    def __add__(self, other):
        array = zeros(self.shape) if other is 0 else copy(other)
        array[self.idx] += self.val
        return array

    def __radd__(self, other):
        return self.__add__(other)

class SparseArrayNode(Node):
    __add__  = P(SparseArray.__add__ , grad_add)
    __radd__ = P(SparseArray.__radd__, reverse_args(grad_add))

Node.type_mappings[SparseArray] = SparseArrayNode
<|endoftext|>


<commit_before>import sys, random, copy

'''
Small-Step, big-step, and denotational semantics of the 'SIMPLE' language
as per the book:
  [Stuart, Tom. Understanding Computation: From Simple Machines to Impossible Programs. " O'Reilly Media, Inc.", 2013],
but implemented in Python instead of Ruby, along with slight modifications for even simpler expressions
of the reduction functions.
'''

# i-o kludge
def env_get(env, name, descr = ''):
  if name not in env:
    if '-randin' in sys.argv:
      val = random.randint(0, 100)
    else:
      val_str = raw_input(' > {}{} : '.format(name, ' ({})'.format(descr) if descr else ''))
      val = int(val_str) if val_str.isdigit() else (val_str[0].lower() == 't')
    env[name] = val
  return env[name]
def env_set(env, name, val):
  env[name] = val
  return val

# SIMPLE syntax elements
SIMPLE = ['b', 'i',
          'not', 'and', 'or', 'eq', 'lt', 'succ', 'add', 'mul',
          'var', 'set', 'if', 'seq', 'while']

# commonalities between semantics
def expr(redu, *kids):
  return { 'redu':redu, 'kids':list(kids) }
def is_expr(e):
  return type(e) == type({})
def expr_dup(e):
  return copy.deepcopy(e)
def redu_expr(env, e, redu_func):
  while is_expr(e):
    e = redu_func(env, e)
  return (e, env)
def redu_tbl(prefix):
  tbl = {}
  for x in SIMPLE:
    tbl[x] = globals().get('{}_{}'.format(prefix, x), None)
  return tbl

# operational semantics
#  big-step semantics
def bs_redu_expr_once(env, e):
  return e['redu'](env, [bs_redu_expr_once(env, sub_exp) for sub_exp in e['kids']])
bs_redu_expr = lambda env, e: redu_expr(env, e, bs_redu_expr_once)
bs_redu_tbl = lambda: redu_tbl('os')
#  small-step semantics
def ss_redu_expr_once(env, e):
  for i,child in enumerate(e['kids']):
    if is_expr(child):
      e['kids'][i] = ss_redu_expr_once(env, child)
      return e
  return e['redu'](env, e['kids'])
ss_redu_expr = lambda env, e: redu_expr(env, e, ss_redu_expr_once)
ss_redu_tbl = lambda: redu_tbl('os')
#  reduction functions
os_b = lambda val: expr(lambda env, kids: bool(val))
os_i = lambda val: expr(lambda env, kids: int(val))
os_not = lambda child: expr(lambda env, kids: (not kids[0]), child)
os_and = lambda l,r: expr(lambda env, kids: (kids[0] and kids[1]), l, r)
os_or = lambda l,r: expr(lambda env, kids: (kids[0] or kids[1]), l, r)
os_eq = lambda l,r: expr(lambda env, kids: (kids[0] == kids[1]), l, r)
os_lt = lambda l,r: expr(lambda env, kids: (kids[0] < kids[1]), l, r)
os_succ = lambda child: expr(lambda env, kids: (kids[0] + 1), child)
os_add = lambda l,r: expr(lambda env, kids: (kids[0] + kids[1]), l, r)
os_mul = lambda l,r: expr(lambda env, kids: (kids[0] * kids[1]), l, r)
os_var = lambda name,descr='': expr(lambda env, kids: env_get(env, name, descr))
os_set = lambda name,r: expr(lambda env, kids: env_set(env, name, kids[0]), r)
os_if = lambda c,l,r: expr(lambda env, kids: (l if kids[0] else r), c)
os_seq = lambda *exprs: expr(lambda env, kids: ('done seq'), *exprs)
os_while = lambda c,bdy: os_if(expr_dup(c), os_seq(expr_dup(bdy), expr(lambda env,kids: os_while(c,bdy))), os_seq())

#denotational semantics (to python)
def dspy_redu_expr_once(env, e):
  return e['redu'](env, [bs_redu_expr_once(env, sub_exp) for sub_exp in e['kids']])
dspy_redu_expr = lambda env, e: redu_expr(env, e, dspy_redu_expr_once)
dspy_redu_tbl = lambda: redu_tbl('dspy')
#  reduction functions
dspy_b = lambda val: ( lambda env, kids: '{}'.format(bool(val)) )
dspy_i = lambda val: ( lambda env, kids: '{}'.format(int(val)) )
dspy_not = lambda env, kids: '(not ({}))'.format(kids[0])
dspy_and = lambda env, kids: '(({}) and ({}))'.format(kids[0], kids[1])
dspy_or = lambda env, kids: '(({}) or ({}))'.format(kids[0], kids[1])
dspy_eq = lambda env, kids: '(({}) == ({}))'.format(kids[0], kids[1])
dspy_lt = lambda env, kids: '(({}) < ({}))'.format(kids[0], kids[1])
dspy_succ = lambda env, kids: '(({})+1)'.format(kids[0])
dspy_var = lambda name: ( lambda env, kids: '{}'.format(name) )
dspy_set = lambda name: ( lambda env, kids: '{} = ({})'.format(name, kids[0]) )
'''
dspy_concat(strings):
  return '[&]'.join(strings)
dspy_if = lambda l, r: ( lambda env, kids: expr(kids[0], l, r) )
'''

# tests
def test1():
  for tbl, redu_expr in [(ss_redu_tbl(), ss_redu_expr), (bs_redu_tbl(), bs_redu_expr), (dspy_redu_tbl(), dspy_redu_expr)][0:2]:
    print tbl['b'](True)
    print redu_expr({}, tbl['b'](True) )
    print redu_expr({}, tbl['not'](tbl['b'](True)) )
    expr_1 = tbl['lt'](tbl['i'](4), tbl['i'](5))
    expr_2 = tbl['lt'](tbl['i'](5), tbl['i'](4))
    expr_3 = tbl['or'](expr_1, expr_2)
    print redu_expr({}, expr_3 )
    print redu_expr({}, tbl['not'](tbl['var']('x', 'test boolean')))
    print redu_expr({}, os_set('y', tbl['succ'](tbl['var']('x'))))
    expr_1 = os_set('y', tbl['succ'](tbl['var']('x')))
    expr_2 = os_set('z', tbl['succ'](tbl['var']('y')))
    print redu_expr({}, tbl['seq'](expr_1, expr_2))
    expr_c = tbl['lt'](tbl['i'](4), tbl['i'](5))
    print redu_expr({}, tbl['if'](expr_c, tbl['i'](6), tbl['i'](-6)) )
    expr_c = tbl['lt'](tbl['var']('i'), tbl['i'](5))
    expr_bdy = os_set('i', tbl['succ'](tbl['var']('i')))
    print redu_expr({'i':0}, os_while(expr_c, expr_bdy))
    print ''
if '-test' in sys.argv:
  test1()
<commit_msg>Added first version of denotational semantics (to python)<commit_after>import sys, random, copy

'''
Small-Step, big-step, and denotational semantics of the 'SIMPLE' language
as per the book:
  [Stuart, Tom. Understanding Computation: From Simple Machines to Impossible Programs. " O'Reilly Media, Inc.", 2013],
but implemented in Python instead of Ruby, along with slight modifications for even simpler expressions
of the reduction functions.
'''

# i-o kludge
def env_get(env, name, descr = ''):
  if name not in env:
    if '-randin' in sys.argv:
      val = random.randint(0, 100)
    else:
      val_str = raw_input(' > {}{} : '.format(name, ' ({})'.format(descr) if descr else ''))
      val = int(val_str) if val_str.isdigit() else (val_str[0].lower() == 't')
    env[name] = val
  return env[name]
def env_set(env, name, val):
  env[name] = val
  return val

# SIMPLE syntax elements
SIMPLE = ['b', 'i',
          'not', 'and', 'or', 'eq', 'lt', 'succ', 'add', 'mul',
          'var', 'set', 'if', 'seq', 'while']

# commonalities between semantics
def expr(redu, *kids):
  return { 'redu':redu, 'kids':list(kids) }
def is_expr(e):
  return type(e) == type({})
def expr_dup(e):
  return copy.deepcopy(e)
def redu_expr(env, e, redu_func):
  while is_expr(e):
    e = redu_func(env, e)
  return (e, env)
def redu_tbl(prefix):
  tbl = {}
  for x in SIMPLE:
    tbl[x] = globals().get('{}_{}'.format(prefix, x), None)
  return tbl

# operational semantics
#  big-step semantics
def bs_redu_expr_once(env, e):
  return e['redu'](env, [bs_redu_expr_once(env, sub_exp) for sub_exp in e['kids']])
bs_redu_expr = lambda env, e: redu_expr(env, e, bs_redu_expr_once)
bs_redu_tbl = lambda: redu_tbl('os')
#  small-step semantics
def ss_redu_expr_once(env, e):
  for i,child in enumerate(e['kids']):
    if is_expr(child):
      e['kids'][i] = ss_redu_expr_once(env, child)
      return e
  return e['redu'](env, e['kids'])
ss_redu_expr = lambda env, e: redu_expr(env, e, ss_redu_expr_once)
ss_redu_tbl = lambda: redu_tbl('os')
#  reduction functions
os_b = lambda val: expr(lambda env, kids: bool(val))
os_i = lambda val: expr(lambda env, kids: int(val))
os_not = lambda child: expr(lambda env, kids: (not kids[0]), child)
os_and = lambda l,r: expr(lambda env, kids: (kids[0] and kids[1]), l,r)
os_or = lambda l,r: expr(lambda env, kids: (kids[0] or kids[1]), l,r)
os_eq = lambda l,r: expr(lambda env, kids: (kids[0] == kids[1]), l,r)
os_lt = lambda l,r: expr(lambda env, kids: (kids[0] < kids[1]), l,r)
os_succ = lambda child: expr(lambda env, kids: (kids[0] + 1), child)
os_add = lambda l,r: expr(lambda env, kids: (kids[0] + kids[1]), l,r)
os_mul = lambda l,r: expr(lambda env, kids: (kids[0] * kids[1]), l,r)
os_var = lambda name,descr='': expr(lambda env, kids: env_get(env, name, descr))
os_set = lambda name,r: expr(lambda env, kids: env_set(env, name, kids[0]), r)
os_if = lambda c,l,r: expr(lambda env, kids: (l if kids[0] else r), c)
os_seq = lambda *exprs: expr(lambda env, kids: ('done seq'), *exprs)
os_while = lambda c,bdy: os_if(expr_dup(c), os_seq(expr_dup(bdy), expr(lambda env,kids: os_while(c,bdy))), os_seq())

#denotational semantics (to python)
def dspy_redu_expr_once(env, e):
  return e['redu'](env, [bs_redu_expr_once(env, sub_exp) for sub_exp in e['kids']])
dspy_redu_expr = lambda env, e: redu_expr(env, e, dspy_redu_expr_once)
dspy_redu_tbl = lambda: redu_tbl('dspy')
#  reduction functions
dspy_b = lambda val: expr(lambda env, kids: '{}'.format(bool(val)))
dspy_i = lambda val: expr(lambda env, kids: '{}'.format(int(val)))
dspy_not = lambda child: expr(lambda env, kids: '(not ({}))'.format(kids[0]), child)
dspy_and = lambda l,r: expr(lambda env, kids: '(({}) and ({}))'.format(kids[0], kids[1]), l,r)
dspy_or = lambda l,r: expr(lambda env, kids: '(({}) or ({}))'.format(kids[0], kids[1]), l,r)
dspy_eq = lambda l,r: expr(lambda env, kids: '(({}) == ({}))'.format(kids[0], kids[1]), l,r)
dspy_lt = lambda l,r: expr(lambda env, kids: '(({}) < ({}))'.format(kids[0], kids[1]), l,r)
dspy_succ = lambda child: expr(lambda env, kids: '(({})+1)'.format(kids[0]), child)
dspy_var = lambda name,descr='': expr(lambda env, kids: '{}'.format(name) )
dspy_set = lambda name,r: expr(lambda env, kids: '{} = ({})'.format(name, kids[0]), r)
dspy_if = lambda c,l,r: expr(lambda env, kids: 'if {}: {{ {} }} else: {{ {} }}'.format(kids[0], kids[1], kids[2]), c,l,r)
dspy_seq = lambda *exprs: expr(lambda env, kids: '\n{};\n'.format('; \n'.join(kids)), *exprs)
dspy_while = lambda c,bdy: expr(lambda env, kids: 'while {}: {{ {} }}'.format(kids[0], kids[1]), c,bdy)

# tests
def test1():
  for tbl, redu_expr in [(ss_redu_tbl(), ss_redu_expr), (bs_redu_tbl(), bs_redu_expr), (dspy_redu_tbl(), dspy_redu_expr)][2:]:
    print tbl['b'](True)
    print redu_expr({}, tbl['b'](True) )
    print redu_expr({}, tbl['not'](tbl['b'](True)) )
    expr_1 = tbl['lt'](tbl['i'](4), tbl['i'](5))
    expr_2 = tbl['lt'](tbl['i'](5), tbl['i'](4))
    expr_3 = tbl['or'](expr_1, expr_2)
    print redu_expr({}, expr_3 )
    print redu_expr({}, tbl['not'](tbl['var']('x', 'test boolean')))
    print redu_expr({}, tbl['set']('y', tbl['succ'](tbl['var']('x'))))
    expr_1 = tbl['set']('y', tbl['succ'](tbl['var']('x')))
    expr_2 = tbl['set']('z', tbl['succ'](tbl['var']('y')))
    print redu_expr({}, tbl['seq'](expr_1, expr_2))
    expr_c = tbl['lt'](tbl['i'](4), tbl['i'](5))
    print redu_expr({}, tbl['if'](expr_c, tbl['i'](6), tbl['i'](-6)) )
    expr_c = tbl['lt'](tbl['var']('i'), tbl['i'](5))
    expr_bdy = tbl['set']('i', tbl['succ'](tbl['var']('i')))
    print redu_expr({'i':0}, tbl['while'](expr_c, expr_bdy))
    print ''
if '-test' in sys.argv:
  test1()
<|endoftext|>


<commit_before>#
#
# class UBMatrix(Module):
#     # list of 3 lists of length 3
     pass
#
#
# class ImageStack(Module):
#     # list of 2-d images
#     pass
<commit_msg>MNT: Fixed comment block<commit_after>#
#
# class UBMatrix(Module):
#     # list of 3 lists of length 3
#      pass
#
#
# class ImageStack(Module):
#     # list of 2-d images
#     pass
<|endoftext|>


<commit_before>from grammar import \
        grammar, name_definition, author_definition, summary_definition, \
        description_definition, modules_definition, extension_definition

class Extension(object):
    def __init__(self, name, src):
        self.name = name
        self.src = src

    def __repr__(self):
        return "Extension %s (sources are %s)" % (self.name, ",".join(self.src))

def _module_name(t):
    return ".".join(t)

class AST(object):
    # Not really an AST...
    def __init__(self):
        # Metadata
        self.name = None
        self.author = None
        self.description = None
        self.summary = None

        # Extensions
        self.extensions = []

        # Packages
        self.pkg = []

        # Modules
        self.py_modules = []

    def parse_name(self, s, loc, toks):
        self.name = toks.asDict()['name']

    def parse_summary(self, s, loc, toks):
        self.summary = " ".join(toks.asDict()['summary'])

    def parse_description(self, s, loc, toks):
        self.description = "\n".join(toks.asDict()['description'])

    def parse_author(self, s, loc, toks):
        self.author = " ".join(toks.asDict()['author'])

    def parse_modules(self, s, loc, toks):
        pass

    def parse_src(self, s, loc, toks):
        pass

    def parse_extension(self, s, loc, toks):
        d = toks.asDict()
        name = _module_name(d['extension_name'])
        src = d['extension_src']
        self.extensions.append(Extension(name, src))

if __name__ == '__main__':
    ast = AST()

    name_definition.setParseAction(ast.parse_name)
    summary_definition.setParseAction(ast.parse_summary)
    description_definition.setParseAction(ast.parse_description)
    author_definition.setParseAction(ast.parse_author)

    extension_definition.setParseAction(ast.parse_extension)

    data = """\
Name: numpy
Description:
    NumPy is a general-purpose array-processing package designed to
    efficiently manipulate large multi-dimensional arrays of arbitrary
    records without sacrificing too much speed for small multi-dimensional
    arrays.  NumPy is built on the Numeric code base and adds features
    introduced by numarray as well as an extended C-API and the ability to
    create arrays of arbitrary type which also makes NumPy suitable for
    interfacing with general-purpose data-base applications.
    .
    There are also basic facilities for discrete fourier transform,
    basic linear algebra and random number generation.
Summary: array processing for numbers, strings, records, and objects.
Modules:
    foo.bar,
    foo.bar2,
    foo.bar3
Author: someone
Extension: _foo.bar
    sources:
        yo
"""
    tokens = grammar.parseString(data)

    print ast.name
    print ast.summary
    print ast.author
    print ast.description
    print ast.extensions
<commit_msg>ENH: handle package section in the AST.<commit_after>from grammar import \
        grammar, name_definition, author_definition, summary_definition, \
        description_definition, modules_definition, extension_definition, \
        package_definition

class Extension(object):
    def __init__(self, name, src):
        self.name = name
        self.src = src

    def __repr__(self):
        return "Extension %s (sources are %s)" % (self.name, ",".join(self.src))

def _module_name(t):
    return ".".join(t)

class AST(object):
    # Not really an AST...
    def __init__(self):
        # Metadata
        self.name = None
        self.author = None
        self.description = None
        self.summary = None

        # Extensions
        self.extensions = []

        # Packages
        self.packages = []

        # Modules
        self.py_modules = []

    def parse_name(self, s, loc, toks):
        self.name = toks.asDict()['name']

    def parse_summary(self, s, loc, toks):
        self.summary = " ".join(toks.asDict()['summary'])

    def parse_description(self, s, loc, toks):
        self.description = "\n".join(toks.asDict()['description'])

    def parse_author(self, s, loc, toks):
        self.author = " ".join(toks.asDict()['author'])

    def parse_modules(self, s, loc, toks):
        pass

    def parse_src(self, s, loc, toks):
        pass

    def parse_package(self, s, loc, toks):
        d = toks.asDict()
        for pkg in d['packages']:
            self.packages.append(_module_name(pkg))

    def parse_extension(self, s, loc, toks):
        d = toks.asDict()
        name = _module_name(d['extension_name'])
        src = d['extension_src']
        self.extensions.append(Extension(name, src))

if __name__ == '__main__':
    ast = AST()

    name_definition.setParseAction(ast.parse_name)
    summary_definition.setParseAction(ast.parse_summary)
    description_definition.setParseAction(ast.parse_description)
    author_definition.setParseAction(ast.parse_author)

    package_definition.setParseAction(ast.parse_package)

    extension_definition.setParseAction(ast.parse_extension)

    data = """\
Name: numpy
Description:
    NumPy is a general-purpose array-processing package designed to
    efficiently manipulate large multi-dimensional arrays of arbitrary
    records without sacrificing too much speed for small multi-dimensional
    arrays.  NumPy is built on the Numeric code base and adds features
    introduced by numarray as well as an extended C-API and the ability to
    create arrays of arbitrary type which also makes NumPy suitable for
    interfacing with general-purpose data-base applications.
    .
    There are also basic facilities for discrete fourier transform,
    basic linear algebra and random number generation.
Summary: array processing for numbers, strings, records, and objects.
Package:
    foo.bar,
    foo.bar2,
    foo.bar3
Modules:
    bar,
    foobar
Author: someone
Extension: _foo.bar
    sources:
        yo
"""
    tokens = grammar.parseString(data)

    print ast.name
    print ast.summary
    print ast.author
    print ast.description

    print ast.packages

    print ast.extensions
<|endoftext|>


<commit_before>'''
Created on Jan 1, 2014

@author: Rob
'''
import unittest

class AtomTest(unittest.TestCase):
    def test_copy_consistency(self):
        from byond.basetypes import Atom, BYONDString, BYONDValue
        atom = Atom('/datum/test',__file__,0)
        atom.properties={
            'dir': BYONDValue(2),
            'name': BYONDString('test datum')
        }
        atom.mapSpecified=['dir','name']

        atom2=atom.copy()

        atom_serialized=atom.dumpPropInfo('test')
        atom2_serialized=atom2.dumpPropInfo('test')

        self.assertEqual(atom_serialized, atom2_serialized)

    def test_serialization(self):
        from byond.basetypes import Atom, BYONDString, BYONDValue
        atom = Atom('/datum/test',__file__,0)
        atom.properties={
            'dir': BYONDValue(2),
            'name': BYONDString('test datum')
        }

        atom_serialized='/datum/test{name="test datum";dir=2}'

        self.assertEqual(str(atom), atom_serialized)

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()<commit_msg>Fix Atom unit tests. (They passed locally, somehow)<commit_after>'''
Created on Jan 1, 2014

@author: Rob
'''
import unittest

class AtomTest(unittest.TestCase):
    def test_copy_consistency(self):
        from byond.basetypes import Atom, BYONDString, BYONDValue
        atom = Atom('/datum/test',__file__,0)

        atom.properties['dir']=BYONDValue(2)
        atom.properties['name']=BYONDString('test datum')

        atom.mapSpecified=['dir','name']

        atom2=atom.copy()

        atom_serialized=atom.dumpPropInfo('test')
        atom2_serialized=atom2.dumpPropInfo('test')

        self.assertEqual(atom_serialized, atom2_serialized)

    def test_serialization(self):
        from byond.basetypes import Atom, BYONDString, BYONDValue
        atom = Atom('/datum/test',__file__,0)

        # Assign properties.  Order is important (orderedDict!)
        atom.properties['dir']=BYONDValue(2)
        atom.properties['name']=BYONDString('test datum')

        # What we expect after running str().
        atom_serialized='/datum/test{dir=2;name="test datum"}'

        # Check it
        self.assertEqual(str(atom), atom_serialized)

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()<|endoftext|>


<commit_before># TODO: move siu.py into its own folder, add this to it (w/ Call Trees)
from typing import _Any, _Union, TypeVar
import inspect

def is_flex_subclass(x, cls):
    if isinstance(x, _Any):
        return True

    return issubclass(x, cls)

def is_dispatch_func_subtype(f, input_cls, output_cls):
    """Returns whether a singledispatch function is subtype of some input and result class.

    A function is a subtype if it is input contravariant, and result covariant.

    Rules for evaluating return types <= output_cls:
        * Any always returns True
        * Union[A, B] returns true if either A or B are covariant
        * f(arg_name:TypeVar) -> TypeVar compares input_cls and output_cls
        * Simple return types checked via issubclass

    Args:
        input_cls - input class for first argument to function
        output_cls - output class for function result

    """
    sig = inspect.signature(f)
    # result annotation
    res_type = sig.return_annotation

    # first parameter annotation
    par0 = next(iter(sig.parameters.values()))
    par_type0 = par0.annotation

    # Case 1: no annotation
    if res_type is None:
        return False

    # Case 2: fancy annotations: Union, generic TypeVar
    if isinstance(res_type, _Union) and hasattr(res_type, '__args__'):
        # passes if any unioned types are subclasses
        sub_types = res_type.__args__
        return any(map(lambda x: is_flex_subclass(x, output_cls), sub_types))
    elif isinstance(res_type, TypeVar):
        if res_type == par_type0:
            # using a generic type variable as first arg and result
            # return type must be covariant on input_cls
            return issubclass(input_cls, output_cls) and res_type.__covariant__
        else:
            raise TypeError("Generic type used as result, but not as first parameter")

    return is_flex_subclass(res_type, output_cls)
<commit_msg>fix typing lib compatibilty with py3.7<commit_after># TODO: move siu.py into its own folder, add this to it (w/ Call Trees)
from typing import Any, Union, TypeVar
import inspect

def is_union(x):
    return getattr(x, '__origin__', None) is Union

def get_union_args(x):
    return getattr(x, '__args__', getattr(x, '__union_args__', None))

def is_flex_subclass(x, cls):
    if x is Any:
        return True

    return issubclass(x, cls)

def is_dispatch_func_subtype(f, input_cls, output_cls):
    """Returns whether a singledispatch function is subtype of some input and result class.

    A function is a subtype if it is input contravariant, and result covariant.

    Rules for evaluating return types <= output_cls:
        * Any always returns True
        * Union[A, B] returns true if either A or B are covariant
        * f(arg_name:TypeVar) -> TypeVar compares input_cls and output_cls
        * Simple return types checked via issubclass

    Args:
        input_cls - input class for first argument to function
        output_cls - output class for function result

    """
    sig = inspect.signature(f)
    # result annotation
    res_type = sig.return_annotation

    # first parameter annotation
    par0 = next(iter(sig.parameters.values()))
    par_type0 = par0.annotation

    # Case 1: no annotation
    if res_type is None:
        return False

    # Case 2: fancy annotations: Union, generic TypeVar
    if is_union(res_type) and get_union_args(res_type):
        sub_types = get_union_args(res_type)
        # passes if any unioned types are subclasses
        return any(map(lambda x: is_flex_subclass(x, output_cls), sub_types))
    elif isinstance(res_type, TypeVar):
        if res_type == par_type0:
            # using a generic type variable as first arg and result
            # return type must be covariant on input_cls
            return issubclass(input_cls, output_cls) and res_type.__covariant__
        else:
            raise TypeError("Generic type used as result, but not as first parameter")

    return is_flex_subclass(res_type, output_cls)
<|endoftext|>


<commit_before>from django.conf.urls import url

from . import views
from . import ApiFunctions

app_name = 'skalvi'
urlpatterns = [
    url(r'^allactivities/', views.allactivities, name='allactivities'),
    url(r'^$', views.index, name='index'),
    url(r'^register/$', views.UserFormView.as_view(), name="register"),
    url(r'^activity/$', views.createActivity.as_view(), name="create-activity"),
    url(r'^activity/(?P<pk>\d+)/$', views.ActivityView.as_view(), name="edit-activity"),
    url(r'^api/skalvi/login/$', views.loginView, name="login"),
    url(r'^api/skalvi/loginF/$', ApiFunctions.loginFacebook, name="loginFacebook"),
    url(r'^api/activities/$', views.getActivities, name="getActivities"),

    url(r'^api/attendingActivities/[a-zA-Z]+/$', views.getAttendingActivities, name="getAAttendingActivities"),

    url(r'^api/activity/(?P<id>[0-9]+)/$', views.getActivity, name="get-activity"),
    url(r'^register/$', views.UserFormView.as_view(), name="register"),
    url(r'^mypage/$', views.MyPageView.as_view(), name='mypage'),

    url(r'^mypage/(?P<id>[a-zA-Z]+)/$', views.MyPageView.as_view(), name='mypage'),

    url(r'^logout_user/$', views.logout_user, name='logout_user'),
    url(r'^choose/$', views.ChooseUserView.as_view(), name='choose'),
    url(r'^selected/$', views.selectedUser, name='selected'),
    url(r'^signupActivity/$', views.signUpActivity, name='signupActivity'),
    url(r'^checkIfSignedUp/$', views.checkIfSingedUp, name='checkIfSignedUp'),
    url(r'^signOfEvent/$', views.signOfEvent, name='signOfEvent'),
    url(r'^registerProfile/$', views.RegisterProfileView.as_view(), name='registerProfile'),
    url(r'^registerProfile/$', views.RegisterProfileView.post, name='registerProfile'),

]
<commit_msg>added api url to populate db with providers from aktordatabasen<commit_after>from django.conf.urls import url

from . import views
from . import ApiFunctions

app_name = 'skalvi'
urlpatterns = [
    url(r'^allactivities/', views.allactivities, name='allactivities'),
    url(r'^$', views.index, name='index'),
    url(r'^register/$', views.UserFormView.as_view(), name="register"),
    url(r'^activity/$', views.createActivity.as_view(), name="create-activity"),
    url(r'^activity/(?P<pk>\d+)/$', views.ActivityView.as_view(), name="edit-activity"),
    url(r'^api/skalvi/login/$', views.loginView, name="login"),
    url(r'^api/skalvi/loginF/$', ApiFunctions.loginFacebook, name="loginFacebook"),
    url(r'^api/activities/$', views.getActivities, name="getActivities"),

    url(r'^api/attendingActivities/[a-zA-Z]+/$', views.getAttendingActivities, name="getAAttendingActivities"),

    url(r'^api/activity/(?P<id>[0-9]+)/$', views.getActivity, name="get-activity"),
    url(r'^register/$', views.UserFormView.as_view(), name="register"),
    url(r'^mypage/$', views.MyPageView.as_view(), name='mypage'),

    url(r'^mypage/(?P<id>[a-zA-Z]+)/$', views.MyPageView.as_view(), name='mypage'),

    url(r'^logout_user/$', views.logout_user, name='logout_user'),
    url(r'^choose/$', views.ChooseUserView.as_view(), name='choose'),
    url(r'^selected/$', views.selectedUser, name='selected'),
    url(r'^signupActivity/$', views.signUpActivity, name='signupActivity'),
    url(r'^checkIfSignedUp/$', views.checkIfSingedUp, name='checkIfSignedUp'),
    url(r'^signOfEvent/$', views.signOfEvent, name='signOfEvent'),
    url(r'^registerProfile/$', views.RegisterProfileView.as_view(), name='registerProfile'),
    url(r'^registerProfile/$', views.RegisterProfileView.post, name='registerProfile'),

    url(r'^api/skalvi/populate/$', ApiFunctions.populate, name="populateDatabase"),

]
<|endoftext|>