<commit_before>import unittest
import numpy
import tensorflow as tf

import model

class TestLSTM(unittest.TestCase):

  def __init__(self, *args, **kwargs):
    super(TestLSTM, self).__init__(*args, **kwargs)

  def test_Trivial(self):
    with tf.Graph().as_default():
      lstm = model.LSTM(length=3, batch_size=1, input_dim=1, output_dim=1)
      loss = lstm.loss
      train_op = lstm.Train(loss, learning_rate=0.001)

      x = [[[1]], [[2]], [[3]]]
      label = [[1]]

      feed_dict_x = list(zip(lstm.inputs, x))
      feed_dict_label = {lstm.label_placeholder: label}

      feed_dict = {}
      feed_dict.update(feed_dict_x)
      feed_dict.update(feed_dict_label)

      with tf.Session() as sess:

        init = tf.initialize_all_variables()
        sess.run(init)

        _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)
<commit_msg>1. add two unittest<commit_after>import unittest
import numpy
import tensorflow as tf

import model

class TestLSTM(unittest.TestCase):

  def __init__(self, *args, **kwargs):
    super(TestLSTM, self).__init__(*args, **kwargs)
    self.learning_rate = 0.001

  def tearDown(self):
    with tf.Graph().as_default():
      lstm = model.LSTM(length=len(self.x),
                        batch_size=len(self.x[0]),
                        input_dim=len(self.x[0][0]),
                        output_dim=len(self.label[0]))
      loss = lstm.loss
      train_op = lstm.Train(loss, learning_rate=self.learning_rate)

      feed_dict_x = list(zip(lstm.inputs, self.x))
      feed_dict_label = {lstm.label_placeholder: self.label}

      feed_dict = {}
      feed_dict.update(feed_dict_x)
      feed_dict.update(feed_dict_label)

      with tf.Session() as sess:

        init = tf.initialize_all_variables()
        sess.run(init)

        for i in range(10):
          _, loss_value = sess.run([train_op, loss], feed_dict=feed_dict)

  def test_1D(self):
    self.x = [[[1]], [[2]], [[3]]]
    self.label = [[1]]

  def test_3D(self):
    self.x = [[[1, 0, 0]], [[0, 1, 0]], [[0, 0, 1]]]
    self.label = [[1, 0, 0, 0, 0]]

  def test_3D_longer_sequence(self):
    self.x = [[[1, 0, 0]], [[0, 1, 0]], [[0, 0, 1]], [[0, 1, 1]]]
    self.label = [[1, 0, 0, 0, 0]]

if __name__ == '__main__':
  unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Created on Mar 11, 2012

@author: moloch

    Copyright 2012 Root the Box

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
'''

import os
import imghdr
import xml.etree.cElementTree as ET

from os import urandom
from uuid import uuid4
from sqlalchemy import Column, ForeignKey
from sqlalchemy.orm import relationship, backref
from sqlalchemy.types import Integer, Unicode, String, Boolean
from models import dbsession
from models.BaseModels import DatabaseObject
from models.Relationships import team_to_box
from models.IpAddress import IpAddress
from models.GameLevel import GameLevel
from models.Corporation import Corporation
from models.SourceCode import SourceCode
from tornado.options import options
from libs.XSSImageCheck import is_xss_image

class Box(DatabaseObject):

    ''' Box definition '''

    uuid = Column(String(36),
                  unique=True,
                  nullable=False,
                  default=lambda: str(uuid4())
                  )

    corporation_id = Column(Integer, ForeignKey('corporation.id'),
                            nullable=False
                            )

    _name = Column(Unicode(32), unique=True, nullable=False)
    _operating_system = Column(Unicode(16))
    _description = Column(Unicode(1024))
    _difficulty = Column(Unicode(16))
    game_level_id = Column(
        Integer, ForeignKey('game_level.id'), nullable=False)
    _avatar = Column(String(64))
    autoformat = Column(Boolean, default=True)

    garbage = Column(String(32),
                     unique=True,
                     nullable=False,
                     default=lambda: urandom(16).encode('hex')
                     )

    teams = relationship("Team",
                         secondary=team_to_box,
                         backref=backref("box", lazy="select")
                         )

    hints = relationship("Hint",
                         backref=backref("box", lazy="select"),
                         cascade="all,delete,delete-orphan"
                         )

    flags = relationship("Flag",
                         backref=backref("box", lazy="select"),
                         cascade="all,delete,delete-orphan"
                         )

    ip_addresses = relationship("IpAddress",
                                backref=backref("box", lazy="select"),
                                cascade="all,delete,delete-orphan"
                                )

    @classmethod
    def all(cls):
        ''' Returns a list of all objects in the database '''
        return dbsession.query(cls).all()

    @classmethod
    def by_id(cls, _id):
        ''' Returns a the object with id of _id '''
        return dbsession.query(cls).filter_by(id=_id).first()

    @classmethod
    def by_uuid(cls, _uuid):
        ''' Return and object based on a uuid '''
        return dbsession.query(cls).filter_by(uuid=unicode(_uuid)).first()

    @classmethod
    def by_name(cls, name):
        ''' Return the box object whose name is "name" '''
        return dbsession.query(cls).filter_by(_name=unicode(name)).first()

    @classmethod
    def by_garbage(cls, _garbage):
        return dbsession.query(cls).filter_by(garbage=_garbage).first()

    @classmethod
    def by_ip_address(cls, ip_addr):
        '''
        Returns a box object based on an ip address, supports both ipv4
        and ipv6
        '''
        ip = dbsession.query(IpAddress).by_address(ip_addr).first()
        return ip.box if ip is not None else None

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        if not 3 <= len(unicode(value)) <= 32:
            raise ValidationError("Name must be 3 - 16 characters")
        self._name = unicode(value)

    @property
    def operating_system(self):
        return self._operating_system if self._operating_system else '?'

    @operating_system.setter
    def operating_system(self, value):
        self._operating_system = unicode(value)

    @property
    def description(self):
        '''
        We have to ensure that the description text is formatted correctly,
        it gets dumped into a <pre> tag which will honor whitespace this will
        split all of the text and insert newlines every 70 chars +2 whitespace
        at be beginning of each line, so the indents line up nicely.
        '''
        if self.autoformat:
            index, step = 0, 70
            ls = [' ']
            if 0 < len(self._description):
                text = self._description.replace('\n', '')
                while index < len(text):
                    ls.append("  " + text[index: index + step])
                    index += step
            else:
                ls.append("  No information on file.")
            ls.append("\n  Operating System: %s\n" % self.operating_system)
            ls.append("  Reported Difficulty: %s\n" % self.difficulty)
            return unicode("\n".join(ls))
        else:
            return self._description

    @description.setter
    def description(self, value):
        if 1024 < len(value):
            raise ValueError("Description must be less than 1024 characters")
        self._description = unicode(value)

    @property
    def difficulty(self):
        return self._difficulty if len(self._difficulty) else u"Unknown"

    @difficulty.setter
    def difficulty(self, value):
        if 16 < len(value):
            raise ValueError("Difficulty must be less than 16 characters")
        self._difficulty = unicode(value)

    @property
    def avatar(self):
        if self._avatar is not None:
            return self._avatar
        else:
            return "default_avatar.jpeg"

    @avatar.setter
    def avatar(self, image_data):
        if self.uuid is None:
            self.uuid = str(uuid4())
        if len(image_data) < (1024 * 1024):
            ext = imghdr.what("", h=image_data)
            if ext in ['png', 'jpeg', 'gif', 'bmp'] and not is_xss_image(image_data):
                if self._avatar is not None and os.path.exists(options.avatar_dir + '/' + self._avatar):
                    os.unlink(options.avatar_dir + self._avatar)
                file_path = str(options.avatar_dir + self.uuid + '.' + ext)
                with open(file_path, 'wb') as fp:
                    fp.write(image_data)
                self._avatar = self.uuid + '.' + ext
            else:
                raise ValidationError(
                    "Invalid image format, avatar must be: .png .jpeg .gif or .bmp")
        else:
            raise ValidationError("The image is too large")

    @property
    def ipv4s(self):
        ''' Return a list of all ipv4 addresses '''
        return filter(lambda ip: ip.version == 4, self.ip_addresses)

    @property
    def ipv6s(self):
        ''' Return a list of all ipv6 addresses '''
        return filter(lambda ip: ip.version == 6, self.ip_addresses)

    @property
    def visable_ips(self):
        return filter(lambda ip: ip.visable is True, self.ip_addresses)

    @property
    def source_code(self):
        return SourceCode.by_box_id(self.id)

    def get_garbage_cfg(self):
        return "[Bot]\nname = %s\ngarbage = %s\n" % (
            self.name.encode('hex'), self.garbage
        )

    def to_xml(self, parent):
        ''' Convert object to XML '''
        box_elem = ET.SubElement(parent, "box")
        box_elem.set("gamelevel", str(self.game_level.number))
        ET.SubElement(box_elem, "name").text = self.name
        ET.SubElement(
            box_elem, "operatingsystem").text = self._operating_system
        ET.SubElement(box_elem, "description").text = self._description
        ET.SubElement(box_elem, "difficulty").text = self._difficulty
        ET.SubElement(box_elem, "garbage").text = self.garbage
        flags_elem = ET.SubElement(box_elem, "flags")
        flags_elem.set("count", str(len(self.flags)))
        for flag in self.flags:
            flag.to_xml(flags_elem)
        hints_elem = ET.SubElement(box_elem, "hints")
        hints_elem.set("count", str(len(self.hints)))
        for hint in self.hints:
            hint.to_xml(hints_elem)
        ips_elem = ET.SubElement(box_elem, "ipaddresses")
        ips_elem.set("count", str(len(self.ip_addresses)))
        for ip in self.ip_addresses:
            ip.to_xml(ips_elem)
        with open(options.avatar_dir + '/' + self.avatar) as _avatar:
            data = _avatar.read()
            ET.SubElement(box_elem, "avatar").text = data.encode('base64')

    def to_dict(self):
        ''' Returns editable data as a dictionary '''
        corp = Corporation.by_id(self.corporation_id)
        game_level = GameLevel.by_id(self.game_level_id)
        return {
            'name': self.name,
            'uuid': self.uuid,
            'corporation': corp.uuid,
            'description': self._description,
            'difficulty': self.difficulty,
            'game_level': game_level.uuid,
        }

    def __repr__(self):
        return u'<Box - name: %s>' % (self.box_name,)

    def __str__(self):
        return self.box_name
<commit_msg>Fixed path bug in Box avatars<commit_after># -*- coding: utf-8 -*-
'''
Created on Mar 11, 2012

@author: moloch

    Copyright 2012 Root the Box

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
'''

import os
import imghdr
import xml.etree.cElementTree as ET

from os import urandom
from uuid import uuid4
from sqlalchemy import Column, ForeignKey
from sqlalchemy.orm import relationship, backref
from sqlalchemy.types import Integer, Unicode, String, Boolean
from models import dbsession
from models.BaseModels import DatabaseObject
from models.Relationships import team_to_box
from models.IpAddress import IpAddress
from models.GameLevel import GameLevel
from models.Corporation import Corporation
from models.SourceCode import SourceCode
from tornado.options import options
from libs.XSSImageCheck import is_xss_image

class Box(DatabaseObject):

    ''' Box definition '''

    uuid = Column(String(36),
                  unique=True,
                  nullable=False,
                  default=lambda: str(uuid4())
                  )

    corporation_id = Column(Integer, ForeignKey('corporation.id'),
                            nullable=False
                            )

    _name = Column(Unicode(32), unique=True, nullable=False)
    _operating_system = Column(Unicode(16))
    _description = Column(Unicode(1024))
    _difficulty = Column(Unicode(16))
    game_level_id = Column(
        Integer, ForeignKey('game_level.id'), nullable=False)
    _avatar = Column(String(64))
    autoformat = Column(Boolean, default=True)

    garbage = Column(String(32),
                     unique=True,
                     nullable=False,
                     default=lambda: urandom(16).encode('hex')
                     )

    teams = relationship("Team",
                         secondary=team_to_box,
                         backref=backref("box", lazy="select")
                         )

    hints = relationship("Hint",
                         backref=backref("box", lazy="select"),
                         cascade="all,delete,delete-orphan"
                         )

    flags = relationship("Flag",
                         backref=backref("box", lazy="select"),
                         cascade="all,delete,delete-orphan"
                         )

    ip_addresses = relationship("IpAddress",
                                backref=backref("box", lazy="select"),
                                cascade="all,delete,delete-orphan"
                                )

    @classmethod
    def all(cls):
        ''' Returns a list of all objects in the database '''
        return dbsession.query(cls).all()

    @classmethod
    def by_id(cls, _id):
        ''' Returns a the object with id of _id '''
        return dbsession.query(cls).filter_by(id=_id).first()

    @classmethod
    def by_uuid(cls, _uuid):
        ''' Return and object based on a uuid '''
        return dbsession.query(cls).filter_by(uuid=unicode(_uuid)).first()

    @classmethod
    def by_name(cls, name):
        ''' Return the box object whose name is "name" '''
        return dbsession.query(cls).filter_by(_name=unicode(name)).first()

    @classmethod
    def by_garbage(cls, _garbage):
        return dbsession.query(cls).filter_by(garbage=_garbage).first()

    @classmethod
    def by_ip_address(cls, ip_addr):
        '''
        Returns a box object based on an ip address, supports both ipv4
        and ipv6
        '''
        ip = dbsession.query(IpAddress).by_address(ip_addr).first()
        return ip.box if ip is not None else None

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        if not 3 <= len(unicode(value)) <= 32:
            raise ValidationError("Name must be 3 - 16 characters")
        self._name = unicode(value)

    @property
    def operating_system(self):
        return self._operating_system if self._operating_system else '?'

    @operating_system.setter
    def operating_system(self, value):
        self._operating_system = unicode(value)

    @property
    def description(self):
        '''
        We have to ensure that the description text is formatted correctly,
        it gets dumped into a <pre> tag which will honor whitespace this will
        split all of the text and insert newlines every 70 chars +2 whitespace
        at be beginning of each line, so the indents line up nicely.
        '''
        if self.autoformat:
            index, step = 0, 70
            ls = [' ']
            if 0 < len(self._description):
                text = self._description.replace('\n', '')
                while index < len(text):
                    ls.append("  " + text[index: index + step])
                    index += step
            else:
                ls.append("  No information on file.")
            ls.append("\n  Operating System: %s\n" % self.operating_system)
            ls.append("  Reported Difficulty: %s\n" % self.difficulty)
            return unicode("\n".join(ls))
        else:
            return self._description

    @description.setter
    def description(self, value):
        if 1024 < len(value):
            raise ValueError("Description must be less than 1024 characters")
        self._description = unicode(value)

    @property
    def difficulty(self):
        return self._difficulty if len(self._difficulty) else u"Unknown"

    @difficulty.setter
    def difficulty(self, value):
        if 16 < len(value):
            raise ValueError("Difficulty must be less than 16 characters")
        self._difficulty = unicode(value)

    @property
    def avatar(self):
        if self._avatar is not None:
            return self._avatar
        else:
            return "default_avatar.jpeg"

    @avatar.setter
    def avatar(self, image_data):
        if self.uuid is None:
            self.uuid = str(uuid4())
        if len(image_data) < (1024 * 1024):
            ext = imghdr.what("", h=image_data)
            if ext in ['png', 'jpeg', 'gif', 'bmp'] and not is_xss_image(image_data):
                if self._avatar is not None and os.path.exists(options.avatar_dir + '/' + self._avatar):
                    os.unlink(options.avatar_dir + '/' + self._avatar)
                file_path = str(options.avatar_dir + '/' + self.uuid + '.' + ext)
                with open(file_path, 'wb') as fp:
                    fp.write(image_data)
                self._avatar = self.uuid + '.' + ext
            else:
                raise ValidationError(
                    "Invalid image format, avatar must be: .png .jpeg .gif or .bmp")
        else:
            raise ValidationError("The image is too large")

    @property
    def ipv4s(self):
        ''' Return a list of all ipv4 addresses '''
        return filter(lambda ip: ip.version == 4, self.ip_addresses)

    @property
    def ipv6s(self):
        ''' Return a list of all ipv6 addresses '''
        return filter(lambda ip: ip.version == 6, self.ip_addresses)

    @property
    def visable_ips(self):
        return filter(lambda ip: ip.visable is True, self.ip_addresses)

    @property
    def source_code(self):
        return SourceCode.by_box_id(self.id)

    def get_garbage_cfg(self):
        return "[Bot]\nname = %s\ngarbage = %s\n" % (
            self.name.encode('hex'), self.garbage
        )

    def to_xml(self, parent):
        ''' Convert object to XML '''
        box_elem = ET.SubElement(parent, "box")
        box_elem.set("gamelevel", str(self.game_level.number))
        ET.SubElement(box_elem, "name").text = self.name
        ET.SubElement(
            box_elem, "operatingsystem").text = self._operating_system
        ET.SubElement(box_elem, "description").text = self._description
        ET.SubElement(box_elem, "difficulty").text = self._difficulty
        ET.SubElement(box_elem, "garbage").text = self.garbage
        flags_elem = ET.SubElement(box_elem, "flags")
        flags_elem.set("count", str(len(self.flags)))
        for flag in self.flags:
            flag.to_xml(flags_elem)
        hints_elem = ET.SubElement(box_elem, "hints")
        hints_elem.set("count", str(len(self.hints)))
        for hint in self.hints:
            hint.to_xml(hints_elem)
        ips_elem = ET.SubElement(box_elem, "ipaddresses")
        ips_elem.set("count", str(len(self.ip_addresses)))
        for ip in self.ip_addresses:
            ip.to_xml(ips_elem)
        with open(options.avatar_dir + '/' + self.avatar) as _avatar:
            data = _avatar.read()
            ET.SubElement(box_elem, "avatar").text = data.encode('base64')

    def to_dict(self):
        ''' Returns editable data as a dictionary '''
        corp = Corporation.by_id(self.corporation_id)
        game_level = GameLevel.by_id(self.game_level_id)
        return {
            'name': self.name,
            'uuid': self.uuid,
            'corporation': corp.uuid,
            'description': self._description,
            'difficulty': self.difficulty,
            'game_level': game_level.uuid,
        }

    def __repr__(self):
        return u'<Box - name: %s>' % (self.box_name,)

    def __str__(self):
        return self.box_name
<|endoftext|>


<commit_before>"""
Line-based plotting commands using MatPlotLib.

Before importing this file, you will probably want to do something
like:

  import matplotlib
  matplotlib.use('TkAgg')

to select a backend, or else select an appropriate one in your
matplotlib.rc file (if any).  There are many backends available for
different GUI or non-GUI uses.

$Id$
"""
__version__='$Revision$'

import matplotlib
### JABHACKALERT: Need to figure out how to use Agg by default, but
### override it with TkAgg, so that the documentation file for this
### file can be generated.  It's apparently not as simple as just
### doing "matplotlib.use('Agg')" here.
import pylab
import re, os
import copy

from numpy.oldnumeric import arange, cos, pi, array, transpose, argmax, argmin

import topo

from topo.base.arrayutils import octave_output
from topo.base.sheet import Sheet
from topo.misc.utils import frange

def windowtitle(title):
    """
    Helper function to set the title of this PyLab plot window to a string.

    At the moment, PyLab does not offer a window-manager-independent
    means for controlling the window title, so what we do is to try
    what should work with Tkinter, and then suppress all errors.  That
    way we should be ok when rendering to a file-based backend, but
    will get nice titles in Tk windows.  If other toolkits are in use,
    the title can be set here using a similar try/except mechanism, or
    else there can be a switch based on the backend type.
    """
    try:
        manager = pylab.get_current_fig_manager()
        manager.window.title(title)
    except:
        pass

def vectorplot(vec,title=None, label=None):
    """
    Simple line plotting for any vector or list of numbers.

    Intended for interactive debugging or analyzing from the command
    prompt.  See MatPlotLib's pylab functions to create more elaborate
    or customized plots; this is just a simple example.

    An optional string can be supplied as a title for the figure, if
    desired.  At present, this is only used for the body of the figure,
    not the
    """
    pylab.plot(vec, label=label)
    pylab.grid(True)
    if (title): windowtitle(title)
    pylab.show._needmain = False
    pylab.show()

def matrixplot(mat,title=None):
    """
    Simple plotting for any matrix as a bitmap with axes.

    Like MatLab's imagesc, scales the values to fit in the range 0 to 1.0.
    Intended for interactive debugging or analyzing from the command
    prompt.  See MatPlotLib's pylab functions to create more elaborate
    or customized plots; this is just a simple example.
    """
    pylab.gray()
    pylab.figure(figsize=(5,5))
    pylab.imshow(mat,interpolation='nearest')
    if (title): windowtitle(title)
    pylab.show._needmain = False
    pylab.show()

def activityplot(sheet,activity=None,title=None,cmap=pylab.cm.Greys):
    """
    Plots the activity in a sheet.

    Gets plot's extent from sheet.bounds.aarect(). Adds a title and
    allows the selection of a colormap.  If activity is not given,
    the sheet's current activity is used.
    """
    l,b,r,t = sheet.bounds.aarect().lbrt()
    if activity is None:
        activity = sheet.activity
    pylab.imshow(activity, extent=(l,r,b,t),cmap=cmap)

def topographic_grid(xsheet_view_name='XPreference',ysheet_view_name='YPreference',axis=[-0.5,0.5,-0.5,0.5]):
    """
    By default, plot the XPreference and YPreference preferences for all
    Sheets for which they are defined, using MatPlotLib.

    If sheet_views other than XPreference and YPreference are desired,
    the names of these can be passed in as arguments.
    """
    for sheet in topo.sim.objects(Sheet).values():
        if ((xsheet_view_name in sheet.sheet_view_dict) and
            (ysheet_view_name in sheet.sheet_view_dict)):

            x = sheet.sheet_view_dict[xsheet_view_name].view()[0]
            y = sheet.sheet_view_dict[ysheet_view_name].view()[0]

            pylab.figure(figsize=(5,5))

            # This one-liner works in Octave, but in matplotlib it
            # results in lines that are all connected across rows and columns,
            # so here we plot each line separately:
            #   pylab.plot(x,y,"k-",transpose(x),transpose(y),"k-")
            # Here, the "k-" means plot in black using solid lines;
            # see matplotlib for more info.
            isint=pylab.isinteractive() # Temporarily make non-interactive for plotting
            pylab.ioff()
            for r,c in zip(y,x):
                pylab.plot(c,r,"k-")
            for r,c in zip(transpose(y),transpose(x)):
                pylab.plot(c,r,"k-")

            pylab.xlabel('x')
            pylab.ylabel('y')
            # Currently sets the input range arbitrarily; should presumably figure out
            # what the actual possible range is for this simulation (which would presumably
            # be the maximum size of any GeneratorSheet?).
            pylab.axis(axis)
            windowtitle('Topographic mapping to '+sheet.name+' at time '+str(topo.sim.time()))

            # Will need to provide a way to save this output
            # when there is no GUI
            #pylab.savefig('simple_plot')

            if isint: pylab.ion()

            # Allow multiple concurrent plots; there may be a
            # cleaner way to do this...
            pylab.show._needmain = False
            pylab.show()

def tuning_curve_data(sheet, x_axis, curve_label, i_value, j_value):
    """
    Collect data for one tuning curve from the curve_dict of each sheet.

    The unit to plot is specified as a matrix index (i_value,j_value).
    There may be many possible curves in the curve_dict; the specific
    one is determined by the x_axis (i.e., what feature to plot), and
    the curve_label (which determines other parameters that specify a
    particular plot).  Returns the feature values (the x_values,
    i.e. locations along the x axis), x_values) and the corresponding
    responses (the y_values).
    """
    x_values= sorted(sheet.curve_dict[x_axis][curve_label].keys())
    y_values=[sheet.curve_dict[x_axis][curve_label][key].view()[0][i_value,j_value] for key in x_values]
    return x_values, y_values

def or_tuning_curve(x_axis,plot_type,unit):
    """
    Plots a tuning curve for orientation which rotates the curve
    values so that minimum y values are at the minimum x value.
    The y_values and labels are rotated by an amount determined by the minmum
    y_value for the first curve plotted (usually the lowest contrast curve).

    The curve datapoints are collected from the curve_dict for each
    sheet, depending on the value of topo.commands.analysis.sheet_name
    and topo.commands.analysis.coordinate (which may be set by the GUI or
    by hand).
    """
    # There may be cases (ie when the lowest contrast curve gives a lot of zero y_values)
    # in which the maximum is not in the centre.
    # Ideally this should be changed so that the preferred orientation is in the centre.
    # This may also be useful for other tuning curve types, not just orientation ie. direction.

    sheet=topo.sim[topo.commands.analysis.sheet_name]
    coordinate=topo.commands.analysis.coordinate
    i_value,j_value=sheet.sheet2matrixidx(coordinate[0],coordinate[1])

    pylab.figure(figsize=(7,7))
    isint=pylab.isinteractive()
    pylab.ioff()
    manager = pylab.get_current_fig_manager()

    pylab.ylabel('Response')
    pylab.xlabel(x_axis.capitalize()+' ('+unit+')')
    pylab.title('Sheet '+topo.commands.analysis.sheet_name+', coordinate(x,y)='+'('+
	 	str(coordinate[0])+','+str(coordinate[1])+')'+' at time '+str(topo.sim.time()))
    manager.window.title(topo.sim.name+': '+x_axis.capitalize()+' Tuning Curve')

    def rotate(seq, n=1):
        n = n % len(seq) # n=hop interval
        return seq[n:] + seq[:n]

    first_curve=True
    for curve_label in sorted(sheet.curve_dict[x_axis].keys()):
	if first_curve==True:
	    x_values, y_values = tuning_curve_data(sheet,x_axis, curve_label, i_value, j_value)
	    min_arg=argmin(y_values)
	    x_min=x_values[min_arg]
	    y_min=y_values[min_arg]
	    y_values=rotate(y_values, n=min_arg)
	    ticks=rotate(x_values, n=min_arg)
	    label_min=ticks[0]
	    x_max=min(x_values)+pi
	    x_values.append(x_max)
	    y_values.append(y_min)
	    first_curve=False
	else:
	    y_values=[sheet.curve_dict[x_axis][curve_label][key].view()[0][i_value,j_value] for key in ticks]
	    y_min=sheet.curve_dict[x_axis][curve_label][x_min].view()[0][i_value,j_value]
	    y_values.append(y_min)
	labels = copy.copy(ticks)
	labels.append(label_min)
	labels= [x*180/pi for x in labels]
	labels_new = [str(int(x)) for x in labels]
	pylab.xticks(x_values, labels_new)
	plot_type(x_values, y_values, label=curve_label)

    if isint: pylab.ion()
    pylab.legend(loc=4)
    pylab.show._needmain = False
    pylab.show()

def tuning_curve(x_axis,plot_type,unit):
    """
    Plots a tuning curve for the appropriate feature type, such as orientation, contrast or size.

    The curve datapoints are collected from the curve_dict for each
    sheet, depending on the value of topo.commands.analysis.sheet_name
    and topo.commands.analysis.coordinate (which may be set by the GUI or
    by hand).
    """

    sheet=topo.sim[topo.commands.analysis.sheet_name]
    coordinate=topo.commands.analysis.coordinate
    i_value,j_value=sheet.sheet2matrixidx(coordinate[0],coordinate[1])

    pylab.figure(figsize=(7,7))
    isint=pylab.isinteractive()
    pylab.ioff()
    manager = pylab.get_current_fig_manager()

    pylab.ylabel('Response')
    pylab.xlabel(x_axis.capitalize()+' ('+unit+')')
    pylab.title('Sheet '+topo.commands.analysis.sheet_name+', coordinate(x,y)='+'('+
	 	str(coordinate[0])+','+str(coordinate[1])+')'+' at time '+str(topo.sim.time()))
    manager.window.title(topo.sim.name+': '+x_axis.capitalize()+' Tuning Curve')

    for curve_label in sorted(sheet.curve_dict[x_axis].keys()):
        x_values, y_values = tuning_curve_data(sheet,x_axis, curve_label, i_value, j_value)
        plot_type(x_values, y_values, label=curve_label)

    if isint: pylab.ion()
    pylab.legend(loc=4)
    pylab.show._needmain = False
    pylab.show()

def plot_cfproj_mapping(dest,proj='Afferent',style='b-'):
    """
    Given a CF sheet receiving a CFProjection, plot
    the mapping of the dests CF centers on the src sheet.
    """
    if isinstance(dest,str):
        from topo import sim
        dest = sim[dest]
    plot_coord_mapping(dest.projections()[proj].coord_mapper,
                       dest,style=style)

def plot_coord_mapping(mapper,sheet,style='b-'):
    """
    Plot a coordinate mapping for a sheet.

    Given a CoordinateMapperFn (as for a CFProjection) and a sheet
    of the projection, plot a grid showing where the sheet's units
    are mapped.
    """

    from pylab import plot,hold,ishold

    xs = sheet.sheet_rows()
    ys = sheet.sheet_cols()

    hold_on = ishold()
    if not hold_on:
        plot()
    hold(True)

    for y in ys:
        pts = [mapper(x,y) for x in xs]
        plot([u for u,v in pts],
             [v for u,v in pts],
             style)

    for x in xs:
        pts = [mapper(x,y) for y in ys]
        plot([u for u,v in pts],
             [v for u,v in pts],
             style)

    hold(hold_on)

<commit_msg>Added histogramplot().<commit_after>"""
Line-based plotting commands using MatPlotLib.

Before importing this file, you will probably want to do something
like:

  import matplotlib
  matplotlib.use('TkAgg')

to select a backend, or else select an appropriate one in your
matplotlib.rc file (if any).  There are many backends available for
different GUI or non-GUI uses.

$Id$
"""
__version__='$Revision$'

import matplotlib
### JABHACKALERT: Need to figure out how to use Agg by default, but
### override it with TkAgg, so that the documentation file for this
### file can be generated.  It's apparently not as simple as just
### doing "matplotlib.use('Agg')" here.
import pylab
import re, os
import copy

from numpy.oldnumeric import arange, cos, pi, array, transpose, argmax, argmin

import topo

from topo.base.arrayutils import octave_output
from topo.base.sheet import Sheet
from topo.misc.utils import frange

def windowtitle(title):
    """
    Helper function to set the title of this PyLab plot window to a string.

    At the moment, PyLab does not offer a window-manager-independent
    means for controlling the window title, so what we do is to try
    what should work with Tkinter, and then suppress all errors.  That
    way we should be ok when rendering to a file-based backend, but
    will get nice titles in Tk windows.  If other toolkits are in use,
    the title can be set here using a similar try/except mechanism, or
    else there can be a switch based on the backend type.
    """
    try:
        manager = pylab.get_current_fig_manager()
        manager.window.title(title)
    except:
        pass

def vectorplot(vec,title=None, label=None):
    """
    Simple line plotting for any vector or list of numbers.

    Intended for interactive debugging or analyzing from the command
    prompt.  See MatPlotLib's pylab functions to create more elaborate
    or customized plots; this is just a simple example.

    An optional string can be supplied as a title for the figure, if
    desired.  At present, this is only used for the body of the figure,
    not the
    """
    pylab.plot(vec, label=label)
    pylab.grid(True)
    if (title): windowtitle(title)
    pylab.show._needmain = False
    pylab.show()

def matrixplot(mat,title=None):
    """
    Simple plotting for any matrix as a bitmap with axes.

    Like MatLab's imagesc, scales the values to fit in the range 0 to 1.0.
    Intended for interactive debugging or analyzing from the command
    prompt.  See MatPlotLib's pylab functions to create more elaborate
    or customized plots; this is just a simple example.
    """
    pylab.gray()
    pylab.figure(figsize=(5,5))
    pylab.imshow(mat,interpolation='nearest')
    if (title): windowtitle(title)
    pylab.show._needmain = False
    pylab.show()

def histogramplot(data,title=None,colors=None,*args,**kw):
    """
    Compute and plot the histogram of the supplied data.

    See help(pylab.hist) for help on the histogram function itself.

    If given, colors is an iterable collection of matplotlib.colors
    (see help (matplotlib.colors) ) specifying the bar colors.

    Example use:
     histplot([1,1,1,2,2,3,4,5],title='hist',colors='rgb',bins=3,normed=1)
    """
    pylab.show._needmain=False
    n,bins,bars = pylab.hist(data,*args,**kw)

    # if len(bars)!=len(colors), then the extra bars won't have their color changed,
    # or the extra colors will be ignored.
    if colors: [bar.set_fc(color) for bar,color in zip(bars,colors)]

    if (title): windowtitle(title)
    pylab.show()

def activityplot(sheet,activity=None,title=None,cmap=pylab.cm.Greys):
    """
    Plots the activity in a sheet.

    Gets plot's extent from sheet.bounds.aarect(). Adds a title and
    allows the selection of a colormap.  If activity is not given,
    the sheet's current activity is used.
    """
    l,b,r,t = sheet.bounds.aarect().lbrt()
    if activity is None:
        activity = sheet.activity
    pylab.imshow(activity, extent=(l,r,b,t),cmap=cmap)

def topographic_grid(xsheet_view_name='XPreference',ysheet_view_name='YPreference',axis=[-0.5,0.5,-0.5,0.5]):
    """
    By default, plot the XPreference and YPreference preferences for all
    Sheets for which they are defined, using MatPlotLib.

    If sheet_views other than XPreference and YPreference are desired,
    the names of these can be passed in as arguments.
    """
    for sheet in topo.sim.objects(Sheet).values():
        if ((xsheet_view_name in sheet.sheet_view_dict) and
            (ysheet_view_name in sheet.sheet_view_dict)):

            x = sheet.sheet_view_dict[xsheet_view_name].view()[0]
            y = sheet.sheet_view_dict[ysheet_view_name].view()[0]

            pylab.figure(figsize=(5,5))

            # This one-liner works in Octave, but in matplotlib it
            # results in lines that are all connected across rows and columns,
            # so here we plot each line separately:
            #   pylab.plot(x,y,"k-",transpose(x),transpose(y),"k-")
            # Here, the "k-" means plot in black using solid lines;
            # see matplotlib for more info.
            isint=pylab.isinteractive() # Temporarily make non-interactive for plotting
            pylab.ioff()
            for r,c in zip(y,x):
                pylab.plot(c,r,"k-")
            for r,c in zip(transpose(y),transpose(x)):
                pylab.plot(c,r,"k-")

            pylab.xlabel('x')
            pylab.ylabel('y')
            # Currently sets the input range arbitrarily; should presumably figure out
            # what the actual possible range is for this simulation (which would presumably
            # be the maximum size of any GeneratorSheet?).
            pylab.axis(axis)
            windowtitle('Topographic mapping to '+sheet.name+' at time '+str(topo.sim.time()))

            # Will need to provide a way to save this output
            # when there is no GUI
            #pylab.savefig('simple_plot')

            if isint: pylab.ion()

            # Allow multiple concurrent plots; there may be a
            # cleaner way to do this...
            pylab.show._needmain = False
            pylab.show()

def tuning_curve_data(sheet, x_axis, curve_label, i_value, j_value):
    """
    Collect data for one tuning curve from the curve_dict of each sheet.

    The unit to plot is specified as a matrix index (i_value,j_value).
    There may be many possible curves in the curve_dict; the specific
    one is determined by the x_axis (i.e., what feature to plot), and
    the curve_label (which determines other parameters that specify a
    particular plot).  Returns the feature values (the x_values,
    i.e. locations along the x axis), x_values) and the corresponding
    responses (the y_values).
    """
    x_values= sorted(sheet.curve_dict[x_axis][curve_label].keys())
    y_values=[sheet.curve_dict[x_axis][curve_label][key].view()[0][i_value,j_value] for key in x_values]
    return x_values, y_values

def or_tuning_curve(x_axis,plot_type,unit):
    """
    Plots a tuning curve for orientation which rotates the curve
    values so that minimum y values are at the minimum x value.
    The y_values and labels are rotated by an amount determined by the minmum
    y_value for the first curve plotted (usually the lowest contrast curve).

    The curve datapoints are collected from the curve_dict for each
    sheet, depending on the value of topo.commands.analysis.sheet_name
    and topo.commands.analysis.coordinate (which may be set by the GUI or
    by hand).
    """
    # There may be cases (ie when the lowest contrast curve gives a lot of zero y_values)
    # in which the maximum is not in the centre.
    # Ideally this should be changed so that the preferred orientation is in the centre.
    # This may also be useful for other tuning curve types, not just orientation ie. direction.

    sheet=topo.sim[topo.commands.analysis.sheet_name]
    coordinate=topo.commands.analysis.coordinate
    i_value,j_value=sheet.sheet2matrixidx(coordinate[0],coordinate[1])

    pylab.figure(figsize=(7,7))
    isint=pylab.isinteractive()
    pylab.ioff()
    manager = pylab.get_current_fig_manager()

    pylab.ylabel('Response')
    pylab.xlabel(x_axis.capitalize()+' ('+unit+')')
    pylab.title('Sheet '+topo.commands.analysis.sheet_name+', coordinate(x,y)='+'('+
	 	str(coordinate[0])+','+str(coordinate[1])+')'+' at time '+str(topo.sim.time()))
    manager.window.title(topo.sim.name+': '+x_axis.capitalize()+' Tuning Curve')

    def rotate(seq, n=1):
        n = n % len(seq) # n=hop interval
        return seq[n:] + seq[:n]

    first_curve=True
    for curve_label in sorted(sheet.curve_dict[x_axis].keys()):
	if first_curve==True:
	    x_values, y_values = tuning_curve_data(sheet,x_axis, curve_label, i_value, j_value)
	    min_arg=argmin(y_values)
	    x_min=x_values[min_arg]
	    y_min=y_values[min_arg]
	    y_values=rotate(y_values, n=min_arg)
	    ticks=rotate(x_values, n=min_arg)
	    label_min=ticks[0]
	    x_max=min(x_values)+pi
	    x_values.append(x_max)
	    y_values.append(y_min)
	    first_curve=False
	else:
	    y_values=[sheet.curve_dict[x_axis][curve_label][key].view()[0][i_value,j_value] for key in ticks]
	    y_min=sheet.curve_dict[x_axis][curve_label][x_min].view()[0][i_value,j_value]
	    y_values.append(y_min)
	labels = copy.copy(ticks)
	labels.append(label_min)
	labels= [x*180/pi for x in labels]
	labels_new = [str(int(x)) for x in labels]
	pylab.xticks(x_values, labels_new)
	plot_type(x_values, y_values, label=curve_label)

    if isint: pylab.ion()
    pylab.legend(loc=4)
    pylab.show._needmain = False
    pylab.show()

def tuning_curve(x_axis,plot_type,unit):
    """
    Plots a tuning curve for the appropriate feature type, such as orientation, contrast or size.

    The curve datapoints are collected from the curve_dict for each
    sheet, depending on the value of topo.commands.analysis.sheet_name
    and topo.commands.analysis.coordinate (which may be set by the GUI or
    by hand).
    """

    sheet=topo.sim[topo.commands.analysis.sheet_name]
    coordinate=topo.commands.analysis.coordinate
    i_value,j_value=sheet.sheet2matrixidx(coordinate[0],coordinate[1])

    pylab.figure(figsize=(7,7))
    isint=pylab.isinteractive()
    pylab.ioff()
    manager = pylab.get_current_fig_manager()

    pylab.ylabel('Response')
    pylab.xlabel(x_axis.capitalize()+' ('+unit+')')
    pylab.title('Sheet '+topo.commands.analysis.sheet_name+', coordinate(x,y)='+'('+
	 	str(coordinate[0])+','+str(coordinate[1])+')'+' at time '+str(topo.sim.time()))
    manager.window.title(topo.sim.name+': '+x_axis.capitalize()+' Tuning Curve')

    for curve_label in sorted(sheet.curve_dict[x_axis].keys()):
        x_values, y_values = tuning_curve_data(sheet,x_axis, curve_label, i_value, j_value)
        plot_type(x_values, y_values, label=curve_label)

    if isint: pylab.ion()
    pylab.legend(loc=4)
    pylab.show._needmain = False
    pylab.show()

def plot_cfproj_mapping(dest,proj='Afferent',style='b-'):
    """
    Given a CF sheet receiving a CFProjection, plot
    the mapping of the dests CF centers on the src sheet.
    """
    if isinstance(dest,str):
        from topo import sim
        dest = sim[dest]
    plot_coord_mapping(dest.projections()[proj].coord_mapper,
                       dest,style=style)

def plot_coord_mapping(mapper,sheet,style='b-'):
    """
    Plot a coordinate mapping for a sheet.

    Given a CoordinateMapperFn (as for a CFProjection) and a sheet
    of the projection, plot a grid showing where the sheet's units
    are mapped.
    """

    from pylab import plot,hold,ishold

    xs = sheet.sheet_rows()
    ys = sheet.sheet_cols()

    hold_on = ishold()
    if not hold_on:
        plot()
    hold(True)

    for y in ys:
        pts = [mapper(x,y) for x in xs]
        plot([u for u,v in pts],
             [v for u,v in pts],
             style)

    for x in xs:
        pts = [mapper(x,y) for y in ys]
        plot([u for u,v in pts],
             [v for u,v in pts],
             style)

    hold(hold_on)

<|endoftext|>


<commit_before>"""
Line-based plotting commands using MatPlotLib.

Before importing this file, you will probably want to do something
like:

  import matplotlib
  matplotlib.use('TkAgg')

to select a backend, or else select an appropriate one in your
matplotlib.rc file (if any).  There are many backends available for
different GUI or non-GUI uses.

$Id$
"""
__version__='$Revision$'

import matplotlib
### JABHACKALERT: Need to figure out how to use Agg by default, but
### override it with TkAgg, so that the documentation file for this
### file can be generated.  It's apparently not as simple as just
### doing "matplotlib.use('Agg')" here.
import pylab
import re, os
import copy

from numpy.oldnumeric import arange, cos, pi, array, transpose, argmax, argmin

import topo

from topo.base.arrayutils import octave_output
from topo.base.sheet import Sheet
from topo.misc.utils import frange

def windowtitle(title):
    """
    Helper function to set the title of this PyLab plot window to a string.

    At the moment, PyLab does not offer a window-manager-independent
    means for controlling the window title, so what we do is to try
    what should work with Tkinter, and then suppress all errors.  That
    way we should be ok when rendering to a file-based backend, but
    will get nice titles in Tk windows.  If other toolkits are in use,
    the title can be set here using a similar try/except mechanism, or
    else there can be a switch based on the backend type.
    """
    try:
        manager = pylab.get_current_fig_manager()
        manager.window.title(title)
    except:
        pass

def vectorplot(vec,title=None, label=None):
    """
    Simple line plotting for any vector or list of numbers.

    Intended for interactive debugging or analyzing from the command
    prompt.  See MatPlotLib's pylab functions to create more elaborate
    or customized plots; this is just a simple example.

    An optional string can be supplied as a title for the figure, if
    desired.  At present, this is only used for the body of the figure,
    not the
    """
    pylab.plot(vec, label=label)
    pylab.grid(True)
    if (title): windowtitle(title)
    pylab.show._needmain = False
    pylab.show()

def matrixplot(mat,title=None):
    """
    Simple plotting for any matrix as a bitmap with axes.

    Like MatLab's imagesc, scales the values to fit in the range 0 to 1.0.
    Intended for interactive debugging or analyzing from the command
    prompt.  See MatPlotLib's pylab functions to create more elaborate
    or customized plots; this is just a simple example.
    """
    pylab.gray()
    pylab.figure(figsize=(5,5))
    pylab.imshow(mat,interpolation='nearest')
    if (title): windowtitle(title)
    pylab.show._needmain = False
    pylab.show()

def histogramplot(data,title=None,colors=None,*args,**kw):
    """
    Compute and plot the histogram of the supplied data.

    See help(pylab.hist) for help on the histogram function itself.

    If given, colors is an iterable collection of matplotlib.colors
    (see help (matplotlib.colors) ) specifying the bar colors.

    Example use:
     histplot([1,1,1,2,2,3,4,5],title='hist',colors='rgb',bins=3,normed=1)
    """
    pylab.show._needmain=False
    n,bins,bars = pylab.hist(data,*args,**kw)

    # if len(bars)!=len(colors), then the extra bars won't have their color changed,
    # or the extra colors will be ignored.
    if colors: [bar.set_fc(color) for bar,color in zip(bars,colors)]

    if (title): windowtitle(title)
    pylab.show()

def activityplot(sheet,activity=None,title=None,cmap=pylab.cm.Greys):
    """
    Plots the activity in a sheet.

    Gets plot's extent from sheet.bounds.aarect(). Adds a title and
    allows the selection of a colormap.  If activity is not given,
    the sheet's current activity is used.
    """
    l,b,r,t = sheet.bounds.aarect().lbrt()
    if activity is None:
        activity = sheet.activity
    pylab.imshow(activity, extent=(l,r,b,t),cmap=cmap)

def topographic_grid(xsheet_view_name='XPreference',ysheet_view_name='YPreference',axis=[-0.5,0.5,-0.5,0.5]):
    """
    By default, plot the XPreference and YPreference preferences for all
    Sheets for which they are defined, using MatPlotLib.

    If sheet_views other than XPreference and YPreference are desired,
    the names of these can be passed in as arguments.
    """
    for sheet in topo.sim.objects(Sheet).values():
        if ((xsheet_view_name in sheet.sheet_view_dict) and
            (ysheet_view_name in sheet.sheet_view_dict)):

            x = sheet.sheet_view_dict[xsheet_view_name].view()[0]
            y = sheet.sheet_view_dict[ysheet_view_name].view()[0]

            pylab.figure(figsize=(5,5))

            # This one-liner works in Octave, but in matplotlib it
            # results in lines that are all connected across rows and columns,
            # so here we plot each line separately:
            #   pylab.plot(x,y,"k-",transpose(x),transpose(y),"k-")
            # Here, the "k-" means plot in black using solid lines;
            # see matplotlib for more info.
            isint=pylab.isinteractive() # Temporarily make non-interactive for plotting
            pylab.ioff()
            for r,c in zip(y,x):
                pylab.plot(c,r,"k-")
            for r,c in zip(transpose(y),transpose(x)):
                pylab.plot(c,r,"k-")

            pylab.xlabel('x')
            pylab.ylabel('y')
            # Currently sets the input range arbitrarily; should presumably figure out
            # what the actual possible range is for this simulation (which would presumably
            # be the maximum size of any GeneratorSheet?).
            pylab.axis(axis)
            windowtitle('Topographic mapping to '+sheet.name+' at time '+str(topo.sim.time()))

            # Will need to provide a way to save this output
            # when there is no GUI
            #pylab.savefig('simple_plot')

            if isint: pylab.ion()

            # Allow multiple concurrent plots; there may be a
            # cleaner way to do this...
            pylab.show._needmain = False
            pylab.show()

def tuning_curve_data(sheet, x_axis, curve_label, i_value, j_value):
    """
    Collect data for one tuning curve from the curve_dict of each sheet.

    The unit to plot is specified as a matrix index (i_value,j_value).
    There may be many possible curves in the curve_dict; the specific
    one is determined by the x_axis (i.e., what feature to plot), and
    the curve_label (which determines other parameters that specify a
    particular plot).  Returns the feature values (the x_values,
    i.e. locations along the x axis), x_values) and the corresponding
    responses (the y_values).
    """
    x_values= sorted(sheet.curve_dict[x_axis][curve_label].keys())
    y_values=[sheet.curve_dict[x_axis][curve_label][key].view()[0][i_value,j_value] for key in x_values]
    return x_values, y_values

def or_tuning_curve(x_axis,plot_type,unit):
    """
    Plots a tuning curve for orientation which rotates the curve
    values so that minimum y values are at the minimum x value.
    The y_values and labels are rotated by an amount determined by the minmum
    y_value for the first curve plotted (usually the lowest contrast curve).

    The curve datapoints are collected from the curve_dict for each
    sheet, depending on the value of topo.commands.analysis.sheet_name
    and topo.commands.analysis.coordinate (which may be set by the GUI or
    by hand).
    """
    # There may be cases (ie when the lowest contrast curve gives a lot of zero y_values)
    # in which the maximum is not in the centre.
    # Ideally this should be changed so that the preferred orientation is in the centre.
    # This may also be useful for other tuning curve types, not just orientation ie. direction.

    sheet=topo.sim[topo.commands.analysis.sheet_name]
    coordinate=topo.commands.analysis.coordinate
    i_value,j_value=sheet.sheet2matrixidx(coordinate[0],coordinate[1])

    pylab.figure(figsize=(7,7))
    isint=pylab.isinteractive()
    pylab.ioff()
    manager = pylab.get_current_fig_manager()

    pylab.ylabel('Response')
    pylab.xlabel(x_axis.capitalize()+' ('+unit+')')
    pylab.title('Sheet '+topo.commands.analysis.sheet_name+', coordinate(x,y)='+'('+
	 	str(coordinate[0])+','+str(coordinate[1])+')'+' at time '+str(topo.sim.time()))
    manager.window.title(topo.sim.name+': '+x_axis.capitalize()+' Tuning Curve')

    def rotate(seq, n=1):
        n = n % len(seq) # n=hop interval
        return seq[n:] + seq[:n]

    first_curve=True
    for curve_label in sorted(sheet.curve_dict[x_axis].keys()):
	if first_curve==True:
	    x_values, y_values = tuning_curve_data(sheet,x_axis, curve_label, i_value, j_value)
	    min_arg=argmin(y_values)
	    x_min=x_values[min_arg]
	    y_min=y_values[min_arg]
	    y_values=rotate(y_values, n=min_arg)
	    ticks=rotate(x_values, n=min_arg)
	    label_min=ticks[0]
	    x_max=min(x_values)+pi
	    x_values.append(x_max)
	    y_values.append(y_min)
	    first_curve=False
	else:
	    y_values=[sheet.curve_dict[x_axis][curve_label][key].view()[0][i_value,j_value] for key in ticks]
	    y_min=sheet.curve_dict[x_axis][curve_label][x_min].view()[0][i_value,j_value]
	    y_values.append(y_min)
	labels = copy.copy(ticks)
	labels.append(label_min)
	labels= [x*180/pi for x in labels]
	labels_new = [str(int(x)) for x in labels]
	pylab.xticks(x_values, labels_new)
	plot_type(x_values, y_values, label=curve_label)

    if isint: pylab.ion()
    pylab.legend(loc=4)
    pylab.show._needmain = False
    pylab.show()

def tuning_curve(x_axis,plot_type,unit):
    """
    Plots a tuning curve for the appropriate feature type, such as orientation, contrast or size.

    The curve datapoints are collected from the curve_dict for each
    sheet, depending on the value of topo.commands.analysis.sheet_name
    and topo.commands.analysis.coordinate (which may be set by the GUI or
    by hand).
    """

    sheet=topo.sim[topo.commands.analysis.sheet_name]
    coordinate=topo.commands.analysis.coordinate
    i_value,j_value=sheet.sheet2matrixidx(coordinate[0],coordinate[1])

    pylab.figure(figsize=(7,7))
    isint=pylab.isinteractive()
    pylab.ioff()
    manager = pylab.get_current_fig_manager()

    pylab.ylabel('Response')
    pylab.xlabel(x_axis.capitalize()+' ('+unit+')')
    pylab.title('Sheet '+topo.commands.analysis.sheet_name+', coordinate(x,y)='+'('+
	 	str(coordinate[0])+','+str(coordinate[1])+')'+' at time '+str(topo.sim.time()))
    manager.window.title(topo.sim.name+': '+x_axis.capitalize()+' Tuning Curve')

    for curve_label in sorted(sheet.curve_dict[x_axis].keys()):
        x_values, y_values = tuning_curve_data(sheet,x_axis, curve_label, i_value, j_value)
        plot_type(x_values, y_values, label=curve_label)

    if isint: pylab.ion()
    pylab.legend(loc=4)
    pylab.show._needmain = False
    pylab.show()

def plot_cfproj_mapping(dest,proj='Afferent',style='b-'):
    """
    Given a CF sheet receiving a CFProjection, plot
    the mapping of the dests CF centers on the src sheet.
    """
    if isinstance(dest,str):
        from topo import sim
        dest = sim[dest]
    plot_coord_mapping(dest.projections()[proj].coord_mapper,
                       dest,style=style)

def plot_coord_mapping(mapper,sheet,style='b-'):
    """
    Plot a coordinate mapping for a sheet.

    Given a CoordinateMapperFn (as for a CFProjection) and a sheet
    of the projection, plot a grid showing where the sheet's units
    are mapped.
    """

    from pylab import plot,hold,ishold

    xs = sheet.sheet_rows()
    ys = sheet.sheet_cols()

    hold_on = ishold()
    if not hold_on:
        plot()
    hold(True)

    for y in ys:
        pts = [mapper(x,y) for x in xs]
        plot([u for u,v in pts],
             [v for u,v in pts],
             style)

    for x in xs:
        pts = [mapper(x,y) for y in ys]
        plot([u for u,v in pts],
             [v for u,v in pts],
             style)

    hold(hold_on)

<commit_msg>Added gradient plot (from JB's octave code).<commit_after>"""
Line-based plotting commands using MatPlotLib.

Before importing this file, you will probably want to do something
like:

  import matplotlib
  matplotlib.use('TkAgg')

to select a backend, or else select an appropriate one in your
matplotlib.rc file (if any).  There are many backends available for
different GUI or non-GUI uses.

$Id$
"""
__version__='$Revision$'

import numpy
import matplotlib
### JABHACKALERT: Need to figure out how to use Agg by default, but
### override it with TkAgg, so that the documentation file for this
### file can be generated.  It's apparently not as simple as just
### doing "matplotlib.use('Agg')" here.
import pylab
import re, os
import copy

from numpy.oldnumeric import arange, cos, pi, array, transpose, argmax, argmin

import topo

from topo.base.arrayutils import octave_output
from topo.base.sheet import Sheet
from topo.misc.utils import frange

def windowtitle(title):
    """
    Helper function to set the title of this PyLab plot window to a string.

    At the moment, PyLab does not offer a window-manager-independent
    means for controlling the window title, so what we do is to try
    what should work with Tkinter, and then suppress all errors.  That
    way we should be ok when rendering to a file-based backend, but
    will get nice titles in Tk windows.  If other toolkits are in use,
    the title can be set here using a similar try/except mechanism, or
    else there can be a switch based on the backend type.
    """
    try:
        manager = pylab.get_current_fig_manager()
        manager.window.title(title)
    except:
        pass

def vectorplot(vec,title=None, label=None):
    """
    Simple line plotting for any vector or list of numbers.

    Intended for interactive debugging or analyzing from the command
    prompt.  See MatPlotLib's pylab functions to create more elaborate
    or customized plots; this is just a simple example.

    An optional string can be supplied as a title for the figure, if
    desired.  At present, this is only used for the body of the figure,
    not the
    """
    pylab.plot(vec, label=label)
    pylab.grid(True)
    if (title): windowtitle(title)
    pylab.show._needmain = False
    pylab.show()

def matrixplot(mat,title=None):
    """
    Simple plotting for any matrix as a bitmap with axes.

    Like MatLab's imagesc, scales the values to fit in the range 0 to 1.0.
    Intended for interactive debugging or analyzing from the command
    prompt.  See MatPlotLib's pylab functions to create more elaborate
    or customized plots; this is just a simple example.
    """
    pylab.gray()
    pylab.figure(figsize=(5,5))
    pylab.imshow(mat,interpolation='nearest')
    if (title): windowtitle(title)
    pylab.show._needmain = False
    pylab.show()

def histogramplot(data,title=None,colors=None,*args,**kw):
    """
    Compute and plot the histogram of the supplied data.

    See help(pylab.hist) for help on the histogram function itself.

    If given, colors is an iterable collection of matplotlib.colors
    (see help (matplotlib.colors) ) specifying the bar colors.

    Example use:
     histplot([1,1,1,2,2,3,4,5],title='hist',colors='rgb',bins=3,normed=1)
    """
    pylab.show._needmain=False
    n,bins,bars = pylab.hist(data,*args,**kw)

    # if len(bars)!=len(colors), then the extra bars won't have their color changed,
    # or the extra colors will be ignored.
    if colors: [bar.set_fc(color) for bar,color in zip(bars,colors)]

    if (title): windowtitle(title)
    pylab.show()

def gradientplot(data,title=None):
    """
    Compute and show the gradient plot of the supplied data.
    """
    # CB: very direct translation of some octave code.
    # Consider division by zero?
    r,c = data.shape
    dx = numpy.diff(data,1)
    dx = dx[1:r-1,1:c-1]
    x = dx.min()
    y = dx.max()

    dx = ((dx-x)/(y-x)-0.5)*2.0

    dy=numpy.diff(data.transpose(),1)
    dy=dy[1:r-1,1:c-1]
    x=dy.min()
    y=dy.max()
    dy=((dy-x)/(y-x)-0.5)*2.0
    matrixplot((cos(dx*2*pi)+cos(dy*2*pi)),title=title)

def activityplot(sheet,activity=None,title=None,cmap=pylab.cm.Greys):
    """
    Plots the activity in a sheet.

    Gets plot's extent from sheet.bounds.aarect(). Adds a title and
    allows the selection of a colormap.  If activity is not given,
    the sheet's current activity is used.
    """
    l,b,r,t = sheet.bounds.aarect().lbrt()
    if activity is None:
        activity = sheet.activity
    pylab.imshow(activity, extent=(l,r,b,t),cmap=cmap)

def topographic_grid(xsheet_view_name='XPreference',ysheet_view_name='YPreference',axis=[-0.5,0.5,-0.5,0.5]):
    """
    By default, plot the XPreference and YPreference preferences for all
    Sheets for which they are defined, using MatPlotLib.

    If sheet_views other than XPreference and YPreference are desired,
    the names of these can be passed in as arguments.
    """
    for sheet in topo.sim.objects(Sheet).values():
        if ((xsheet_view_name in sheet.sheet_view_dict) and
            (ysheet_view_name in sheet.sheet_view_dict)):

            x = sheet.sheet_view_dict[xsheet_view_name].view()[0]
            y = sheet.sheet_view_dict[ysheet_view_name].view()[0]

            pylab.figure(figsize=(5,5))

            # This one-liner works in Octave, but in matplotlib it
            # results in lines that are all connected across rows and columns,
            # so here we plot each line separately:
            #   pylab.plot(x,y,"k-",transpose(x),transpose(y),"k-")
            # Here, the "k-" means plot in black using solid lines;
            # see matplotlib for more info.
            isint=pylab.isinteractive() # Temporarily make non-interactive for plotting
            pylab.ioff()
            for r,c in zip(y,x):
                pylab.plot(c,r,"k-")
            for r,c in zip(transpose(y),transpose(x)):
                pylab.plot(c,r,"k-")

            pylab.xlabel('x')
            pylab.ylabel('y')
            # Currently sets the input range arbitrarily; should presumably figure out
            # what the actual possible range is for this simulation (which would presumably
            # be the maximum size of any GeneratorSheet?).
            pylab.axis(axis)
            windowtitle('Topographic mapping to '+sheet.name+' at time '+str(topo.sim.time()))

            # Will need to provide a way to save this output
            # when there is no GUI
            #pylab.savefig('simple_plot')

            if isint: pylab.ion()

            # Allow multiple concurrent plots; there may be a
            # cleaner way to do this...
            pylab.show._needmain = False
            pylab.show()

def tuning_curve_data(sheet, x_axis, curve_label, i_value, j_value):
    """
    Collect data for one tuning curve from the curve_dict of each sheet.

    The unit to plot is specified as a matrix index (i_value,j_value).
    There may be many possible curves in the curve_dict; the specific
    one is determined by the x_axis (i.e., what feature to plot), and
    the curve_label (which determines other parameters that specify a
    particular plot).  Returns the feature values (the x_values,
    i.e. locations along the x axis), x_values) and the corresponding
    responses (the y_values).
    """
    x_values= sorted(sheet.curve_dict[x_axis][curve_label].keys())
    y_values=[sheet.curve_dict[x_axis][curve_label][key].view()[0][i_value,j_value] for key in x_values]
    return x_values, y_values

def or_tuning_curve(x_axis,plot_type,unit):
    """
    Plots a tuning curve for orientation which rotates the curve
    values so that minimum y values are at the minimum x value.
    The y_values and labels are rotated by an amount determined by the minmum
    y_value for the first curve plotted (usually the lowest contrast curve).

    The curve datapoints are collected from the curve_dict for each
    sheet, depending on the value of topo.commands.analysis.sheet_name
    and topo.commands.analysis.coordinate (which may be set by the GUI or
    by hand).
    """
    # There may be cases (ie when the lowest contrast curve gives a lot of zero y_values)
    # in which the maximum is not in the centre.
    # Ideally this should be changed so that the preferred orientation is in the centre.
    # This may also be useful for other tuning curve types, not just orientation ie. direction.

    sheet=topo.sim[topo.commands.analysis.sheet_name]
    coordinate=topo.commands.analysis.coordinate
    i_value,j_value=sheet.sheet2matrixidx(coordinate[0],coordinate[1])

    pylab.figure(figsize=(7,7))
    isint=pylab.isinteractive()
    pylab.ioff()
    manager = pylab.get_current_fig_manager()

    pylab.ylabel('Response')
    pylab.xlabel(x_axis.capitalize()+' ('+unit+')')
    pylab.title('Sheet '+topo.commands.analysis.sheet_name+', coordinate(x,y)='+'('+
	 	str(coordinate[0])+','+str(coordinate[1])+')'+' at time '+str(topo.sim.time()))
    manager.window.title(topo.sim.name+': '+x_axis.capitalize()+' Tuning Curve')

    def rotate(seq, n=1):
        n = n % len(seq) # n=hop interval
        return seq[n:] + seq[:n]

    first_curve=True
    for curve_label in sorted(sheet.curve_dict[x_axis].keys()):
	if first_curve==True:
	    x_values, y_values = tuning_curve_data(sheet,x_axis, curve_label, i_value, j_value)
	    min_arg=argmin(y_values)
	    x_min=x_values[min_arg]
	    y_min=y_values[min_arg]
	    y_values=rotate(y_values, n=min_arg)
	    ticks=rotate(x_values, n=min_arg)
	    label_min=ticks[0]
	    x_max=min(x_values)+pi
	    x_values.append(x_max)
	    y_values.append(y_min)
	    first_curve=False
	else:
	    y_values=[sheet.curve_dict[x_axis][curve_label][key].view()[0][i_value,j_value] for key in ticks]
	    y_min=sheet.curve_dict[x_axis][curve_label][x_min].view()[0][i_value,j_value]
	    y_values.append(y_min)
	labels = copy.copy(ticks)
	labels.append(label_min)
	labels= [x*180/pi for x in labels]
	labels_new = [str(int(x)) for x in labels]
	pylab.xticks(x_values, labels_new)
	plot_type(x_values, y_values, label=curve_label)

    if isint: pylab.ion()
    pylab.legend(loc=4)
    pylab.show._needmain = False
    pylab.show()

def tuning_curve(x_axis,plot_type,unit):
    """
    Plots a tuning curve for the appropriate feature type, such as orientation, contrast or size.

    The curve datapoints are collected from the curve_dict for each
    sheet, depending on the value of topo.commands.analysis.sheet_name
    and topo.commands.analysis.coordinate (which may be set by the GUI or
    by hand).
    """

    sheet=topo.sim[topo.commands.analysis.sheet_name]
    coordinate=topo.commands.analysis.coordinate
    i_value,j_value=sheet.sheet2matrixidx(coordinate[0],coordinate[1])

    pylab.figure(figsize=(7,7))
    isint=pylab.isinteractive()
    pylab.ioff()
    manager = pylab.get_current_fig_manager()

    pylab.ylabel('Response')
    pylab.xlabel(x_axis.capitalize()+' ('+unit+')')
    pylab.title('Sheet '+topo.commands.analysis.sheet_name+', coordinate(x,y)='+'('+
	 	str(coordinate[0])+','+str(coordinate[1])+')'+' at time '+str(topo.sim.time()))
    manager.window.title(topo.sim.name+': '+x_axis.capitalize()+' Tuning Curve')

    for curve_label in sorted(sheet.curve_dict[x_axis].keys()):
        x_values, y_values = tuning_curve_data(sheet,x_axis, curve_label, i_value, j_value)
        plot_type(x_values, y_values, label=curve_label)

    if isint: pylab.ion()
    pylab.legend(loc=4)
    pylab.show._needmain = False
    pylab.show()

def plot_cfproj_mapping(dest,proj='Afferent',style='b-'):
    """
    Given a CF sheet receiving a CFProjection, plot
    the mapping of the dests CF centers on the src sheet.
    """
    if isinstance(dest,str):
        from topo import sim
        dest = sim[dest]
    plot_coord_mapping(dest.projections()[proj].coord_mapper,
                       dest,style=style)

def plot_coord_mapping(mapper,sheet,style='b-'):
    """
    Plot a coordinate mapping for a sheet.

    Given a CoordinateMapperFn (as for a CFProjection) and a sheet
    of the projection, plot a grid showing where the sheet's units
    are mapped.
    """

    from pylab import plot,hold,ishold

    xs = sheet.sheet_rows()
    ys = sheet.sheet_cols()

    hold_on = ishold()
    if not hold_on:
        plot()
    hold(True)

    for y in ys:
        pts = [mapper(x,y) for x in xs]
        plot([u for u,v in pts],
             [v for u,v in pts],
             style)

    for x in xs:
        pts = [mapper(x,y) for y in ys]
        plot([u for u,v in pts],
             [v for u,v in pts],
             style)

    hold(hold_on)

<|endoftext|>


<commit_before>from utils     import *
from mydicts   import *
from myschemas import *

def getchichar(request,char):
    cdict_name = request.request.get('dict_name', CHICHARDICT)
    chichars_query = Chichar.query(Chichar.chichar == char)
    qresult = chichars_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getallchichars(request):
    dict_name      = request.request.get('dict_name',CHICHARDICT)
    chichars_query = Chichar.query(ancestor=dict_key(dict_name)).order(-Chichar.date)
    chichars       = chichars_query.fetch()
    return chichars

def getallsentences(request):
    sdict_name      = request.request.get('dict_name',SENTENCEDICT)
    sentences_query = Sentence.query(ancestor=dict_key(sdict_name))
    sentences       = sentences_query.fetch()
    return sentences

def getsentence(request,chi):
    sdict_name = request.request.get('dict_name', SENTENCEDICT)
    sentences_query = Sentence.query(Sentence.chichar == chi)
    qresult = sentences_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getword(request,chi):
    sdict_name = request.request.get('dict_name', WORDDICT)
    words_query = Word.query(Word.chichar == chi)
    qresult = words_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getsentencechichars(request,sentence):
    return charsentence2chichars(request,sentence.chichar)

def getwordchichars(request,word):
    return charsentence2chichars(request,word.chichar)

def addchichar(request,chi,translation,pronunciation):
    sdict_name = request.request.get('dict_name', CHICHARDICT)
    ochichar = Chichar(parent=dict_key(sdict_name));
    ochichar.chichar        = chi
    ochichar.translation    = translation
    ochichar.pronunciation  = pronunciation
    ochichar.put()
    return ochichar

def addsentence(request,chi,translation,pronunciation):
    sdict_name = request.request.get('dict_name', SENTENCEDICT)
    osentence = Sentence(parent=dict_key(sdict_name));
    osentence.chichar        = chi
    osentence.translation    = translation
    osentence.pronunciation  = pronunciation
    osentence.put()
    return osentence

def addword(request,chi,translation,pronunciation):
    sdict_name = request.request.get('dict_name', WORDDICT)
    oword = Word(parent=dict_key(sdict_name));
    oword.chichar        = chi
    oword.translation    = translation
    oword.pronunciation  = pronunciation
    oword.put()
    return oword

def asciitoremove():
    return [""," ","\n","\t",",",".","!","4","?"]

def sentence2chars(charstring):
    chars  = lsubstract(list(charstring.strip()),asciitoremove())
    return chars

def sentence2pinyins(charstring):
    pinyins  = lsubstract(list(charstring.strip()),asciitoremove())
    return pinyins

def charsentence2chichars(request,charstring):
    result = []
    for char in sentence2chars(charstring):
        chichar = getchichar(request,char)
        if chichar != None:
            result.append(chichar)
    return result

def chars2pinyins(request,chars):
    result = []
    for char in chars:
        chichar = getchichar(request,char)
        if chichar != None:
            result.append(chichar.pronunciation)
        else:
            result.append("TODO")
    return result

def checkaddchar(request,char,translation,pronunciation):
    char          = char.strip()
    if translation:
        translation   = translation.strip()
    if pronunciation:
        pronunciation = pronunciation.strip()
    ochichar = getchichar(request,char)
    if ochichar == None:
        request.response.write("need to add " + char)
        result = addchichar(request,char,translation,pronunciation)
    else:
        # TODO:check everything is the same
        request.response.write("char " + char + " already known")
        result = ochichar
    return result

def makepronunciation(chi,pinyins):
    index = 0
    result = ""
    for char in list(chi):
        if char in asciitoremove():
            result = result + char
        else:
            result = result + pinyins[index]
            index += 1
    return result

def checkaddsentence(request,chi,translation,pronunciation):
    osentence = getsentence(request,chi)

    if osentence == None:
        chars  = sentence2chars(chi.strip())
        pinyins = iff( pronunciation.strip() == "TODO", chars2pinyins(request,chars), sentence2pinyins(pronunciation))

        if not (len(chars) == len(pinyins)):
            request.response.write("
  error matching char " + "@".join(chars) + " pynyin " + "@".join(pinyins) + "
\n")
            result = None
        else:
            request.response.write("need to add sentence " + chi)
            result = addsentence(request,chi,translation,makepronunciation(chi,pinyins))
            for (char,pinyin) in zip(chars,pinyins):
                chichar = checkaddchar(request,char,None,pinyin)
    else:
        request.response.write("sentence " + chi + " already known")
        result = osentence

    return result

def checkaddword(request,chi,translation,pronunciation):
    oword = getword(request,chi)

    if oword == None:
        chars  = sentence2chars(chi.strip())
        pinyins = iff( pronunciation.strip() == "TODO", chars2pinyins(request,chars), sentence2pinyins(pronunciation))

        if not (len(chars) == len(pinyins)):
            request.response.write("
  error matching char " + "@".join(chars) + " pynyin " + "@".join(pinyins) + "
\n")
            result = None
        else:
            request.response.write("need to add word " + chi)
            result = addword(request,chi,translation,makepronunciation(chi,pinyins))
            for (char,pinyin) in zip(chars,pinyins):
                chichar = checkaddchar(request,char,None,pinyin)
    else:
        request.response.write("word " + chi + " already known")
        result = oword

    return result

def allocatedata(request,chichar,translation,pronunciation):
    result = None
    if len(chichar) == 1:
        # this is a new char
        result = checkaddchar(request,chichar,translation,pronunciation)
    else:
        if len(translation.split(" ")) > 1:
            # this is a sentence
            result = checkaddsentence(request,chichar,translation,pronunciation)
        else:
            result = checkaddword(request,chichar,translation,pronunciation)
    return result

def datatype(data):
    if not data:
        return None
    if isinstance(data,Chichar):
        return "Chichar"
    if isinstance(data,Sentence):
        return "Sentence"
    if isinstance(data,Word):
        return "Word"

def removeduplicatechichars(request):
    array = {}
    for chichar in getallchichars(request):
        if not chichar.chichar in array:
            array[chichar.chichar] = []
        array[chichar.chichar].append(chichar)

    for char in array:
        if len(array[char]) > 1:
            for chichar in array[char]:
                if chichar.translation == None:
                    chichar.key.delete()

def getbook(request,bookname):
    books_query = Book.query(Book.name == bookname)
    qresult = books_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getchapter(request,bookname,chaptername):
    chapters_query = Chapter.query(Chapter.book == bookname, Chapter.name == chaptername)
    qresult = chapters_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getunit(request,chaptername,chichar):
    units_query = Unit.query(Unit.chapter == chaptername, Unit.chichar == chichar)
    qresult = units_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getunitdata(request,unit):
    return ndb.Key(urlsafe=unit.unitkey).get()

<commit_msg>Correction for multiple pinyin<commit_after>from utils     import *
from mydicts   import *
from myschemas import *

def getchichar(request,char):
    cdict_name = request.request.get('dict_name', CHICHARDICT)
    chichars_query = Chichar.query(Chichar.chichar == char)
    qresult = chichars_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getallchichars(request):
    dict_name      = request.request.get('dict_name',CHICHARDICT)
    chichars_query = Chichar.query(ancestor=dict_key(dict_name)).order(-Chichar.date)
    chichars       = chichars_query.fetch()
    return chichars

def getallsentences(request):
    sdict_name      = request.request.get('dict_name',SENTENCEDICT)
    sentences_query = Sentence.query(ancestor=dict_key(sdict_name))
    sentences       = sentences_query.fetch()
    return sentences

def getsentence(request,chi):
    sdict_name = request.request.get('dict_name', SENTENCEDICT)
    sentences_query = Sentence.query(Sentence.chichar == chi)
    qresult = sentences_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getword(request,chi):
    sdict_name = request.request.get('dict_name', WORDDICT)
    words_query = Word.query(Word.chichar == chi)
    qresult = words_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getsentencechichars(request,sentence):
    return charsentence2chichars(request,sentence.chichar)

def getwordchichars(request,word):
    return charsentence2chichars(request,word.chichar)

def addchichar(request,chi,translation,pronunciation):
    sdict_name = request.request.get('dict_name', CHICHARDICT)
    ochichar = Chichar(parent=dict_key(sdict_name));
    ochichar.chichar        = chi
    ochichar.translation    = translation
    ochichar.pronunciation  = pronunciation
    ochichar.put()
    return ochichar

def addsentence(request,chi,translation,pronunciation):
    sdict_name = request.request.get('dict_name', SENTENCEDICT)
    osentence = Sentence(parent=dict_key(sdict_name));
    osentence.chichar        = chi
    osentence.translation    = translation
    osentence.pronunciation  = pronunciation
    osentence.put()
    return osentence

def addword(request,chi,translation,pronunciation):
    sdict_name = request.request.get('dict_name', WORDDICT)
    oword = Word(parent=dict_key(sdict_name));
    oword.chichar        = chi
    oword.translation    = translation
    oword.pronunciation  = pronunciation
    oword.put()
    return oword

def asciitoremove():
    return [""," ","\n","\t",",",".","!","4","?"]

def sentence2chars(charstring):
    chars  = lsubstract(list(charstring.strip()),asciitoremove())
    return chars

def sentence2pinyins(charstring):
    pinyins  = lsubstract(list(charstring.strip()),asciitoremove())
    return pinyins

def sentence2pinyins2(charstring):
    pinyins  = lsubstract(charstring.strip().split(" "),asciitoremove())
    return pinyins

def charsentence2chichars(request,charstring):
    result = []
    for char in sentence2chars(charstring):
        chichar = getchichar(request,char)
        if chichar != None:
            result.append(chichar)
    return result

def chars2pinyins(request,chars):
    result = []
    for char in chars:
        chichar = getchichar(request,char)
        if chichar != None:
            result.append(chichar.pronunciation)
        else:
            result.append("TODO")
    return result

def checkaddchar(request,char,translation,pronunciation):
    char          = char.strip()
    if translation:
        translation   = translation.strip()
    if pronunciation:
        pronunciation = pronunciation.strip()
    ochichar = getchichar(request,char)
    if ochichar == None:
        request.response.write("need to add " + char)
        result = addchichar(request,char,translation,pronunciation)
    else:
        # TODO:check everything is the same
        request.response.write("char " + char + " already known")
        result = ochichar
    return result

def makepronunciation(chi,pinyins):
    index = 0
    result = ""
    for char in list(chi):
        if char in asciitoremove():
            result = result + char
        else:
            result = result + pinyins[index]
            index += 1
    return result

def checkaddsentence(request,chi,translation,pronunciation):
    osentence = getsentence(request,chi)

    if osentence == None:
        chars  = sentence2chars(chi.strip())
        pinyins = iff( pronunciation.strip() == "TODO", chars2pinyins(request,chars), sentence2pinyins2(pronunciation))

        if not (len(chars) == len(pinyins)):
            request.response.write("
  error matching char " + "@".join(chars) + " pynyin " + "@".join(pinyins) + "
\n")
            result = None
        else:
            request.response.write("need to add sentence " + chi)
            result = addsentence(request,chi,translation,makepronunciation(chi,pinyins))
            for (char,pinyin) in zip(chars,pinyins):
                chichar = checkaddchar(request,char,None,pinyin)
    else:
        request.response.write("sentence " + chi + " already known")
        result = osentence

    return result

def checkaddword(request,chi,translation,pronunciation):
    oword = getword(request,chi)

    if oword == None:
        chars  = sentence2chars(chi.strip())
        pinyins = iff( pronunciation.strip() == "TODO", chars2pinyins(request,chars), sentence2pinyins2(pronunciation))

        if not (len(chars) == len(pinyins)):
            request.response.write("
  error matching char " + "@".join(chars) + " pynyin " + "@".join(pinyins) + "
\n")
            result = None
        else:
            request.response.write("
need to add word " + chi + "
")
            result = addword(request,chi,translation,makepronunciation(chi,pinyins))
            for (char,pinyin) in zip(chars,pinyins):
                chichar = checkaddchar(request,char,None,pinyin)
    else:
        request.response.write("
word " + chi + " already known
")
        result = oword

    return result

def allocatedata(request,chichar,translation,pronunciation):
    result = None
    if len(chichar) == 1:
        # this is a new char
        result = checkaddchar(request,chichar,translation,pronunciation)
    else:
        if len(translation.split(" ")) > 1:
            # this is a sentence
            result = checkaddsentence(request,chichar,translation,pronunciation)
        else:
            result = checkaddword(request,chichar,translation,pronunciation)
    return result

def datatype(data):
    if not data:
        return None
    if isinstance(data,Chichar):
        return "Chichar"
    if isinstance(data,Sentence):
        return "Sentence"
    if isinstance(data,Word):
        return "Word"

def removeduplicatechichars(request):
    array = {}
    for chichar in getallchichars(request):
        if not chichar.chichar in array:
            array[chichar.chichar] = []
        array[chichar.chichar].append(chichar)

    for char in array:
        if len(array[char]) > 1:
            for chichar in array[char]:
                if chichar.translation == None:
                    chichar.key.delete()

def getbook(request,bookname):
    books_query = Book.query(Book.name == bookname)
    qresult = books_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getchapter(request,bookname,chaptername):
    chapters_query = Chapter.query(Chapter.book == bookname, Chapter.name == chaptername)
    qresult = chapters_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getunit(request,chaptername,chichar):
    units_query = Unit.query(Unit.chapter == chaptername, Unit.chichar == chichar)
    qresult = units_query.fetch(1)
    if len(qresult) > 0:
        return qresult[0]
    return None

def getunitdata(request,unit):
    return ndb.Key(urlsafe=unit.unitkey).get()

<|endoftext|>