<commit_before>import re
import os
import logging
from time import sleep
from datetime import datetime
import cPickle as pickle

import pymongo
from pylons import c
from ming import Document, Session, Field
from ming import schema as S
from ming.orm.base import mapper, session, state
from ming.orm.mapped_class import MappedClass, MappedClassMeta
from ming.orm.property import FieldProperty, ForeignIdProperty, RelationProperty
from pymongo.errors import OperationFailure

from pyforge.lib.helpers import nonce, push_config, push_context
from .session import ProjectSession
from .session import main_doc_session, main_orm_session
from .session import project_doc_session, project_orm_session
from .session import artifact_orm_session
from .types import ArtifactReference, ArtifactReferenceType

log = logging.getLogger(__name__)

def gen_message_id():
    parts = c.project.url().split('/')[1:-1]
    return '%s.%s@%s.sourceforge.net' % (nonce(40),
                                         c.app.config.options['mount_point'],
                                         '.'.join(reversed(parts)))

class ArtifactLink(MappedClass):
    class __mongometa__:
        session = project_orm_session
        name='artifact_link'
        indexes = [
            ('link', 'project_id') ]

    core_re = r'''(\[
            (?:(?P<project_id>.*?):)?      # optional project ID
            (?:(?P<app_id>.*?):)?      # optional plugin ID
            (?P<artifact_id>.*)             # artifact ID
    \])'''

    re_link_1 = re.compile(r'\s' + core_re, re.VERBOSE)
    re_link_2 = re.compile(r'^' +  core_re, re.VERBOSE)

    _id = FieldProperty(str)
    link = FieldProperty(str)
    project_id = ForeignIdProperty('Project')
    plugin_name = FieldProperty(str)
    mount_point = FieldProperty(str)
    url = FieldProperty(str)
    artifact_reference = FieldProperty(ArtifactReferenceType)

    @classmethod
    def add(cls, artifact):
        aid = artifact.index_id()
        entry = cls.query.get(_id=aid)
        if entry is None:
            entry = cls(_id=aid)
        entry.link=artifact.shorthand_id()
        entry.project_id=artifact.project_id
        entry.plugin_name=artifact.app_config.plugin_name
        entry.mount_point=artifact.app_config.options.mount_point
        entry.url=artifact.url()
        entry.artifact_reference = artifact.dump_ref()

    @classmethod
    def remove(cls, artifact):
        mapper(cls).remove(dict(_id=artifact.index_id()))

    @classmethod
    def lookup(cls, link):
        from .project import Project
        #
        # Parse the link syntax
        #
        m = cls.re_link_1.match(link)
        if m is None: m = cls.re_link_2.match(link)
        if m is None: return None
        groups = m.groupdict()
        project_id = groups.get('project_id', None)
        app_id = groups.get('app_id', None)
        artifact_id = groups.get('artifact_id', None)
        if app_id is None:
            app_id = project_id
            project_id = None

        #
        # Find the projects to search
        #
        if project_id is None:
            projects = list(c.project.parent_iter())
        elif project_id.startswith('/'):
            projects = Project.query.find(dict(shortname=project_id[1:])).all()
        else:
            project_id = os.path.normpath(
                os.path.join('/' + c.project.shortname, project_id))
            projects = Project.query.find(dict(shortname=project_id[1:])).all()
        if not projects: return None
        #
        # Find the app_id to search
        #
        if app_id is None:
            app_id = c.app.config.options.mount_point
        #
        # Actually search the projects
        #
        with push_config(c, project=projects[0]):
            for p in projects:
                links = cls.query.find(dict(project_id=p._id, link=artifact_id)).all()
                for l in links:
                    if app_id == l.mount_point: return l
                for l in links:
                    if app_id == l.plugin_name: return l
        return None

class Artifact(MappedClass):
    class __mongometa__:
        session = artifact_orm_session
        name='artifact'
    type_s = 'Generic Artifact'

    # Artifact base schema
    _id = FieldProperty(S.ObjectId)
    app_config_id = ForeignIdProperty('AppConfig', if_missing=lambda:c.app.config._id)
    plugin_verson = FieldProperty(
        S.Object,
        { str: str },
        if_missing=lambda:{c.app.config.plugin_name:c.app.__version__})
    acl = FieldProperty({str:[S.ObjectId]})
    tags = FieldProperty([dict(tag=str, count=int)])
    references = FieldProperty([ArtifactReferenceType])
    backreferences = FieldProperty({str:ArtifactReferenceType})
    app_config = RelationProperty('AppConfig')

    def dump_ref(self):
        '''Return a pickle-serializable reference to an artifact'''
        d = ArtifactReference(dict(
                project_id=self.app_config.project._id,
                mount_point=self.app_config.options.mount_point,
                artifact_type=pymongo.bson.Binary(pickle.dumps(self.__class__)),
                artifact_id=self._id))
        return d

    def add_tags(self, tags):
        'Update the tags collection to reflect new tags added'
        cur_tags = dict((t['tag'], t['count']) for t in self.tags)
        for t in tags:
            c = cur_tags.get(t, 0)
            c += 1
            cur_tags[t] = c
        self.tags = [ dict(tag=k, count=v) for k,v in cur_tags.iteritems() ]

    def remove_tags(self, tags):
        'Update the tags collection to reflect tags removed'
        cur_tags = dict((t['tag'], t['count']) for t in self.tags)
        for t in tags:
            c = cur_tags.get(t, 1)
            c -= 1
            if c:
                cur_tags[t] = c
            else:
                cur_tags.pop(t, None)
        self.tags = [ dict(tag=k, count=v) for k,v in cur_tags.iteritems() ]

    @property
    def project(self):
        return self.app_config.project

    @property
    def project_id(self):
        return self.app_config.project_id

    @property
    def app(self):
        ac = self.app_config
        return self.app_config.load()(self.project, self.app_config)

    def give_access(self, *access_types, **kw):
        user = kw.pop('user', c.user)
        project = kw.pop('project', c.project)
        with push_config(c, project=project):
            project_role_id = user.project_role()._id
        for at in access_types:
            l = self.acl.setdefault(at, [])
            if project_role_id not in l:
                l.append(project_role_id)

    def revoke_access(self, *access_types, **kw):
        user = kw.pop('user', c.user)
        project = kw.pop('project', c.project)
        with push_config(c, project=project):
            project_role_id = user.project_role()._id
        for at in access_types:
            l = self.acl.setdefault(at, [])
            if project_role_id in l:
                l.remove(project_role_id)

    def index_id(self):
        id = '%s.%s#%s' % (
            self.__class__.__module__,
            self.__class__.__name__,
            self._id)
        return id

    def index(self):
        project = self.project
        return dict(
            id=self.index_id(),
            title_s='Artifact %s' % self._id,
            project_id_s=project._id,
            project_name_t=project.name,
            project_shortname_t=project.shortname,
            plugin_name_s=self.app_config.plugin_name,
            mount_point_s=self.app_config.options.mount_point,
            is_history_b=False,
            url_s=self.url(),
            type_s=self.type_s,
            snippet_s='')

    def url(self):
        raise NotImplementedError, 'url' # pragma no cover

    def shorthand_id(self):
        '''How to refer to this artifact within the app instance context.

        For a wiki page, it might be the title.  For a ticket, it might be the
        ticket number.  For a discussion, it might be the message ID.  Generally
        this should have a strong correlation to the URL.
        '''
        return str(self._id) # pragma no cover

class Snapshot(Artifact):
    class __mongometa__:
        session = artifact_orm_session
        name='artifact_snapshot'

    _id = FieldProperty(S.ObjectId)
    artifact_id = FieldProperty(S.ObjectId)
    artifact_class = FieldProperty(str)
    version = FieldProperty(S.Int, if_missing=0)
    author = FieldProperty(dict(
            id=S.ObjectId,
            username=str,
            display_name=str))
    timestamp = FieldProperty(datetime)
    data = FieldProperty(None)

    def index(self):
        result = Artifact.index(self)
        result.update(
            version_i=self.version,
            author_username_t=self.author.username,
            author_display_name_t=self.author.display_name,
            timestamp_dt=self.timestamp,
            is_history_b=True)
        return result

    def original(self):
        raise NotImplemented, 'original' # pragma no cover

    def shorthand_id(self):
        return '%s#%s' % (self.original().shorthand_id(), self.version)

    def __getattr__(self, name):
        return getattr(self.data, name)

class VersionedArtifact(Artifact):
    class __mongometa__:
        session = artifact_orm_session
        name='versioned_artifact'
        history_class = Snapshot

    version = FieldProperty(S.Int, if_missing=0)

    def commit(self):
        '''Save off a snapshot of the artifact and increment the version #'''
        self.version += 1
        data = dict(
            artifact_id=self._id,
            artifact_class='%s.%s' % (
                self.__class__.__module__,
                self.__class__.__name__),
            version=self.version,
            author=dict(
                id=c.user._id,
                username=c.user.username,
                display_name=c.user.display_name),
            timestamp=datetime.utcnow(),
            data=state(self).document.deinstrumented_clone())
        ss = self.__mongometa__.history_class(**data)
        log.info('Snapshot version %s of %s',
                 self.version, self.__class__)
        return ss

    def get_version(self, n):
        if n < 0:
            n = self.version + n + 1
        ss = self.__mongometa__.history_class.query.get(
            artifact_id=self._id,
            artifact_class='%s.%s' % (
                self.__class__.__module__,
                self.__class__.__name__),
            version=n)
        if ss is None:
            raise IndexError, n
        return ss

    def revert(self, version):
        ss = self.get_version(version)
        old_version = self.version
        for k,v in ss.data.iteritems():
            setattr(self, k, v)
        self.version = old_version

    def history(self):
        HC = self.__mongometa__.history_class
        q = HC.query.find(dict(artifact_id=self._id)).sort('version', pymongo.DESCENDING)
        return q

class Message(Artifact):
    class __mongometa__:
        session = artifact_orm_session
        name='message'
        indexes = [ 'slug', 'parent_id' ]

    _id=FieldProperty(str, if_missing=gen_message_id)
    slug=FieldProperty(str, if_missing=nonce)
    parent_id=FieldProperty(str)
    app_id=FieldProperty(S.ObjectId, if_missing=lambda:c.app.config._id)
    timestamp=FieldProperty(datetime, if_missing=datetime.utcnow)
    author_id=FieldProperty(S.ObjectId, if_missing=lambda:c.user._id)
    text=FieldProperty(str, if_missing='')

    def author(self):
        from .auth import User
        return User.query.get(_id=self.author_id) or User.anonymous

    def reply(self):
        new_id = gen_message_id()
        new_slug = self.slug + '/' + nonce()
        new_args = dict(
            state(self).document,
            _id=new_id,
            slug=new_slug,
            parent_id=self._id,
            timestamp=datetime.utcnow(),
            author_id=c.user._id)
        return self.__class__(**new_args)

    def descendants(self):
        q = self.query.find(dict(slug={'$gt':self.slug})).sort('slug')
        for msg in q:
            if msg.slug.startswith(self.slug):
                yield msg
            else:
                break

    def replies(self):
        return self.query.find(dict(parent_id=self._id))

    def index(self):
        result = Artifact.index(self)
        author = self.author()
        result.update(
            author_user_name_t=author.username,
            author_display_name_t=author.display_name,
            timestamp_dt=self.timestamp,
            text=self.text,
            type_s='Generic Message')
        return result

    def shorthand_id(self):
        return self.slug

<commit_msg>FORGE-302: Index Artifact tags for search and index id for Snapshot<commit_after>import re
import os
import logging
from time import sleep
from datetime import datetime
import cPickle as pickle

import pymongo
from pylons import c
from ming import Document, Session, Field
from ming import schema as S
from ming.orm.base import mapper, session, state
from ming.orm.mapped_class import MappedClass, MappedClassMeta
from ming.orm.property import FieldProperty, ForeignIdProperty, RelationProperty
from pymongo.errors import OperationFailure

from pyforge.lib.helpers import nonce, push_config, push_context
from .session import ProjectSession
from .session import main_doc_session, main_orm_session
from .session import project_doc_session, project_orm_session
from .session import artifact_orm_session
from .types import ArtifactReference, ArtifactReferenceType

log = logging.getLogger(__name__)

def gen_message_id():
    parts = c.project.url().split('/')[1:-1]
    return '%s.%s@%s.sourceforge.net' % (nonce(40),
                                         c.app.config.options['mount_point'],
                                         '.'.join(reversed(parts)))

class ArtifactLink(MappedClass):
    class __mongometa__:
        session = project_orm_session
        name='artifact_link'
        indexes = [
            ('link', 'project_id') ]

    core_re = r'''(\[
            (?:(?P<project_id>.*?):)?      # optional project ID
            (?:(?P<app_id>.*?):)?      # optional plugin ID
            (?P<artifact_id>.*)             # artifact ID
    \])'''

    re_link_1 = re.compile(r'\s' + core_re, re.VERBOSE)
    re_link_2 = re.compile(r'^' +  core_re, re.VERBOSE)

    _id = FieldProperty(str)
    link = FieldProperty(str)
    project_id = ForeignIdProperty('Project')
    plugin_name = FieldProperty(str)
    mount_point = FieldProperty(str)
    url = FieldProperty(str)
    artifact_reference = FieldProperty(ArtifactReferenceType)

    @classmethod
    def add(cls, artifact):
        aid = artifact.index_id()
        entry = cls.query.get(_id=aid)
        if entry is None:
            entry = cls(_id=aid)
        entry.link=artifact.shorthand_id()
        entry.project_id=artifact.project_id
        entry.plugin_name=artifact.app_config.plugin_name
        entry.mount_point=artifact.app_config.options.mount_point
        entry.url=artifact.url()
        entry.artifact_reference = artifact.dump_ref()

    @classmethod
    def remove(cls, artifact):
        mapper(cls).remove(dict(_id=artifact.index_id()))

    @classmethod
    def lookup(cls, link):
        from .project import Project
        #
        # Parse the link syntax
        #
        m = cls.re_link_1.match(link)
        if m is None: m = cls.re_link_2.match(link)
        if m is None: return None
        groups = m.groupdict()
        project_id = groups.get('project_id', None)
        app_id = groups.get('app_id', None)
        artifact_id = groups.get('artifact_id', None)
        if app_id is None:
            app_id = project_id
            project_id = None

        #
        # Find the projects to search
        #
        if project_id is None:
            projects = list(c.project.parent_iter())
        elif project_id.startswith('/'):
            projects = Project.query.find(dict(shortname=project_id[1:])).all()
        else:
            project_id = os.path.normpath(
                os.path.join('/' + c.project.shortname, project_id))
            projects = Project.query.find(dict(shortname=project_id[1:])).all()
        if not projects: return None
        #
        # Find the app_id to search
        #
        if app_id is None:
            app_id = c.app.config.options.mount_point
        #
        # Actually search the projects
        #
        with push_config(c, project=projects[0]):
            for p in projects:
                links = cls.query.find(dict(project_id=p._id, link=artifact_id)).all()
                for l in links:
                    if app_id == l.mount_point: return l
                for l in links:
                    if app_id == l.plugin_name: return l
        return None

class Artifact(MappedClass):
    class __mongometa__:
        session = artifact_orm_session
        name='artifact'
    type_s = 'Generic Artifact'

    # Artifact base schema
    _id = FieldProperty(S.ObjectId)
    app_config_id = ForeignIdProperty('AppConfig', if_missing=lambda:c.app.config._id)
    plugin_verson = FieldProperty(
        S.Object,
        { str: str },
        if_missing=lambda:{c.app.config.plugin_name:c.app.__version__})
    acl = FieldProperty({str:[S.ObjectId]})
    tags = FieldProperty([dict(tag=str, count=int)])
    references = FieldProperty([ArtifactReferenceType])
    backreferences = FieldProperty({str:ArtifactReferenceType})
    app_config = RelationProperty('AppConfig')

    def dump_ref(self):
        '''Return a pickle-serializable reference to an artifact'''
        d = ArtifactReference(dict(
                project_id=self.app_config.project._id,
                mount_point=self.app_config.options.mount_point,
                artifact_type=pymongo.bson.Binary(pickle.dumps(self.__class__)),
                artifact_id=self._id))
        return d

    def add_tags(self, tags):
        'Update the tags collection to reflect new tags added'
        cur_tags = dict((t['tag'], t['count']) for t in self.tags)
        for t in tags:
            c = cur_tags.get(t, 0)
            c += 1
            cur_tags[t] = c
        self.tags = [ dict(tag=k, count=v) for k,v in cur_tags.iteritems() ]

    def remove_tags(self, tags):
        'Update the tags collection to reflect tags removed'
        cur_tags = dict((t['tag'], t['count']) for t in self.tags)
        for t in tags:
            c = cur_tags.get(t, 1)
            c -= 1
            if c:
                cur_tags[t] = c
            else:
                cur_tags.pop(t, None)
        self.tags = [ dict(tag=k, count=v) for k,v in cur_tags.iteritems() ]

    @property
    def project(self):
        return self.app_config.project

    @property
    def project_id(self):
        return self.app_config.project_id

    @property
    def app(self):
        ac = self.app_config
        return self.app_config.load()(self.project, self.app_config)

    def give_access(self, *access_types, **kw):
        user = kw.pop('user', c.user)
        project = kw.pop('project', c.project)
        with push_config(c, project=project):
            project_role_id = user.project_role()._id
        for at in access_types:
            l = self.acl.setdefault(at, [])
            if project_role_id not in l:
                l.append(project_role_id)

    def revoke_access(self, *access_types, **kw):
        user = kw.pop('user', c.user)
        project = kw.pop('project', c.project)
        with push_config(c, project=project):
            project_role_id = user.project_role()._id
        for at in access_types:
            l = self.acl.setdefault(at, [])
            if project_role_id in l:
                l.remove(project_role_id)

    def index_id(self):
        id = '%s.%s#%s' % (
            self.__class__.__module__,
            self.__class__.__name__,
            self._id)
        return id

    def index(self):
        project = self.project
        return dict(
            id=self.index_id(),
            title_s='Artifact %s' % self._id,
            project_id_s=project._id,
            project_name_t=project.name,
            project_shortname_t=project.shortname,
            plugin_name_s=self.app_config.plugin_name,
            mount_point_s=self.app_config.options.mount_point,
            is_history_b=False,
            url_s=self.url(),
            type_s=self.type_s,
            tags_t=','.join(t.tag for t in self.tags),
            snippet_s='')

    def url(self):
        raise NotImplementedError, 'url' # pragma no cover

    def shorthand_id(self):
        '''How to refer to this artifact within the app instance context.

        For a wiki page, it might be the title.  For a ticket, it might be the
        ticket number.  For a discussion, it might be the message ID.  Generally
        this should have a strong correlation to the URL.
        '''
        return str(self._id) # pragma no cover

class Snapshot(Artifact):
    class __mongometa__:
        session = artifact_orm_session
        name='artifact_snapshot'

    _id = FieldProperty(S.ObjectId)
    artifact_id = FieldProperty(S.ObjectId)
    artifact_class = FieldProperty(str)
    version = FieldProperty(S.Int, if_missing=0)
    author = FieldProperty(dict(
            id=S.ObjectId,
            username=str,
            display_name=str))
    timestamp = FieldProperty(datetime)
    data = FieldProperty(None)

    def index(self):
        result = Artifact.index(self)
        result.update(self.original().index())
        result.update(
            id=self.index_id(),
            version_i=self.version,
            author_username_t=self.author.username,
            author_display_name_t=self.author.display_name,
            timestamp_dt=self.timestamp,
            is_history_b=True)
        return result

    def original(self):
        raise NotImplemented, 'original' # pragma no cover

    def shorthand_id(self):
        return '%s#%s' % (self.original().shorthand_id(), self.version)

    def __getattr__(self, name):
        return getattr(self.data, name)

class VersionedArtifact(Artifact):
    class __mongometa__:
        session = artifact_orm_session
        name='versioned_artifact'
        history_class = Snapshot

    version = FieldProperty(S.Int, if_missing=0)

    def commit(self):
        '''Save off a snapshot of the artifact and increment the version #'''
        self.version += 1
        data = dict(
            artifact_id=self._id,
            artifact_class='%s.%s' % (
                self.__class__.__module__,
                self.__class__.__name__),
            version=self.version,
            author=dict(
                id=c.user._id,
                username=c.user.username,
                display_name=c.user.display_name),
            timestamp=datetime.utcnow(),
            data=state(self).document.deinstrumented_clone())
        ss = self.__mongometa__.history_class(**data)
        log.info('Snapshot version %s of %s',
                 self.version, self.__class__)
        return ss

    def get_version(self, n):
        if n < 0:
            n = self.version + n + 1
        ss = self.__mongometa__.history_class.query.get(
            artifact_id=self._id,
            artifact_class='%s.%s' % (
                self.__class__.__module__,
                self.__class__.__name__),
            version=n)
        if ss is None:
            raise IndexError, n
        return ss

    def revert(self, version):
        ss = self.get_version(version)
        old_version = self.version
        for k,v in ss.data.iteritems():
            setattr(self, k, v)
        self.version = old_version

    def history(self):
        HC = self.__mongometa__.history_class
        q = HC.query.find(dict(artifact_id=self._id)).sort('version', pymongo.DESCENDING)
        return q

class Message(Artifact):
    class __mongometa__:
        session = artifact_orm_session
        name='message'
        indexes = [ 'slug', 'parent_id' ]

    _id=FieldProperty(str, if_missing=gen_message_id)
    slug=FieldProperty(str, if_missing=nonce)
    parent_id=FieldProperty(str)
    app_id=FieldProperty(S.ObjectId, if_missing=lambda:c.app.config._id)
    timestamp=FieldProperty(datetime, if_missing=datetime.utcnow)
    author_id=FieldProperty(S.ObjectId, if_missing=lambda:c.user._id)
    text=FieldProperty(str, if_missing='')

    def author(self):
        from .auth import User
        return User.query.get(_id=self.author_id) or User.anonymous

    def reply(self):
        new_id = gen_message_id()
        new_slug = self.slug + '/' + nonce()
        new_args = dict(
            state(self).document,
            _id=new_id,
            slug=new_slug,
            parent_id=self._id,
            timestamp=datetime.utcnow(),
            author_id=c.user._id)
        return self.__class__(**new_args)

    def descendants(self):
        q = self.query.find(dict(slug={'$gt':self.slug})).sort('slug')
        for msg in q:
            if msg.slug.startswith(self.slug):
                yield msg
            else:
                break

    def replies(self):
        return self.query.find(dict(parent_id=self._id))

    def index(self):
        result = Artifact.index(self)
        author = self.author()
        result.update(
            author_user_name_t=author.username,
            author_display_name_t=author.display_name,
            timestamp_dt=self.timestamp,
            text=self.text,
            type_s='Generic Message')
        return result

    def shorthand_id(self):
        return self.slug

<|endoftext|>


<commit_before>"""
    Copyright (c) 2013, Los Alamos National Security, LLC
    All rights reserved.

    Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
    following conditions are met:

    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following
      disclaimer.
    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
      following disclaimer in the documentation and/or other materials provided with the distribution.
    * Neither the name of Los Alamos National Security, LLC nor the names of its contributors may be used to endorse or
      promote products derived from this software without specific prior written permission.

    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
    THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

import random
from multiprocessing import Pool, Event
from datetime import datetime
from collections import namedtuple

# Note: We use a global multiprocessing.Event to deal with a KeyboardInterrupt. This idea comes from
# http://stackoverflow.com/questions/14579474/multiprocessing-pool-spawning-new-childern-after-terminate-on-linux-python2-7.
# This is not necessary when running under Python 3, but to keep 2.7 compatability, I'm leaving it in.
terminating = Event()

# HarmonySearchResults is a struct-like object that we'll use to attach the results of the search.
# namedtuples are lightweight and trivial to extend should more results be desired in the future. Right now, we're just
# keeping track of the total elapsed clock time, the best harmony found, the fitness for that harmony, and the harmony memory,
# which allows you to see the top harmonies.
HarmonySearchResults = namedtuple('HarmonySearchResults', ['elapsed_time', 'best_harmony', 'best_fitness', 'harmony_memories', 'harmony_histories'])

def harmony_search(objective_function, num_processes, num_iterations, initial_harmonies=None):
    """
        Here, we use multiprocessing.Pool to do multiple harmony searches simultaneously. Since HS is stochastic (unless random_seed is set),
        multiple runs can find different results. We run the specified number of iterations on the specified number of processes and return
        an instance of HarmonySearchResults.
    """
    pool = Pool(num_processes)
    try:
        start = datetime.now()
        pool_results = [pool.apply_async(worker, args=(objective_function, initial_harmonies,)) for i in range(num_iterations)]
        pool.close()  # no more tasks will be submitted to the pool
        pool.join()  # wait for all tasks to finish before moving on
        end = datetime.now()
        elapsed_time = end - start

        # find best harmony from all iterations
        best_harmony = None
        best_fitness = float('-inf') if objective_function.maximize() else float('+inf')
        harmony_memories = list()
        harmony_histories = list()
        for result in pool_results:
            harmony, fitness, harmony_memory, harmony_history = result.get()  # multiprocessing.pool.AsyncResult is returned for each process, so we need to call get() to pull out the value
            if (objective_function.maximize() and fitness > best_fitness) or (not objective_function.maximize() and fitness < best_fitness):
                best_harmony = harmony
                best_fitness = fitness
            harmony_memories.append(harmony_memory)
            harmony_histories.append(harmony_history)

        return HarmonySearchResults(elapsed_time=elapsed_time, best_harmony=best_harmony, best_fitness=best_fitness,\
                                    harmony_memories=harmony_memories, harmony_histories=harmony_histories)
    except KeyboardInterrupt:
        pool.terminate()

def worker(objective_function, initial_harmonies=None):
    """
        This is just a dummy function to make multiprocessing work with a class. It also checks/sets the global multiprocessing.Event to prevent
        new processes from starting work on a KeyboardInterrupt.
    """
    try:
        if not terminating.is_set():
            hs = HarmonySearch(objective_function)
            return hs.run(initial_harmonies=initial_harmonies)
    except KeyboardInterrupt:
        terminating.set()  # set the Event to true to prevent the other processes from doing any work

class HarmonySearch(object):

    """
        This class implements the harmony search (HS) global optimization algorithm. In general, what you'll do is this:

        1. Implement an objective function that inherits from ObjectiveFunctionInterface.
        2. Initialize HarmonySearch with this objective function (e.g., hs = HarmonySearch(objective_function)).
        3. Run HarmonySearch (e.g., results = hs.run()).
    """

    def __init__(self, objective_function):
        """
            Initialize HS with the specified objective function. Note that this objective function must implement ObjectiveFunctionInterface.
        """
        self._obj_fun = objective_function

    def run(self, initial_harmonies=None):
        """
            This is the main HS loop. It initializes the harmony memory and then continually generates new harmonies
            until the stopping criterion (max_imp iterations) is reached.
        """
        # set optional random seed
        if self._obj_fun.use_random_seed():
            random.seed(self._obj_fun.get_random_seed())

        # harmony_memory stores the best hms harmonies
        self._harmony_memory = list()

        # harmony_history stores all hms harmonies every generation (or n generations)
        self._harmony_history = list()

        # fill harmony_memory using random parameter values by default, but with initial_harmonies if provided
	self._initialize(initial_harmonies)

        # create max_imp improvisations
        generation = 0
        num_imp = 0
        while(num_imp < self._obj_fun.get_max_imp()):
            # generate new harmony
            harmony = list()
            for i in range(0, self._obj_fun.get_num_parameters()):
                if random.random() < self._obj_fun.get_hmcr():
                    self._memory_consideration(harmony, i)
                    if random.random() < self._obj_fun.get_par():
                        self._pitch_adjustment(harmony, i)
                else:
                    self._random_selection(harmony, i)
            fitness = self._obj_fun.get_fitness(harmony)
            self._update_harmony_memory(harmony, fitness)
            num_imp += 1

            # save harmonies every nth improvisations (i.e., one 'generation')
            if num_imp % self._obj_fun.get_hms() == 0:
                generation += 1
                harmony_list = {'gen': generation, 'harmonies': self._harmony_memory}
                self._harmony_history.append(harmony_list)

        # return best harmony
        best_harmony = None
        best_fitness = float('-inf') if self._obj_fun.maximize() else float('+inf')
        for harmony, fitness in self._harmony_memory:
            if (self._obj_fun.maximize() and fitness > best_fitness) or (not self._obj_fun.maximize() and fitness < best_fitness):
                best_harmony = harmony
                best_fitness = fitness
        return best_harmony, best_fitness, self._harmony_memory, self._harmony_history

    def _initialize(self, initial_harmonies=None):
        """
            Initialize harmony_memory, the matrix (list of lists) containing the various harmonies (solution vectors). Note
            that we aren't actually doing any matrix operations, so a library like NumPy isn't necessary here. The matrix
            merely stores previous harmonies.

            If harmonies are provided, then use them instead of randomly initializing them.

            Populate harmony_history with initial harmony memory.
        """
        if initial_harmonies is not None:
            if len(initial_harmonies) != self._obj_fun.get_hms():
                raise ValueError('Number of initial harmonies does not equal to the harmony memory size.')
            for i in range(len(initial_harmonies)):
                if len(initial_harmonies[i]) != self._obj_fun.get_num_parameters():
                    raise ValueError('Number of dimensions in initial harmonies does not match that defined.')

        if initial_harmonies is None:
            initial_harmonies = list()
            for i in range(0, self._obj_fun.get_hms()):
                harmony = list()
                for j in range(0, self._obj_fun.get_num_parameters()):
                    self._random_selection(harmony, j)
                initial_harmonies.append(harmony)

        for i in range(0, self._obj_fun.get_hms()):
            fitness = self._obj_fun.get_fitness(initial_harmonies[i])
            self._harmony_memory.append((initial_harmonies[i], fitness))

	harmony_list = {'gen': 0, 'harmonies': self._harmony_memory}
	self._harmony_history.append(harmony_list)

    def _random_selection(self, harmony, i):
        """
            Choose a note according to get_value(). Remember that even if a note is not variable, get_value() must still
            return a valid value.
        """
        harmony.append(self._obj_fun.get_value(i))

    def _memory_consideration(self, harmony, i):
        """
            Randomly choose a note previously played.
        """
        memory_index = random.randint(0, self._obj_fun.get_hms() - 1)
        harmony.append(self._harmony_memory[memory_index][0][i])

    def _pitch_adjustment(self, harmony, i):
        """
            If variable, randomly adjust the pitch up or down by some amount. This is the only place in the algorithm where there
            is an explicit difference between continuous and discrete variables.

            The probability of adjusting the pitch either up or down is fixed at 0.5. The maximum pitch adjustment proportion (mpap)
            and maximum pitch adjustment index (mpai) determine the maximum amount the pitch may change for continuous and discrete
            variables, respectively.

            For example, suppose that it is decided via coin flip that the pitch will be adjusted down. Also suppose that mpap is set to 0.25.
            This means that the maximum value the pitch can be dropped will be 25% of the difference between the lower bound and the current
            pitch. mpai functions similarly, only it relies on indices of the possible values instead.
        """
        if(self._obj_fun.is_variable(i)):
            if self._obj_fun.is_discrete(i):
                current_index = self._obj_fun.get_index(i, harmony[i])
                # discrete variable
                if random.random() < 0.5:
                    # adjust pitch down
                    harmony[i] = self._obj_fun.get_value(i, current_index - random.randint(0, min(self._obj_fun.get_mpai(), current_index)))
                else:
                    # adjust pitch up
                    harmony[i] = self._obj_fun.get_value(i, current_index + random.randint(0, min(self._obj_fun.get_mpai(), self._obj_fun.get_num_discrete_values(i) - current_index - 1)))
            else:
                # continuous variable
                if random.random() < 0.5:
                    # adjust pitch down
                    harmony[i] -= (harmony[i] - self._obj_fun.get_lower_bound(i)) * random.random() * self._obj_fun.get_mpap()
                else:
                    # adjust pitch up
                    harmony[i] += (self._obj_fun.get_upper_bound(i) - harmony[i]) * random.random() * self._obj_fun.get_mpap()

    def _update_harmony_memory(self, considered_harmony, considered_fitness):
        """
            Update the harmony memory if necessary with the given harmony. If the given harmony is better than the worst
            harmony in memory, replace it. This function doesn't allow duplicate harmonies in memory.
        """
        if (considered_harmony, considered_fitness) not in self._harmony_memory:
            worst_index = None
            worst_fitness = float('+inf') if self._obj_fun.maximize() else float('-inf')
            for i, (harmony, fitness) in enumerate(self._harmony_memory):
                if (self._obj_fun.maximize() and fitness < worst_fitness) or (not self._obj_fun.maximize() and fitness > worst_fitness):
                    worst_index = i
                    worst_fitness = fitness
            if (self._obj_fun.maximize() and considered_fitness > worst_fitness) or (not self._obj_fun.maximize() and considered_fitness < worst_fitness):
                self._harmony_memory[worst_index] = (considered_harmony, considered_fitness)
<commit_msg>Refactor to avoid calling self._obj_fun.get_num_parameters() multiple times, as it could be more involved<commit_after>"""
    Copyright (c) 2013, Los Alamos National Security, LLC
    All rights reserved.

    Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
    following conditions are met:

    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following
      disclaimer.
    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
      following disclaimer in the documentation and/or other materials provided with the distribution.
    * Neither the name of Los Alamos National Security, LLC nor the names of its contributors may be used to endorse or
      promote products derived from this software without specific prior written permission.

    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
    DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
    SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
    WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
    THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

import random
from multiprocessing import Pool, Event
from datetime import datetime
from collections import namedtuple

# Note: We use a global multiprocessing.Event to deal with a KeyboardInterrupt. This idea comes from
# http://stackoverflow.com/questions/14579474/multiprocessing-pool-spawning-new-childern-after-terminate-on-linux-python2-7.
# This is not necessary when running under Python 3, but to keep 2.7 compatability, I'm leaving it in.
terminating = Event()

# HarmonySearchResults is a struct-like object that we'll use to attach the results of the search.
# namedtuples are lightweight and trivial to extend should more results be desired in the future. Right now, we're just
# keeping track of the total elapsed clock time, the best harmony found, the fitness for that harmony, and the harmony memory,
# which allows you to see the top harmonies.
HarmonySearchResults = namedtuple('HarmonySearchResults', ['elapsed_time', 'best_harmony', 'best_fitness', 'harmony_memories', 'harmony_histories'])

def harmony_search(objective_function, num_processes, num_iterations, initial_harmonies=None):
    """
        Here, we use multiprocessing.Pool to do multiple harmony searches simultaneously. Since HS is stochastic (unless random_seed is set),
        multiple runs can find different results. We run the specified number of iterations on the specified number of processes and return
        an instance of HarmonySearchResults.
    """
    pool = Pool(num_processes)
    try:
        start = datetime.now()
        pool_results = [pool.apply_async(worker, args=(objective_function, initial_harmonies,)) for i in range(num_iterations)]
        pool.close()  # no more tasks will be submitted to the pool
        pool.join()  # wait for all tasks to finish before moving on
        end = datetime.now()
        elapsed_time = end - start

        # find best harmony from all iterations
        best_harmony = None
        best_fitness = float('-inf') if objective_function.maximize() else float('+inf')
        harmony_memories = list()
        harmony_histories = list()
        for result in pool_results:
            harmony, fitness, harmony_memory, harmony_history = result.get()  # multiprocessing.pool.AsyncResult is returned for each process, so we need to call get() to pull out the value
            if (objective_function.maximize() and fitness > best_fitness) or (not objective_function.maximize() and fitness < best_fitness):
                best_harmony = harmony
                best_fitness = fitness
            harmony_memories.append(harmony_memory)
            harmony_histories.append(harmony_history)

        return HarmonySearchResults(elapsed_time=elapsed_time, best_harmony=best_harmony, best_fitness=best_fitness,\
                                    harmony_memories=harmony_memories, harmony_histories=harmony_histories)
    except KeyboardInterrupt:
        pool.terminate()

def worker(objective_function, initial_harmonies=None):
    """
        This is just a dummy function to make multiprocessing work with a class. It also checks/sets the global multiprocessing.Event to prevent
        new processes from starting work on a KeyboardInterrupt.
    """
    try:
        if not terminating.is_set():
            hs = HarmonySearch(objective_function)
            return hs.run(initial_harmonies=initial_harmonies)
    except KeyboardInterrupt:
        terminating.set()  # set the Event to true to prevent the other processes from doing any work

class HarmonySearch(object):

    """
        This class implements the harmony search (HS) global optimization algorithm. In general, what you'll do is this:

        1. Implement an objective function that inherits from ObjectiveFunctionInterface.
        2. Initialize HarmonySearch with this objective function (e.g., hs = HarmonySearch(objective_function)).
        3. Run HarmonySearch (e.g., results = hs.run()).
    """

    def __init__(self, objective_function):
        """
            Initialize HS with the specified objective function. Note that this objective function must implement ObjectiveFunctionInterface.
        """
        self._obj_fun = objective_function

    def run(self, initial_harmonies=None):
        """
            This is the main HS loop. It initializes the harmony memory and then continually generates new harmonies
            until the stopping criterion (max_imp iterations) is reached.
        """
        # set optional random seed
        if self._obj_fun.use_random_seed():
            random.seed(self._obj_fun.get_random_seed())

        # harmony_memory stores the best hms harmonies
        self._harmony_memory = list()

        # harmony_history stores all hms harmonies every generation (or n generations)
        self._harmony_history = list()

        # fill harmony_memory using random parameter values by default, but with initial_harmonies if provided
	self._initialize(initial_harmonies)

        # create max_imp improvisations
        generation = 0
        num_imp = 0
        while(num_imp < self._obj_fun.get_max_imp()):
            # generate new harmony
            harmony = list()
            for i in range(0, self._obj_fun.get_num_parameters()):
                if random.random() < self._obj_fun.get_hmcr():
                    self._memory_consideration(harmony, i)
                    if random.random() < self._obj_fun.get_par():
                        self._pitch_adjustment(harmony, i)
                else:
                    self._random_selection(harmony, i)
            fitness = self._obj_fun.get_fitness(harmony)
            self._update_harmony_memory(harmony, fitness)
            num_imp += 1

            # save harmonies every nth improvisations (i.e., one 'generation')
            if num_imp % self._obj_fun.get_hms() == 0:
                generation += 1
                harmony_list = {'gen': generation, 'harmonies': self._harmony_memory}
                self._harmony_history.append(harmony_list)

        # return best harmony
        best_harmony = None
        best_fitness = float('-inf') if self._obj_fun.maximize() else float('+inf')
        for harmony, fitness in self._harmony_memory:
            if (self._obj_fun.maximize() and fitness > best_fitness) or (not self._obj_fun.maximize() and fitness < best_fitness):
                best_harmony = harmony
                best_fitness = fitness
        return best_harmony, best_fitness, self._harmony_memory, self._harmony_history

    def _initialize(self, initial_harmonies=None):
        """
            Initialize harmony_memory, the matrix (list of lists) containing the various harmonies (solution vectors). Note
            that we aren't actually doing any matrix operations, so a library like NumPy isn't necessary here. The matrix
            merely stores previous harmonies.

            If harmonies are provided, then use them instead of randomly initializing them.

            Populate harmony_history with initial harmony memory.
        """
        if initial_harmonies is not None:
            if len(initial_harmonies) != self._obj_fun.get_hms():
                raise ValueError('Number of initial harmonies does not equal to the harmony memory size.')

            num_parameters = self._obj_fun.get_num_parameters
            for i in range(len(initial_harmonies)):
                num_parameters_initial_harmonies = len(initial_harmonies[i])
                if num_parameters_initial_harmonies != num_parameters:
                    raise ValueError('Number of parameters in initial harmonies does not match that defined.')

        if initial_harmonies is None:
            initial_harmonies = list()
            for i in range(0, self._obj_fun.get_hms()):
                harmony = list()
                for j in range(0, self._obj_fun.get_num_parameters()):
                    self._random_selection(harmony, j)
                initial_harmonies.append(harmony)

        for i in range(0, self._obj_fun.get_hms()):
            fitness = self._obj_fun.get_fitness(initial_harmonies[i])
            self._harmony_memory.append((initial_harmonies[i], fitness))

	harmony_list = {'gen': 0, 'harmonies': self._harmony_memory}
	self._harmony_history.append(harmony_list)

    def _random_selection(self, harmony, i):
        """
            Choose a note according to get_value(). Remember that even if a note is not variable, get_value() must still
            return a valid value.
        """
        harmony.append(self._obj_fun.get_value(i))

    def _memory_consideration(self, harmony, i):
        """
            Randomly choose a note previously played.
        """
        memory_index = random.randint(0, self._obj_fun.get_hms() - 1)
        harmony.append(self._harmony_memory[memory_index][0][i])

    def _pitch_adjustment(self, harmony, i):
        """
            If variable, randomly adjust the pitch up or down by some amount. This is the only place in the algorithm where there
            is an explicit difference between continuous and discrete variables.

            The probability of adjusting the pitch either up or down is fixed at 0.5. The maximum pitch adjustment proportion (mpap)
            and maximum pitch adjustment index (mpai) determine the maximum amount the pitch may change for continuous and discrete
            variables, respectively.

            For example, suppose that it is decided via coin flip that the pitch will be adjusted down. Also suppose that mpap is set to 0.25.
            This means that the maximum value the pitch can be dropped will be 25% of the difference between the lower bound and the current
            pitch. mpai functions similarly, only it relies on indices of the possible values instead.
        """
        if(self._obj_fun.is_variable(i)):
            if self._obj_fun.is_discrete(i):
                current_index = self._obj_fun.get_index(i, harmony[i])
                # discrete variable
                if random.random() < 0.5:
                    # adjust pitch down
                    harmony[i] = self._obj_fun.get_value(i, current_index - random.randint(0, min(self._obj_fun.get_mpai(), current_index)))
                else:
                    # adjust pitch up
                    harmony[i] = self._obj_fun.get_value(i, current_index + random.randint(0, min(self._obj_fun.get_mpai(), self._obj_fun.get_num_discrete_values(i) - current_index - 1)))
            else:
                # continuous variable
                if random.random() < 0.5:
                    # adjust pitch down
                    harmony[i] -= (harmony[i] - self._obj_fun.get_lower_bound(i)) * random.random() * self._obj_fun.get_mpap()
                else:
                    # adjust pitch up
                    harmony[i] += (self._obj_fun.get_upper_bound(i) - harmony[i]) * random.random() * self._obj_fun.get_mpap()

    def _update_harmony_memory(self, considered_harmony, considered_fitness):
        """
            Update the harmony memory if necessary with the given harmony. If the given harmony is better than the worst
            harmony in memory, replace it. This function doesn't allow duplicate harmonies in memory.
        """
        if (considered_harmony, considered_fitness) not in self._harmony_memory:
            worst_index = None
            worst_fitness = float('+inf') if self._obj_fun.maximize() else float('-inf')
            for i, (harmony, fitness) in enumerate(self._harmony_memory):
                if (self._obj_fun.maximize() and fitness < worst_fitness) or (not self._obj_fun.maximize() and fitness > worst_fitness):
                    worst_index = i
                    worst_fitness = fitness
            if (self._obj_fun.maximize() and considered_fitness > worst_fitness) or (not self._obj_fun.maximize() and considered_fitness < worst_fitness):
                self._harmony_memory[worst_index] = (considered_harmony, considered_fitness)
<|endoftext|>


<commit_before>import itertools
import random

from climin import KrylovSubspaceDescent, HessianFree, Rprop

import pylab
import scipy
import theano
import theano.tensor as T
import zeitgeist.data
import zeitgeist.util as util
from zeitgeist.model import transfermap
import numpy as np

# Hyper parameters.

n_inpt = 784
n_hidden = 300
n_hidden1 = 500
n_hidden2 = 500
n_hidden3 = 2000
n_output = 10

initial_damping = 15

# Expressions for the deep network.
def deep_mlp(insize, hiddensize1, hiddensize2, hiddensize3, outsize, transferfunc='tanh', outfunc='id'):
    P = util.ParameterSet(
        inweights=(insize, hiddensize1),
        hiddenbias1=hiddensize1,
        hiddenweights1 = (hiddensize1, hiddensize2),
        hiddenbias2=hiddensize2,
        hiddenweights2 = (hiddensize2, hiddensize3),
        hiddenbias3=hiddensize3,
        outweights=(hiddensize3, outsize),
        outbias=outsize)

    P.randomize(0.1)

    inpt = T.matrix('inpt')
    hidden1_in = T.dot(inpt, P.inweights)
    hidden1_in += P.hiddenbias1

    nonlinear = transfermap[transferfunc]
    hidden1 = nonlinear(hidden1_in)

    hidden2_in = T.dot(hidden1, P.hiddenweights1)
    hidden2_in += P.hiddenbias2
    hidden2 = nonlinear(hidden2_in)

    hidden3_in = T.dot(hidden2, P.hiddenweights2)
    hidden3_in += P.hiddenbias3
    hidden3 = nonlinear(hidden3_in)

    output_in = T.dot(hidden3, P.outweights)
    output_in += P.outbias
    output = output_in
    output = transfermap[outfunc](output_in)

    exprs = {'inpt': inpt,
             'hidden1-in': hidden1_in,
             'hidden1': hidden1,
             'hidden2-in': hidden2_in,
             'hidden2': hidden2,
             'hidden3-in': hidden3_in,
             'hidden3': hidden3,
             'output-in': output_in,
             'output': output}
    return exprs, P

exprs, P = deep_mlp(n_inpt, n_hidden1, n_hidden2, n_hidden3, n_output, transferfunc='sig', outfunc='softmax')
# To make the passing of the parameters explicit, we need to substitute it
# later with the givens parameter.
par_sub = T.vector()

# Some tensor for constructing the loss. the loss will only be defined on
# the end of the sequences.
inpt = exprs['inpt']
target = T.matrix('target')
output = exprs['output']

# Shorthand to create a cross entropy expression
def cross_entropy(a, b):
    epsilon = 0
    return -(a * T.log(b+epsilon)).mean()

# Vector for the expression of the Hessian vector product, where this will
# be the vector.
p = T.vector('p')

# The loss and its gradient.
loss = cross_entropy(target, output)
lossgrad = T.grad(loss, P.flat)

# Expression for the Gauss-Newton matrix.
Jp = T.Rop(output, P.flat, p)
HJp = T.grad(T.sum(T.grad(loss, output) * Jp),
             output, consider_constant=[Jp])
Hp = T.grad(T.sum(HJp * output), P.flat, consider_constant=[HJp, Jp])

# Functions.
givens = {P.flat: par_sub}
f = theano.function([par_sub, inpt, target], loss, givens=givens)
fprime = theano.function([par_sub, inpt, target], lossgrad, givens=givens)
f_Hp = theano.function([par_sub, p, inpt, target], Hp, givens=givens)
f_predict = theano.function([par_sub, inpt], exprs['output'], givens=givens)

# Build a dataset.
#MNIST
import cPickle, gzip

# Load the dataset
with gzip.open('mnist.pkl.gz','rb') as MNISTfile:
    train_set, valid_set, test_set = cPickle.load(MNISTfile)

X, labels = train_set
N, _ = X.shape
Y = np.zeros((N, 10))
for i in range(N):
    Y[i][labels[i]] = 1

#sparse initialization
P['hiddenbias1'][:] = scipy.zeros(n_hidden1)
P['hiddenbias2'][:] = scipy.zeros(n_hidden2)
P['hiddenbias3'][:] = scipy.zeros(n_hidden3)
P['outbias'][:] = scipy.zeros(n_output)

P['inweights'][:,:] = np.random.randn(n_inpt, n_hidden1)
P['hiddenweights1'][:,:] = np.random.randn(n_hidden1, n_hidden2)
P['hiddenweights2'][:,:] = np.random.randn(n_hidden2, n_hidden3)
P['outweights'][:,:] = np.random.randn(n_hidden3, n_output)

def sparse_initialization(a, b, s, MaxNonZeroPerColumn = 15):
    for j in range(b):
        perm = np.random.permutation(a)
        P[s][perm[MaxNonZeroPerColumn:], j] *= 0

sparse_initialization( n_inpt, n_hidden1, 'inweights')
sparse_initialization( n_hidden1, n_hidden2, 'hiddenweights1')
sparse_initialization( n_hidden2, n_hidden3, 'hiddenweights2')
sparse_initialization( n_hidden3, n_output, 'outweights')

#minibatches for HF: whole dataset
args = (([X, Y], {}) for _ in itertools.repeat(()))

#minibatches for cg: 5000
batchsize = 5000
n_batches = N/batchsize
random_numbers = (random.randint(0, n_batches - 1) for _ in itertools.count())
idxs = ((r * batchsize, (r + 1) * batchsize) for r in random_numbers)
minibatches = ((X[lower:upper], Y[lower:upper]) for lower, upper in idxs)
cg_args = ((m, {}) for m in minibatches)

print '#pars:', P.data.size

import chopmunk

@chopmunk.coroutine
def print_Hp_min_max():
    while True:
        info = (yield)
        if 'Hp' not in info:
            continue
        absed = abs(info['Hp'])
        print 'min max std mean', absed.min(), absed.max(), absed.std(), absed.mean()

logger = chopmunk.prettyprint_sink()
filesink = chopmunk.file_sink('mnist.log')
filesink = chopmunk.jsonify(filesink)
logger = chopmunk.broadcast(logger, filesink, print_Hp_min_max())
logfunc = logger.send

optimizer = 'hf'

if optimizer == 'ksd':
    opt = KrylovSubspaceDescent(
        P.data, f, fprime, f_Hp, n_bases=10,
        args=args, logfunc=logfunc)
elif optimizer == 'rprop':
    opt = Rprop(P.data, f, fprime, args=args, logfunc=logfunc)
elif optimizer == 'hf':
    opt = HessianFree(
        P.data, f, fprime, f_Hp, args=args, cg_args=cg_args,
        initial_damping=initial_damping,
        precond='martens',
        logfunc=logfunc)

for i, info in enumerate(opt):
    X, Y = info['args']
    loss = f(P.data, X, Y)
    print 'loss', loss
    if i > 30:
        break

#pylab.plot(steps)
#pylab.plot(losses)
#pylab.show()
#
#pylab.plot(f_predict(X)[:, 0, 0])
#pylab.plot(Z[:, 0, 0])
#pylab.show()

## P = f_predict(P.data, TX)[-5:, :3, 0]
## print (P > 0.5).astype('uint8')
## print P
<commit_msg>MNIST: plotting of losses for a shallow NN<commit_after>import itertools
import random

from climin import KrylovSubspaceDescent, HessianFree, Rprop

import pylab
import scipy
import theano
import theano.tensor as T
import zeitgeist.data
import zeitgeist.util as util
from zeitgeist.model import transfermap
import numpy as np

# Hyper parameters.

n_inpt = 784
n_hidden = 300
n_hidden1 = 500
n_hidden2 = 500
n_hidden3 = 2000
n_output = 10

initial_damping = 15

#Expressions for a one-layer network
def mlp(insize, hiddensize, outsize, transferfunc='tanh', outfunc='id'):
    P = util.ParameterSet(
        inweights=(insize, hiddensize),
        hiddenbias=hiddensize,
        outweights=(hiddensize, outsize),
        outbias=outsize)

    P.randomize(0.1)

    inpt = T.matrix('inpt')
    hidden_in = T.dot(inpt, P.inweights)
    hidden_in += P.hiddenbias

    nonlinear = transfermap[transferfunc]
    hidden = nonlinear(hidden_in)
    output_in = T.dot(hidden, P.outweights)
    output_in += P.outbias
    output = output_in
    output = transfermap[outfunc](output_in)

    exprs = {'inpt': inpt,
             'hidden-in': hidden_in,
             'hidden': hidden,
             'output-in': output_in,
             'output': output}
    return exprs, P

# Expressions for the deep network.
def deep_mlp(insize, hiddensize1, hiddensize2, hiddensize3, outsize, transferfunc='tanh', outfunc='id'):
    P = util.ParameterSet(
        inweights=(insize, hiddensize1),
        hiddenbias1=hiddensize1,
        hiddenweights1 = (hiddensize1, hiddensize2),
        hiddenbias2=hiddensize2,
        hiddenweights2 = (hiddensize2, hiddensize3),
        hiddenbias3=hiddensize3,
        outweights=(hiddensize3, outsize),
        outbias=outsize)

    P.randomize(0.1)

    inpt = T.matrix('inpt')
    hidden1_in = T.dot(inpt, P.inweights)
    hidden1_in += P.hiddenbias1

    nonlinear = transfermap[transferfunc]
    hidden1 = nonlinear(hidden1_in)

    hidden2_in = T.dot(hidden1, P.hiddenweights1)
    hidden2_in += P.hiddenbias2
    hidden2 = nonlinear(hidden2_in)

    hidden3_in = T.dot(hidden2, P.hiddenweights2)
    hidden3_in += P.hiddenbias3
    hidden3 = nonlinear(hidden3_in)

    output_in = T.dot(hidden3, P.outweights)
    output_in += P.outbias
    output = output_in
    output = transfermap[outfunc](output_in)

    exprs = {'inpt': inpt,
             'hidden1-in': hidden1_in,
             'hidden1': hidden1,
             'hidden2-in': hidden2_in,
             'hidden2': hidden2,
             'hidden3-in': hidden3_in,
             'hidden3': hidden3,
             'output-in': output_in,
             'output': output}
    return exprs, P

#exprs, P = deep_mlp(n_inpt, n_hidden1, n_hidden2, n_hidden3, n_output, transferfunc='sig', outfunc='softmax')
exprs, P = mlp(n_inpt, n_hidden, n_output, transferfunc='sig', outfunc='softmax')
# To make the passing of the parameters explicit, we need to substitute it
# later with the givens parameter.
par_sub = T.vector()

# Some tensor for constructing the loss. the loss will only be defined on
# the end of the sequences.
inpt = exprs['inpt']
target = T.matrix('target')
output = exprs['output']

# Shorthand to create a cross entropy expression
def cross_entropy(a, b):
    epsilon = 0
    return -(a * T.log(b+epsilon)).mean()

# Vector for the expression of the Hessian vector product, where this will
# be the vector.
p = T.vector('p')

# The loss and its gradient.
loss = cross_entropy(target, output)
lossgrad = T.grad(loss, P.flat)

# Expression for the Gauss-Newton matrix.
Jp = T.Rop(output, P.flat, p)
HJp = T.grad(T.sum(T.grad(loss, output) * Jp),
             output, consider_constant=[Jp])
Hp = T.grad(T.sum(HJp * output), P.flat, consider_constant=[HJp, Jp])

# Functions.
givens = {P.flat: par_sub}
f = theano.function([par_sub, inpt, target], loss, givens=givens)
fprime = theano.function([par_sub, inpt, target], lossgrad, givens=givens)
f_Hp = theano.function([par_sub, p, inpt, target], Hp, givens=givens)
f_predict = theano.function([par_sub, inpt], exprs['output'], givens=givens)

# Build a dataset.
#MNIST
import cPickle, gzip

# Load the dataset
with gzip.open('mnist.pkl.gz','rb') as MNISTfile:
    train_set, valid_set, test_set = cPickle.load(MNISTfile)

X, labels = train_set
N, _ = X.shape
Y = np.zeros((N, 10))
for i in range(N):
    Y[i][labels[i]] = 1

#sparse initialization
## P['hiddenbias1'][:] = scipy.zeros(n_hidden1)
## P['hiddenbias2'][:] = scipy.zeros(n_hidden2)
## P['hiddenbias3'][:] = scipy.zeros(n_hidden3)
## P['outbias'][:] = scipy.zeros(n_output)

## P['inweights'][:,:] = np.random.randn(n_inpt, n_hidden1)
## P['hiddenweights1'][:,:] = np.random.randn(n_hidden1, n_hidden2)
## P['hiddenweights2'][:,:] = np.random.randn(n_hidden2, n_hidden3)
## P['outweights'][:,:] = np.random.randn(n_hidden3, n_output)

## def sparse_initialization(a, b, s, MaxNonZeroPerColumn = 15):
##     for j in range(b):
##         perm = np.random.permutation(a)
##         P[s][perm[MaxNonZeroPerColumn:], j] *= 0

## sparse_initialization( n_inpt, n_hidden1, 'inweights')
## sparse_initialization( n_hidden1, n_hidden2, 'hiddenweights1')
## sparse_initialization( n_hidden2, n_hidden3, 'hiddenweights2')
## sparse_initialization( n_hidden3, n_output, 'outweights')

#minibatches for HF: whole dataset
args = (([X, Y], {}) for _ in itertools.repeat(()))

#minibatches for cg: 5000
batchsize = 5000
n_batches = N/batchsize
random_numbers = (random.randint(0, n_batches - 1) for _ in itertools.count())
idxs = ((r * batchsize, (r + 1) * batchsize) for r in random_numbers)
minibatches = ((X[lower:upper], Y[lower:upper]) for lower, upper in idxs)
cg_args = ((m, {}) for m in minibatches)

print '#pars:', P.data.size

import chopmunk

@chopmunk.coroutine
def print_Hp_min_max():
    while True:
        info = (yield)
        if 'Hp' not in info:
            continue
        absed = abs(info['Hp'])
        print 'min max std mean', absed.min(), absed.max(), absed.std(), absed.mean()

logger = chopmunk.prettyprint_sink()
filesink = chopmunk.file_sink('mnist.log')
filesink = chopmunk.jsonify(filesink)
logger = chopmunk.broadcast(logger, filesink, print_Hp_min_max())
logfunc = logger.send

optimizer = 'hf'

if optimizer == 'ksd':
    opt = KrylovSubspaceDescent(
        P.data, f, fprime, f_Hp, n_bases=10,
        args=args, logfunc=logfunc)
elif optimizer == 'rprop':
    opt = Rprop(P.data, f, fprime, args=args, logfunc=logfunc)
elif optimizer == 'hf':
    opt = HessianFree(
        P.data, f, fprime, f_Hp, args=args, cg_args=cg_args,
        initial_damping=initial_damping,
        precond='martens',
        logfunc=logfunc)

lossTab = scipy.empty(100)

for i, info in enumerate(opt):
    X, Y = info['args']
    loss = f(P.data, X, Y)
    print 'iteration', i
    print 'loss', loss
    lossTab[i] = loss
    logfunc({'loss':loss})

    if i > 100:
        break

pylab.plot(lossTab)
pylab.show()

#pylab.plot(steps)
#pylab.plot(losses)
#pylab.show()
#
#pylab.plot(f_predict(X)[:, 0, 0])
#pylab.plot(Z[:, 0, 0])
#pylab.show()

## P = f_predict(P.data, TX)[-5:, :3, 0]
## print (P > 0.5).astype('uint8')
## print P
<|endoftext|>


<commit_before>#
# Autogenerated by Thrift
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
#  @generated
#

from __future__ import absolute_import
import six
from thrift.util.Recursive import fix_spec
from thrift.Thrift import *
from thrift.protocol.TProtocol import TProtocolException

from .ttypes import *
import module.ttypes
import includes.ttypes

from thrift.Thrift import TProcessor
import pprint
import warnings
from thrift import Thrift
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.protocol import TCompactProtocol
from thrift.protocol import THeaderProtocol
try:
  from thrift.protocol import fastproto
except:
  fastproto = None

all_structs = []
UTF8STRINGS = bool(0) or sys.version_info.major >= 3

from thrift.util.Decorators import *

class Iface:
  def query(self, s=None, i=None):
    """
    Parameters:
     - s
     - i
    """
    pass

class ContextIface:
  def query(self, handler_ctx, s=None, i=None):
    """
    Parameters:
     - s
     - i
    """
    pass

# HELPER FUNCTIONS AND STRUCTURES

class query_args:
  """
  Attributes:
   - s
   - i
  """

  thrift_spec = None
  thrift_field_annotations = None
  thrift_struct_annotations = None
  __init__ = None
  @staticmethod
  def isUnion():
    return False

  def read(self, iprot):
    if (isinstance(iprot, TBinaryProtocol.TBinaryProtocolAccelerated) or (isinstance(iprot, THeaderProtocol.THeaderProtocol) and iprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_BINARY_PROTOCOL)) and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastproto is not None:
      fastproto.decode(self, iprot.trans, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=0)
      self.checkRequired()
      return
    if (isinstance(iprot, TCompactProtocol.TCompactProtocolAccelerated) or (isinstance(iprot, THeaderProtocol.THeaderProtocol) and iprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_COMPACT_PROTOCOL)) and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastproto is not None:
      fastproto.decode(self, iprot.trans, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=2)
      self.checkRequired()
      return
    iprot.readStructBegin()
    while True:
      (fname, ftype, fid) = iprot.readFieldBegin()
      if ftype == TType.STOP:
        break
      if fid == 1:
        if ftype == TType.STRUCT:
          self.s = module.ttypes.MyStruct()
          self.s.read(iprot)
        else:
          iprot.skip(ftype)
      elif fid == 2:
        if ftype == TType.STRUCT:
          self.i = includes.ttypes.Included()
          self.i.read(iprot)
        else:
          iprot.skip(ftype)
      else:
        iprot.skip(ftype)
      iprot.readFieldEnd()
    iprot.readStructEnd()
    self.checkRequired()

  def checkRequired(self):
    return

  def write(self, oprot):
    if (isinstance(oprot, TBinaryProtocol.TBinaryProtocolAccelerated) or (isinstance(oprot, THeaderProtocol.THeaderProtocol) and oprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_BINARY_PROTOCOL)) and self.thrift_spec is not None and fastproto is not None:
      oprot.trans.write(fastproto.encode(self, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=0))
      return
    if (isinstance(oprot, TCompactProtocol.TCompactProtocolAccelerated) or (isinstance(oprot, THeaderProtocol.THeaderProtocol) and oprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_COMPACT_PROTOCOL)) and self.thrift_spec is not None and fastproto is not None:
      oprot.trans.write(fastproto.encode(self, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=2))
      return
    oprot.writeStructBegin('query_args')
    if self.s != None:
      oprot.writeFieldBegin('s', TType.STRUCT, 1)
      self.s.write(oprot)
      oprot.writeFieldEnd()
    if self.i != None:
      oprot.writeFieldBegin('i', TType.STRUCT, 2)
      self.i.write(oprot)
      oprot.writeFieldEnd()
    oprot.writeFieldStop()
    oprot.writeStructEnd()

  def __repr__(self):
    L = []
    for key, value in six.iteritems(self.__dict__):
      padding = ' ' * 4
      value = pprint.pformat(value, indent=0)
      value = padding.join(value.splitlines(True))
      L.append('    %s=%s' % (key, value))
    return "%s(\n%s)" % (self.__class__.__name__, ",\n".join(L))

  def __eq__(self, other):
    if not isinstance(other, self.__class__):
      return False

    return self.__dict__ == other.__dict__

  def __ne__(self, other):
    return not (self == other)

  # Override the __hash__ function for Python3 - t10434117
  if not six.PY2:
    __hash__ = object.__hash__

all_structs.append(query_args)
query_args.thrift_spec = (
  None, # 0
  (1, TType.STRUCT, 's', [module.ttypes.MyStruct, module.ttypes.MyStruct.thrift_spec, False], None, 2, ), # 1
  (2, TType.STRUCT, 'i', [includes.ttypes.Included, includes.ttypes.Included.thrift_spec, False], None, 2, ), # 2
)

query_args.thrift_struct_annotations = {
}
query_args.thrift_field_annotations = {
}

def query_args__init__(self, s=None, i=None,):
  self.s = s
  self.i = i

query_args.__init__ = query_args__init__

class query_result:

  thrift_spec = None
  thrift_field_annotations = None
  thrift_struct_annotations = None
  @staticmethod
  def isUnion():
    return False

  def read(self, iprot):
    if (isinstance(iprot, TBinaryProtocol.TBinaryProtocolAccelerated) or (isinstance(iprot, THeaderProtocol.THeaderProtocol) and iprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_BINARY_PROTOCOL)) and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastproto is not None:
      fastproto.decode(self, iprot.trans, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=0)
      self.checkRequired()
      return
    if (isinstance(iprot, TCompactProtocol.TCompactProtocolAccelerated) or (isinstance(iprot, THeaderProtocol.THeaderProtocol) and iprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_COMPACT_PROTOCOL)) and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastproto is not None:
      fastproto.decode(self, iprot.trans, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=2)
      self.checkRequired()
      return
    iprot.readStructBegin()
    while True:
      (fname, ftype, fid) = iprot.readFieldBegin()
      if ftype == TType.STOP:
        break
      else:
        iprot.skip(ftype)
      iprot.readFieldEnd()
    iprot.readStructEnd()
    self.checkRequired()

  def checkRequired(self):
    return

  def write(self, oprot):
    if (isinstance(oprot, TBinaryProtocol.TBinaryProtocolAccelerated) or (isinstance(oprot, THeaderProtocol.THeaderProtocol) and oprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_BINARY_PROTOCOL)) and self.thrift_spec is not None and fastproto is not None:
      oprot.trans.write(fastproto.encode(self, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=0))
      return
    if (isinstance(oprot, TCompactProtocol.TCompactProtocolAccelerated) or (isinstance(oprot, THeaderProtocol.THeaderProtocol) and oprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_COMPACT_PROTOCOL)) and self.thrift_spec is not None and fastproto is not None:
      oprot.trans.write(fastproto.encode(self, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=2))
      return
    oprot.writeStructBegin('query_result')
    oprot.writeFieldStop()
    oprot.writeStructEnd()

  def __repr__(self):
    L = []
    for key, value in six.iteritems(self.__dict__):
      padding = ' ' * 4
      value = pprint.pformat(value, indent=0)
      value = padding.join(value.splitlines(True))
      L.append('    %s=%s' % (key, value))
    return "%s(\n%s)" % (self.__class__.__name__, ",\n".join(L))

  def __eq__(self, other):
    if not isinstance(other, self.__class__):
      return False

    return self.__dict__ == other.__dict__

  def __ne__(self, other):
    return not (self == other)

  # Override the __hash__ function for Python3 - t10434117
  if not six.PY2:
    __hash__ = object.__hash__

all_structs.append(query_result)
query_result.thrift_spec = (
)

query_result.thrift_struct_annotations = {
}
query_result.thrift_field_annotations = {
}

class Client(Iface):
  def __init__(self, iprot, oprot=None):
    self._iprot = self._oprot = iprot
    if oprot != None:
      self._oprot = oprot
    self._seqid = 0

  def query(self, s=None, i=None):
    """
    Parameters:
     - s
     - i
    """
    self.send_query(s, i)
    self.recv_query()

  def send_query(self, s=None, i=None):
    self._oprot.writeMessageBegin('query', TMessageType.CALL, self._seqid)
    args = query_args()
    args.s = s
    args.i = i
    args.write(self._oprot)
    self._oprot.writeMessageEnd()
    self._oprot.trans.flush()

  def recv_query(self, ):
    (fname, mtype, rseqid) = self._iprot.readMessageBegin()
    if mtype == TMessageType.EXCEPTION:
      x = TApplicationException()
      x.read(self._iprot)
      self._iprot.readMessageEnd()
      raise x
    result = query_result()
    result.read(self._iprot)
    self._iprot.readMessageEnd()
    return

class Processor(Iface, TProcessor):
  _onewayMethods = ()

  def __init__(self, handler):
    TProcessor.__init__(self)
    self._handler = handler
    self._processMap = {}
    self._processMap["query"] = Processor.process_query

  def onewayMethods(self):
    l = []
    l.extend(Processor._onewayMethods)
    return tuple(l)

  @process_main()
  def process(self,): pass

  @process_method(query_args, oneway=False)
  def process_query(self, args, handler_ctx):
    result = query_result()
    try:
      self._handler.query(args.s, args.i)
    except:
      ex = sys.exc_info()[1]
      self._event_handler.handlerError(handler_ctx, 'query', ex)
      result = Thrift.TApplicationException(message=str(ex))
    return result

Iface._processor_type = Processor

class ContextProcessor(ContextIface, TProcessor):
  _onewayMethods = ()

  def __init__(self, handler):
    TProcessor.__init__(self)
    self._handler = handler
    self._processMap = {}
    self._processMap["query"] = ContextProcessor.process_query

  def onewayMethods(self):
    l = []
    l.extend(ContextProcessor._onewayMethods)
    return tuple(l)

  @process_main()
  def process(self,): pass

  @process_method(query_args, oneway=False)
  def process_query(self, args, handler_ctx):
    result = query_result()
    try:
      self._handler.query(handler_ctx, args.s, args.i)
    except:
      ex = sys.exc_info()[1]
      self._event_handler.handlerError(handler_ctx, 'query', ex)
      result = Thrift.TApplicationException(message=str(ex))
    return result

ContextIface._processor_type = ContextProcessor

fix_spec(all_structs)
del all_structs

<commit_msg>Test Fixture for python client context handlers<commit_after>#
# Autogenerated by Thrift
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
#  @generated
#

from __future__ import absolute_import
import six
from thrift.util.Recursive import fix_spec
from thrift.Thrift import *
from thrift.protocol.TProtocol import TProtocolException

from .ttypes import *
import module.ttypes
import includes.ttypes

from thrift.Thrift import TProcessor
import pprint
import warnings
from thrift import Thrift
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.protocol import TCompactProtocol
from thrift.protocol import THeaderProtocol
try:
  from thrift.protocol import fastproto
except:
  fastproto = None

all_structs = []
UTF8STRINGS = bool(0) or sys.version_info.major >= 3

from thrift.util.Decorators import *

class Iface:
  def query(self, s=None, i=None):
    """
    Parameters:
     - s
     - i
    """
    pass

class ContextIface:
  def query(self, handler_ctx, s=None, i=None):
    """
    Parameters:
     - s
     - i
    """
    pass

# HELPER FUNCTIONS AND STRUCTURES

class query_args:
  """
  Attributes:
   - s
   - i
  """

  thrift_spec = None
  thrift_field_annotations = None
  thrift_struct_annotations = None
  __init__ = None
  @staticmethod
  def isUnion():
    return False

  def read(self, iprot):
    if (isinstance(iprot, TBinaryProtocol.TBinaryProtocolAccelerated) or (isinstance(iprot, THeaderProtocol.THeaderProtocol) and iprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_BINARY_PROTOCOL)) and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastproto is not None:
      fastproto.decode(self, iprot.trans, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=0)
      self.checkRequired()
      return
    if (isinstance(iprot, TCompactProtocol.TCompactProtocolAccelerated) or (isinstance(iprot, THeaderProtocol.THeaderProtocol) and iprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_COMPACT_PROTOCOL)) and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastproto is not None:
      fastproto.decode(self, iprot.trans, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=2)
      self.checkRequired()
      return
    iprot.readStructBegin()
    while True:
      (fname, ftype, fid) = iprot.readFieldBegin()
      if ftype == TType.STOP:
        break
      if fid == 1:
        if ftype == TType.STRUCT:
          self.s = module.ttypes.MyStruct()
          self.s.read(iprot)
        else:
          iprot.skip(ftype)
      elif fid == 2:
        if ftype == TType.STRUCT:
          self.i = includes.ttypes.Included()
          self.i.read(iprot)
        else:
          iprot.skip(ftype)
      else:
        iprot.skip(ftype)
      iprot.readFieldEnd()
    iprot.readStructEnd()
    self.checkRequired()

  def checkRequired(self):
    return

  def write(self, oprot):
    if (isinstance(oprot, TBinaryProtocol.TBinaryProtocolAccelerated) or (isinstance(oprot, THeaderProtocol.THeaderProtocol) and oprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_BINARY_PROTOCOL)) and self.thrift_spec is not None and fastproto is not None:
      oprot.trans.write(fastproto.encode(self, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=0))
      return
    if (isinstance(oprot, TCompactProtocol.TCompactProtocolAccelerated) or (isinstance(oprot, THeaderProtocol.THeaderProtocol) and oprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_COMPACT_PROTOCOL)) and self.thrift_spec is not None and fastproto is not None:
      oprot.trans.write(fastproto.encode(self, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=2))
      return
    oprot.writeStructBegin('query_args')
    if self.s != None:
      oprot.writeFieldBegin('s', TType.STRUCT, 1)
      self.s.write(oprot)
      oprot.writeFieldEnd()
    if self.i != None:
      oprot.writeFieldBegin('i', TType.STRUCT, 2)
      self.i.write(oprot)
      oprot.writeFieldEnd()
    oprot.writeFieldStop()
    oprot.writeStructEnd()

  def __repr__(self):
    L = []
    for key, value in six.iteritems(self.__dict__):
      padding = ' ' * 4
      value = pprint.pformat(value, indent=0)
      value = padding.join(value.splitlines(True))
      L.append('    %s=%s' % (key, value))
    return "%s(\n%s)" % (self.__class__.__name__, ",\n".join(L))

  def __eq__(self, other):
    if not isinstance(other, self.__class__):
      return False

    return self.__dict__ == other.__dict__

  def __ne__(self, other):
    return not (self == other)

  # Override the __hash__ function for Python3 - t10434117
  if not six.PY2:
    __hash__ = object.__hash__

all_structs.append(query_args)
query_args.thrift_spec = (
  None, # 0
  (1, TType.STRUCT, 's', [module.ttypes.MyStruct, module.ttypes.MyStruct.thrift_spec, False], None, 2, ), # 1
  (2, TType.STRUCT, 'i', [includes.ttypes.Included, includes.ttypes.Included.thrift_spec, False], None, 2, ), # 2
)

query_args.thrift_struct_annotations = {
}
query_args.thrift_field_annotations = {
}

def query_args__init__(self, s=None, i=None,):
  self.s = s
  self.i = i

query_args.__init__ = query_args__init__

class query_result:

  thrift_spec = None
  thrift_field_annotations = None
  thrift_struct_annotations = None
  @staticmethod
  def isUnion():
    return False

  def read(self, iprot):
    if (isinstance(iprot, TBinaryProtocol.TBinaryProtocolAccelerated) or (isinstance(iprot, THeaderProtocol.THeaderProtocol) and iprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_BINARY_PROTOCOL)) and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastproto is not None:
      fastproto.decode(self, iprot.trans, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=0)
      self.checkRequired()
      return
    if (isinstance(iprot, TCompactProtocol.TCompactProtocolAccelerated) or (isinstance(iprot, THeaderProtocol.THeaderProtocol) and iprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_COMPACT_PROTOCOL)) and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastproto is not None:
      fastproto.decode(self, iprot.trans, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=2)
      self.checkRequired()
      return
    iprot.readStructBegin()
    while True:
      (fname, ftype, fid) = iprot.readFieldBegin()
      if ftype == TType.STOP:
        break
      else:
        iprot.skip(ftype)
      iprot.readFieldEnd()
    iprot.readStructEnd()
    self.checkRequired()

  def checkRequired(self):
    return

  def write(self, oprot):
    if (isinstance(oprot, TBinaryProtocol.TBinaryProtocolAccelerated) or (isinstance(oprot, THeaderProtocol.THeaderProtocol) and oprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_BINARY_PROTOCOL)) and self.thrift_spec is not None and fastproto is not None:
      oprot.trans.write(fastproto.encode(self, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=0))
      return
    if (isinstance(oprot, TCompactProtocol.TCompactProtocolAccelerated) or (isinstance(oprot, THeaderProtocol.THeaderProtocol) and oprot.get_protocol_id() == THeaderProtocol.THeaderProtocol.T_COMPACT_PROTOCOL)) and self.thrift_spec is not None and fastproto is not None:
      oprot.trans.write(fastproto.encode(self, [self.__class__, self.thrift_spec, False], utf8strings=UTF8STRINGS, protoid=2))
      return
    oprot.writeStructBegin('query_result')
    oprot.writeFieldStop()
    oprot.writeStructEnd()

  def __repr__(self):
    L = []
    for key, value in six.iteritems(self.__dict__):
      padding = ' ' * 4
      value = pprint.pformat(value, indent=0)
      value = padding.join(value.splitlines(True))
      L.append('    %s=%s' % (key, value))
    return "%s(\n%s)" % (self.__class__.__name__, ",\n".join(L))

  def __eq__(self, other):
    if not isinstance(other, self.__class__):
      return False

    return self.__dict__ == other.__dict__

  def __ne__(self, other):
    return not (self == other)

  # Override the __hash__ function for Python3 - t10434117
  if not six.PY2:
    __hash__ = object.__hash__

all_structs.append(query_result)
query_result.thrift_spec = (
)

query_result.thrift_struct_annotations = {
}
query_result.thrift_field_annotations = {
}

class Client(Iface):
  def __enter__(self):
    return self

  def __exit__(self, type, value, tb):
    self._iprot.trans.close()
    if self._iprot is not self._oprot:
      self._oprot.trans.close()

  def __init__(self, iprot, oprot=None):
    self._iprot = self._oprot = iprot
    if oprot != None:
      self._oprot = oprot
    self._seqid = 0

  def query(self, s=None, i=None):
    """
    Parameters:
     - s
     - i
    """
    self.send_query(s, i)
    self.recv_query()

  def send_query(self, s=None, i=None):
    self._oprot.writeMessageBegin('query', TMessageType.CALL, self._seqid)
    args = query_args()
    args.s = s
    args.i = i
    args.write(self._oprot)
    self._oprot.writeMessageEnd()
    self._oprot.trans.flush()

  def recv_query(self, ):
    (fname, mtype, rseqid) = self._iprot.readMessageBegin()
    if mtype == TMessageType.EXCEPTION:
      x = TApplicationException()
      x.read(self._iprot)
      self._iprot.readMessageEnd()
      raise x
    result = query_result()
    result.read(self._iprot)
    self._iprot.readMessageEnd()
    return

class Processor(Iface, TProcessor):
  _onewayMethods = ()

  def __init__(self, handler):
    TProcessor.__init__(self)
    self._handler = handler
    self._processMap = {}
    self._processMap["query"] = Processor.process_query

  def onewayMethods(self):
    l = []
    l.extend(Processor._onewayMethods)
    return tuple(l)

  @process_main()
  def process(self,): pass

  @process_method(query_args, oneway=False)
  def process_query(self, args, handler_ctx):
    result = query_result()
    try:
      self._handler.query(args.s, args.i)
    except:
      ex = sys.exc_info()[1]
      self._event_handler.handlerError(handler_ctx, 'query', ex)
      result = Thrift.TApplicationException(message=str(ex))
    return result

Iface._processor_type = Processor

class ContextProcessor(ContextIface, TProcessor):
  _onewayMethods = ()

  def __init__(self, handler):
    TProcessor.__init__(self)
    self._handler = handler
    self._processMap = {}
    self._processMap["query"] = ContextProcessor.process_query

  def onewayMethods(self):
    l = []
    l.extend(ContextProcessor._onewayMethods)
    return tuple(l)

  @process_main()
  def process(self,): pass

  @process_method(query_args, oneway=False)
  def process_query(self, args, handler_ctx):
    result = query_result()
    try:
      self._handler.query(handler_ctx, args.s, args.i)
    except:
      ex = sys.exc_info()[1]
      self._event_handler.handlerError(handler_ctx, 'query', ex)
      result = Thrift.TApplicationException(message=str(ex))
    return result

ContextIface._processor_type = ContextProcessor

fix_spec(all_structs)
del all_structs

<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Application: Vimba - CMS
# Module: SimpleNews
# Copyright (c) 2010 Vimba inc. All rights reserved.
# Created by Francois Lebel on 20-03-2010.

from django.db import models
from django.utils.translation import ugettext_lazy as _
from tagging.fields import TagField
from vcms.apps.www.models import Language, Page, PageElementPosition
from vcms.apps.simpleannouncements.models import Announcement

APP_SLUGS = "simplenews"

class Blog(Announcement):
    class Meta:
        verbose_name_plural = _("Blogs")
        get_latest_by = ['-date_created']
        ordering = ['-date_created']

class BlogPageModule(PageElementPosition):
    from vcms.apps.www.models import DashboardPage as DP
    page = models.ForeignKey(DP)
    tags = TagField()
    title = models.CharField(max_length="60", help_text=_("Max 60 characters"), verbose_name=_("Title"))

    def __unicode__(self):
        return self.page.name
<commit_msg>Fixed the APP_SLUG for SimpleBlogs.<commit_after># -*- coding: UTF-8 -*-
# Application: Vimba - CMS
# Module: SimpleNews
# Copyright (c) 2010 Vimba inc. All rights reserved.
# Created by Francois Lebel on 20-03-2010.

from django.db import models
from django.utils.translation import ugettext_lazy as _
from tagging.fields import TagField
from vcms.apps.www.models import Language, Page, PageElementPosition
from vcms.apps.simpleannouncements.models import Announcement

APP_SLUGS = "simpleblogs"

class Blog(Announcement):
    class Meta:
        verbose_name_plural = _("Blogs")
        get_latest_by = ['-date_created']
        ordering = ['-date_created']

class BlogPageModule(PageElementPosition):
    from vcms.apps.www.models import DashboardPage as DP
    page = models.ForeignKey(DP)
    tags = TagField()
    title = models.CharField(max_length="60", help_text=_("Max 60 characters"), verbose_name=_("Title"))

    def __unicode__(self):
        return self.page.name
<|endoftext|>