<commit_before># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2003, 2004 Edgewall Software
# Copyright (C) 2003, 2004 Jonas Borgstrm <jonas@edgewall.com>
#
# Trac is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Trac is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# Author: Jonas Borgstrm <jonas@edgewall.com>

import re
import time
import string

import perm
from util import TracError, escape, shorten_line, add_dictlist_to_hdf
from Module import Module

class Search(Module):
    template_name = 'search.cs'

    RESULTS_PER_PAGE = 10

    def query_to_sql(self, q, name):
        self.log.debug("Query: %s" % q)
        if q[0] == q[-1] == "'" or q[0] == q[-1] == '"':
            sql_q = "%s like '%%%s%%'" % (name, q[1:-1].replace('\'',
                                                                '\'\''))
        else:
            q = q.replace('\'', '\'\'')
            keywords = q.split(' ')
            x = map(lambda x, name=name: name + ' LIKE \'%' + x + '%\'', keywords)
            sql_q = string.join(x, ' AND ')
        self.log.debug("SQL Condition: %s" % sql_q)
        return sql_q

    def shorten_result(self, text='', keywords=[], maxlen=240, fuzz=60):
        if not text: text = ''
        text_low = text.lower()
        beg = -1
        for k in keywords:
            i = text_low.find(k.lower())
            if (i > -1 and i < beg) or beg == -1:
                beg = i
        excerpt_beg = 0
        if beg > fuzz:
            for sep in ".:;= ":
                eb = text.find(sep, beg - fuzz, beg - 1)
                if eb > -1:
                    eb += 1
                    break
            else:
                eb = beg - fuzz
            excerpt_beg = eb
        if excerpt_beg < 0: excerpt_beg = 0
        msg = text[excerpt_beg:beg+maxlen]
        if beg > fuzz:
            msg = '... ' + msg
        if beg < len(text)-maxlen:
            msg = msg + ' ...'
        return msg

    def perform_query (self, query, changeset, tickets, wiki, page=0):
        keywords = query.split(' ')

        if changeset:
            changeset = self.perm.has_permission(perm.CHANGESET_VIEW)
        if tickets:
            tickets = self.perm.has_permission(perm.TICKET_VIEW)
        if wiki:
            wiki = self.perm.has_permission(perm.WIKI_VIEW)

        if changeset == tickets == wiki == 0:
            return []

        if len(keywords) == 1:
            kwd = keywords[0]
            redir = None
            # Prepending a '!' disables quickjump feature
            if kwd[0] == '!':
                keywords[0] = kwd[1:]
                query = query[1:]
                self.req.hdf.setValue('search.q', query)
            # Ticket quickjump
            elif kwd[0] == '#' and kwd[1:].isdigit():
                redir = self.env.href.ticket(kwd[1:])
            # Changeset quickjump
            elif kwd[0] == '[' and kwd[-1] == ']' and kwd[1:-1].isdigit():
                redir = self.env.href.changeset(kwd[1:-1])
            # Report quickjump
            elif kwd[0] == '{' and kwd[-1] == '}' and kwd[1:-1].isdigit():
                redir = self.env.href.report(kwd[1:-1])
            elif kwd[0].isupper() and kwd[1].islower():
                r = "((^|(?<=[^A-Za-z]))[!]?[A-Z][a-z/]+(?:[A-Z][a-z/]+)+)"
                if re.match (r, kwd):
                    redir = self.env.href.wiki(kwd)
            if redir:
                self.req.hdf.setValue('search.q', '')
                self.req.redirect(redir)
            elif len(query) < 3:
                raise TracError('Search query too short. '
                                'Query must be at least 3 characters long.',
                                'Search Error')

        cursor = self.db.cursor ()

        q = []
        if changeset:
            q.append('SELECT 1 as type, message AS title, message, author, '
                     ' \'\' AS keywords, rev AS data, time,0 AS ver'
                     ' FROM revision WHERE %s OR %s' %
                     (self.query_to_sql(query, 'message'),
                      self.query_to_sql(query, 'author')))
        if tickets:
            q.append('SELECT DISTINCT 2 as type, a.summary AS title, '
                     ' a.description AS message, a.reporter AS author, a.keywords as keywords,'
                     ' a.id AS data, a.time as time, 0 AS ver'
                     ' FROM ticket a LEFT JOIN ticket_change b ON a.id = b.ticket'
                     ' WHERE (b.field=\'comment\' AND %s ) OR'
                     ' %s OR %s OR %s OR %s OR %s' %
                      (self.query_to_sql(query, 'b.newvalue'),
                       self.query_to_sql(query, 'summary'),
                       self.query_to_sql(query, 'keywords'),
                       self.query_to_sql(query, 'description'),
                       self.query_to_sql(query, 'reporter'),
                       self.query_to_sql(query, 'cc')))
        if wiki:
            q.append('SELECT 3 as type, text AS title, text AS message,'
                     ' author, \'\' AS keywords, w1.name AS data, time,'
                     ' w1.version as ver'
                     ' FROM wiki w1, '
                     ' (SELECT name,max(version) AS ver '
                     '    FROM wiki GROUP BY name) w2'
                     ' WHERE w1.version = w2.ver AND w1.name = w2.name  AND'
                     ' (%s OR %s OR %s) ' %
                     (self.query_to_sql(query, 'w1.name'),
                      self.query_to_sql(query, 'w1.author'),
                      self.query_to_sql(query, 'w1.text')))

        if not q: return []

        q_str = string.join(q, ' UNION ALL ')
        q_str += ' ORDER BY 7 DESC LIMIT %d OFFSET %d' % \
                 (self.RESULTS_PER_PAGE, self.RESULTS_PER_PAGE * page)

        self.log.debug("SQL Query: %s" % q_str)
        cursor.execute(q_str)

        # Make the data more HDF-friendly
        info = []
        while 1:
            row = cursor.fetchone()
            if not row:
                break
            msg = row['message']
            t = time.localtime(int(row['time']))
            item = {'type': int(row['type']),
                    'keywords': row['keywords'] or '',
                    'data': row['data'],
                    'title': row['title'],
                    'datetime' : time.strftime('%c', t),
                    'author': row['author']}
            if item['type'] == 1:
                item['changeset_href'] = self.env.href.changeset(int(row['data']))
            elif item['type'] == 2:
                item['ticket_href'] = self.env.href.ticket(int(row['data']))
            elif item['type'] == 3:
                item['wiki_href'] = self.env.href.wiki(row['data'])

            item['shortmsg'] = escape(shorten_line(msg))
            item['message'] = escape(self.shorten_result(msg, keywords))
            info.append(item)
        return info

    def render (self):
        self.perm.assert_permission(perm.SEARCH_VIEW)
        self.req.hdf.setValue('title', 'Search')
        self.req.hdf.setValue('search.ticket', 'checked')
        self.req.hdf.setValue('search.changeset', 'checked')
        self.req.hdf.setValue('search.wiki', 'checked')
        self.req.hdf.setValue('search.results_per_page', str(self.RESULTS_PER_PAGE))

        if self.args.has_key('q'):
            query = self.args.get('q')
            self.req.hdf.setValue('title', 'Search Results')
            self.req.hdf.setValue('search.q', query.replace('"', """))
            tickets = self.args.has_key('ticket')
            changesets = self.args.has_key('changeset')
            wiki = self.args.has_key('wiki')

            # If no search options chosen, choose all
            if not (tickets or changesets or wiki):
                tickets = changesets = wiki = 1

            if self.args.has_key('page'):
                page = int(self.args.get('page'))
                self.req.hdf.setValue('search.result.page', str(page))
            else:
                page = 0
            if not tickets:
                self.req.hdf.setValue('search.ticket', '')
            if not changesets:
                self.req.hdf.setValue('search.changeset', '')
            if not wiki:
                self.req.hdf.setValue('search.wiki', '')
            info = self.perform_query(query, changesets, tickets, wiki, page)
            self.req.hdf.setValue('search.result.count', str(len(info)))
            if len(info) == self.RESULTS_PER_PAGE:
                self.req.hdf.setValue('search.result.more', '1')
            add_dictlist_to_hdf(info, self.req.hdf, 'search.result')

<commit_msg>Escape HTML in search results. Fixes #836.<commit_after># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2003, 2004 Edgewall Software
# Copyright (C) 2003, 2004 Jonas Borgstrm <jonas@edgewall.com>
#
# Trac is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Trac is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# Author: Jonas Borgstrm <jonas@edgewall.com>

import re
import time
import string

import perm
from util import TracError, escape, shorten_line, add_dictlist_to_hdf
from Module import Module

class Search(Module):
    template_name = 'search.cs'

    RESULTS_PER_PAGE = 10

    def query_to_sql(self, q, name):
        self.log.debug("Query: %s" % q)
        if q[0] == q[-1] == "'" or q[0] == q[-1] == '"':
            sql_q = "%s like '%%%s%%'" % (name, q[1:-1].replace('\'',
                                                                '\'\''))
        else:
            q = q.replace('\'', '\'\'')
            keywords = q.split(' ')
            x = map(lambda x, name=name: name + ' LIKE \'%' + x + '%\'', keywords)
            sql_q = string.join(x, ' AND ')
        self.log.debug("SQL Condition: %s" % sql_q)
        return sql_q

    def shorten_result(self, text='', keywords=[], maxlen=240, fuzz=60):
        if not text: text = ''
        text_low = text.lower()
        beg = -1
        for k in keywords:
            i = text_low.find(k.lower())
            if (i > -1 and i < beg) or beg == -1:
                beg = i
        excerpt_beg = 0
        if beg > fuzz:
            for sep in ".:;= ":
                eb = text.find(sep, beg - fuzz, beg - 1)
                if eb > -1:
                    eb += 1
                    break
            else:
                eb = beg - fuzz
            excerpt_beg = eb
        if excerpt_beg < 0: excerpt_beg = 0
        msg = text[excerpt_beg:beg+maxlen]
        if beg > fuzz:
            msg = '... ' + msg
        if beg < len(text)-maxlen:
            msg = msg + ' ...'
        return msg

    def perform_query (self, query, changeset, tickets, wiki, page=0):
        keywords = query.split(' ')

        if changeset:
            changeset = self.perm.has_permission(perm.CHANGESET_VIEW)
        if tickets:
            tickets = self.perm.has_permission(perm.TICKET_VIEW)
        if wiki:
            wiki = self.perm.has_permission(perm.WIKI_VIEW)

        if changeset == tickets == wiki == 0:
            return []

        if len(keywords) == 1:
            kwd = keywords[0]
            redir = None
            # Prepending a '!' disables quickjump feature
            if kwd[0] == '!':
                keywords[0] = kwd[1:]
                query = query[1:]
                self.req.hdf.setValue('search.q', query)
            # Ticket quickjump
            elif kwd[0] == '#' and kwd[1:].isdigit():
                redir = self.env.href.ticket(kwd[1:])
            # Changeset quickjump
            elif kwd[0] == '[' and kwd[-1] == ']' and kwd[1:-1].isdigit():
                redir = self.env.href.changeset(kwd[1:-1])
            # Report quickjump
            elif kwd[0] == '{' and kwd[-1] == '}' and kwd[1:-1].isdigit():
                redir = self.env.href.report(kwd[1:-1])
            elif kwd[0].isupper() and kwd[1].islower():
                r = "((^|(?<=[^A-Za-z]))[!]?[A-Z][a-z/]+(?:[A-Z][a-z/]+)+)"
                if re.match (r, kwd):
                    redir = self.env.href.wiki(kwd)
            if redir:
                self.req.hdf.setValue('search.q', '')
                self.req.redirect(redir)
            elif len(query) < 3:
                raise TracError('Search query too short. '
                                'Query must be at least 3 characters long.',
                                'Search Error')

        cursor = self.db.cursor ()

        q = []
        if changeset:
            q.append('SELECT 1 as type, message AS title, message, author, '
                     ' \'\' AS keywords, rev AS data, time,0 AS ver'
                     ' FROM revision WHERE %s OR %s' %
                     (self.query_to_sql(query, 'message'),
                      self.query_to_sql(query, 'author')))
        if tickets:
            q.append('SELECT DISTINCT 2 as type, a.summary AS title, '
                     ' a.description AS message, a.reporter AS author, a.keywords as keywords,'
                     ' a.id AS data, a.time as time, 0 AS ver'
                     ' FROM ticket a LEFT JOIN ticket_change b ON a.id = b.ticket'
                     ' WHERE (b.field=\'comment\' AND %s ) OR'
                     ' %s OR %s OR %s OR %s OR %s' %
                      (self.query_to_sql(query, 'b.newvalue'),
                       self.query_to_sql(query, 'summary'),
                       self.query_to_sql(query, 'keywords'),
                       self.query_to_sql(query, 'description'),
                       self.query_to_sql(query, 'reporter'),
                       self.query_to_sql(query, 'cc')))
        if wiki:
            q.append('SELECT 3 as type, text AS title, text AS message,'
                     ' author, \'\' AS keywords, w1.name AS data, time,'
                     ' w1.version as ver'
                     ' FROM wiki w1, '
                     ' (SELECT name,max(version) AS ver '
                     '    FROM wiki GROUP BY name) w2'
                     ' WHERE w1.version = w2.ver AND w1.name = w2.name  AND'
                     ' (%s OR %s OR %s) ' %
                     (self.query_to_sql(query, 'w1.name'),
                      self.query_to_sql(query, 'w1.author'),
                      self.query_to_sql(query, 'w1.text')))

        if not q: return []

        q_str = string.join(q, ' UNION ALL ')
        q_str += ' ORDER BY 7 DESC LIMIT %d OFFSET %d' % \
                 (self.RESULTS_PER_PAGE, self.RESULTS_PER_PAGE * page)

        self.log.debug("SQL Query: %s" % q_str)
        cursor.execute(q_str)

        # Make the data more HDF-friendly
        info = []
        while 1:
            row = cursor.fetchone()
            if not row:
                break
            msg = row['message']
            t = time.localtime(int(row['time']))
            item = {'type': int(row['type']),
                    'keywords': row['keywords'] or '',
                    'data': row['data'],
                    'title': escape(row['title'] or ''),
                    'datetime' : time.strftime('%c', t),
                    'author': escape(row['author'])}
            if item['type'] == 1:
                item['changeset_href'] = self.env.href.changeset(int(row['data']))
            elif item['type'] == 2:
                item['ticket_href'] = self.env.href.ticket(int(row['data']))
            elif item['type'] == 3:
                item['wiki_href'] = self.env.href.wiki(row['data'])

            item['shortmsg'] = escape(shorten_line(msg))
            item['message'] = escape(self.shorten_result(msg, keywords))
            info.append(item)
        return info

    def render (self):
        self.perm.assert_permission(perm.SEARCH_VIEW)
        self.req.hdf.setValue('title', 'Search')
        self.req.hdf.setValue('search.ticket', 'checked')
        self.req.hdf.setValue('search.changeset', 'checked')
        self.req.hdf.setValue('search.wiki', 'checked')
        self.req.hdf.setValue('search.results_per_page', str(self.RESULTS_PER_PAGE))

        if self.args.has_key('q'):
            query = self.args.get('q')
            self.req.hdf.setValue('title', 'Search Results')
            self.req.hdf.setValue('search.q', query.replace('"', """))
            tickets = self.args.has_key('ticket')
            changesets = self.args.has_key('changeset')
            wiki = self.args.has_key('wiki')

            # If no search options chosen, choose all
            if not (tickets or changesets or wiki):
                tickets = changesets = wiki = 1

            if self.args.has_key('page'):
                page = int(self.args.get('page'))
                self.req.hdf.setValue('search.result.page', str(page))
            else:
                page = 0
            if not tickets:
                self.req.hdf.setValue('search.ticket', '')
            if not changesets:
                self.req.hdf.setValue('search.changeset', '')
            if not wiki:
                self.req.hdf.setValue('search.wiki', '')
            info = self.perform_query(query, changesets, tickets, wiki, page)
            self.req.hdf.setValue('search.result.count', str(len(info)))
            if len(info) == self.RESULTS_PER_PAGE:
                self.req.hdf.setValue('search.result.more', '1')
            add_dictlist_to_hdf(info, self.req.hdf, 'search.result')

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2005-2009 Edgewall Software
# Copyright (C) 2005-2006 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from glob import glob
import imp
import os.path
import pkg_resources
from pkg_resources import working_set, DistributionNotFound, VersionConflict, \
                          UnknownExtra
import sys

from trac.util import get_doc, get_module_path, get_sources, get_pkginfo
from trac.util.text import exception_to_unicode, to_unicode

__all__ = ['load_components']

def _enable_plugin(env, module):
    """Enable the given plugin module if it wasn't disabled explicitly."""
    if env.is_component_enabled(module) is None:
        env.enable_component(module)

def load_eggs(entry_point_name):
    """Loader that loads any eggs on the search path and `sys.path`."""
    def _load_eggs(env, search_path, auto_enable=None):
        # Note that the following doesn't seem to support unicode search_path
        distributions, errors = working_set.find_plugins(
            pkg_resources.Environment(search_path)
        )
        for dist in distributions:
            if dist not in working_set:
                env.log.debug('Adding plugin %s from %s', dist, dist.location)
                working_set.add(dist)

        def _log_error(item, e):
            ue = exception_to_unicode(e)
            if isinstance(e, DistributionNotFound):
                env.log.debug('Skipping "%s": ("%s" not found)', item, ue)
            elif isinstance(e, VersionConflict):
                env.log.error('Skipping "%s": (version conflict "%s")',
                              item, ue)
            elif isinstance(e, UnknownExtra):
                env.log.error('Skipping "%s": (unknown extra "%s")', item, ue)
            elif isinstance(e, ImportError):
                env.log.error('Skipping "%s": (can\'t import "%s")', item, ue)
            else:
                env.log.error('Skipping "%s": %s)', item,
                              exception_to_unicode(e, traceback=True))

        for dist, e in errors.iteritems():
            _log_error(dist, e)

        for entry in sorted(working_set.iter_entry_points(entry_point_name),
                            key=lambda entry: entry.name):
            env.log.debug('Loading %s from %s', entry.name, entry.dist.location)
            try:
                entry.load(require=True)
            except Exception, e:
                _log_error(entry, e)
            else:
                if os.path.dirname(entry.dist.location) == auto_enable:
                    _enable_plugin(env, entry.module_name)
    return _load_eggs

def load_py_files():
    """Loader that look for Python source files in the plugins directories,
    which simply get imported, thereby registering them with the component
    manager if they define any components.
    """
    def _load_py_files(env, search_path, auto_enable=None):
        for path in search_path:
            plugin_files = glob(os.path.join(path, '*.py'))
            for plugin_file in plugin_files:
                try:
                    plugin_name = os.path.basename(plugin_file[:-3])
                    env.log.debug('Loading file plugin %s from %s' % \
                                  (plugin_name, plugin_file))
                    if plugin_name not in sys.modules:
                        module = imp.load_source(plugin_name, plugin_file)
                    if path == auto_enable:
                        _enable_plugin(env, plugin_name)
                except Exception, e:
                    env.log.error('Failed to load plugin from %s: %s',
                                  plugin_file,
                                  exception_to_unicode(e, traceback=True))

    return _load_py_files

def get_plugins_dir(env):
    """Return the path to the `plugins` directory of the environment."""
    plugins_dir = os.path.realpath(os.path.join(env.path, 'plugins'))
    return os.path.normcase(plugins_dir)

def load_components(env, extra_path=None, loaders=(load_eggs('trac.plugins'),
                                                   load_py_files())):
    """Load all plugin components found on the given search path."""
    plugins_dir = get_plugins_dir(env)
    search_path = [plugins_dir]
    if extra_path:
        search_path += list(extra_path)

    for loadfunc in loaders:
        loadfunc(env, search_path, auto_enable=plugins_dir)

def get_plugin_info(env, include_core=False):
    """Return package information about Trac core and installed plugins."""
    path_sources = {}

    def find_distribution(module):
        name = module.__name__
        path = get_module_path(module)
        sources = path_sources.get(path)
        if sources is None:
            sources = path_sources[path] = get_sources(path)
        dist = sources.get(name.replace('.', '/') + '.py')
        if dist is None:
            dist = sources.get(name.replace('.', '/') + '/__init__.py')
        if dist is None:
            # This is a plain Python source file, not an egg
            dist = pkg_resources.Distribution(project_name=name,
                                              version='',
                                              location=module.__file__)
        return dist

    plugins_dir = get_plugins_dir(env)
    plugins = {}
    from trac.core import ComponentMeta
    for component in ComponentMeta._components:
        module = sys.modules[component.__module__]

        dist = find_distribution(module)
        plugin_filename = None
        if os.path.realpath(os.path.dirname(dist.location)) == plugins_dir:
            plugin_filename = os.path.basename(dist.location)

        if dist.project_name not in plugins:
            readonly = True
            if plugin_filename and os.access(dist.location,
                                             os.F_OK + os.W_OK):
                readonly = False
            # retrieve plugin metadata
            info = get_pkginfo(dist)
            if not info:
                info = {}
                for k in ('author', 'author_email', 'home_page', 'url',
                          'license', 'trac'):
                    v = getattr(module, k, '')
                    if v and isinstance(v, basestring):
                        if k == 'home_page' or k == 'url':
                            k = 'home_page'
                            v = v.replace('$', '').replace('URL: ', '')
                        if k == 'author':
                            v = to_unicode(v)
                        info[k] = v
            else:
                # Info found; set all those fields to "None" that have the
                # value "UNKNOWN" as this is the value for fields that
                # aren't specified in "setup.py"
                for k in info:
                    if info[k] == 'UNKNOWN':
                        info[k] = ''
                    elif k == 'author':
                        # Must be encoded as unicode as otherwise Genshi
                        # may raise a "UnicodeDecodeError".
                        info[k] = to_unicode(info[k])

            # retrieve plugin version info
            version = dist.version
            if not version:
                version = (getattr(module, 'version', '') or
                           getattr(module, 'revision', ''))
                # special handling for "$Rev$" strings
                version = version.replace('$', '').replace('Rev: ', 'r')
            plugins[dist.project_name] = {
                'name': dist.project_name, 'version': version,
                'path': dist.location, 'plugin_filename': plugin_filename,
                'readonly': readonly, 'info': info, 'modules': {},
            }
        modules = plugins[dist.project_name]['modules']
        if module.__name__ not in modules:
            summary, description = get_doc(module)
            plugins[dist.project_name]['modules'][module.__name__] = {
                'summary': summary, 'description': description,
                'components': {},
            }
        full_name = module.__name__ + '.' + component.__name__
        summary, description = get_doc(component)
        c = component
        if c in env and not issubclass(c, env.__class__):
            c = component(env)
        modules[module.__name__]['components'][component.__name__] = {
            'full_name': full_name,
            'summary': summary, 'description': description,
            'enabled': env.is_component_enabled(component),
            'required': getattr(c, 'required', False),
        }
    if not include_core:
        for name in plugins.keys():
            if name.lower() == 'trac':
                plugins.pop(name)
    return sorted(plugins.itervalues(),
                  key=lambda p: (p['name'].lower() != 'trac',
                                 p['name'].lower()))

def match_plugins_to_frames(plugins, frames):
    """Add a `frame_idx` element to plugin information as returned by
    `get_plugin_info()`, containing the index of the highest frame in the
    list that was located in the plugin.
    """
    egg_frames = [(i, f) for i, f in enumerate(frames)
                  if f['filename'].startswith('build/')]

    def find_egg_frame_index(plugin):
        for dist in pkg_resources.find_distributions(plugin['path'],
                                                     only=True):
            try:
                sources = dist.get_metadata('SOURCES.txt')
                for src in sources.splitlines():
                    if src.endswith('.py'):
                        nsrc = src.replace('\\', '/')
                        for i, f in egg_frames:
                            if f['filename'].endswith(nsrc):
                                plugin['frame_idx'] = i
                                return
            except KeyError:
                pass    # Metadata not found

    for plugin in plugins:
        base, ext = os.path.splitext(plugin['path'])
        if ext == '.egg' and egg_frames:
            find_egg_frame_index(plugin)
        else:
            for i, f in enumerate(frames):
                if f['filename'].startswith(base):
                    plugin['frame_idx'] = i
                    break
<commit_msg>0.13dev: Merged [9925] from 0.11-stable.<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2005-2009 Edgewall Software
# Copyright (C) 2005-2006 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from glob import glob
import imp
import os.path
import pkg_resources
from pkg_resources import working_set, DistributionNotFound, VersionConflict, \
                          UnknownExtra
import sys

from trac.util import get_doc, get_module_path, get_sources, get_pkginfo
from trac.util.text import exception_to_unicode, to_unicode

__all__ = ['load_components']

def _enable_plugin(env, module):
    """Enable the given plugin module if it wasn't disabled explicitly."""
    if env.is_component_enabled(module) is None:
        env.enable_component(module)

def load_eggs(entry_point_name):
    """Loader that loads any eggs on the search path and `sys.path`."""
    def _load_eggs(env, search_path, auto_enable=None):
        # Note that the following doesn't seem to support unicode search_path
        distributions, errors = working_set.find_plugins(
            pkg_resources.Environment(search_path)
        )
        for dist in distributions:
            if dist not in working_set:
                env.log.debug('Adding plugin %s from %s', dist, dist.location)
                working_set.add(dist)

        def _log_error(item, e):
            ue = exception_to_unicode(e)
            if isinstance(e, DistributionNotFound):
                env.log.debug('Skipping "%s": ("%s" not found)', item, ue)
            elif isinstance(e, VersionConflict):
                env.log.error('Skipping "%s": (version conflict "%s")',
                              item, ue)
            elif isinstance(e, UnknownExtra):
                env.log.error('Skipping "%s": (unknown extra "%s")', item, ue)
            elif isinstance(e, ImportError):
                env.log.error('Skipping "%s": (can\'t import "%s")', item, ue)
            else:
                env.log.error('Skipping "%s": %s)', item,
                              exception_to_unicode(e, traceback=True))

        for dist, e in errors.iteritems():
            _log_error(dist, e)

        for entry in sorted(working_set.iter_entry_points(entry_point_name),
                            key=lambda entry: entry.name):
            env.log.debug('Loading %s from %s', entry.name, entry.dist.location)
            try:
                entry.load(require=True)
            except Exception, e:
                _log_error(entry, e)
            else:
                if os.path.dirname(entry.dist.location) == auto_enable:
                    _enable_plugin(env, entry.module_name)
    return _load_eggs

def load_py_files():
    """Loader that look for Python source files in the plugins directories,
    which simply get imported, thereby registering them with the component
    manager if they define any components.
    """
    def _load_py_files(env, search_path, auto_enable=None):
        for path in search_path:
            plugin_files = glob(os.path.join(path, '*.py'))
            for plugin_file in plugin_files:
                try:
                    plugin_name = os.path.basename(plugin_file[:-3])
                    env.log.debug('Loading file plugin %s from %s' % \
                                  (plugin_name, plugin_file))
                    if plugin_name not in sys.modules:
                        module = imp.load_source(plugin_name, plugin_file)
                    if path == auto_enable:
                        _enable_plugin(env, plugin_name)
                except Exception, e:
                    env.log.error('Failed to load plugin from %s: %s',
                                  plugin_file,
                                  exception_to_unicode(e, traceback=True))

    return _load_py_files

def get_plugins_dir(env):
    """Return the path to the `plugins` directory of the environment."""
    plugins_dir = os.path.realpath(os.path.join(env.path, 'plugins'))
    return os.path.normcase(plugins_dir)

def load_components(env, extra_path=None, loaders=(load_eggs('trac.plugins'),
                                                   load_py_files())):
    """Load all plugin components found on the given search path."""
    plugins_dir = get_plugins_dir(env)
    search_path = [plugins_dir]
    if extra_path:
        search_path += list(extra_path)

    for loadfunc in loaders:
        loadfunc(env, search_path, auto_enable=plugins_dir)

def get_plugin_info(env, include_core=False):
    """Return package information about Trac core and installed plugins."""
    path_sources = {}

    def find_distribution(module):
        name = module.__name__
        path = get_module_path(module)
        sources = path_sources.get(path)
        if sources is None:
            sources = path_sources[path] = get_sources(path)
        dist = sources.get(name.replace('.', '/') + '.py')
        if dist is None:
            dist = sources.get(name.replace('.', '/') + '/__init__.py')
        if dist is None:
            # This is a plain Python source file, not an egg
            dist = pkg_resources.Distribution(project_name=name,
                                              version='',
                                              location=module.__file__)
        return dist

    plugins_dir = get_plugins_dir(env)
    plugins = {}
    from trac.core import ComponentMeta
    for component in ComponentMeta._components:
        module = sys.modules[component.__module__]

        dist = find_distribution(module)
        plugin_filename = None
        if os.path.realpath(os.path.dirname(dist.location)) == plugins_dir:
            plugin_filename = os.path.basename(dist.location)

        if dist.project_name not in plugins:
            readonly = True
            if plugin_filename and os.access(dist.location,
                                             os.F_OK + os.W_OK):
                readonly = False
            # retrieve plugin metadata
            info = get_pkginfo(dist)
            if not info:
                info = {}
                for k in ('author', 'author_email', 'home_page', 'url',
                          'license', 'trac'):
                    v = getattr(module, k, '')
                    if v and isinstance(v, basestring):
                        if k == 'home_page' or k == 'url':
                            k = 'home_page'
                            v = v.replace('$', '').replace('URL: ', '')
                        else:
                            v = to_unicode(v)
                        info[k] = v
            else:
                # Info found; set all those fields to "None" that have the
                # value "UNKNOWN" as this is the value for fields that
                # aren't specified in "setup.py"
                for k in info:
                    if info[k] == 'UNKNOWN':
                        info[k] = ''
                    else:
                        # Must be encoded as unicode as otherwise Genshi
                        # may raise a "UnicodeDecodeError".
                        info[k] = to_unicode(info[k])

            # retrieve plugin version info
            version = dist.version
            if not version:
                version = (getattr(module, 'version', '') or
                           getattr(module, 'revision', ''))
                # special handling for "$Rev$" strings
                version = version.replace('$', '').replace('Rev: ', 'r')
            plugins[dist.project_name] = {
                'name': dist.project_name, 'version': version,
                'path': dist.location, 'plugin_filename': plugin_filename,
                'readonly': readonly, 'info': info, 'modules': {},
            }
        modules = plugins[dist.project_name]['modules']
        if module.__name__ not in modules:
            summary, description = get_doc(module)
            plugins[dist.project_name]['modules'][module.__name__] = {
                'summary': summary, 'description': description,
                'components': {},
            }
        full_name = module.__name__ + '.' + component.__name__
        summary, description = get_doc(component)
        c = component
        if c in env and not issubclass(c, env.__class__):
            c = component(env)
        modules[module.__name__]['components'][component.__name__] = {
            'full_name': full_name,
            'summary': summary, 'description': description,
            'enabled': env.is_component_enabled(component),
            'required': getattr(c, 'required', False),
        }
    if not include_core:
        for name in plugins.keys():
            if name.lower() == 'trac':
                plugins.pop(name)
    return sorted(plugins.itervalues(),
                  key=lambda p: (p['name'].lower() != 'trac',
                                 p['name'].lower()))

def match_plugins_to_frames(plugins, frames):
    """Add a `frame_idx` element to plugin information as returned by
    `get_plugin_info()`, containing the index of the highest frame in the
    list that was located in the plugin.
    """
    egg_frames = [(i, f) for i, f in enumerate(frames)
                  if f['filename'].startswith('build/')]

    def find_egg_frame_index(plugin):
        for dist in pkg_resources.find_distributions(plugin['path'],
                                                     only=True):
            try:
                sources = dist.get_metadata('SOURCES.txt')
                for src in sources.splitlines():
                    if src.endswith('.py'):
                        nsrc = src.replace('\\', '/')
                        for i, f in egg_frames:
                            if f['filename'].endswith(nsrc):
                                plugin['frame_idx'] = i
                                return
            except KeyError:
                pass    # Metadata not found

    for plugin in plugins:
        base, ext = os.path.splitext(plugin['path'])
        if ext == '.egg' and egg_frames:
            find_egg_frame_index(plugin)
        else:
            for i, f in enumerate(frames):
                if f['filename'].startswith(base):
                    plugin['frame_idx'] = i
                    break
<|endoftext|>


<commit_before>from tastypie.resources import ModelResource
from tracker.models import Member, Report
from tastypie import fields

class MemberResource(ModelResource):

    reports = fields.ToManyField('tracker.api.ReportResource', 'report_set')

    class Meta:
        queryset = Member.objects.all()
        resource_name = 'member'
        excludes = ['race', 'status_id', 'service', 'margin_2010', 'social_networks', 'facebook_10', 'facebook_status', 'twitter_10', 'twitter_status', 'index_10']
        allowed_methods = ['get']

class ReportResource(ModelResource):
    member = fields.ToOneField(MemberResource, 'members')

    class Meta:
        queryset = Report.objects.all()
        resource_name = 'report'
<commit_msg>Member is a ToOne relation<commit_after>from tastypie.resources import ModelResource
from tracker.models import Member, Report
from tastypie import fields

class MemberResource(ModelResource):

    reports = fields.ToManyField('tracker.api.ReportResource', 'report_set')

    class Meta:
        queryset = Member.objects.all()
        resource_name = 'member'
        excludes = ['race', 'status_id', 'service', 'margin_2010', 'social_networks', 'facebook_10', 'facebook_status', 'twitter_10', 'twitter_status', 'index_10']
        allowed_methods = ['get']

class ReportResource(ModelResource):
    member = fields.ToOneField(MemberResource, 'member')

    class Meta:
        queryset = Report.objects.all()
        resource_name = 'report'
<|endoftext|>


<commit_before>#!/opt/local/bin/python

# Copyright (c) 2009 Sean M. Graham <www.sean-graham.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

import os
import time
import sys
import getopt
import ConfigParser

from datetime import date
from appscript import *

def usage():
    print( "Usage: trackupdate.py [arguments]" )
    print( """
This is a simple script which polls iTunes every 10 seconds and writes information about the current track to Nicecast's "NowPlaying.txt" file.

Arguments:
    -w  --wiki      directory to place wiki-formatted table of songs (optional)
    -e  --episode   the episode number (optional, only used in wiki text)
    -h  --help      show this help page

Example:
    ./trackupdate.py --wiki ~/aatemp/
""")

wfh = None
wikiFilePath = ""
wikiArchiveURL = ""
ignoreAlbum = ""
createWikiText = False
trackArtist = ""
trackName  = ""
trackAlbum = ""
trackTime = ""
episodeNumber = "XX"

def main(argv):
    # process config file
    try:
        config = ConfigParser.ConfigParser()
        config.read(os.path.expanduser('~/.trackupdaterc'))
    except ConfigParser.MissingSectionHeaderError:
        print "Warning: Invalid config file, no [trackupdate] section."

    global wikiFilePath
    global wikiArchiveURL
    global ignoreAlbum
    global createWikiText

    try:
        wikiFilePath = config.get('wiki', 'wikiTextDirectory')
        wikiArchiveURL = config.get('wiki', 'wikiArchiveURL')
        ignoreAlbum = config.get('trackupdate', 'ignoreAlbum')
    except ConfigParser.NoSectionError:
        pass
    except ConfigParser.NoOptionError:
        pass

    if(wikiFilePath != ""):
        createWikiText = True

    # process command-line arguments
    if(len(argv) > 0):
        try:
            opts, args = getopt.getopt(sys.argv[1:], "hw:e:", ["help",
                                                               "wiki=",
                                                               "episode="])
        except getopt.GetoptError, err:
            # print help information and exit:
            print str(err) # will print something like "option -a not recognized"
            usage()
            sys.exit(2)

        for o, a in opts:
            if o in ("-w", "--wiki"):
                createWikiText = True
                wikiFilePath = a
            elif o in ("-e", "--episode"):
                global episodeNumber

                episodeNumber = a
            elif o in ("-h", "--help"):
                usage()
                sys.exit()
            else:
                assert False, "unhandled option"

    if(wikiFilePath.endswith('/') != True):
        wikiFilePath += "/"

    try:
        iTunes = app('iTunes')

        # prepare wiki text file (if the user requested it)
        if( createWikiText == True ):
            initWikiFile()

        while(1):
            if(iTunes.player_state() == k.playing):
                processCurrentTrack(iTunes.current_track)

            time.sleep(10.0)

    except KeyboardInterrupt,SystemExit:
        global wfh
        if( (createWikiText == True) and (wfh != None) ):
            wfh.write("|}\n")
            wfh.close()

def processCurrentTrack(currentTrack):
    global trackArtist
    global trackName
    global trackAlbum
    global trackTime

    iArtist = currentTrack.artist.get().encode("utf-8")
    iName = currentTrack.name.get().encode("utf-8")
    iAlbum = currentTrack.album.get().encode("utf-8")
    iTime = currentTrack.time.get().encode("utf-8")

    # make sure the track has actually changed
    if( (iArtist != trackArtist) or (iName != trackName) ):
        trackArtist = iArtist
        trackName  = iName
        trackAlbum = iAlbum
        trackTime = iTime

        # if the album name matches the blacklist name don't do anything
        if( trackAlbum == ignoreAlbum ):
            return

        fh = open(os.path.expanduser(
            '~/Library/Application Support/Nicecast/NowPlaying.txt'), 'w')
        fh.write("Title: " + trackName + '\n')
        fh.write("Artist: " + trackArtist  + '\n')
        fh.write("Album: " + trackAlbum  + '\n')
        fh.write("Time: " + trackTime  + '\n')
        fh.close()

        global createWikiText
        if( createWikiText == True ):
            global wfh
            wfh.write("|" + trackName + '\n')
            wfh.write("|" + trackArtist + '\n')
            wfh.write("|" + trackAlbum + '\n')
            wfh.write("|-\n")

        print trackArtist + " - " + trackName

def initWikiFile():
    global wikiFilePath
    global wikiArchiveURL

    dateString = date.today().strftime("%Y%m%d")

    filename = os.path.expanduser(wikiFilePath + dateString +
                                  "-wikitext.txt")

    # if the file already exists, delete it
    if(os.access(filename, os.F_OK)):
        os.unlink(filename)

    global wfh
    global episodeNumber
    wfh = open(filename, 'a')

    wfh.write("=== ")

    if(wikiArchiveURL != ""):
        wfh.write("[" + date.today().strftime(wikiArchiveURL) + " ")

    wfh.write("Show #" + episodeNumber + " - ")

    # compute the suffix
    day = date.today().day
    if 4 <= day <= 20 or 24 <= day <= 30:
        suffix = "th"
    else:
        suffix = ["st", "nd", "rd"][day % 10 - 1]

    wfh.write(date.today().strftime("%A, %B %d"))
    wfh.write(suffix)
    wfh.write(date.today().strftime(", %Y"))

    if(wikiArchiveURL != ""):
        wfh.write("]")

    wfh.write(" ===\n")

    wfh.write("""{| border=1 cellspacing=0 cellpadding=5
|'''Song'''
|'''Artist'''
|'''Album'''
|-
""")

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>got sick of the 'global' syntax so I turned it into a class.<commit_after>#!/opt/local/bin/python

# Copyright (c) 2009 Sean M. Graham <www.sean-graham.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

import os
import time
import sys
import getopt
import ConfigParser

from datetime import date
from appscript import *

class TrackUpdate:
    wfh = None
    wikiFilePath = ""
    wikiArchiveURL = ""
    ignoreAlbum = ""
    createWikiText = False
    trackArtist = ""
    trackName  = ""
    trackAlbum = ""
    trackTime = ""
    episodeNumber = "XX"

    def usage(self):
        print( "Usage: trackupdate.py [arguments]" )
        print( """
    This is a simple script which polls iTunes every 10 seconds and writes information about the current track to Nicecast's "NowPlaying.txt" file.

    Arguments:
        -w  --wiki      directory to place wiki-formatted table of songs (optional)
        -e  --episode   the episode number (optional, only used in wiki text)
        -h  --help      show this help page

    Example:
        ./trackupdate.py --wiki ~/aatemp/
    """)

    def __init__(self,argv):
        # process config file
        try:
            config = ConfigParser.ConfigParser()
            config.read(os.path.expanduser('~/.trackupdaterc'))
        except ConfigParser.MissingSectionHeaderError:
            print "Warning: Invalid config file, no [trackupdate] section."

        try:
            self.wikiFilePath = config.get('wiki', 'wikiTextDirectory')
            self.wikiArchiveURL = config.get('wiki', 'self.wikiArchiveURL')
            self.ignoreAlbum = config.get('trackupdate', 'self.ignoreAlbum')
        except ConfigParser.NoSectionError:
            pass
        except ConfigParser.NoOptionError:
            pass

        if(self.wikiFilePath != ""):
            self.createWikiText = True

        # process command-line arguments
        if(len(argv) > 0):
            try:
                opts, args = getopt.getopt(sys.argv[1:], "hw:e:", ["help",
                                                                   "wiki=",
                                                                   "episode="])
            except getopt.GetoptError, err:
                # print help information and exit:
                print str(err) # will print something like "option -a not recognized"
                usage()
                sys.exit(2)

            for o, a in opts:
                if o in ("-w", "--wiki"):
                    self.createWikiText = True
                    self.wikiFilePath = a
                elif o in ("-e", "--episode"):
                    self.episodeNumber = a
                elif o in ("-h", "--help"):
                    usage()
                    sys.exit()
                else:
                    assert False, "unhandled option"

        if(self.wikiFilePath.endswith('/') != True):
            self.wikiFilePath += "/"

        try:
            iTunes = app('iTunes')

            # prepare wiki text file (if the user requested it)
            if( self.createWikiText == True ):
                self.initWikiFile()

            while(1):
                if(iTunes.player_state() == k.playing):
                    self.processCurrentTrack(iTunes.current_track)

                time.sleep(10.0)

        except KeyboardInterrupt,SystemExit:
            if( (self.createWikiText == True) and (self.wfh != None) ):
                self.wfh.write("|}\n")
                self.wfh.close()

    def processCurrentTrack(self, currentTrack):
        iArtist = currentTrack.artist.get().encode("utf-8")
        iName = currentTrack.name.get().encode("utf-8")
        iAlbum = currentTrack.album.get().encode("utf-8")
        iTime = currentTrack.time.get().encode("utf-8")

        # make sure the track has actually changed
        if( (iArtist != self.trackArtist) or (iName != self.trackName) ):
            self.trackArtist = iArtist
            self.trackName  = iName
            self.trackAlbum = iAlbum
            self.trackTime = iTime

            # if the album name matches the blacklist name don't do anything
            if( self.trackAlbum == self.ignoreAlbum ):
                return

            fh = open(os.path.expanduser(
                '~/Library/Application Support/Nicecast/NowPlaying.txt'), 'w')
            fh.write("Title: " + self.trackName + '\n')
            fh.write("Artist: " + self.trackArtist  + '\n')
            fh.write("Album: " + self.trackAlbum  + '\n')
            fh.write("Time: " + self.trackTime  + '\n')
            fh.close()

            if( self.createWikiText == True ):
                self.wfh.write("|" + self.trackName + '\n')
                self.wfh.write("|" + self.trackArtist + '\n')
                self.wfh.write("|" + self.trackAlbum + '\n')
                self.wfh.write("|-\n")

            print self.trackArtist + " - " + self.trackName

    def initWikiFile(self):
        dateString = date.today().strftime("%Y%m%d")

        filename = os.path.expanduser(self.wikiFilePath + dateString +
                                      "-wikitext.txt")

        # if the file already exists, delete it
        if(os.access(filename, os.F_OK)):
            os.unlink(filename)

        self.wfh = open(filename, 'a')

        self.wfh.write("=== ")

        if(self.wikiArchiveURL != ""):
            self.wfh.write("[" + date.today().strftime(self.wikiArchiveURL) + " ")

        self.wfh.write("Show #" + self.episodeNumber + " - ")

        # compute the suffix
        day = date.today().day
        if 4 <= day <= 20 or 24 <= day <= 30:
            suffix = "th"
        else:
            suffix = ["st", "nd", "rd"][day % 10 - 1]

        self.wfh.write(date.today().strftime("%A, %B %d"))
        self.wfh.write(suffix)
        self.wfh.write(date.today().strftime(", %Y"))

        if(self.wikiArchiveURL != ""):
            self.wfh.write("]")

        self.wfh.write(" ===\n")

        self.wfh.write("{| border=1 cellspacing=0 cellpadding=5\n")
        self.wfh.write("|'''Song'''\n")
        self.wfh.write("|'''Artist'''\n")
        self.wfh.write("|'''Album'''\n")
        self.wfh.write("|-\n")

if __name__ == "__main__":
    trackUpdate = TrackUpdate(sys.argv[1:])
<|endoftext|>


<commit_before>from __future__ import print_function

import argparse
from collections import OrderedDict

from blocks.algorithms import GradientDescent, Adam, CompositeRule, StepClipping
from blocks.extensions import Timing, Printing, FinishAfter, ProgressBar
from blocks.extensions.monitoring import TrainingDataMonitoring
from blocks.extensions.training import SharedVariableModifier
from blocks.main_loop import MainLoop
from blocks.monitoring import aggregation
from blocks.monitoring.evaluators import AggregationBuffer
from fuel.datasets.hdf5 import H5PYDataset
from fuel.schemes import SequentialScheme, ShuffledScheme
from fuel.streams import DataStream
from numpy import load, array, ones
from theano import function, shared

from custom_blocks import PadAndAddMasks, EarlyStopping
from network import *

from prepare_lk import ALPHABET_FILE, OUT_FILE_NAME as DATA_FILE_LOC
from util import StateComputer
from martin_test_module import OverrideStateReset

parser = argparse.ArgumentParser()
parser.add_argument("-c", "--clipping", default=1.0, type=float, help="Step clipping threshold (default: 1)")
parser.add_argument("-t", "--type", default="simple", help="Type of RNN ("+NetworkType.SIMPLE_RNN+", " +
                                                           NetworkType.GRU+" or "+NetworkType.LSTM+")")
parser.add_argument("-e", "--epochs", default=0, type=int, help="Stop after this many epochs (default: 0 -- means it will" +
                                                                " run forever until you cancel manually)")
parser.add_argument("-d", "--dimensions", default="512,512,512", type=str, help="Configure number of layers and dimensions by" +
                                                                                " just enumerating dimensions (e.g. 100, 100 for" +
                                                                                " a two-layered network with dims 100)")
args = parser.parse_args()

dimensions = [int(d) for d in args.dimensions.split(',')]
print('Dimensions:', dimensions)

nkwargs = {'network_type': args.type, 'reset_states': True}
if dimensions:
    nkwargs['hidden_dims'] = dimensions

network = Network(**nkwargs)

cross_ent = network.cost
generator = network.generator
cost_model = network.cost_model

# do init_state stuff
initial_states = network.initial_states
#init_state_2 = initial_states[2]

# data
train_data = H5PYDataset(DATA_FILE_LOC, which_sets=("train",), load_in_memory=True)
valid_data = H5PYDataset(DATA_FILE_LOC, which_sets=("valid",), load_in_memory=True)

# see custom_blocks for the transformer
data_stream = PadAndAddMasks(DataStream.default_stream(dataset=train_data, iteration_scheme=ShuffledScheme(train_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)
data_stream_valid = PadAndAddMasks(DataStream.default_stream(dataset=valid_data, iteration_scheme=SequentialScheme(valid_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)

# monitor:
# - training cost every 200 batches (computed along the way, so cheap to do), as well as gradient and step lengths to
#   detect exploding gradient problems
# - validation cost once every epoch

char2ix = load(ALPHABET_FILE).item()
ix2char = {}
for k, v in char2ix.items():
    ix2char[v] = k

sc = StateComputer(network.cost_model, ix2char)
state_to_compare = list(filter(lambda x: x.name == 'sequencegenerator_cost_matrix_states#2', sc.state_variables))[0]  # notice: python2 filter seems to return a list, but anyway
states_to_compare = list(filter(lambda x: 'sequencegenerator_cost_matrix_states' in x.name, sc.state_variables))

# The thing that I feed into the parameters argument I copied from some blocks-examples thing. the good old computation
# graph and then cg.parameters should work as well.
# On the step rule: This means gradient clipping (threshold passed in as a command line argument) followed by Adam
# performed on the clipped gradient.
# If you don't know what gradient clipping is: Define threshold T; if the length of the gradient is > T, scale it such
# that its length is equal to T. This serves to make exploding gradients less severe.
algorithm = GradientDescent(cost=cross_ent, parameters=cost_model.parameters,
                            step_rule=CompositeRule(components=[StepClipping(threshold=args.clipping), Adam()]),
                            on_unused_sources="ignore")

#OverrideStateReset(OrderedDict({init_state_2 : state_to_compare[0][-1]}))

aggr = AggregationBuffer(variables=[state_to_compare], use_take_last=True)
aggr.initialize_aggregators()

def modifier_function(iterations_done, old_value):
    """
    Note about this method: The accumulator seems to get the state vector in reversed order
    :param iterations_done:
    :param old_value:
    :return:
    """
    values = aggr.get_aggregated_values()
    new_value = values[state_to_compare.name]
    aggr.initialize_aggregators()  # TODO what's the purpose of that? I observed them do it in the monitoring extensions after every request
    #new_value = ones(10, dtype='float32')
    #print('OLD ..:', old_value)
    #print('NEW in: ', new_value[-1][0], new_value[0][0], sep='\n')
    return new_value[0][0]

init_state_modifier = SharedVariableModifier(network.transitions[-1].initial_state_, function=modifier_function, after_batch=True)

#state_function = function([state_to_compare], initial_states[2], updates=[(init_state_2, state_to_compare[0][-1])]) #TODO look at this, this is how it basically works!

monitor_grad = TrainingDataMonitoring(variables=[cross_ent, aggregation.mean(algorithm.total_gradient_norm),
                                                 aggregation.mean(algorithm.total_step_norm)]+initial_states+[state_to_compare], after_epoch=True,
                                      prefix="training")

early_stopping = EarlyStopping(variables=[cross_ent], data_stream=data_stream_valid,
                               path="seqgen_" + args.type + "_" + "_".join([str(d) for d in network.hidden_dims]) + ".pkl",
                               tolerance=4, prefix="validation")

main_loop = MainLoop(algorithm=algorithm, data_stream=data_stream, model=cost_model,
                     extensions=[monitor_grad, FinishAfter(after_n_epochs=args.epochs), ProgressBar(),
                                 Timing(), Printing(), init_state_modifier])

main_loop.algorithm.add_updates(aggr.accumulation_updates)

# remove update
# updates = main_loop.algorithm.updates
# init_state_update = list(filter(lambda u: u[0] == init_state_2, updates))[0]
# updates.remove(init_state_update)
# print('REMOVED UPDATE:', init_state_update)
# print('\nUPDATES:', main_loop.algorithm.updates,'\n')

main_loop.run()
<commit_msg>I am confused<commit_after>from __future__ import print_function

import argparse
from collections import OrderedDict

from blocks.algorithms import GradientDescent, Adam, CompositeRule, StepClipping
from blocks.extensions import Timing, Printing, FinishAfter, ProgressBar
from blocks.extensions.monitoring import TrainingDataMonitoring
from blocks.extensions.training import SharedVariableModifier
from blocks.main_loop import MainLoop
from blocks.monitoring import aggregation
from blocks.monitoring.evaluators import AggregationBuffer
from fuel.datasets.hdf5 import H5PYDataset
from fuel.schemes import SequentialScheme, ShuffledScheme
from fuel.streams import DataStream
from numpy import load, array, ones
from theano import function, shared

from custom_blocks import PadAndAddMasks, EarlyStopping
from network import *

from prepare_lk import ALPHABET_FILE, OUT_FILE_NAME as DATA_FILE_LOC
from util import StateComputer
from martin_test_module import OverrideStateReset

parser = argparse.ArgumentParser()
parser.add_argument("-c", "--clipping", default=1.0, type=float, help="Step clipping threshold (default: 1)")
parser.add_argument("-t", "--type", default="simple", help="Type of RNN ("+NetworkType.SIMPLE_RNN+", " +
                                                           NetworkType.GRU+" or "+NetworkType.LSTM+")")
parser.add_argument("-e", "--epochs", default=0, type=int, help="Stop after this many epochs (default: 0 -- means it will" +
                                                                " run forever until you cancel manually)")
parser.add_argument("-d", "--dimensions", default="512,512,512", type=str, help="Configure number of layers and dimensions by" +
                                                                                " just enumerating dimensions (e.g. 100, 100 for" +
                                                                                " a two-layered network with dims 100)")
args = parser.parse_args()

dimensions = [int(d) for d in args.dimensions.split(',')]
print('Dimensions:', dimensions)

nkwargs = {'network_type': args.type, 'reset_states': True}
if dimensions:
    nkwargs['hidden_dims'] = dimensions

network = Network(**nkwargs)

cross_ent = network.cost
generator = network.generator
cost_model = network.cost_model

# do init_state stuff
initial_states = network.initial_states
#init_state_2 = initial_states[2]

# data
train_data = H5PYDataset(DATA_FILE_LOC, which_sets=("train",), load_in_memory=True)
valid_data = H5PYDataset(DATA_FILE_LOC, which_sets=("valid",), load_in_memory=True)

# see custom_blocks for the transformer
data_stream = PadAndAddMasks(DataStream.default_stream(dataset=train_data, iteration_scheme=ShuffledScheme(train_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)
data_stream_valid = PadAndAddMasks(DataStream.default_stream(dataset=valid_data, iteration_scheme=SequentialScheme(valid_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)

# monitor:
# - training cost every 200 batches (computed along the way, so cheap to do), as well as gradient and step lengths to
#   detect exploding gradient problems
# - validation cost once every epoch

char2ix = load(ALPHABET_FILE).item()
ix2char = {}
for k, v in char2ix.items():
    ix2char[v] = k

sc = StateComputer(network.cost_model, ix2char)
state_to_compare = list(filter(lambda x: x.name == 'sequencegenerator_cost_matrix_states#2', sc.state_variables))[0]  # notice: python2 filter seems to return a list, but anyway
states_to_compare = list(filter(lambda x: 'sequencegenerator_cost_matrix_states' in x.name, sc.state_variables))

# The thing that I feed into the parameters argument I copied from some blocks-examples thing. the good old computation
# graph and then cg.parameters should work as well.
# On the step rule: This means gradient clipping (threshold passed in as a command line argument) followed by Adam
# performed on the clipped gradient.
# If you don't know what gradient clipping is: Define threshold T; if the length of the gradient is > T, scale it such
# that its length is equal to T. This serves to make exploding gradients less severe.
algorithm = GradientDescent(cost=cross_ent, parameters=cost_model.parameters,
                            step_rule=CompositeRule(components=[StepClipping(threshold=args.clipping), Adam()]),
                            on_unused_sources="ignore")

#OverrideStateReset(OrderedDict({init_state_2 : state_to_compare[0][-1]}))

aggr = AggregationBuffer(variables=[state_to_compare], use_take_last=True)
aggr.initialize_aggregators()

def modifier_function(iterations_done, old_value):
    """
    Note about this method: The accumulator seems to get the state vector in reversed order
    :param iterations_done:
    :param old_value:
    :return:
    """
    values = aggr.get_aggregated_values()
    new_value = values[state_to_compare.name]
    aggr.initialize_aggregators()  # TODO what's the purpose of that? I observed them do it in the monitoring extensions after every request
    #new_value = ones(10, dtype='float32')
    #print('OLD ..:', old_value)
    #print('NEW in: ', new_value[-1][0], new_value[0][0], sep='\n')
    return new_value[-1][0]

init_state_modifier = SharedVariableModifier(network.transitions[-1].initial_state_, function=modifier_function, after_batch=True)

#state_function = function([state_to_compare], initial_states[2], updates=[(init_state_2, state_to_compare[0][-1])]) #TODO look at this, this is how it basically works!

monitor_grad = TrainingDataMonitoring(variables=[cross_ent, aggregation.mean(algorithm.total_gradient_norm),
                                                 aggregation.mean(algorithm.total_step_norm)]+initial_states+[state_to_compare], after_epoch=True,
                                      prefix="training")

early_stopping = EarlyStopping(variables=[cross_ent], data_stream=data_stream_valid,
                               path="seqgen_" + args.type + "_" + "_".join([str(d) for d in network.hidden_dims]) + ".pkl",
                               tolerance=4, prefix="validation")

main_loop = MainLoop(algorithm=algorithm, data_stream=data_stream, model=cost_model,
                     extensions=[monitor_grad, FinishAfter(after_n_epochs=args.epochs), ProgressBar(),
                                 Timing(), Printing(), init_state_modifier])

main_loop.algorithm.add_updates(aggr.accumulation_updates)

# remove update
# updates = main_loop.algorithm.updates
# init_state_update = list(filter(lambda u: u[0] == init_state_2, updates))[0]
# updates.remove(init_state_update)
# print('REMOVED UPDATE:', init_state_update)
# print('\nUPDATES:', main_loop.algorithm.updates,'\n')

main_loop.run()
<|endoftext|>


<commit_before>import argparse

from blocks.algorithms import GradientDescent, Adam, CompositeRule, StepClipping
from blocks.extensions import Timing, Printing, FinishAfter, ProgressBar
from blocks.extensions.monitoring import TrainingDataMonitoring
from blocks.extensions.training import SharedVariableModifier
from blocks.main_loop import MainLoop
from blocks.monitoring import aggregation
from blocks.monitoring.evaluators import AggregationBuffer
from fuel.datasets.hdf5 import H5PYDataset
from fuel.schemes import SequentialScheme, ShuffledScheme
from fuel.streams import DataStream
from numpy import load, array
from theano import function

from custom_blocks import PadAndAddMasks, EarlyStopping
from network import *

from prepare_lk import ALPHABET_FILE, OUT_FILE_NAME as DATA_FILE_LOC
from util import StateComputer

parser = argparse.ArgumentParser()
parser.add_argument("-c", "--clipping", default=1.0, type=float, help="Step clipping threshold (default: 1)")
parser.add_argument("-t", "--type", default="simple", help="Type of RNN ("+NetworkType.SIMPLE_RNN+", " +
                                                           NetworkType.GRU+" or "+NetworkType.LSTM+")")
parser.add_argument("-e", "--epochs", default=0, type=int, help="Stop after this many epochs (default: 0 -- means it will" +
                                                                " run forever until you cancel manually)")
parser.add_argument("-d", "--dimensions", default="512,512,512", type=str, help="Configure number of layers and dimensions by" +
                                                                                " just enumerating dimensions (e.g. 100, 100 for" +
                                                                                " a two-layered network with dims 100)")
args = parser.parse_args()

dimensions = [int(d) for d in args.dimensions.split(',')]
print('Dimensions:', dimensions)

nkwargs = {'network_type': args.type}
if dimensions:
    nkwargs['hidden_dims'] = dimensions

network = Network(**nkwargs)

cross_ent = network.cost
generator = network.generator
cost_model = network.cost_model

# do init_state stuff
initial_states = network.initial_states

# The thing that I feed into the parameters argument I copied from some blocks-examples thing. the good old computation
# graph and then cg.parameters should work as well.
# On the step rule: This means gradient clipping (threshold passed in as a command line argument) followed by Adam
# performed on the clipped gradient.
# If you don't know what gradient clipping is: Define threshold T; if the length of the gradient is > T, scale it such
# that its length is equal to T. This serves to make exploding gradients less severe.
algorithm = GradientDescent(cost=cross_ent, parameters=cost_model.parameters,
                            step_rule=CompositeRule(components=[StepClipping(threshold=args.clipping), Adam()]),
                            on_unused_sources="ignore")

# data
train_data = H5PYDataset(DATA_FILE_LOC, which_sets=("train",), load_in_memory=True)
valid_data = H5PYDataset(DATA_FILE_LOC, which_sets=("valid",), load_in_memory=True)

# see custom_blocks for the transformer
data_stream = PadAndAddMasks(DataStream.default_stream(dataset=train_data, iteration_scheme=ShuffledScheme(train_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)
data_stream_valid = PadAndAddMasks(DataStream.default_stream(dataset=valid_data, iteration_scheme=SequentialScheme(valid_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)

# monitor:
# - training cost every 200 batches (computed along the way, so cheap to do), as well as gradient and step lengths to
#   detect exploding gradient problems
# - validation cost once every epoch

char2ix = load(ALPHABET_FILE).item()
ix2char = {}
for k, v in char2ix.items():
    ix2char[v] = k

sc = StateComputer(network.cost_model, ix2char)
state_to_compare = list(filter(lambda x: x.name == 'sequencegenerator_cost_matrix_states#2', sc.state_variables))[0]  # notice: python2 filter seems to return a list, but anyway

def modifier_function(iterations_done):
    aggr = AggregationBuffer(state_to_compare, use_take_last=True)
    values = aggr.get_aggregated_values()
    print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', values[state_to_compare.name])
    return values[state_to_compare.name][0][-1][0]

init_state_modifier = SharedVariableModifier(initial_states[2], num_args=1, function=modifier_function)

monitor_grad = TrainingDataMonitoring(variables=[cross_ent, aggregation.mean(algorithm.total_gradient_norm),
                                                 aggregation.mean(algorithm.total_step_norm), initial_states[2], state_to_compare], after_epoch=True,
                                      prefix="training")
#monitor_init_states = TrainingDataMonitoring(variables=[network.initial_states[2]], after_epoch=True, prefix='training')

early_stopping = EarlyStopping(variables=[cross_ent], data_stream=data_stream_valid,
                               path="seqgen_" + args.type + "_" + "_".join([str(d) for d in network.hidden_dims]) + ".pkl",
                               tolerance=4, prefix="validation")

main_loop = MainLoop(algorithm=algorithm, data_stream=data_stream, model=cost_model,
                     extensions=[init_state_modifier, early_stopping, monitor_grad, FinishAfter(after_n_epochs=args.epochs), ProgressBar(),
                                 Timing(), Printing()])

main_loop.run()
<commit_msg>introduced a marker<commit_after>import argparse

from blocks.algorithms import GradientDescent, Adam, CompositeRule, StepClipping
from blocks.extensions import Timing, Printing, FinishAfter, ProgressBar
from blocks.extensions.monitoring import TrainingDataMonitoring
from blocks.extensions.training import SharedVariableModifier
from blocks.main_loop import MainLoop
from blocks.monitoring import aggregation
from blocks.monitoring.evaluators import AggregationBuffer
from fuel.datasets.hdf5 import H5PYDataset
from fuel.schemes import SequentialScheme, ShuffledScheme
from fuel.streams import DataStream
from numpy import load, array
from theano import function

from custom_blocks import PadAndAddMasks, EarlyStopping
from network import *

from prepare_lk import ALPHABET_FILE, OUT_FILE_NAME as DATA_FILE_LOC
from util import StateComputer

parser = argparse.ArgumentParser()
parser.add_argument("-c", "--clipping", default=1.0, type=float, help="Step clipping threshold (default: 1)")
parser.add_argument("-t", "--type", default="simple", help="Type of RNN ("+NetworkType.SIMPLE_RNN+", " +
                                                           NetworkType.GRU+" or "+NetworkType.LSTM+")")
parser.add_argument("-e", "--epochs", default=0, type=int, help="Stop after this many epochs (default: 0 -- means it will" +
                                                                " run forever until you cancel manually)")
parser.add_argument("-d", "--dimensions", default="512,512,512", type=str, help="Configure number of layers and dimensions by" +
                                                                                " just enumerating dimensions (e.g. 100, 100 for" +
                                                                                " a two-layered network with dims 100)")
args = parser.parse_args()

dimensions = [int(d) for d in args.dimensions.split(',')]
print('Dimensions:', dimensions)

nkwargs = {'network_type': args.type}
if dimensions:
    nkwargs['hidden_dims'] = dimensions

network = Network(**nkwargs)

cross_ent = network.cost
generator = network.generator
cost_model = network.cost_model

# do init_state stuff
initial_states = network.initial_states

# The thing that I feed into the parameters argument I copied from some blocks-examples thing. the good old computation
# graph and then cg.parameters should work as well.
# On the step rule: This means gradient clipping (threshold passed in as a command line argument) followed by Adam
# performed on the clipped gradient.
# If you don't know what gradient clipping is: Define threshold T; if the length of the gradient is > T, scale it such
# that its length is equal to T. This serves to make exploding gradients less severe.
algorithm = GradientDescent(cost=cross_ent, parameters=cost_model.parameters,
                            step_rule=CompositeRule(components=[StepClipping(threshold=args.clipping), Adam()]),
                            on_unused_sources="ignore")

# data
train_data = H5PYDataset(DATA_FILE_LOC, which_sets=("train",), load_in_memory=True)
valid_data = H5PYDataset(DATA_FILE_LOC, which_sets=("valid",), load_in_memory=True)

# see custom_blocks for the transformer
data_stream = PadAndAddMasks(DataStream.default_stream(dataset=train_data, iteration_scheme=ShuffledScheme(train_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)
data_stream_valid = PadAndAddMasks(DataStream.default_stream(dataset=valid_data, iteration_scheme=SequentialScheme(valid_data.num_examples,
                                                                                batch_size=1)), produces_examples=False)

# monitor:
# - training cost every 200 batches (computed along the way, so cheap to do), as well as gradient and step lengths to
#   detect exploding gradient problems
# - validation cost once every epoch

char2ix = load(ALPHABET_FILE).item()
ix2char = {}
for k, v in char2ix.items():
    ix2char[v] = k

sc = StateComputer(network.cost_model, ix2char)
state_to_compare = list(filter(lambda x: x.name == 'sequencegenerator_cost_matrix_states#2', sc.state_variables))[0]  # notice: python2 filter seems to return a list, but anyway

def modifier_function(iterations_done):
    aggr = AggregationBuffer(state_to_compare, use_take_last=True)
    values = aggr.get_aggregated_values()
    print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
    print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>', values[state_to_compare.name])
    return values[state_to_compare.name][0][-1][0]

init_state_modifier = SharedVariableModifier(initial_states[2], num_args=1, function=modifier_function)

monitor_grad = TrainingDataMonitoring(variables=[cross_ent, aggregation.mean(algorithm.total_gradient_norm),
                                                 aggregation.mean(algorithm.total_step_norm), initial_states[2], state_to_compare], after_epoch=True,
                                      prefix="training")
#monitor_init_states = TrainingDataMonitoring(variables=[network.initial_states[2]], after_epoch=True, prefix='training')

early_stopping = EarlyStopping(variables=[cross_ent], data_stream=data_stream_valid,
                               path="seqgen_" + args.type + "_" + "_".join([str(d) for d in network.hidden_dims]) + ".pkl",
                               tolerance=4, prefix="validation")

main_loop = MainLoop(algorithm=algorithm, data_stream=data_stream, model=cost_model,
                     extensions=[init_state_modifier, early_stopping, monitor_grad, FinishAfter(after_n_epochs=args.epochs), ProgressBar(),
                                 Timing(), Printing()])

main_loop.run()
<|endoftext|>