<commit_before># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2003, 2004 Edgewall Software
# Copyright (C) 2003, 2004 Jonas Borgstrm <jonas@edgewall.com>
#
# Trac is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Trac is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# Author: Jonas Borgstrm <jonas@edgewall.com>

import os
import sys
import time
import util
from StringIO import StringIO
from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED

import svn
import svn.delta
import svn.fs

import Diff
import perm
import Module
from WikiFormatter import wiki_to_html

class BaseDiffEditor(svn.delta.Editor):
    """
    Base class for diff renderers.
    """

    def __init__(self, old_root, new_root, rev, req, args, env):
        self.old_root = old_root
        self.new_root = new_root
        self.rev = rev
        self.req = req
        self.args = args
        self.env = env

    def open_directory(self, path, parent_baton, base_revision, dir_pool):
        return [path, path, dir_pool]

    def open_file(self, path, parent_baton, base_revision, file_pool):
        return [path, path, file_pool]

        if file_baton:
            self.print_diff(*file_baton)

class HtmlDiffEditor(BaseDiffEditor):
    """
    Generates a htmlized unified diff of the changes for a given changeset.
    the output is written to stdout.
    """

    def __init__(self, old_root, new_root, rev, req, args, env):
        BaseDiffEditor.__init__(self, old_root, new_root, rev, req, args, env)
        self.prev_path = None
        self.fileno = -1
        self.prefix = None

    def _check_next(self, old_path, new_path, pool):
        if self.prev_path == (old_path or new_path):
            return

        self.fileno += 1
        self.prev_path = old_path or new_path

        self.prefix = 'changeset.changes.%d' % (self.fileno)
        if old_path:
            old_rev = svn.fs.node_created_rev(self.old_root, old_path, pool)
            self.req.hdf.setValue('%s.rev.old' % self.prefix, str(old_rev))
            self.req.hdf.setValue('%s.browser_href.old' % self.prefix,
                                  self.env.href.browser(old_path, old_rev))
        if new_path:
            new_rev = svn.fs.node_created_rev(self.new_root, new_path, pool)
            self.req.hdf.setValue('%s.rev.new' % self.prefix, str(new_rev))
            self.req.hdf.setValue('%s.browser_href.new' % self.prefix,
                                  self.env.href.browser(new_path, new_rev))

    def add_directory(self, path, parent_baton, copyfrom_path,
                      copyfrom_revision, dir_pool):
        self._check_next(None, path, dir_pool)

    def delete_entry(self, path, revision, parent_baton, pool):
        self._check_next(path, None, pool)

    def change_dir_prop(self, dir_baton, name, value, dir_pool):
        if not dir_baton:
            return
        (old_path, new_path, pool) = dir_baton
        self._check_next(old_path, new_path, dir_pool)

        prefix = '%s.props.%s' % (self.prefix, name)
        if old_path:
            old_value = svn.fs.node_prop(self.old_root, old_path, name, dir_pool)
            if old_value:
                self.req.hdf.setValue(prefix + '.old', old_value)
        if value:
            self.req.hdf.setValue(prefix + '.new', value)

    def add_file(self, path, parent_baton, copyfrom_path, copyfrom_revision,
                 file_pool):
        self._check_next(None, path, file_pool)

    def apply_textdelta(self, file_baton, base_checksum):
        if not file_baton:
            return
        (old_path, new_path, pool) = file_baton
        self._check_next(old_path, new_path, pool)

        # Try to figure out the charset used. We assume that both the old
        # and the new version uses the same charset, not always the case
        # but that's all we can do...
        mime_type = svn.fs.node_prop(self.new_root, new_path,
                                     svn.util.SVN_PROP_MIME_TYPE, pool)
        # We don't have to guess if the charset is specified in the
        # svn:mime-type property
        ctpos = mime_type and mime_type.find('charset=') or -1
        if ctpos >= 0:
            charset = mime_type[ctpos + 8:]
            self.log.debug("Charset %s selected" % charset)
        else:
            charset = self.env.get_config('trac', 'default_charset',
                                          'iso-8859-15')

        # Start up the diff process
        options = Diff.get_options(self.env, self.req, self.args, 1)
        differ = svn.fs.FileDiff(self.old_root, old_path, self.new_root,
                                 new_path, pool, options)
        differ.get_files()
        pobj = differ.get_pipe()

        tabwidth = int(self.env.get_config('diff', 'tab_width', '8'))
        builder = Diff.HDFBuilder(self.req.hdf, '%s.diff' % self.prefix, tabwidth)
        while 1:
            line = pobj.readline()
            if not line:
                break
            builder.writeline(util.to_utf8(line, charset))
        builder.close()
        pobj.close()
        # svn.fs.FileDiff creates a child process and there is no waitpid()
        # calls to eliminate zombies (this is a problem especially when
        # mod_python is used.
        if sys.platform[:3] != "win" and sys.platform != "os2emx":
            try:
                os.waitpid(-1, 0)
            except OSError: pass

    def change_file_prop(self, file_baton, name, value, file_pool):
        if not file_baton:
            return
        (old_path, new_path, pool) = file_baton
        self._check_next(old_path, new_path, file_pool)

        prefix = '%s.props.%s' % (self.prefix, name)
        if old_path:
            old_value = svn.fs.node_prop(self.old_root, old_path, name, file_pool)
            if old_value:
                self.req.hdf.setValue(prefix + '.old', old_value)
        if value:
            self.req.hdf.setValue(prefix + '.new', value)

class UnifiedDiffEditor(BaseDiffEditor):
    """
    Generates a unified diff of the changes for a given changeset.
    the output is written to stdout.
    """

    def apply_textdelta(self, file_baton, base_checksum):
        if not file_baton:
            return
        (old_path, new_path, pool) = file_baton
        options = ['-u']
        options.append('-L')
        options.append("%s\t(revision %d)" % (old_path, self.rev-1))
        options.append('-L')
        options.append("%s\t(revision %d)" % (new_path, self.rev))

        differ = svn.fs.FileDiff(self.old_root, old_path,
                                 self.new_root, new_path, pool, options)
        differ.get_files()
        pobj = differ.get_pipe()
        line = pobj.readline()
        while line:
            self.req.write(line)
            line = pobj.readline()
        pobj.close()
        if sys.platform[:3] != "win" and sys.platform != "os2emx":
            try:
                os.waitpid(-1, 0)
            except OSError: pass

class ZipDiffEditor(BaseDiffEditor):
    """
    Generates a ZIP archive containing the modified and added files.
    """

    def __init__(self, old_root, new_root, rev, req, args, env):
        BaseDiffEditor.__init__(self, old_root, new_root, rev, req, args, env)
        self.buffer = StringIO()
        self.zip = ZipFile(self.buffer, 'w', ZIP_DEFLATED)

    def add_file(self, path, parent_baton, copyfrom_path,
                 copyfrom_revision, file_pool):
        self._add_file_to_zip(path, file_pool)

    def open_file(self, path, parent_baton, base_revision, file_pool):
        self._add_file_to_zip(path, file_pool)

    def close_edit(self):
        self.zip.close()
        self.req.write(self.buffer.getvalue())

    def _add_file_to_zip(self, path, pool):
        fd = svn.fs.file_contents(self.new_root, path, pool)
        info = ZipInfo()
        info.filename = path
        date = svn.fs.revision_prop(svn.fs.root_fs(self.new_root), self.rev,
                                    svn.util.SVN_PROP_REVISION_DATE, pool)
        date = svn.util.svn_time_from_cstring(date, pool) / 1000000
        date = time.localtime(date)
        info.date_time = date[:6]
        info.compress_type = ZIP_DEFLATED
        data = ""
        while 1:
            chunk = svn.util.svn_stream_read(fd, 512)
            if not chunk:
                break
            data = data + chunk
        self.zip.writestr(info, data)

class Changeset (Module.Module):
    template_name = 'changeset.cs'
    perm = None
    fs_ptr = None
    pool = None

    def get_changeset_info (self, rev):
        cursor = self.db.cursor ()
        cursor.execute ('SELECT time, author, message FROM revision ' +
                        'WHERE rev=%d', rev)
        row = cursor.fetchone()
        if not row:
            raise util.TracError('Changeset %d does not exist.' % rev,
                            'Invalid Changset')
        return row

    def get_change_info (self, rev):
        cursor = self.db.cursor ()
        cursor.execute ('SELECT name, change FROM node_change ' +
                        'WHERE rev=%d', rev)
        info = []
        while 1:
            row = cursor.fetchone()
            if not row:
                break
            info.append({'name': row['name'],
                         'change': row['change'],
                         'log_href': self.env.href.log(row['name'])})
        return info

    def render(self):
        self.perm.assert_permission (perm.CHANGESET_VIEW)

        self.add_link('alternate', '?format=diff', 'Unified Diff',
            'text/plain', 'diff')
        self.add_link('alternate', '?format=zip', 'Zip Archive',
            'application/zip', 'zip')

        youngest_rev = svn.fs.youngest_rev(self.fs_ptr, self.pool)
        if self.args.has_key('rev'):
            self.rev = int(self.args.get('rev'))
        else:
            self.rev = youngest_rev

        Diff.get_options(self.env, self.req, self.args, 1)
        if self.args.has_key('update'):
            self.req.redirect(self.env.href.changeset(self.rev))

        change_info = self.get_change_info (self.rev)
        changeset_info = self.get_changeset_info (self.rev)

        self.req.hdf.setValue('title', '[%d] (changeset)' % self.rev)
        self.req.hdf.setValue('changeset.time',
                              time.asctime(time.localtime(int(changeset_info['time']))))
        author = changeset_info['author'] or 'anonymous'
        self.req.hdf.setValue('changeset.author', util.escape(author))
        self.req.hdf.setValue('changeset.message',
                              wiki_to_html(util.wiki_escape_newline(
                                           changeset_info['message']),
                                           self.req.hdf, self.env, self.db))
        self.req.hdf.setValue('changeset.revision', str(self.rev))
        util.add_to_hdf(change_info, self.req.hdf, 'changeset.changes')

        self.req.hdf.setValue('changeset.href',
                              self.env.href.changeset(self.rev))
        if self.rev > 1:
            self.add_link('first', self.env.href.changeset(1), 'Changeset 1')
            self.add_link('prev', self.env.href.changeset(self.rev - 1),
                          'Changeset %d' % (self.rev - 1))
        if self.rev < youngest_rev:
            self.add_link('next', self.env.href.changeset(self.rev + 1),
                          'Changeset %d' % (self.rev + 1))
            self.add_link('last', self.env.href.changeset(youngest_rev),
                          'Changeset %d' % youngest_rev)

    def render_diffs(self, editor_class=HtmlDiffEditor):
        """
        generates a unified diff of the changes for a given changeset.
        the output is written to stdout.
        """
        try:
            old_root = svn.fs.revision_root(self.fs_ptr, int(self.rev) - 1, self.pool)
            new_root = svn.fs.revision_root(self.fs_ptr, int(self.rev), self.pool)
        except svn.core.SubversionException:
            raise util.TracError('Invalid revision number: %d' % int(self.rev))

        editor = editor_class(old_root, new_root, int(self.rev), self.req,
                              self.args, self.env)
        e_ptr, e_baton = svn.delta.make_editor(editor, self.pool)

        def authz_cb(root, path, pool):
            return self.authzperm.has_permission(path) and 1 or 0
        svn.repos.svn_repos_dir_delta(old_root, '', '',
                                      new_root, '', e_ptr, e_baton, authz_cb,
                                      0, 1, 0, 1, self.pool)

    def display(self):
        """Pretty HTML view of the changeset"""
        self.render_diffs()
        Module.Module.display(self)

    def display_diff(self):
        """Raw Unified Diff version"""
        self.req.send_response(200)
        self.req.send_header('Content-Type', 'text/plain;charset=utf-8')
        self.req.send_header('Content-Disposition', 'filename=Changeset%d.diff' % self.rev)
        self.req.end_headers()
        self.render_diffs(UnifiedDiffEditor)

    def display_zip(self):
        """ZIP archive with all the added and/or modified files."""
        self.req.send_response(200)
        self.req.send_header('Content-Type', 'application/zip')
        self.req.send_header('Content-Disposition', 'filename=Changeset%d.zip' % self.rev)
        self.req.end_headers()
        self.render_diffs(ZipDiffEditor)

    def display_hdf(self):
        self.render_diffs()
        Module.Module.display_hdf(self)
<commit_msg>Remove refactoring left-over. Closes #997.<commit_after># -*- coding: iso8859-1 -*-
#
# Copyright (C) 2003, 2004 Edgewall Software
# Copyright (C) 2003, 2004 Jonas Borgstrm <jonas@edgewall.com>
#
# Trac is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# Trac is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# Author: Jonas Borgstrm <jonas@edgewall.com>

import os
import sys
import time
import util
from StringIO import StringIO
from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED

import svn
import svn.delta
import svn.fs

import Diff
import perm
import Module
from WikiFormatter import wiki_to_html

class BaseDiffEditor(svn.delta.Editor):
    """
    Base class for diff renderers.
    """

    def __init__(self, old_root, new_root, rev, req, args, env):
        self.old_root = old_root
        self.new_root = new_root
        self.rev = rev
        self.req = req
        self.args = args
        self.env = env

    def open_directory(self, path, parent_baton, base_revision, dir_pool):
        return [path, path, dir_pool]

    def open_file(self, path, parent_baton, base_revision, file_pool):
        return [path, path, file_pool]

class HtmlDiffEditor(BaseDiffEditor):
    """
    Generates a htmlized unified diff of the changes for a given changeset.
    the output is written to stdout.
    """

    def __init__(self, old_root, new_root, rev, req, args, env):
        BaseDiffEditor.__init__(self, old_root, new_root, rev, req, args, env)
        self.prev_path = None
        self.fileno = -1
        self.prefix = None

    def _check_next(self, old_path, new_path, pool):
        if self.prev_path == (old_path or new_path):
            return

        self.fileno += 1
        self.prev_path = old_path or new_path

        self.prefix = 'changeset.changes.%d' % (self.fileno)
        if old_path:
            old_rev = svn.fs.node_created_rev(self.old_root, old_path, pool)
            self.req.hdf.setValue('%s.rev.old' % self.prefix, str(old_rev))
            self.req.hdf.setValue('%s.browser_href.old' % self.prefix,
                                  self.env.href.browser(old_path, old_rev))
        if new_path:
            new_rev = svn.fs.node_created_rev(self.new_root, new_path, pool)
            self.req.hdf.setValue('%s.rev.new' % self.prefix, str(new_rev))
            self.req.hdf.setValue('%s.browser_href.new' % self.prefix,
                                  self.env.href.browser(new_path, new_rev))

    def add_directory(self, path, parent_baton, copyfrom_path,
                      copyfrom_revision, dir_pool):
        self._check_next(None, path, dir_pool)

    def delete_entry(self, path, revision, parent_baton, pool):
        self._check_next(path, None, pool)

    def change_dir_prop(self, dir_baton, name, value, dir_pool):
        if not dir_baton:
            return
        (old_path, new_path, pool) = dir_baton
        self._check_next(old_path, new_path, dir_pool)

        prefix = '%s.props.%s' % (self.prefix, name)
        if old_path:
            old_value = svn.fs.node_prop(self.old_root, old_path, name, dir_pool)
            if old_value:
                self.req.hdf.setValue(prefix + '.old', old_value)
        if value:
            self.req.hdf.setValue(prefix + '.new', value)

    def add_file(self, path, parent_baton, copyfrom_path, copyfrom_revision,
                 file_pool):
        self._check_next(None, path, file_pool)

    def apply_textdelta(self, file_baton, base_checksum):
        if not file_baton:
            return
        (old_path, new_path, pool) = file_baton
        self._check_next(old_path, new_path, pool)

        # Try to figure out the charset used. We assume that both the old
        # and the new version uses the same charset, not always the case
        # but that's all we can do...
        mime_type = svn.fs.node_prop(self.new_root, new_path,
                                     svn.util.SVN_PROP_MIME_TYPE, pool)
        # We don't have to guess if the charset is specified in the
        # svn:mime-type property
        ctpos = mime_type and mime_type.find('charset=') or -1
        if ctpos >= 0:
            charset = mime_type[ctpos + 8:]
            self.log.debug("Charset %s selected" % charset)
        else:
            charset = self.env.get_config('trac', 'default_charset',
                                          'iso-8859-15')

        # Start up the diff process
        options = Diff.get_options(self.env, self.req, self.args, 1)
        differ = svn.fs.FileDiff(self.old_root, old_path, self.new_root,
                                 new_path, pool, options)
        differ.get_files()
        pobj = differ.get_pipe()

        tabwidth = int(self.env.get_config('diff', 'tab_width', '8'))
        builder = Diff.HDFBuilder(self.req.hdf, '%s.diff' % self.prefix, tabwidth)
        while 1:
            line = pobj.readline()
            if not line:
                break
            builder.writeline(util.to_utf8(line, charset))
        builder.close()
        pobj.close()
        # svn.fs.FileDiff creates a child process and there is no waitpid()
        # calls to eliminate zombies (this is a problem especially when
        # mod_python is used.
        if sys.platform[:3] != "win" and sys.platform != "os2emx":
            try:
                os.waitpid(-1, 0)
            except OSError: pass

    def change_file_prop(self, file_baton, name, value, file_pool):
        if not file_baton:
            return
        (old_path, new_path, pool) = file_baton
        self._check_next(old_path, new_path, file_pool)

        prefix = '%s.props.%s' % (self.prefix, name)
        if old_path:
            old_value = svn.fs.node_prop(self.old_root, old_path, name, file_pool)
            if old_value:
                self.req.hdf.setValue(prefix + '.old', old_value)
        if value:
            self.req.hdf.setValue(prefix + '.new', value)

class UnifiedDiffEditor(BaseDiffEditor):
    """
    Generates a unified diff of the changes for a given changeset.
    the output is written to stdout.
    """

    def apply_textdelta(self, file_baton, base_checksum):
        if not file_baton:
            return
        (old_path, new_path, pool) = file_baton
        options = ['-u']
        options.append('-L')
        options.append("%s\t(revision %d)" % (old_path, self.rev-1))
        options.append('-L')
        options.append("%s\t(revision %d)" % (new_path, self.rev))

        differ = svn.fs.FileDiff(self.old_root, old_path,
                                 self.new_root, new_path, pool, options)
        differ.get_files()
        pobj = differ.get_pipe()
        line = pobj.readline()
        while line:
            self.req.write(line)
            line = pobj.readline()
        pobj.close()
        if sys.platform[:3] != "win" and sys.platform != "os2emx":
            try:
                os.waitpid(-1, 0)
            except OSError: pass

class ZipDiffEditor(BaseDiffEditor):
    """
    Generates a ZIP archive containing the modified and added files.
    """

    def __init__(self, old_root, new_root, rev, req, args, env):
        BaseDiffEditor.__init__(self, old_root, new_root, rev, req, args, env)
        self.buffer = StringIO()
        self.zip = ZipFile(self.buffer, 'w', ZIP_DEFLATED)

    def add_file(self, path, parent_baton, copyfrom_path,
                 copyfrom_revision, file_pool):
        self._add_file_to_zip(path, file_pool)

    def open_file(self, path, parent_baton, base_revision, file_pool):
        self._add_file_to_zip(path, file_pool)

    def close_edit(self):
        self.zip.close()
        self.req.write(self.buffer.getvalue())

    def _add_file_to_zip(self, path, pool):
        fd = svn.fs.file_contents(self.new_root, path, pool)
        info = ZipInfo()
        info.filename = path
        date = svn.fs.revision_prop(svn.fs.root_fs(self.new_root), self.rev,
                                    svn.util.SVN_PROP_REVISION_DATE, pool)
        date = svn.util.svn_time_from_cstring(date, pool) / 1000000
        date = time.localtime(date)
        info.date_time = date[:6]
        info.compress_type = ZIP_DEFLATED
        data = ""
        while 1:
            chunk = svn.util.svn_stream_read(fd, 512)
            if not chunk:
                break
            data = data + chunk
        self.zip.writestr(info, data)

class Changeset (Module.Module):
    template_name = 'changeset.cs'
    perm = None
    fs_ptr = None
    pool = None

    def get_changeset_info (self, rev):
        cursor = self.db.cursor ()
        cursor.execute ('SELECT time, author, message FROM revision ' +
                        'WHERE rev=%d', rev)
        row = cursor.fetchone()
        if not row:
            raise util.TracError('Changeset %d does not exist.' % rev,
                            'Invalid Changset')
        return row

    def get_change_info (self, rev):
        cursor = self.db.cursor ()
        cursor.execute ('SELECT name, change FROM node_change ' +
                        'WHERE rev=%d', rev)
        info = []
        while 1:
            row = cursor.fetchone()
            if not row:
                break
            info.append({'name': row['name'],
                         'change': row['change'],
                         'log_href': self.env.href.log(row['name'])})
        return info

    def render(self):
        self.perm.assert_permission (perm.CHANGESET_VIEW)

        self.add_link('alternate', '?format=diff', 'Unified Diff',
            'text/plain', 'diff')
        self.add_link('alternate', '?format=zip', 'Zip Archive',
            'application/zip', 'zip')

        youngest_rev = svn.fs.youngest_rev(self.fs_ptr, self.pool)
        if self.args.has_key('rev'):
            self.rev = int(self.args.get('rev'))
        else:
            self.rev = youngest_rev

        Diff.get_options(self.env, self.req, self.args, 1)
        if self.args.has_key('update'):
            self.req.redirect(self.env.href.changeset(self.rev))

        change_info = self.get_change_info (self.rev)
        changeset_info = self.get_changeset_info (self.rev)

        self.req.hdf.setValue('title', '[%d] (changeset)' % self.rev)
        self.req.hdf.setValue('changeset.time',
                              time.asctime(time.localtime(int(changeset_info['time']))))
        author = changeset_info['author'] or 'anonymous'
        self.req.hdf.setValue('changeset.author', util.escape(author))
        self.req.hdf.setValue('changeset.message',
                              wiki_to_html(util.wiki_escape_newline(
                                           changeset_info['message']),
                                           self.req.hdf, self.env, self.db))
        self.req.hdf.setValue('changeset.revision', str(self.rev))
        util.add_to_hdf(change_info, self.req.hdf, 'changeset.changes')

        self.req.hdf.setValue('changeset.href',
                              self.env.href.changeset(self.rev))
        if self.rev > 1:
            self.add_link('first', self.env.href.changeset(1), 'Changeset 1')
            self.add_link('prev', self.env.href.changeset(self.rev - 1),
                          'Changeset %d' % (self.rev - 1))
        if self.rev < youngest_rev:
            self.add_link('next', self.env.href.changeset(self.rev + 1),
                          'Changeset %d' % (self.rev + 1))
            self.add_link('last', self.env.href.changeset(youngest_rev),
                          'Changeset %d' % youngest_rev)

    def render_diffs(self, editor_class=HtmlDiffEditor):
        """
        generates a unified diff of the changes for a given changeset.
        the output is written to stdout.
        """
        try:
            old_root = svn.fs.revision_root(self.fs_ptr, int(self.rev) - 1, self.pool)
            new_root = svn.fs.revision_root(self.fs_ptr, int(self.rev), self.pool)
        except svn.core.SubversionException:
            raise util.TracError('Invalid revision number: %d' % int(self.rev))

        editor = editor_class(old_root, new_root, int(self.rev), self.req,
                              self.args, self.env)
        e_ptr, e_baton = svn.delta.make_editor(editor, self.pool)

        def authz_cb(root, path, pool):
            return self.authzperm.has_permission(path) and 1 or 0
        svn.repos.svn_repos_dir_delta(old_root, '', '',
                                      new_root, '', e_ptr, e_baton, authz_cb,
                                      0, 1, 0, 1, self.pool)

    def display(self):
        """Pretty HTML view of the changeset"""
        self.render_diffs()
        Module.Module.display(self)

    def display_diff(self):
        """Raw Unified Diff version"""
        self.req.send_response(200)
        self.req.send_header('Content-Type', 'text/plain;charset=utf-8')
        self.req.send_header('Content-Disposition', 'filename=Changeset%d.diff' % self.rev)
        self.req.end_headers()
        self.render_diffs(UnifiedDiffEditor)

    def display_zip(self):
        """ZIP archive with all the added and/or modified files."""
        self.req.send_response(200)
        self.req.send_header('Content-Type', 'application/zip')
        self.req.send_header('Content-Disposition', 'filename=Changeset%d.zip' % self.rev)
        self.req.end_headers()
        self.render_diffs(ZipDiffEditor)

    def display_hdf(self):
        self.render_diffs()
        Module.Module.display_hdf(self)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import logging
import os
import datetime
import re
import itertools
import sys
import lib.argconfig_parse as argconfig_parse
import lib.alchemy as alchemy
import shutil
from lib.config_parser import get_xml_handlers

xmlhandlers = get_xml_handlers('process.cfg')
logfile = "./" + 'xml-parsing.log'
logging.basicConfig(filename=logfile, level=logging.DEBUG)
commit_frequency = alchemy.get_config().get('parse').get('commit_frequency')

class Patobj(object): pass

def list_files(patentroot, xmlregex):
    """
    Returns listing of all files within patentroot
    whose filenames match xmlregex
    """
    files = [patentroot+'/'+fi for fi in os.listdir(patentroot)
             if re.search(xmlregex, fi, re.I) is not None]
    if not files:
        logging.error("No files matching {0} found in {1}".format(XMLREGEX, PATENTROOT))
        sys.exit(1)
    return files

def _get_date(filename, dateformat='ipg%y%m%d.xml'):
    """
    Given a [filename], returns the expanded year.
    The optional [dateformat] argument allows for different file formats
    """
    filename = re.search(r'ipg\d{6}',filename)
    if not filename: return 'default'
    filename = filename.group() + '.xml'
    dateobj = datetime.datetime.strptime(filename, dateformat)
    return int(dateobj.strftime('%Y%m%d')) # returns YYYYMMDD

def _get_parser(date):
    """
    Given a [date], returns the class of parser needed
    to parse it
    """
    for daterange in xmlhandlers.iterkeys():
        if daterange[0] <= date <= daterange[1]:
            return xmlhandlers[daterange]
    return xmlhandlers['default']

def extract_xml_strings(filename):
    """
    Given a string [filename], opens the file and returns a generator
    that yields tuples. A tuple is of format (year, xmldoc string). A tuple
    is returned for every valid XML doc in [filename]
    """
    # search for terminating XML tag
    endtag_regex = re.compile('^<!DOCTYPE (.*) SYSTEM')
    endtag = ''
    with open(filename, 'r') as f:
        doc = '' # (re)initialize current XML doc to empty string
        for line in f:
            doc += line
            endtag = endtag_regex.findall(line) if not endtag else endtag
            if not endtag: continue
            terminate = re.compile('^</{0}>'.format(endtag[0]))
            if terminate.findall(line):
                yield (_get_date(filename), doc)
                endtag = ''
                doc = ''

def parse_files(filelist):
    """
    Given a list of files, extracts the XML strings from each and returns a
    flat iterable of all of them.
    """
    if not filelist: return []
    parsed = itertools.imap(extract_xml_strings, filelist)
    return itertools.chain.from_iterable(parsed)

def parse_patent(xmltuple):
    """
    Parses an xml string given as [xmltuple] with the appropriate parser (given
    by the first part of the tuple). Returns list of objects
    to be inserted into the database using SQLAlchemy
    """
    if not xmltuple: return
    try:
        date, xml = xmltuple # extract out the parts of the tuple
        patent = _get_parser(date).PatentGrant(xml, True)
    except Exception as inst:
        logging.error(inst)
        logging.error("  - Error parsing patent: %s" % (xml[:400]))
    del xmltuple
    patobj = Patobj()
    for attr in ['pat','patent','app','assignee_list','inventor_list','lawyer_list',
                 'us_relation_list','us_classifications','ipcr_classifications',
                 'citation_list']:
        patobj.__dict__[attr] = getattr(patent,attr)
    return patobj

def parse_patents(xmltuples):
    """
    Given a list of xml strings as [xmltuples], parses them
    all and returns a flat iterator of patSQL.*XML instances
    """
    if not xmltuples: return []
    return map(parse_patent, xmltuples)

def database_commit(patobjects, commit_frequency=commit_frequency):
    """
    takes in a list of Patent objects (from parse_patents above) and commits
    them to the database. This method is designed to be used sequentially to
    account for db concurrency.  The optional argument `commit_frequency`
    determines the frequency with which we commit the objects to the database.
    If set to 0, it will commit after all patobjects have been added.  Setting
    `commit_frequency` to be low (but not 0) is helpful for low memory machines.
    """
    i = 0
    num_objs = len(patobjects) # compute this once
    for patobj in patobjects:
        i += 1
        alchemy.add(patobj)
        if commit_frequency and (i % commit_frequency == 0 or i == num_objs):
            alchemy.commit()
    if not commit_frequency:
        alchemy.commit()

# TODO: this should only move alchemy.sqlite3
def move_tables(output_directory):
    """
    Moves the output sqlite3 files to the output directory
    """
    if output_directory == ".":
        return
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)
    dbtype = alchemy.config.get('global', 'database')
    dbfile = alchemy.config.get(dbtype, 'database')
    try:
        shutil.move(dbfile,
                    '{0}/{1}'.format(output_directory, dbfile))
    except:
        print 'Database file {0} does not exist'.format(dbfile)

def main(patentroot, xmlregex, verbosity, output_directory='.'):
    logfile = "./" + 'xml-parsing.log'
    logging.basicConfig(filename=logfile, level=verbosity)

    logging.info("Starting parse on {0} on directory {1}".format(str(datetime.datetime.today()), patentroot))
    files = list_files(patentroot, xmlregex)

    logging.info("Found all files matching {0} in directory {1}".format(xmlregex, patentroot))
    parsed_xmls = parse_files(files)
    logging.info("Extracted all individual XML files")
    patobjects = parse_patents(parsed_xmls)
    database_commit(patobjects)
    move_tables(output_directory)

    logging.info("SQL tables moved to {0}".format(output_directory))
    logging.info("Parse completed at {0}".format(str(datetime.datetime.today())))

if __name__ == '__main__':
    args = argconfig_parse.ArgHandler(sys.argv[1:])

    XMLREGEX = args.get_xmlregex()
    PATENTROOT = args.get_patentroot()
    VERBOSITY = args.get_verbosity()
    PATENTOUTPUTDIR = args.get_output_directory()

    main(PATENTROOT, XMLREGEX, VERBOSITY, PATENTOUTPUTDIR)
<commit_msg>add claims text to patobj to be committed<commit_after>#!/usr/bin/env python

import logging
import os
import datetime
import re
import itertools
import sys
import lib.argconfig_parse as argconfig_parse
import lib.alchemy as alchemy
import shutil
from lib.config_parser import get_xml_handlers

xmlhandlers = get_xml_handlers('process.cfg')
logfile = "./" + 'xml-parsing.log'
logging.basicConfig(filename=logfile, level=logging.DEBUG)
commit_frequency = alchemy.get_config().get('parse').get('commit_frequency')

class Patobj(object): pass

def list_files(patentroot, xmlregex):
    """
    Returns listing of all files within patentroot
    whose filenames match xmlregex
    """
    files = [patentroot+'/'+fi for fi in os.listdir(patentroot)
             if re.search(xmlregex, fi, re.I) is not None]
    if not files:
        logging.error("No files matching {0} found in {1}".format(XMLREGEX, PATENTROOT))
        sys.exit(1)
    return files

def _get_date(filename, dateformat='ipg%y%m%d.xml'):
    """
    Given a [filename], returns the expanded year.
    The optional [dateformat] argument allows for different file formats
    """
    filename = re.search(r'ipg\d{6}',filename)
    if not filename: return 'default'
    filename = filename.group() + '.xml'
    dateobj = datetime.datetime.strptime(filename, dateformat)
    return int(dateobj.strftime('%Y%m%d')) # returns YYYYMMDD

def _get_parser(date):
    """
    Given a [date], returns the class of parser needed
    to parse it
    """
    for daterange in xmlhandlers.iterkeys():
        if daterange[0] <= date <= daterange[1]:
            return xmlhandlers[daterange]
    return xmlhandlers['default']

def extract_xml_strings(filename):
    """
    Given a string [filename], opens the file and returns a generator
    that yields tuples. A tuple is of format (year, xmldoc string). A tuple
    is returned for every valid XML doc in [filename]
    """
    # search for terminating XML tag
    endtag_regex = re.compile('^<!DOCTYPE (.*) SYSTEM')
    endtag = ''
    with open(filename, 'r') as f:
        doc = '' # (re)initialize current XML doc to empty string
        for line in f:
            doc += line
            endtag = endtag_regex.findall(line) if not endtag else endtag
            if not endtag: continue
            terminate = re.compile('^</{0}>'.format(endtag[0]))
            if terminate.findall(line):
                yield (_get_date(filename), doc)
                endtag = ''
                doc = ''

def parse_files(filelist):
    """
    Given a list of files, extracts the XML strings from each and returns a
    flat iterable of all of them.
    """
    if not filelist: return []
    parsed = itertools.imap(extract_xml_strings, filelist)
    return itertools.chain.from_iterable(parsed)

def parse_patent(xmltuple):
    """
    Parses an xml string given as [xmltuple] with the appropriate parser (given
    by the first part of the tuple). Returns list of objects
    to be inserted into the database using SQLAlchemy
    """
    if not xmltuple: return
    try:
        date, xml = xmltuple # extract out the parts of the tuple
        patent = _get_parser(date).PatentGrant(xml, True)
    except Exception as inst:
        logging.error(inst)
        logging.error("  - Error parsing patent: %s" % (xml[:400]))
        return
    del xmltuple
    patobj = Patobj()
    for attr in ['pat','patent','app','assignee_list','inventor_list','lawyer_list',
                 'us_relation_list','us_classifications','ipcr_classifications',
                 'citation_list','claims']:
        patobj.__dict__[attr] = getattr(patent, attr)
    return patobj

def parse_patents(xmltuples):
    """
    Given a list of xml strings as [xmltuples], parses them
    all and returns a flat iterator of patSQL.*XML instances
    """
    if not xmltuples: return []
    return map(parse_patent, xmltuples)

def database_commit(patobjects, commit_frequency=commit_frequency):
    """
    takes in a list of Patent objects (from parse_patents above) and commits
    them to the database. This method is designed to be used sequentially to
    account for db concurrency.  The optional argument `commit_frequency`
    determines the frequency with which we commit the objects to the database.
    If set to 0, it will commit after all patobjects have been added.  Setting
    `commit_frequency` to be low (but not 0) is helpful for low memory machines.
    """
    i = 0
    num_objs = len(patobjects) # compute this once
    for patobj in patobjects:
        i += 1
        alchemy.add(patobj)
        if commit_frequency and (i % commit_frequency == 0 or i == num_objs):
            alchemy.commit()
    if not commit_frequency:
        alchemy.commit()

# TODO: this should only move alchemy.sqlite3
def move_tables(output_directory):
    """
    Moves the output sqlite3 files to the output directory
    """
    if output_directory == ".":
        return
    if not os.path.exists(output_directory):
        os.makedirs(output_directory)
    dbtype = alchemy.config.get('global', 'database')
    dbfile = alchemy.config.get(dbtype, 'database')
    try:
        shutil.move(dbfile,
                    '{0}/{1}'.format(output_directory, dbfile))
    except:
        print 'Database file {0} does not exist'.format(dbfile)

def main(patentroot, xmlregex, verbosity, output_directory='.'):
    logfile = "./" + 'xml-parsing.log'
    logging.basicConfig(filename=logfile, level=verbosity)

    logging.info("Starting parse on {0} on directory {1}".format(str(datetime.datetime.today()), patentroot))
    files = list_files(patentroot, xmlregex)

    logging.info("Found all files matching {0} in directory {1}".format(xmlregex, patentroot))
    parsed_xmls = parse_files(files)
    logging.info("Extracted all individual XML files")
    patobjects = parse_patents(parsed_xmls)
    database_commit(patobjects)
    move_tables(output_directory)

    logging.info("SQL tables moved to {0}".format(output_directory))
    logging.info("Parse completed at {0}".format(str(datetime.datetime.today())))

if __name__ == '__main__':
    args = argconfig_parse.ArgHandler(sys.argv[1:])

    XMLREGEX = args.get_xmlregex()
    PATENTROOT = args.get_patentroot()
    VERBOSITY = args.get_verbosity()
    PATENTOUTPUTDIR = args.get_output_directory()

    main(PATENTROOT, XMLREGEX, VERBOSITY, PATENTOUTPUTDIR)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2003-2009 Edgewall Software
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.

from HTMLParser import HTMLParser
import re

from genshi import Markup, HTML, escape, unescape
from genshi.core import stripentities, striptags, START, END
from genshi.builder import Element, ElementFactory, Fragment, tag
from genshi.filters.html import HTMLSanitizer
from genshi.input import ParseError
try:
    from babel.support import LazyProxy
except ImportError:
    LazyProxy = None

from trac.core import TracError
from trac.util.text import to_unicode

__all__ = ['Deuglifier', 'FormTokenInjector', 'TracHTMLSanitizer', 'escape',
           'find_element', 'html', 'plaintext', 'to_fragment', 'unescape']

class TracHTMLSanitizer(HTMLSanitizer):
    """Sanitize HTML constructions which are potentially vector of
    phishing or XSS attacks, in user-supplied HTML.

    See also `genshi.HTMLSanitizer`_.

    .. _genshi.HTMLSanitizer:
       http://genshi.edgewall.org/wiki/Documentation/filters.html#html-sanitizer
    """

    SAFE_CSS = frozenset([
        # CSS 3 properties <http://www.w3.org/TR/CSS/#properties>
        'background', 'background-attachment', 'background-color',
        'background-image', 'background-position', 'background-repeat',
        'border', 'border-bottom', 'border-bottom-color',
        'border-bottom-style', 'border-bottom-left-radius',
        'border-bottom-right-radius', 'border-bottom-width',
        'border-collapse', 'border-color', 'border-left', 'border-left-color',
        'border-left-style', 'border-left-width', 'border-radius',
        'border-right', 'border-right-color', 'border-right-style',
        'border-right-width', 'border-spacing', 'border-style', 'border-top',
        'border-top-color', 'border-top-left-radius', 'border-top-right-radius',
        'border-top-style', 'border-top-width', 'border-width', 'bottom',
        'caption-side', 'clear', 'clip', 'color', 'content',
        'counter-increment', 'counter-reset', 'cursor', 'direction',
        'display', 'empty-cells', 'float', 'font', 'font-family', 'font-size',
        'font-style', 'font-variant', 'font-weight', 'height', 'left',
        'letter-spacing', 'line-height', 'list-style', 'list-style-image',
        'list-style-position', 'list-style-type', 'margin', 'margin-bottom',
        'margin-left', 'margin-right', 'margin-top', 'max-height', 'max-width',
        'min-height', 'min-width', 'opacity', 'orphans', 'outline',
        'outline-color', 'outline-style', 'outline-width', 'overflow',
        'padding', 'padding-bottom', 'padding-left', 'padding-right',
        'padding-top', 'page-break-after', 'page-break-before',
        'page-break-inside', 'position', 'quotes', 'right', 'table-layout',
        'text-align', 'text-decoration', 'text-indent', 'text-transform',
        'top', 'unicode-bidi', 'vertical-align', 'visibility', 'white-space',
        'widows', 'width', 'word-spacing', 'z-index',
    ])

    def __init__(self, safe_schemes=HTMLSanitizer.SAFE_SCHEMES,
                 safe_css=SAFE_CSS):
        safe_attrs = HTMLSanitizer.SAFE_ATTRS | frozenset(['style'])
        safe_schemes = frozenset(safe_schemes)
        super(TracHTMLSanitizer, self).__init__(safe_attrs=safe_attrs,
                                                safe_schemes=safe_schemes)
        self.safe_css = frozenset(safe_css)

    # IE6 <http://heideri.ch/jso/#80>
    _EXPRESSION_SEARCH = re.compile(
        u'[eE\uFF25\uFF45]'         # FULLWIDTH LATIN CAPITAL LETTER E
                                    # FULLWIDTH LATIN SMALL LETTER E
        u'[xX\uFF38\uFF58]'         # FULLWIDTH LATIN CAPITAL LETTER X
                                    # FULLWIDTH LATIN SMALL LETTER X
        u'[pP\uFF30\uFF50]'         # FULLWIDTH LATIN CAPITAL LETTER P
                                    # FULLWIDTH LATIN SMALL LETTER P
        u'[rR\u0280\uFF32\uFF52]'   # LATIN LETTER SMALL CAPITAL R
                                    # FULLWIDTH LATIN CAPITAL LETTER R
                                    # FULLWIDTH LATIN SMALL LETTER R
        u'[eE\uFF25\uFF45]'         # FULLWIDTH LATIN CAPITAL LETTER E
                                    # FULLWIDTH LATIN SMALL LETTER E
        u'[sS\uFF33\uFF53]{2}'      # FULLWIDTH LATIN CAPITAL LETTER S
                                    # FULLWIDTH LATIN SMALL LETTER S
        u'[iI\u026A\uFF29\uFF49]'   # LATIN LETTER SMALL CAPITAL I
                                    # FULLWIDTH LATIN CAPITAL LETTER I
                                    # FULLWIDTH LATIN SMALL LETTER I
        u'[oO\uFF2F\uFF4F]'         # FULLWIDTH LATIN CAPITAL LETTER O
                                    # FULLWIDTH LATIN SMALL LETTER O
        u'[nN\u0274\uFF2E\uFF4E]'   # LATIN LETTER SMALL CAPITAL N
                                    # FULLWIDTH LATIN CAPITAL LETTER N
                                    # FULLWIDTH LATIN SMALL LETTER N
    ).search

    # IE6 <http://openmya.hacker.jp/hasegawa/security/expression.txt>
    #     7) Particular bit of Unicode characters
    _URL_FINDITER = re.compile(
        u'[Uu][Rr\u0280][Ll\u029F]\s*\(([^)]+)').finditer

    def sanitize_css(self, text):
        decls = []
        text = self._strip_css_comments(self._replace_unicode_escapes(text))
        for decl in filter(None, text.split(';')):
            decl = decl.strip()
            if not decl:
                continue
            try:
                prop, value = decl.split(':', 1)
            except ValueError:
                continue
            if not self.is_safe_css(prop.strip().lower(), value.strip()):
                continue
            is_evil = False
            if self._EXPRESSION_SEARCH(decl):
                is_evil = True
            for match in self._URL_FINDITER(decl):
                if not self.is_safe_uri(match.group(1)):
                    is_evil = True
                    break
            if not is_evil:
                decls.append(decl.strip())
        return decls

    def __call__(self, stream):
        """Remove input type="password" elements from the stream
        """
        suppress = False
        for kind, data, pos in super(TracHTMLSanitizer, self).__call__(stream):
            if kind is START:
                tag, attrs = data
                if (tag == 'input' and
                    attrs.get('type', '').lower() == 'password'):
                    suppress = True
                else:
                    yield kind, data, pos
            elif kind is END:
                if not suppress:
                    yield kind, data, pos
                suppress = False
            else:
                yield kind, data, pos

    def is_safe_css(self, prop, value):
        """Determine whether the given css property declaration is to be
        considered safe for inclusion in the output.
        """
        if prop not in self.safe_css:
            return False
        # Position can be used for phishing, 'static' excepted
        if prop == 'position':
            return value.lower() == 'static'
        # Negative margins can be used for phishing
        if prop.startswith('margin'):
            return '-' not in value
        return True

    _NORMALIZE_NEWLINES = re.compile(r'\r\n').sub
    _UNICODE_ESCAPE = re.compile(
        r"""\\([0-9a-fA-F]{1,6})\s?|\\([^\r\n\f0-9a-fA-F'"{};:()#*])""",
        re.UNICODE).sub

    def _replace_unicode_escapes(self, text):
        def _repl(match):
            t = match.group(1)
            if t:
                code = int(t, 16)
                chr = unichr(code)
                if code <= 0x1f:
                    # replace space character because IE ignores control
                    # characters
                    chr = ' '
                elif chr == '\\':
                    chr = r'\\'
                return chr
            t = match.group(2)
            if t == '\\':
                return r'\\'
            else:
                return t
        return self._UNICODE_ESCAPE(_repl,
                                    self._NORMALIZE_NEWLINES('\n', text))

    _CSS_COMMENTS = re.compile(r'/\*.*?\*/').sub

    def _strip_css_comments(self, text):
        """Replace comments with space character instead of superclass which
        removes comments to avoid problems when nested comments.
        """
        return self._CSS_COMMENTS(' ', text)

class Deuglifier(object):
    """Help base class used for cleaning up HTML riddled with ```` tags and replace them with appropriate ````.

    The subclass must define a `rules()` static method returning a
    list of regular expression fragments, each defining a capture
    group in which the name will be reused for the span's class. Two
    special group names, ``font`` and ``endfont`` are used to emit
    ```` and ````, respectively.
    """
    def __new__(cls):
        self = object.__new__(cls)
        if not hasattr(cls, '_compiled_rules'):
            cls._compiled_rules = re.compile('(?:%s)' % '|'.join(cls.rules()))
        self._compiled_rules = cls._compiled_rules
        return self

    def format(self, indata):
        return re.sub(self._compiled_rules, self.replace, indata)

    def replace(self, fullmatch):
        for mtype, match in fullmatch.groupdict().items():
            if match:
                if mtype == 'font':
                    return ''
                elif mtype == 'endfont':
                    return ''
                return '' % mtype

class FormTokenInjector(HTMLParser):
    """Identify and protect forms from CSRF attacks.

    This filter works by adding a input type=hidden field to POST forms.
    """
    def __init__(self, form_token, out):
        HTMLParser.__init__(self)
        self.out = out
        self.token = form_token

    def handle_starttag(self, tag, attrs):
        self.out.write(self.get_starttag_text())
        if tag.lower() == 'form':
            for name, value in attrs:
                if name.lower() == 'method' and value.lower() == 'post':
                    self.out.write('<input type="hidden" name="__FORM_TOKEN"'
                                   ' value="%s"/>' % self.token)
                    break

    def handle_startendtag(self, tag, attrs):
        self.out.write(self.get_starttag_text())

    def handle_charref(self, name):
        self.out.write('&#%s;' % name)

    def handle_entityref(self, name):
        self.out.write('&%s;' % name)

    def handle_comment(self, data):
        self.out.write('' % data)

    def handle_decl(self, data):
        self.out.write('<!%s>' % data)

    def handle_pi(self, data):
        self.out.write('<?%s?>' % data)

    def handle_data(self, data):
        self.out.write(data)

    def handle_endtag(self, tag):
        self.out.write('</' + tag + '>')

class TransposingElementFactory(ElementFactory):
    """A `genshi.builder.ElementFactory` which applies `func` to the
    named attributes before creating a `genshi.builder.Element`.
    """

    def __init__(self, func, namespace=None):
        ElementFactory.__init__(self, namespace=namespace)
        self.func = func

    def __getattr__(self, name):
        return ElementFactory.__getattr__(self, self.func(name))

html = TransposingElementFactory(str.lower)

def plaintext(text, keeplinebreaks=True):
    """Extract the text elements from (X)HTML content

    :param text: `unicode` or `genshi.builder.Fragment`
    :param keeplinebreaks: optionally keep linebreaks
    """
    if isinstance(text, Fragment):
        text = text.generate().render('text', encoding=None)
    else:
        text = stripentities(striptags(text))
    if not keeplinebreaks:
        text = text.replace(u'\n', u' ')
    return text

def find_element(frag, attr=None, cls=None, tag=None):
    """Return the first element in the fragment having the given attribute,
    class or tag, using a preorder depth-first search.
    """
    if isinstance(frag, Element):
        if attr is not None and attr in frag.attrib:
            return frag
        if cls is not None and cls in frag.attrib.get('class', '').split():
            return frag
        if tag is not None and tag == frag.tag:
            return frag
    if isinstance(frag, Fragment):
        for child in frag.children:
            elt = find_element(child, attr, cls, tag)
            if elt is not None:
                return elt

def expand_markup(stream, ctxt=None):
    """A Genshi stream filter for expanding `genshi.Markup` events.

    Note: Expansion may not be possible if the fragment is badly
    formed, or partial.
    """
    for event in stream:
        if isinstance(event[1], Markup):
            try:
                for subevent in HTML(event[1]):
                    yield subevent
            except ParseError:
                yield event
        else:
            yield event

def to_fragment(input):
    """Convert input to a `Fragment` object."""

    if isinstance(input, TracError):
        input = input.message
    if LazyProxy and isinstance(input, LazyProxy):
        input = input.value
    if isinstance(input, Fragment):
        return input
    return tag(to_unicode(input))

html = TransposingElementFactory(str.lower)
<commit_msg>1.0.2dev: follow-up to r12322, removed duplicated `html` symbol from `trac.util.html`<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2003-2009 Edgewall Software
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.

from HTMLParser import HTMLParser
import re

from genshi import Markup, HTML, escape, unescape
from genshi.core import stripentities, striptags, START, END
from genshi.builder import Element, ElementFactory, Fragment, tag
from genshi.filters.html import HTMLSanitizer
from genshi.input import ParseError
try:
    from babel.support import LazyProxy
except ImportError:
    LazyProxy = None

from trac.core import TracError
from trac.util.text import to_unicode

__all__ = ['Deuglifier', 'FormTokenInjector', 'TracHTMLSanitizer', 'escape',
           'find_element', 'html', 'plaintext', 'to_fragment', 'unescape']

class TracHTMLSanitizer(HTMLSanitizer):
    """Sanitize HTML constructions which are potentially vector of
    phishing or XSS attacks, in user-supplied HTML.

    See also `genshi.HTMLSanitizer`_.

    .. _genshi.HTMLSanitizer:
       http://genshi.edgewall.org/wiki/Documentation/filters.html#html-sanitizer
    """

    SAFE_CSS = frozenset([
        # CSS 3 properties <http://www.w3.org/TR/CSS/#properties>
        'background', 'background-attachment', 'background-color',
        'background-image', 'background-position', 'background-repeat',
        'border', 'border-bottom', 'border-bottom-color',
        'border-bottom-style', 'border-bottom-left-radius',
        'border-bottom-right-radius', 'border-bottom-width',
        'border-collapse', 'border-color', 'border-left', 'border-left-color',
        'border-left-style', 'border-left-width', 'border-radius',
        'border-right', 'border-right-color', 'border-right-style',
        'border-right-width', 'border-spacing', 'border-style', 'border-top',
        'border-top-color', 'border-top-left-radius', 'border-top-right-radius',
        'border-top-style', 'border-top-width', 'border-width', 'bottom',
        'caption-side', 'clear', 'clip', 'color', 'content',
        'counter-increment', 'counter-reset', 'cursor', 'direction',
        'display', 'empty-cells', 'float', 'font', 'font-family', 'font-size',
        'font-style', 'font-variant', 'font-weight', 'height', 'left',
        'letter-spacing', 'line-height', 'list-style', 'list-style-image',
        'list-style-position', 'list-style-type', 'margin', 'margin-bottom',
        'margin-left', 'margin-right', 'margin-top', 'max-height', 'max-width',
        'min-height', 'min-width', 'opacity', 'orphans', 'outline',
        'outline-color', 'outline-style', 'outline-width', 'overflow',
        'padding', 'padding-bottom', 'padding-left', 'padding-right',
        'padding-top', 'page-break-after', 'page-break-before',
        'page-break-inside', 'position', 'quotes', 'right', 'table-layout',
        'text-align', 'text-decoration', 'text-indent', 'text-transform',
        'top', 'unicode-bidi', 'vertical-align', 'visibility', 'white-space',
        'widows', 'width', 'word-spacing', 'z-index',
    ])

    def __init__(self, safe_schemes=HTMLSanitizer.SAFE_SCHEMES,
                 safe_css=SAFE_CSS):
        safe_attrs = HTMLSanitizer.SAFE_ATTRS | frozenset(['style'])
        safe_schemes = frozenset(safe_schemes)
        super(TracHTMLSanitizer, self).__init__(safe_attrs=safe_attrs,
                                                safe_schemes=safe_schemes)
        self.safe_css = frozenset(safe_css)

    # IE6 <http://heideri.ch/jso/#80>
    _EXPRESSION_SEARCH = re.compile(
        u'[eE\uFF25\uFF45]'         # FULLWIDTH LATIN CAPITAL LETTER E
                                    # FULLWIDTH LATIN SMALL LETTER E
        u'[xX\uFF38\uFF58]'         # FULLWIDTH LATIN CAPITAL LETTER X
                                    # FULLWIDTH LATIN SMALL LETTER X
        u'[pP\uFF30\uFF50]'         # FULLWIDTH LATIN CAPITAL LETTER P
                                    # FULLWIDTH LATIN SMALL LETTER P
        u'[rR\u0280\uFF32\uFF52]'   # LATIN LETTER SMALL CAPITAL R
                                    # FULLWIDTH LATIN CAPITAL LETTER R
                                    # FULLWIDTH LATIN SMALL LETTER R
        u'[eE\uFF25\uFF45]'         # FULLWIDTH LATIN CAPITAL LETTER E
                                    # FULLWIDTH LATIN SMALL LETTER E
        u'[sS\uFF33\uFF53]{2}'      # FULLWIDTH LATIN CAPITAL LETTER S
                                    # FULLWIDTH LATIN SMALL LETTER S
        u'[iI\u026A\uFF29\uFF49]'   # LATIN LETTER SMALL CAPITAL I
                                    # FULLWIDTH LATIN CAPITAL LETTER I
                                    # FULLWIDTH LATIN SMALL LETTER I
        u'[oO\uFF2F\uFF4F]'         # FULLWIDTH LATIN CAPITAL LETTER O
                                    # FULLWIDTH LATIN SMALL LETTER O
        u'[nN\u0274\uFF2E\uFF4E]'   # LATIN LETTER SMALL CAPITAL N
                                    # FULLWIDTH LATIN CAPITAL LETTER N
                                    # FULLWIDTH LATIN SMALL LETTER N
    ).search

    # IE6 <http://openmya.hacker.jp/hasegawa/security/expression.txt>
    #     7) Particular bit of Unicode characters
    _URL_FINDITER = re.compile(
        u'[Uu][Rr\u0280][Ll\u029F]\s*\(([^)]+)').finditer

    def sanitize_css(self, text):
        decls = []
        text = self._strip_css_comments(self._replace_unicode_escapes(text))
        for decl in filter(None, text.split(';')):
            decl = decl.strip()
            if not decl:
                continue
            try:
                prop, value = decl.split(':', 1)
            except ValueError:
                continue
            if not self.is_safe_css(prop.strip().lower(), value.strip()):
                continue
            is_evil = False
            if self._EXPRESSION_SEARCH(decl):
                is_evil = True
            for match in self._URL_FINDITER(decl):
                if not self.is_safe_uri(match.group(1)):
                    is_evil = True
                    break
            if not is_evil:
                decls.append(decl.strip())
        return decls

    def __call__(self, stream):
        """Remove input type="password" elements from the stream
        """
        suppress = False
        for kind, data, pos in super(TracHTMLSanitizer, self).__call__(stream):
            if kind is START:
                tag, attrs = data
                if (tag == 'input' and
                    attrs.get('type', '').lower() == 'password'):
                    suppress = True
                else:
                    yield kind, data, pos
            elif kind is END:
                if not suppress:
                    yield kind, data, pos
                suppress = False
            else:
                yield kind, data, pos

    def is_safe_css(self, prop, value):
        """Determine whether the given css property declaration is to be
        considered safe for inclusion in the output.
        """
        if prop not in self.safe_css:
            return False
        # Position can be used for phishing, 'static' excepted
        if prop == 'position':
            return value.lower() == 'static'
        # Negative margins can be used for phishing
        if prop.startswith('margin'):
            return '-' not in value
        return True

    _NORMALIZE_NEWLINES = re.compile(r'\r\n').sub
    _UNICODE_ESCAPE = re.compile(
        r"""\\([0-9a-fA-F]{1,6})\s?|\\([^\r\n\f0-9a-fA-F'"{};:()#*])""",
        re.UNICODE).sub

    def _replace_unicode_escapes(self, text):
        def _repl(match):
            t = match.group(1)
            if t:
                code = int(t, 16)
                chr = unichr(code)
                if code <= 0x1f:
                    # replace space character because IE ignores control
                    # characters
                    chr = ' '
                elif chr == '\\':
                    chr = r'\\'
                return chr
            t = match.group(2)
            if t == '\\':
                return r'\\'
            else:
                return t
        return self._UNICODE_ESCAPE(_repl,
                                    self._NORMALIZE_NEWLINES('\n', text))

    _CSS_COMMENTS = re.compile(r'/\*.*?\*/').sub

    def _strip_css_comments(self, text):
        """Replace comments with space character instead of superclass which
        removes comments to avoid problems when nested comments.
        """
        return self._CSS_COMMENTS(' ', text)

class Deuglifier(object):
    """Help base class used for cleaning up HTML riddled with ```` tags and replace them with appropriate ````.

    The subclass must define a `rules()` static method returning a
    list of regular expression fragments, each defining a capture
    group in which the name will be reused for the span's class. Two
    special group names, ``font`` and ``endfont`` are used to emit
    ```` and ````, respectively.
    """
    def __new__(cls):
        self = object.__new__(cls)
        if not hasattr(cls, '_compiled_rules'):
            cls._compiled_rules = re.compile('(?:%s)' % '|'.join(cls.rules()))
        self._compiled_rules = cls._compiled_rules
        return self

    def format(self, indata):
        return re.sub(self._compiled_rules, self.replace, indata)

    def replace(self, fullmatch):
        for mtype, match in fullmatch.groupdict().items():
            if match:
                if mtype == 'font':
                    return ''
                elif mtype == 'endfont':
                    return ''
                return '' % mtype

class FormTokenInjector(HTMLParser):
    """Identify and protect forms from CSRF attacks.

    This filter works by adding a input type=hidden field to POST forms.
    """
    def __init__(self, form_token, out):
        HTMLParser.__init__(self)
        self.out = out
        self.token = form_token

    def handle_starttag(self, tag, attrs):
        self.out.write(self.get_starttag_text())
        if tag.lower() == 'form':
            for name, value in attrs:
                if name.lower() == 'method' and value.lower() == 'post':
                    self.out.write('<input type="hidden" name="__FORM_TOKEN"'
                                   ' value="%s"/>' % self.token)
                    break

    def handle_startendtag(self, tag, attrs):
        self.out.write(self.get_starttag_text())

    def handle_charref(self, name):
        self.out.write('&#%s;' % name)

    def handle_entityref(self, name):
        self.out.write('&%s;' % name)

    def handle_comment(self, data):
        self.out.write('' % data)

    def handle_decl(self, data):
        self.out.write('<!%s>' % data)

    def handle_pi(self, data):
        self.out.write('<?%s?>' % data)

    def handle_data(self, data):
        self.out.write(data)

    def handle_endtag(self, tag):
        self.out.write('</' + tag + '>')

class TransposingElementFactory(ElementFactory):
    """A `genshi.builder.ElementFactory` which applies `func` to the
    named attributes before creating a `genshi.builder.Element`.
    """

    def __init__(self, func, namespace=None):
        ElementFactory.__init__(self, namespace=namespace)
        self.func = func

    def __getattr__(self, name):
        return ElementFactory.__getattr__(self, self.func(name))

html = TransposingElementFactory(str.lower)

def plaintext(text, keeplinebreaks=True):
    """Extract the text elements from (X)HTML content

    :param text: `unicode` or `genshi.builder.Fragment`
    :param keeplinebreaks: optionally keep linebreaks
    """
    if isinstance(text, Fragment):
        text = text.generate().render('text', encoding=None)
    else:
        text = stripentities(striptags(text))
    if not keeplinebreaks:
        text = text.replace(u'\n', u' ')
    return text

def find_element(frag, attr=None, cls=None, tag=None):
    """Return the first element in the fragment having the given attribute,
    class or tag, using a preorder depth-first search.
    """
    if isinstance(frag, Element):
        if attr is not None and attr in frag.attrib:
            return frag
        if cls is not None and cls in frag.attrib.get('class', '').split():
            return frag
        if tag is not None and tag == frag.tag:
            return frag
    if isinstance(frag, Fragment):
        for child in frag.children:
            elt = find_element(child, attr, cls, tag)
            if elt is not None:
                return elt

def expand_markup(stream, ctxt=None):
    """A Genshi stream filter for expanding `genshi.Markup` events.

    Note: Expansion may not be possible if the fragment is badly
    formed, or partial.
    """
    for event in stream:
        if isinstance(event[1], Markup):
            try:
                for subevent in HTML(event[1]):
                    yield subevent
            except ParseError:
                yield event
        else:
            yield event

def to_fragment(input):
    """Convert input to a `Fragment` object."""

    if isinstance(input, TracError):
        input = input.message
    if LazyProxy and isinstance(input, LazyProxy):
        input = input.value
    if isinstance(input, Fragment):
        return input
    return tag(to_unicode(input))
<|endoftext|>


<commit_before>
# stdlib imports
import re
import time

# trac imports
import trac
from trac.core import *
from trac.util.html import html
from trac.web import IRequestHandler
from trac.web.chrome import INavigationContributor, ITemplateProvider
from trac.web.chrome import add_ctxtnav, add_stylesheet, add_script

def fmt_timestamp(seconds):
    millis = int(seconds * 1000) % 1000
    localtime = time.localtime(seconds)
    text = []
    text.append(time.strftime('%m/%d/%y  %H:%M:%S', localtime))
    text.append('.%03d' % millis)
    return "".join(text)

class TracSqlPlugin(Component):
    implements(INavigationContributor, IRequestHandler, ITemplateProvider)

    # INavigationContributor methods

    def get_active_navigation_item(self, req):
        return 'sql'

    def get_navigation_items(self, req):
        if not req.perm.has_permission('TRAC_ADMIN'):
            return
        yield 'mainnav', 'sql', html.A('SQL', href=req.href.sql())

    # ITemplateProvider methods

    def get_htdocs_dirs(self):
        from pkg_resources import resource_filename
        return [('sql', resource_filename(__name__, 'htdocs'))]

    def get_templates_dirs(self):
        from pkg_resources import resource_filename
        return [resource_filename(__name__, 'templates')]

    # IRequestHandler methods

    def match_request(self, req):
        import re
        match = re.match(r'/sql(?:(/.*))?', req.path_info)
        if match:
            path, = match.groups()
            req.args['path'] = path or '/'
            return True

    def process_request(self, req):
        req.perm.require('TRAC_ADMIN')

        path = req.args.get('path', '')

        data = {}

        db = self.env.get_db_cnx()
        cursor = db.cursor()

        db_str = self.env.config.get('trac', 'database')
        db_type, db_path = db_str.split(':', 1)
        assert db_type in ('sqlite', 'mysql', 'postgres'), \
                            'Unsupported database "%s"' % db_type
        self.db_type = db_type

        # Include trac wiki stylesheet
        add_stylesheet(req, 'common/css/wiki.css')

        # Include trac stats stylesheet
        add_stylesheet(req, 'sql/common.css')

        # Include javascript libraries
        add_script(req, 'stats/jquery-1.4.2.min.js')

        # Include context navigation links
        add_ctxtnav(req, 'Query', req.href.sql())
        add_ctxtnav(req, 'Schema', req.href.sql('schema'))

        if path == '/':
            result = self._process(req, cursor, data)
            cursor.close()
            return result

        elif path == '/schema':
            result = self._process_schema(req, cursor, data)
            cursor.close()
            return result

        else:
            cursor.close()
            raise ValueError, "unknown path '%s'" % path

    def _process(self, req, cursor, data):

        if self.db_type == 'mysql':
            cursor.execute('set SQL_SELECT_LIMIT=1000')
        else:
            pass

        sql = req.args.get('query', '')
        raw = req.args.get('raw', '')

        cols = rows = []
        took = 0
        error = None

        if re.search('.*delete|drop|insert|replace|set|update.*', sql,
                     re.IGNORECASE):
            error = "Query must be read-only!"

        elif sql.strip():
            try:
                start = time.time()
                cursor.execute(sql)
                cols = map(lambda x: x[0], cursor.description)
                rows = cursor.fetchall()[:1000]
                took = '%.3f' % (time.time() - start)
            except BaseException, e:
                error = e.message

        if not raw:

            format = {
                'path' : lambda x: html.A(x, href=req.href.browser(x)),
                'rev' : lambda x: html.A(x, href=req.href.changeset(x)),
                'time' : lambda x: fmt_timestamp(x),
            }

            if trac.__version__.startswith('0.12'):
                format['time'] = lambda x: fmt_timestamp(x/1000000.)

            format['base_path'] = format['path']
            format['base_rev'] = format['rev']
            format['changetime'] = format['time']

            default = lambda x: x

            formats = [format.get(col, default) for col in cols]
            for i, row in enumerate(rows):
                rows[i] = [fmt(col) for fmt, col in zip(formats, row)]

        data['query'] = sql
        data['error'] = error
        data['cols'] = cols
        data['rows'] = rows
        data['took'] = took
        data['raw'] = raw

        return 'sql.html', data, None

    def _process_schema(self, req, cursor, data):

        if self.db_type == 'mysql':
            sql = 'show tables'
        elif self.db_type == 'sqlite':
            sql = 'SELECT name FROM sqlite_master WHERE type = "table"'
        else:
            assert False, "Unsupported db_type: %s" % self.db_type

        cursor.execute(sql)
        rows = cursor.fetchall()

        table = req.args.get('table', '')
        valid = False

        tables = []
        for x, in sorted(rows):
            if x == table:
                valid = True
                tables.append(html.B(x))
            else:
                tables.append(html.A(x, href=req.href.sql('schema', table=x)))

        if table and valid:
            if self.db_type == 'mysql':
                sql = 'describe %s' % table
            elif self.db_type == 'sqlite':
                sql = 'PRAGMA table_info("%s")' % table
            else:
                assert False, "Unsupported db_type: %s" % self.db_type

            cursor.execute(sql)
            cols = map(lambda x: x[0], cursor.description)
            rows = cursor.fetchall()

            cursor.execute("select count(*) from %s" % table)
            count, = cursor.fetchall()[0]
        else:
            cols = rows = []
            count = 0

        data['tables'] = tables
        data['cols'] = cols
        data['rows'] = rows
        data['table'] = table
        data['count'] = count

        # FIXME: Add index list?
        # FIXME: Add foreign key list?

        return 'schema.html', data, None

<commit_msg>More formatting options, including rendering wiki text.<commit_after>
# stdlib imports
import re
import time

# trac imports
import trac
from trac.core import *
from trac.util.html import html
from trac.web import IRequestHandler
from trac.web.chrome import INavigationContributor, ITemplateProvider
from trac.web.chrome import add_ctxtnav, add_stylesheet, add_script

def fmt_timestamp(seconds):
    millis = int(seconds * 1000) % 1000
    localtime = time.localtime(seconds)
    text = []
    text.append(time.strftime('%m/%d/%y  %H:%M:%S', localtime))
    text.append('.%03d' % millis)
    return "".join(text)

class TracSqlPlugin(Component):
    implements(INavigationContributor, IRequestHandler, ITemplateProvider)

    # INavigationContributor methods

    def get_active_navigation_item(self, req):
        return 'sql'

    def get_navigation_items(self, req):
        if not req.perm.has_permission('TRAC_ADMIN'):
            return
        yield 'mainnav', 'sql', html.A('SQL', href=req.href.sql())

    # ITemplateProvider methods

    def get_htdocs_dirs(self):
        from pkg_resources import resource_filename
        return [('sql', resource_filename(__name__, 'htdocs'))]

    def get_templates_dirs(self):
        from pkg_resources import resource_filename
        return [resource_filename(__name__, 'templates')]

    # IRequestHandler methods

    def match_request(self, req):
        import re
        match = re.match(r'/sql(?:(/.*))?', req.path_info)
        if match:
            path, = match.groups()
            req.args['path'] = path or '/'
            return True

    def process_request(self, req):
        req.perm.require('TRAC_ADMIN')

        path = req.args.get('path', '')

        data = {}

        db = self.env.get_db_cnx()
        cursor = db.cursor()

        db_str = self.env.config.get('trac', 'database')
        db_type, db_path = db_str.split(':', 1)
        assert db_type in ('sqlite', 'mysql', 'postgres'), \
                            'Unsupported database "%s"' % db_type
        self.db_type = db_type

        # Include trac wiki stylesheet
        add_stylesheet(req, 'common/css/wiki.css')

        # Include trac stats stylesheet
        add_stylesheet(req, 'sql/common.css')

        # Include javascript libraries
        add_script(req, 'stats/jquery-1.4.2.min.js')

        # Include context navigation links
        add_ctxtnav(req, 'Query', req.href.sql())
        add_ctxtnav(req, 'Schema', req.href.sql('schema'))

        if path == '/':
            result = self._process(req, cursor, data)
            cursor.close()
            return result

        elif path == '/schema':
            result = self._process_schema(req, cursor, data)
            cursor.close()
            return result

        else:
            cursor.close()
            raise ValueError, "unknown path '%s'" % path

    def _process(self, req, cursor, data):

        if self.db_type == 'mysql':
            cursor.execute('set SQL_SELECT_LIMIT=1000')
        else:
            pass

        sql = req.args.get('query', '')
        raw = req.args.get('raw', '')

        cols = rows = []
        took = 0
        error = None

        if re.search('.*delete|drop|insert|replace|set|update.*', sql,
                     re.IGNORECASE):
            error = "Query must be read-only!"

        elif sql.strip():
            try:
                start = time.time()
                cursor.execute(sql)
                cols = map(lambda x: x[0], cursor.description)
                rows = cursor.fetchall()[:1000]
                took = '%.3f' % (time.time() - start)
            except BaseException, e:
                error = e.message

        if not raw:

            format = {
                'path' : lambda x: html.A(x, href=req.href.browser(x)),
                'rev' : lambda x: html.A(x, href=req.href.changeset(x)),
                'ticket' : lambda x: html.A(x, href=req.href.ticket(x)),
                'query' : lambda x: html.FONT(x, face="monospace"),
                'time' : lambda x: fmt_timestamp(x),
            }

            if trac.__version__.startswith('0.12'):
                format['time'] = lambda x: fmt_timestamp(x/1000000.)

            format['base_path'] = format['path']
            format['base_rev'] = format['rev']
            format['changetime'] = format['time']

            def format_wiki_text(text):
                from trac.mimeview.api import Mimeview
                mimeview = Mimeview(self.env)
                mimetype = 'text/x-trac-wiki'
                return mimeview.render(req, mimetype, text)

            if re.search('.*from wiki.*', sql, re.IGNORECASE|re.MULTILINE):
                format['name'] = lambda x: html.A(x, href=req.href.wiki(x))
                format['text'] = format_wiki_text
            elif re.search('.*from ticket.*', sql, re.IGNORECASE|re.MULTILINE):
                format['id'] = lambda x: html.A(x, href=req.href.ticket(x))
                format['component'] = lambda x: html.A(x, href=req.href.query(component=x))
                format['severity'] = lambda x: html.A(x, href=req.href.query(severity=x))
                format['type'] = lambda x: html.A(x, href=req.href.query(type=x))
                format['milestone'] = lambda x: html.A(x, href=req.href.query(milestone=x))
                format['version'] = lambda x: html.A(x, href=req.href.query(version=x))
                format['status'] = lambda x: html.A(x, href=req.href.query(status=x))
                format['owner'] = lambda x: html.A(x, href=req.href.query(owner=x))
                format['reporter'] = lambda x: html.A(x, href=req.href.query(reporter=x))
                format['priority'] = lambda x: html.A(x, href=req.href.query(priority=x))
                format['resolution'] = lambda x: html.A(x, href=req.href.query(resolution=x))
            elif re.search('.*from report.*', sql, re.IGNORECASE|re.MULTILINE):
                format['id'] = lambda x: html.A(x, href=req.href.report(x))

            default = lambda x: x

            formats = [format.get(col, default) for col in cols]
            for i, row in enumerate(rows):
                rows[i] = [fmt(col) for fmt, col in zip(formats, row)]

        data['query'] = sql
        data['error'] = error
        data['cols'] = cols
        data['rows'] = rows
        data['took'] = took
        data['raw'] = raw

        return 'sql.html', data, None

    def _process_schema(self, req, cursor, data):

        if self.db_type == 'mysql':
            sql = 'show tables'
        elif self.db_type == 'sqlite':
            sql = 'SELECT name FROM sqlite_master WHERE type = "table"'
        else:
            assert False, "Unsupported db_type: %s" % self.db_type

        cursor.execute(sql)
        rows = cursor.fetchall()

        table = req.args.get('table', '')
        valid = False

        tables = []
        for x, in sorted(rows):
            if x == table:
                valid = True
                tables.append(html.B(x))
            else:
                tables.append(html.A(x, href=req.href.sql('schema', table=x)))

        if table and valid:
            if self.db_type == 'mysql':
                sql = 'describe %s' % table
            elif self.db_type == 'sqlite':
                sql = 'PRAGMA table_info("%s")' % table
            else:
                assert False, "Unsupported db_type: %s" % self.db_type

            cursor.execute(sql)
            cols = map(lambda x: x[0], cursor.description)
            rows = cursor.fetchall()

            cursor.execute("select count(*) from %s" % table)
            count, = cursor.fetchall()[0]
        else:
            cols = rows = []
            count = 0

        data['tables'] = tables
        data['cols'] = cols
        data['rows'] = rows
        data['table'] = table
        data['count'] = count

        # FIXME: Add index list?
        # FIXME: Add foreign key list?

        return 'schema.html', data, None

<|endoftext|>


<commit_before>#Copyright (C) 2016 Paolo Galeone <nessuno@nerdz.eu>
# Based on Tensorflow cifar10_train.py file
# https://github.com/tensorflow/tensorflow/blob/r0.11/tensorflow/models/image/cifar10/cifar10_train.py
#
#This Source Code Form is subject to the terms of the Mozilla Public
#License, v. 2.0. If a copy of the MPL was not distributed with this
#file, you can obtain one at http://mozilla.org/MPL/2.0/.
#Exhibit B is not attached; this software is compatible with the
#licenses expressed under Section 1.12 of the MPL v2.
""" Train model with a single GPU. Evaluate it on the second one"""

import argparse
import importlib
import sys
from datetime import datetime
import os.path
import time
import math

import numpy as np
import tensorflow as tf
import evaluate
from models import utils
from decay import supervised_parameter_decay

def train():
    """Train model"""
    with tf.Graph().as_default(), tf.device('/gpu:0'):
        global_step = tf.Variable(0, trainable=False)

        # Get images and labels for CIFAR-10.
        images, labels = DATASET.distorted_inputs(BATCH_SIZE)

        # Build a Graph that computes the logits predictions from the
        # inference model.
        keep_prob_, logits = MODEL.get_model(
            images, DATASET.NUM_CLASSES, train_phase=True)

        # Calculate loss.
        loss = MODEL.loss(logits, labels)

        # Decay the learning rate exponentially based on the number of steps.
        learning_rate = tf.train.exponential_decay(
            INITIAL_LEARNING_RATE,
            global_step,
            STEPS_PER_EPOCH,
            LEARNING_RATE_DECAY_FACTOR,
            staircase=True)

        utils.log(tf.scalar_summary('learning_rate', learning_rate))
        train_op = tf.train.MomentumOptimizer(learning_rate, MOMENTUM).minimize(
            loss, global_step=global_step)

        # Create the train saver.
        variables_to_save = tf.trainable_variables() + [global_step]
        train_saver = tf.train.Saver(variables_to_save)
        # Create the best model saver.
        best_saver = tf.train.Saver(variables_to_save)

        # Train accuracy ops
        top_k_op = tf.nn.in_top_k(logits, labels, 1)
        train_accuracy = tf.reduce_mean(tf.cast(top_k_op, tf.float32))
        # General validation summary
        accuracy_value_ = tf.placeholder(tf.float32, shape=())
        accuracy_summary = tf.scalar_summary('accuracy', accuracy_value_)

        # Initialize decay_keep_prob op
        # va placeholder required for supervised_parameter_decay
        validation_accuracy_ = tf.placeholder(
            tf.float32, shape=(), name="validation_accuracy_")
        decay_keep_prob = supervised_parameter_decay(
            validation_accuracy_, initial_parameter_value=MAX_KEEP_PROB)
        keep_prob_summary = tf.scalar_summary('keep_prob', decay_keep_prob)

        # read collection after keep_prob_decay that adds
        # the keep_prob summary
        train_summaries = tf.merge_summary(
            tf.get_collection_ref('train_summaries'))

        # Build an initialization operation to run below.
        init = tf.initialize_all_variables()

        # Start running operations on the Graph.
        with tf.Session(config=tf.ConfigProto(
                allow_soft_placement=True)) as sess:
            sess.run(init)

            # Start the queue runners.
            tf.train.start_queue_runners(sess=sess)
            train_log = tf.train.SummaryWriter(LOG_DIR + "/train", sess.graph)
            validation_log = tf.train.SummaryWriter(LOG_DIR + "/validation",
                                                    sess.graph)

            # Extract previous global step value
            old_gs = sess.run(global_step)

            # set initial keep_prob
            keep_prob = MAX_KEEP_PROB
            # set best_validation_accuracy, used by best_saver
            best_validation_accuracy = 0.0

            # Restart from where we were
            for step in range(old_gs, MAX_STEPS):
                start_time = time.time()
                _, loss_value, summary_lines = sess.run(
                    [train_op, loss, train_summaries],
                    feed_dict={keep_prob_: keep_prob})
                duration = time.time() - start_time

                assert not np.isnan(
                    loss_value), 'Model diverged with loss = NaN'

                # update logs every 10 iterations
                if step % 10 == 0:
                    num_examples_per_step = BATCH_SIZE
                    examples_per_sec = num_examples_per_step / duration
                    sec_per_batch = float(duration)

                    format_str = ('{}: step {}, loss = {:.2f} '
                                  '({:.1f} examples/sec; {:.3f} sec/batch)')
                    print(
                        format_str.format(datetime.now(), step, loss_value,
                                          examples_per_sec, sec_per_batch))
                    # log train values
                    train_log.add_summary(summary_lines, global_step=step)

                # Save the model checkpoint at the end of every epoch
                # evaluate train and validation performance
                if (step > 0 and
                        step % STEPS_PER_EPOCH == 0) or (step + 1) == MAX_STEPS:
                    checkpoint_path = os.path.join(LOG_DIR, 'model.ckpt')
                    train_saver.save(sess, checkpoint_path, global_step=step)

                    # validation accuracy
                    validation_accuracy_value = evaluate.get_validation_accuracy(
                        LOG_DIR, DATASET.NUM_CLASSES)
                    summary_line = sess.run(
                        accuracy_summary,
                        feed_dict={accuracy_value_: validation_accuracy_value})
                    validation_log.add_summary(summary_line, global_step=step)

                    # update keep_prob using new validation accuracy
                    keep_prob, summary_line = sess.run(
                        [decay_keep_prob, keep_prob_summary],
                        feed_dict={
                            validation_accuracy_: validation_accuracy_value
                        })
                    train_log.add_summary(summary_line, global_step=step)

                    # train accuracy
                    train_accuracy_value = sess.run(
                        train_accuracy, feed_dict={keep_prob_: 1.0})
                    summary_line = sess.run(
                        accuracy_summary,
                        feed_dict={accuracy_value_: train_accuracy_value})
                    train_log.add_summary(summary_line, global_step=step)

                    print(('{}: train accuracy = {:.3f}\n'
                           'validation accuracy = {:.3f}\n'
                           'keep_prob = {:.2f}').format(datetime.now(
                           ), train_accuracy_value, validation_accuracy_value,
                                                        keep_prob))
                    # save best model
                    if validation_accuracy_value > best_validation_accuracy:
                        best_validation_accuracy = validation_accuracy_value
                        # fixed global_step, the best model is only one
                        best_saver.save(
                            sess,
                            os.path.join(BEST_MODEL_DIR, 'model.ckpt'),
                            global_step=0)

def main():
    """main function"""
    DATASET.maybe_download_and_extract()
    if tf.gfile.Exists(LOG_DIR):
        tf.gfile.DeleteRecursively(LOG_DIR)
    tf.gfile.MakeDirs(LOG_DIR)
    if not tf.gfile.Exists(BEST_MODEL_DIR):
        tf.gfile.MakeDirs(BEST_MODEL_DIR)
    train()
    return 0

if __name__ == '__main__':
    # CLI arguments
    PARSER = argparse.ArgumentParser(description="Train the model")
    PARSER.add_argument("--model", required=True)
    PARSER.add_argument("--dataset", required=True)
    ARGS = PARSER.parse_args()

    # Load required model and dataset
    MODEL = importlib.import_module("models." + ARGS.model)
    DATASET = importlib.import_module("inputs." + ARGS.dataset)

    # Training constants
    BATCH_SIZE = 128
    STEPS_PER_EPOCH = math.ceil(DATASET.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN /
                                BATCH_SIZE)
    MAX_EPOCH = 300
    MAX_STEPS = STEPS_PER_EPOCH * MAX_EPOCH

    MOMENTUM = 0.9

    # Learning rate decay constants
    INITIAL_LEARNING_RATE = 1e-2
    NUM_EPOCHS_PER_DECAY = 25  # Epochs after which learning rate decays.
    LEARNING_RATE_DECAY_FACTOR = 0.1  # Learning rate decay factor.

    CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
    LOG_DIR = os.path.join(CURRENT_DIR, 'log', MODEL.NAME, 'kp_decay')
    BEST_MODEL_DIR = os.path.join(LOG_DIR, 'best')

    MAX_KEEP_PROB = 1.0

    sys.exit(main())
<commit_msg>train_kp_decay.py: fix decay steps number<commit_after>#Copyright (C) 2016 Paolo Galeone <nessuno@nerdz.eu>
# Based on Tensorflow cifar10_train.py file
# https://github.com/tensorflow/tensorflow/blob/r0.11/tensorflow/models/image/cifar10/cifar10_train.py
#
#This Source Code Form is subject to the terms of the Mozilla Public
#License, v. 2.0. If a copy of the MPL was not distributed with this
#file, you can obtain one at http://mozilla.org/MPL/2.0/.
#Exhibit B is not attached; this software is compatible with the
#licenses expressed under Section 1.12 of the MPL v2.
""" Train model with a single GPU. Evaluate it on the second one"""

import argparse
import importlib
import sys
from datetime import datetime
import os.path
import time
import math

import numpy as np
import tensorflow as tf
import evaluate
from models import utils
from decay import supervised_parameter_decay

def train():
    """Train model"""
    with tf.Graph().as_default(), tf.device('/gpu:0'):
        global_step = tf.Variable(0, trainable=False)

        # Get images and labels for CIFAR-10.
        images, labels = DATASET.distorted_inputs(BATCH_SIZE)

        # Build a Graph that computes the logits predictions from the
        # inference model.
        keep_prob_, logits = MODEL.get_model(
            images, DATASET.NUM_CLASSES, train_phase=True)

        # Calculate loss.
        loss = MODEL.loss(logits, labels)

        # Decay the learning rate exponentially based on the number of steps.
        learning_rate = tf.train.exponential_decay(
            INITIAL_LEARNING_RATE,
            global_step,
            STEPS_PER_DECAY,
            LEARNING_RATE_DECAY_FACTOR,
            staircase=True)

        utils.log(tf.scalar_summary('learning_rate', learning_rate))
        train_op = tf.train.MomentumOptimizer(learning_rate, MOMENTUM).minimize(
            loss, global_step=global_step)

        # Create the train saver.
        variables_to_save = tf.trainable_variables() + [global_step]
        train_saver = tf.train.Saver(variables_to_save)
        # Create the best model saver.
        best_saver = tf.train.Saver(variables_to_save)

        # Train accuracy ops
        top_k_op = tf.nn.in_top_k(logits, labels, 1)
        train_accuracy = tf.reduce_mean(tf.cast(top_k_op, tf.float32))
        # General validation summary
        accuracy_value_ = tf.placeholder(tf.float32, shape=())
        accuracy_summary = tf.scalar_summary('accuracy', accuracy_value_)

        # Initialize decay_keep_prob op
        # va placeholder required for supervised_parameter_decay
        validation_accuracy_ = tf.placeholder(
            tf.float32, shape=(), name="validation_accuracy_")
        decay_keep_prob = supervised_parameter_decay(
            validation_accuracy_, initial_parameter_value=MAX_KEEP_PROB)
        keep_prob_summary = tf.scalar_summary('keep_prob', decay_keep_prob)

        # read collection after keep_prob_decay that adds
        # the keep_prob summary
        train_summaries = tf.merge_summary(
            tf.get_collection_ref('train_summaries'))

        # Build an initialization operation to run below.
        init = tf.initialize_all_variables()

        # Start running operations on the Graph.
        with tf.Session(config=tf.ConfigProto(
                allow_soft_placement=True)) as sess:
            sess.run(init)

            # Start the queue runners.
            tf.train.start_queue_runners(sess=sess)
            train_log = tf.train.SummaryWriter(LOG_DIR + "/train", sess.graph)
            validation_log = tf.train.SummaryWriter(LOG_DIR + "/validation",
                                                    sess.graph)

            # Extract previous global step value
            old_gs = sess.run(global_step)

            # set initial keep_prob
            keep_prob = MAX_KEEP_PROB
            # set best_validation_accuracy, used by best_saver
            best_validation_accuracy = 0.0

            # Restart from where we were
            for step in range(old_gs, MAX_STEPS):
                start_time = time.time()
                _, loss_value, summary_lines = sess.run(
                    [train_op, loss, train_summaries],
                    feed_dict={keep_prob_: keep_prob})
                duration = time.time() - start_time

                assert not np.isnan(
                    loss_value), 'Model diverged with loss = NaN'

                # update logs every 10 iterations
                if step % 10 == 0:
                    num_examples_per_step = BATCH_SIZE
                    examples_per_sec = num_examples_per_step / duration
                    sec_per_batch = float(duration)

                    format_str = ('{}: step {}, loss = {:.2f} '
                                  '({:.1f} examples/sec; {:.3f} sec/batch)')
                    print(
                        format_str.format(datetime.now(), step, loss_value,
                                          examples_per_sec, sec_per_batch))
                    # log train values
                    train_log.add_summary(summary_lines, global_step=step)

                # Save the model checkpoint at the end of every epoch
                # evaluate train and validation performance
                if (step > 0 and
                        step % STEPS_PER_EPOCH == 0) or (step + 1) == MAX_STEPS:
                    checkpoint_path = os.path.join(LOG_DIR, 'model.ckpt')
                    train_saver.save(sess, checkpoint_path, global_step=step)

                    # validation accuracy
                    validation_accuracy_value = evaluate.get_validation_accuracy(
                        LOG_DIR, DATASET.NUM_CLASSES)
                    summary_line = sess.run(
                        accuracy_summary,
                        feed_dict={accuracy_value_: validation_accuracy_value})
                    validation_log.add_summary(summary_line, global_step=step)

                    # update keep_prob using new validation accuracy
                    keep_prob, summary_line = sess.run(
                        [decay_keep_prob, keep_prob_summary],
                        feed_dict={
                            validation_accuracy_: validation_accuracy_value
                        })
                    train_log.add_summary(summary_line, global_step=step)

                    # train accuracy
                    train_accuracy_value = sess.run(
                        train_accuracy, feed_dict={keep_prob_: 1.0})
                    summary_line = sess.run(
                        accuracy_summary,
                        feed_dict={accuracy_value_: train_accuracy_value})
                    train_log.add_summary(summary_line, global_step=step)

                    print(('{}: train accuracy = {:.3f}\n'
                           'validation accuracy = {:.3f}\n'
                           'keep_prob = {:.2f}').format(datetime.now(
                           ), train_accuracy_value, validation_accuracy_value,
                                                        keep_prob))
                    # save best model
                    if validation_accuracy_value > best_validation_accuracy:
                        best_validation_accuracy = validation_accuracy_value
                        # fixed global_step, the best model is only one
                        best_saver.save(
                            sess,
                            os.path.join(BEST_MODEL_DIR, 'model.ckpt'),
                            global_step=0)

def main():
    """main function"""
    DATASET.maybe_download_and_extract()
    if tf.gfile.Exists(LOG_DIR):
        tf.gfile.DeleteRecursively(LOG_DIR)
    tf.gfile.MakeDirs(LOG_DIR)
    if not tf.gfile.Exists(BEST_MODEL_DIR):
        tf.gfile.MakeDirs(BEST_MODEL_DIR)
    train()
    return 0

if __name__ == '__main__':
    # CLI arguments
    PARSER = argparse.ArgumentParser(description="Train the model")
    PARSER.add_argument("--model", required=True)
    PARSER.add_argument("--dataset", required=True)
    ARGS = PARSER.parse_args()

    # Load required model and dataset
    MODEL = importlib.import_module("models." + ARGS.model)
    DATASET = importlib.import_module("inputs." + ARGS.dataset)

    # Training constants
    BATCH_SIZE = 128
    STEPS_PER_EPOCH = math.ceil(DATASET.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN /
                                BATCH_SIZE)
    MAX_EPOCH = 300
    MAX_STEPS = STEPS_PER_EPOCH * MAX_EPOCH

    MOMENTUM = 0.9

    # Learning rate decay constants
    INITIAL_LEARNING_RATE = 1e-2
    NUM_EPOCHS_PER_DECAY = 25  # Epochs after which learning rate decays.
    LEARNING_RATE_DECAY_FACTOR = 0.1  # Learning rate decay factor.
    STEPS_PER_DECAY = STEPS_PER_EPOCH * NUM_EPOCHS_PER_DECAY

    CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
    LOG_DIR = os.path.join(CURRENT_DIR, 'log', MODEL.NAME, 'kp_decay')
    BEST_MODEL_DIR = os.path.join(LOG_DIR, 'best')

    MAX_KEEP_PROB = 1.0

    sys.exit(main())
<|endoftext|>