<commit_before>###
# Copyright (c) 2002-2004, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

"""
The Lookup plugin handles looking up various values by their key.
"""

__revision__ = "$Id$"

import supybot.plugins as plugins

import os
import re
import sys
import sets
import getopt
import string

import supybot.conf as conf
import supybot.utils as utils
import supybot.privmsgs as privmsgs
import supybot.registry as registry
import supybot.callbacks as callbacks

try:
    import sqlite
except ImportError:
    raise callbacks.Error, 'You need to have PySQLite installed to use this ' \
                           'plugin.  Download it at <http://pysqlite.sf.net/>'

def configure(advanced):
    from supybot.questions import output, expect, anything, something, yn
    conf.registerPlugin('Lookup', True)
    lookups = conf.supybot.plugins.Lookup.lookups
    output("""This module allows you to define commands that do a simple key
              lookup and return some simple value.  It has a command "add"
              that takes a command name and a file from the data dir and adds a
              command with that name that responds with the mapping from that
              file. The file itself should be composed of lines
              of the form key:value.""")
    while yn('Would you like to add a file?'):
        filename = something('What\'s the filename?')
        try:
            fd = file(filename)
        except EnvironmentError, e:
            output('I couldn\'t open that file: %s' % e)
            continue
        counter = 1
        try:
            for line in fd:
                line = line.rstrip('\r\n')
                if not line or line.startswith('#'):
                    continue
                (key, value) = line.split(':', 1)
                counter += 1
        except ValueError:
            output('That\'s not a valid file; '
                   'line #%s is malformed.' % counter)
            continue
        command = something('What would you like the command to be?')
        conf.registerGlobalValue(lookups,command, registry.String(filename,''))

conf.registerPlugin('Lookup')
conf.registerGroup(conf.supybot.plugins.Lookup, 'lookups')

class LookupDB(plugins.DBHandler):
    def makeDb(self, filename):
        return sqlite.connect(filename)

class Lookup(callbacks.Privmsg):
    def __init__(self):
        callbacks.Privmsg.__init__(self)
        self.lookupDomains = sets.Set()
        dataDir = conf.supybot.directories.data()
        self.dbHandler = LookupDB(name=os.path.join(dataDir, 'Lookup'))
        for (name, value) in registry._cache.iteritems():
            name = name.lower()
            if name.startswith('supybot.plugins.lookup.lookups.'):
                name = name[len('supybot.plugins.lookup.lookups.'):]
                if '.' in name:
                    continue
                self.addRegistryValue(name, value)
        group = conf.supybot.plugins.Lookup.lookups
        for (name, value) in group.getValues(fullNames=False):
            name = name.lower() # Just in case.
            filename = value()
            try:
                self.addDatabase(name, filename)
                self.addCommand(name)
            except Exception, e:
                self.log.warning('Couldn\'t add lookup %s: %s', name, e)

    def _shrink(self, s):
        return utils.ellipsisify(s, 50)

    def die(self):
        self.dbHandler.die()

    def remove(self, irc, msg, args):
        """<name>

        Removes the lookup for <name>.
        """
        name = privmsgs.getArgs(args)
        name = callbacks.canonicalName(name)
        if name not in self.lookupDomains:
            irc.error('That\'s not a valid lookup to remove.')
            return
        db = self.dbHandler.getDb()
        cursor = db.cursor()
        try:
            cursor.execute("""DROP TABLE %s""" % name)
            db.commit()
            delattr(self.__class__, name)
            irc.replySuccess()
        except sqlite.DatabaseError:
            irc.error('No such lookup exists.')
    remove = privmsgs.checkCapability(remove, 'admin')

    _splitRe = re.compile(r'(?<!\\):')
    def add(self, irc, msg, args):
        """<name> <filename>

        Adds a lookup for <name> with the key/value pairs specified in the
        colon-delimited file specified by <filename>.  <filename> is searched
        for in conf.supybot.directories.data.  If <name> is not singular, we
        try to make it singular before creating the command.
        """
        (name, filename) = privmsgs.getArgs(args, required=2)
        name = utils.depluralize(name)
        name = callbacks.canonicalName(name)
        if hasattr(self, name):
            s = 'I already have a command in this plugin named %s' % name
            irc.error(s)
            return
        db = self.dbHandler.getDb()
        cursor = db.cursor()
        try:
            cursor.execute("""SELECT * FROM %s LIMIT 1""" % name)
            self.addCommand(name)
        except sqlite.DatabaseError:
            try:
                self.addDatabase(name, filename)
            except EnvironmentError, e:
                irc.error('Could not open %s: %s' % (filename, e.args[1]))
                return
            self.addCommand(name)
        self.addRegistryValue(name, filename)
        irc.replySuccess('Lookup %s added.' % name)
    add = privmsgs.checkCapability(add, 'admin')

    def addRegistryValue(self, name, filename):
        v = registry.String(filename, '')
        conf.supybot.plugins.Lookup.lookups.register(name, v)

    def addDatabase(self, name, filename):
        db = self.dbHandler.getDb()
        cursor = db.cursor()
        dataDir = conf.supybot.directories.data()
        filename = os.path.join(dataDir, filename)
        fd = file(filename)
        try:
            cursor.execute("""SELECT COUNT(*) FROM %s""" % name)
        except sqlite.DatabaseError:
            cursor.execute("CREATE TABLE %s (key TEXT, value TEXT)" % name)
            sql = "INSERT INTO %s VALUES (%%s, %%s)" % name
            for line in utils.nonCommentNonEmptyLines(fd):
                line = line.rstrip('\r\n')
                try:
                    (key, value) = self._splitRe.split(line, 1)
                    key = key.replace('\\:', ':')
                except ValueError:
                    cursor.execute("""DROP TABLE %s""" % name)
                    s = 'Invalid line in %s: %r' % (filename, line)
                    raise callbacks.Error, s
                cursor.execute(sql, key, value)
            cursor.execute("CREATE INDEX %s_keys ON %s (key)" % (name, name))
            db.commit()

    def addCommand(self, name):
        def f(self, irc, msg, args):
            args.insert(0, name)
            self._lookup(irc, msg, args)
        db = self.dbHandler.getDb()
        cursor = db.cursor()
        cursor.execute("""SELECT COUNT(*) FROM %s""" % name)
        rows = int(cursor.fetchone()[0])
        docstring = """[<key>]

        If <key> is given, looks up <key> in the %s database.  Otherwise,
        returns a random key: value pair from the database.  There are
        %s in the database.
        """ % (name, utils.nItems(name, rows))
        f = utils.changeFunctionName(f, name, docstring)
        self.lookupDomains.add(name)
        setattr(self.__class__, name, f)

    _sqlTrans = string.maketrans('*?', '%_')
    def search(self, irc, msg, args):
        """[--{regexp}=<value>] [--values] <name> <glob>

        Searches the domain <name> for lookups matching <glob>.  If --regexp
        is given, its associated value is taken as a regexp and matched
        against the lookups.  If --values is given, search the values rather
        than the keys.
        """
        column = 'key'
        while '--values' in args:
            column = 'value'
            args.remove('--values')
        (options, rest) = getopt.getopt(args, '', ['regexp='])
        (name, globs) = privmsgs.getArgs(rest, optional=1)
        db = self.dbHandler.getDb()
        criteria = []
        formats = []
        predicateName = 'p'
        for (option, arg) in options:
            if option == '--regexp':
                criteria.append('%s(%s)' % (predicateName, column))
                try:
                    r = utils.perlReToPythonRe(arg)
                except ValueError, e:
                    irc.error('%r is not a valid regular expression' %
                              arg)
                    return
                def p(s, r=r):
                    return int(bool(r.search(s)))
                db.create_function(predicateName, 1, p)
                predicateName += 'p'
        for glob in globs.split():
            if '?' not in glob and '*' not in glob:
                glob = '*%s*' % glob
            criteria.append('%s LIKE %%s' % column)
            formats.append(glob.translate(self._sqlTrans))
        if not criteria:
            raise callbacks.ArgumentError
        #print 'criteria: %s' % repr(criteria)
        #print 'formats: %s' % repr(formats)
        cursor = db.cursor()
        sql = """SELECT key, value FROM %s WHERE %s""" % \
              (name, ' AND '.join(criteria))
        #print 'sql: %s' % sql
        cursor.execute(sql, formats)
        if cursor.rowcount == 0:
            irc.reply('No entries in %s matched that query.' % name)
        else:
            lookups = ['%s: %s' % (item[0], self._shrink(item[1]))
                       for item in cursor.fetchall()]
            irc.reply(utils.commaAndify(lookups))

    def _lookup(self, irc, msg, args):
        """<name> <key>

        Looks up the value of <key> in the domain <name>.
        """
        (name, key) = privmsgs.getArgs(args, optional=1)
        db = self.dbHandler.getDb()
        cursor = db.cursor()
        if key:
            sql = """SELECT value FROM %s WHERE key LIKE %%s""" % name
            try:
                cursor.execute(sql, key)
            except sqlite.DatabaseError, e:
                if 'no such table' in str(e):
                    irc.error('I don\'t have a domain %s' % name)
                else:
                    irc.error(str(e))
                return
            if cursor.rowcount == 0:
                irc.error('I couldn\'t find %s in %s.' % (key, name))
            elif cursor.rowcount == 1:
                irc.reply(cursor.fetchone()[0])
            else:
                values = [t[0] for t in cursor.fetchall()]
                irc.reply('%s could be %s' % (key, ', or '.join(values)))
        else:
            sql = """SELECT key, value FROM %s
                     ORDER BY random() LIMIT 1""" % name
            try:
                cursor.execute(sql)
            except sqlite.DatabaseError, e:
                if 'no such table' in str(e):
                    irc.error('I don\'t have a domain %r' % name)
                else:
                    irc.error(str(e))
                return
            (key, value) = cursor.fetchone()
            irc.reply('%s: %s' % (key, value))

Class = Lookup

# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
<commit_msg>RFE 871111 : Added --nokey option Fixed a bug in the remove command where lookup specific registry values were not being removed along with the lookup Database abstraction<commit_after>###
# Copyright (c) 2002-2004, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

"""
The Lookup plugin handles looking up various values by their key.
"""

import supybot

__revision__ = "$Id$"
__contributors__ = {
    supybot.authors.skorobeus: ['--nokey parameter', 'database abstraction'],
    }

import supybot.plugins as plugins

import os
import re
import sys
import sets
import getopt
import string

import supybot.dbi as dbi
import supybot.conf as conf
import supybot.utils as utils
import supybot.privmsgs as privmsgs
import supybot.registry as registry
import supybot.callbacks as callbacks

try:
    import sqlite
except ImportError:
    raise callbacks.Error, 'You need to have PySQLite installed to use this ' \
                           'plugin.  Download it at <http://pysqlite.sf.net/>'

def configure(advanced):
    from supybot.questions import output, expect, anything, something, yn
    conf.registerPlugin('Lookup', True)
    lookups = conf.supybot.plugins.Lookup.lookups
    output("""This module allows you to define commands that do a simple key
              lookup and return some simple value.  It has a command "add"
              that takes a command name and a file from the data dir and adds a
              command with that name that responds with the mapping from that
              file. The file itself should be composed of lines
              of the form key:value.""")
    while yn('Would you like to add a file?'):
        filename = something('What\'s the filename?')
        try:
            fd = file(filename)
        except EnvironmentError, e:
            output('I couldn\'t open that file: %s' % e)
            continue
        counter = 1
        try:
            for line in fd:
                line = line.rstrip('\r\n')
                if not line or line.startswith('#'):
                    continue
                (key, value) = line.split(':', 1)
                counter += 1
        except ValueError:
            output('That\'s not a valid file; '
                   'line #%s is malformed.' % counter)
            continue
        command = something('What would you like the command to be?')
        conf.registerGlobalValue(lookups,command, registry.String(filename,''))
        nokeyVal = yn('Would you like the key to be shown for random \
                        responses?')
        conf.registerGlobalValue(lookups.get(command), 'nokey',
                                    registry.Boolean(nokeyVal, ''))

conf.registerPlugin('Lookup')
conf.registerGroup(conf.supybot.plugins.Lookup, 'lookups')

class SqliteLookupDB(object):
    def __init__(self, filename):
        try:
            import sqlite
        except ImportError:
            raise callbacks.Error, 'You need to have PySQLite installed to '\
                                   'use this plugin.  Download it at '\
                                   '<http://pysqlite.sf.net/>'
        self.filename = filename
        try:
            self.db = sqlite.connect(self.filename)
        except sqlite.DatabaseError, e:
            raise dbi.InvalidDBError, str(e)

    def close(self):
        self.db.close()

    def getRecordCount(self, tableName):
        cursor = self.db.cursor()
        cursor.execute("""SELECT COUNT(*) FROM %s""" % tableName)
        rows = int(cursor.fetchone()[0])
        if rows == 0:
            raise dbi.NoRecordError
        return rows

    def checkLookup(self, name):
        cursor = self.db.cursor()
        sql = "SELECT name FROM sqlite_master \
                WHERE type='table' \
                AND name='%s'" % name
        cursor.execute(sql)
        if cursor.rowcount == 0:
            return False
        else:
            return True

    def addLookup(self, name, fd, splitRe):
        cursor = self.db.cursor()
        cursor.execute("CREATE TABLE %s (key TEXT, value TEXT)" % name)
        sql = "INSERT INTO %s VALUES (%%s, %%s)" % name
        for line in utils.nonCommentNonEmptyLines(fd):
            line = line.rstrip('\r\n')
            try:
                (key, value) = splitRe.split(line, 1)
                key = key.replace('\\:', ':')
            except ValueError:
                cursor.execute("""DROP TABLE %s""" % name)
                s = 'Invalid line in %s: %r' % (filename, line)
                raise callbacks.Error, s
            cursor.execute(sql, key, value)
        cursor.execute("CREATE INDEX %s_keys ON %s (key)" % (name, name))
        self.db.commit()

    def dropLookup(self, name):
        cursor = self.db.cursor()
        if self.checkLookup(name):
            cursor.execute("""DROP TABLE %s""" % name)
            self.db.commit()
        else:
            raise dbi.NoRecordError

    def getResults(self, name, key):
        cursor = self.db.cursor()
        sql = """SELECT value FROM %s WHERE key LIKE %%s""" % name
        cursor.execute(sql, key)
        if cursor.rowcount == 0:
            raise dbi.NoRecordError
        else:
            return cursor.fetchall()

    def getRandomResult(self, name, key):
        cursor = self.db.cursor()
        sql = """SELECT key, value FROM %s
                 ORDER BY random() LIMIT 1""" % name
        cursor.execute(sql)
        if cursor.rowcount == 0:
            raise dbi.NoRecordError
        else:
            return cursor.fetchone()

    _sqlTrans = string.maketrans('*?', '%_')
    def searchResults(self, name, options, globs, column):
        cursor = self.db.cursor()
        criteria = []
        formats = []
        predicateName = 'p'
        for (option, arg) in options:
            if option == '--regexp':
                criteria.append('%s(%s)' % (predicateName, column))
                try:
                    r = utils.perlReToPythonRe(arg)
                except ValueError, e:
                    irc.error('%r is not a valid regular expression' %
                              arg)
                    return
                def p(s, r=r):
                    return int(bool(r.search(s)))
                self.db.create_function(predicateName, 1, p)
                predicateName += 'p'
        for glob in globs.split():
            if '?' not in glob and '*' not in glob:
                glob = '*%s*' % glob
            criteria.append('%s LIKE %%s' % column)
            formats.append(glob.translate(self._sqlTrans))
        if not criteria:
            raise callbacks.ArgumentError
        #print 'criteria: %s' % repr(criteria)
        #print 'formats: %s' % repr(formats)
        sql = """SELECT key, value FROM %s WHERE %s""" % \
              (name, ' AND '.join(criteria))
        #print 'sql: %s' % sql
        cursor.execute(sql, formats)
        if cursor.rowcount == 0:
            raise dbi.NoRecordError
        else:
            return cursor.fetchall()

LookupDB = plugins.DB('Lookup',
                        {'sqlite': SqliteLookupDB,
                        }
                     )

class Lookup(callbacks.Privmsg):
    def __init__(self):
        callbacks.Privmsg.__init__(self)
        self.lookupDomains = sets.Set()
        try:
            self.db = LookupDB()
        except Exception:
            self.log.exception('Error loading %s:', self.filename)
            raise # So it doesn't get loaded without its database.
        for (name, value) in registry._cache.iteritems():
            name = name.lower()
            if name.startswith('supybot.plugins.lookup.lookups.'):
                name = name[len('supybot.plugins.lookup.lookups.'):]
                if '.' in name:
                    continue
                self.addRegistryValue(name, value)
        group = conf.supybot.plugins.Lookup.lookups
        for (name, value) in group.getValues(fullNames=False):
            name = name.lower() # Just in case.
            filename = value()
            try:
                if not self.db.checkLookup(name):
                    self.addDatabase(name, filename)
                self.addCommand(name)
            except Exception, e:
                self.log.warning('Couldn\'t add lookup %s: %s', name, e)

    def _shrink(self, s):
        return utils.ellipsisify(s, 50)

    def die(self):
        self.db.close()

    def remove(self, irc, msg, args):
        """<name>

        Removes the lookup for <name>.
        """
        name = privmsgs.getArgs(args)
        name = callbacks.canonicalName(name)
        if name not in self.lookupDomains:
            irc.error('That\'s not a valid lookup to remove.')
            return
        try:
            self.db.dropLookup(name)
            delattr(self.__class__, name)
            self.delRegistryValues(name)
            irc.replySuccess()
        except dbi.NoRecordError:
            irc.error('No such lookup exists.')
    remove = privmsgs.checkCapability(remove, 'admin')

    _splitRe = re.compile(r'(?<!\\):')
    def add(self, irc, msg, args):
        """[--nokey] <name> <filename>

        Adds a lookup for <name> with the key/value pairs specified in the
        colon-delimited file specified by <filename>.  <filename> is searched
        for in conf.supybot.directories.data.  If <name> is not singular, we
        try to make it singular before creating the command.  If the --nokey
        option is specified, the new lookup will display only the value when
        queried, and will omit the key from the response.
        """
        opts = ['nokey']
        (optlist, rest) = getopt.getopt(args, '', opts)
        (name, filename) = privmsgs.getArgs(rest, required=2)
        nokey = False
        for (option, argument) in optlist:
            option = option.lstrip('-')
            if option == 'nokey':
                nokey = True
        #print 'nokey: %s' % nokey
        name = utils.depluralize(name)
        name = callbacks.canonicalName(name)
        if hasattr(self, name):
            s = 'I already have a command in this plugin named %s' % name
            irc.error(s)
            return
        if not self.db.checkLookup(name):
            try:
                self.addDatabase(name, filename)
            except EnvironmentError, e:
                irc.error('Could not open %s: %s' % (filename, e.args[1]))
                return
        self.addCommand(name)
        self.addRegistryValue(name, filename, nokey)
        irc.replySuccess('Lookup %s added.' % name)
    add = privmsgs.checkCapability(add, 'admin')

    def addRegistryValue(self, name, filename, nokey = False):
        group = conf.supybot.plugins.Lookup.lookups
        conf.registerGlobalValue(group, name, registry.String(filename, ''))
        #print 'nokey: %s' % nokey
        conf.registerGlobalValue(group.get(name), 'nokey', registry.Boolean(nokey, ''))

    def delRegistryValues(self, name):
        group = conf.supybot.plugins.Lookup.lookups
        group.unregister(name)

    def addDatabase(self, name, filename):
        dataDir = conf.supybot.directories.data()
        filename = os.path.join(dataDir, filename)
        fd = file(filename)
        self.db.addLookup(name, fd, self._splitRe)

    def addCommand(self, name):
        def f(self, irc, msg, args):
            args.insert(0, name)
            self._lookup(irc, msg, args)
        rows = self.db.getRecordCount(name)
        docstring = """[<key>]

        If <key> is given, looks up <key> in the %s database.  Otherwise,
        returns a random key: value pair from the database.  There are
        %s in the database.
        """ % (name, utils.nItems(name, rows))
        f = utils.changeFunctionName(f, name, docstring)
        self.lookupDomains.add(name)
        setattr(self.__class__, name, f)

    def search(self, irc, msg, args):
        """[--{regexp}=<value>] [--values] <name> <glob>

        Searches the domain <name> for lookups matching <glob>.  If --regexp
        is given, its associated value is taken as a regexp and matched
        against the lookups.  If --values is given, search the values rather
        than the keys.
        """
        column = 'key'
        while '--values' in args:
            column = 'value'
            args.remove('--values')
        (options, rest) = getopt.getopt(args, '', ['regexp='])
        (name, globs) = privmsgs.getArgs(rest, optional=1)
        try:
            results = self.db.searchResults(name, options, globs, column)
        except dbi.NoRecordError:
            irc.reply('No entries in %s matched that query.' % name)
        lookups = ['%s: %s' % (item[0], self._shrink(item[1]))
                   for item in results]
        irc.reply(utils.commaAndify(lookups))

    def _lookup(self, irc, msg, args):
        """<name> <key>

        Looks up the value of <key> in the domain <name>.
        """
        (name, key) = privmsgs.getArgs(args, optional=1)
        if self.db.checkLookup(name):
            results = []
            if key:
                try:
                    results = self.db.getResults(name, key)
                except dbi.NoRecordError:
                    irc.error('I couldn\'t find %s in %s.' % (key, name))
                    return
                if len(results) == 1:
                    irc.reply(results[0][0])
                else:
                    values = [t[0] for t in results]
                    irc.reply('%s could be %s' % (key, ', or '.join(values)))
            else:
                (key, value) = self.db.getRandomResult(name, key)
                nokeyRegKey = 'lookups.%s.nokey' % name
                if not self.registryValue(nokeyRegKey):
                    irc.reply('%s: %s' % (key, value))
                else:
                    irc.reply('%s' % value)
        else:
            irc.error('I don\'t have a domain %s' % name)
            return

Class = Lookup

# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
<|endoftext|>


<commit_before>""" Read Manatee files"""
from __future__ import print_function

from collections import defaultdict
import os

import mirtop.libs.logger as mylog
from mirtop.bam.bam import intersect
from mirtop.bam import filter
from mirtop.mirna.realign import isomir, reverse_complement, make_id, hits
from mirtop.gff.body import paste_columns, variant_with_nt
# from mirtop.mirna import mapper
from mirtop.gff.classgff import feature
from mirtop.gff import body
from mirtop.mirna.annotate import annotate

logger = mylog.getLogger(__name__)

def read_file(fn, database, args):
    """
    Read Manatee file and convert to mirtop GFF format.

    Args:
        *fn(str)*: file name with Manatee output information.

        *database(str)*: database name.

        *args(namedtuple)*: arguments from command line.
            See *mirtop.libs.parse.add_subparser_gff()*.

    Returns:
        *reads (nested dicts)*:gff_list has the format as
            defined in *mirtop.gff.body.read()*.

    """
    reads = defaultdict(dict)
    sample = os.path.splitext(os.path.basename(fn))[0]
    precursors = args.precursors
    bed_fn = os.path.join(args.out, os.path.basename(fn) + ".bed")
    sep = " " if args.out_format == "gtf" else "="
    seen = set()
    with open(fn, 'r') as handle:
        _bed(handle, bed_fn)
    intersect_fn = intersect(bed_fn, args.gtf)
    for line in intersect_fn:
        data = _analyze_line(line, precursors, database, sample, sep, args)
        if data:
            start = data["start"]
            chrom = data["chrom"]
            key = "%s:%s:%s" % (chrom, start, data["line"][0])
            if start not in reads[chrom]:
                reads[chrom][start] = []
            if key not in seen:
                seen.add(key)
                reads[chrom][start].append(data["line"])
    return reads

def _analyze_line(line, precursors, database, sample, sep, args):
    start_idx = 10
    end_idx = 11
    attr_idx = 15
    if str(line).find("miRNA_primary_transcript") < 0: # only working with mirbase
        return None

    query_name = line[3]
    sequence = line[4]
    logger.debug(("READ::line name:{0}").format(line))
    if sequence and sequence.find("N") > -1:
        return None

    chrom = line[attr_idx].strip().split("Name=")[-1]
    start = line[1]
    end = line[2]
    strand = line[5]
    counts = float(line[6])
    Filter = "Pass"
    reads = dict()
    if not start:
        return None
    if strand == "+":
        start = int(start) - int(line[start_idx]) + 1
    else:
        start = int(line[end_idx]) - int(end)
    iso = isomir()
    iso.align = line
    iso.set_pos(start, len(sequence))
    logger.debug("READ::From BAM start %s end %s at chrom %s" % (iso.start, iso.end, chrom))
    if len(precursors[chrom]) < start + len(sequence):
        logger.debug("READ::%s start + %s sequence size are bigger than"
                     " size precursor %s" % (
                                             chrom,
                                             len(sequence),
                                             len(precursors[chrom])))
    iso.subs, iso.add, iso.cigar = filter.tune(
        sequence, precursors[chrom],
        start, None)
    logger.debug("READ::After tune start %s end %s" % (iso.start, iso.end))
    logger.debug("READ::iso add %s iso subs %s" % (iso.add, iso.subs))

    idu = make_id(sequence)
    reads[query_name] = hits()
    reads[query_name].set_sequence(sequence)
    reads[query_name].counts = counts
    reads[query_name].sequence = sequence
    reads[query_name].set_precursor(chrom, iso)
    reads = annotate(reads, args.matures, args.precursors, quiet=False)
    gff_line = body.create(reads, args.database, sample, args, quiet=False)
    line = gff_line[chrom][start][0][4]
    logger.debug("READ::line:%s" % line)
    if args.add_extra:
        extra = variant_with_nt(line, args.precursors,
                                args.matures)
        line = "%s Changes %s;" % (line, extra)

    line = paste_columns(feature(line), sep=sep)
    return {'chrom': chrom,
            'start': start,
            'line': [idu, chrom, counts, sample, line]}

def _bed(handle, bed_fn):
    with open(bed_fn, 'w') as outh:
        for line in handle:
            if line.startswith("@"):
                continue
            cols = line.strip().split()
            if cols[2]=="*":
                logger.debug("READ::Sequence not mapped: %s" % cols[0])
                continue
            query_name = cols[0]
            query_sequence = cols[9]
            counts = cols[14]
            start = int(cols[3])
            strand = cols[1]
            chrom = cols[2]
            # is there no hits
            # is the sequence always matching the read, assuming YES now
            # if not current or query_name!=current:
            query_sequence = query_sequence if not strand=="-" else reverse_complement(query_sequence)
            # logger.debug(("READ::Read name:{0} and Read sequence:{1}").format(line.query_name, sequence))
            if query_sequence and query_sequence.find("N") > -1:
                continue
            end = start + len(query_sequence) - 1
            bed_line = "\t".join(map(str, [chrom, start, end, query_name,
                                           query_sequence, strand, counts]))
            outh.write(bed_line + '\n')
<commit_msg>keep bed file<commit_after>""" Read Manatee files"""
from __future__ import print_function

from collections import defaultdict
import os

import mirtop.libs.logger as mylog
from mirtop.bam.bam import intersect
from mirtop.bam import filter
from mirtop.mirna.realign import isomir, reverse_complement, make_id, hits
from mirtop.gff.body import paste_columns, variant_with_nt
# from mirtop.mirna import mapper
from mirtop.gff.classgff import feature
from mirtop.gff import body
from mirtop.mirna.annotate import annotate

logger = mylog.getLogger(__name__)

def read_file(fn, database, args):
    """
    Read Manatee file and convert to mirtop GFF format.

    Args:
        *fn(str)*: file name with Manatee output information.

        *database(str)*: database name.

        *args(namedtuple)*: arguments from command line.
            See *mirtop.libs.parse.add_subparser_gff()*.

    Returns:
        *reads (nested dicts)*:gff_list has the format as
            defined in *mirtop.gff.body.read()*.

    """
    reads = defaultdict(dict)
    sample = os.path.splitext(os.path.basename(fn))[0]
    precursors = args.precursors
    bed_fn = os.path.join(args.out, os.path.basename(fn) + ".bed")
    sep = " " if args.out_format == "gtf" else "="
    seen = set()
    with open(fn, 'r') as handle:
        if not os.path.exists(bed_fn):
            _bed(handle, bed_fn)
    intersect_fn = intersect(bed_fn, args.gtf)
    for line in intersect_fn:
        data = _analyze_line(line, precursors, database, sample, sep, args)
        if data:
            start = data["start"]
            chrom = data["chrom"]
            key = "%s:%s" % (data['mirna'], data["name"])
            if start not in reads[chrom]:
                reads[chrom][start] = []
            if key not in seen:
                seen.add(key)
                reads[chrom][start].append(data["line"])
    return reads

def _analyze_line(line, precursors, database, sample, sep, args):
    start_idx = 10
    end_idx = 11
    attr_idx = 15
    query_name = line[3]
    sequence = line[4]
    if str(line).find("miRNA_primary_transcript") < 0: # only working with mirbase
        return None

    logger.debug(("READ::line name:{0}").format(line))
    if sequence and sequence.find("N") > -1:
        return None

    chrom = line[attr_idx].strip().split("Name=")[-1]
    start = line[1]
    end = line[2]
    strand = line[5]
    counts = float(line[6])
    Filter = "Pass"
    reads = dict()
    if not start:
        return None
    if strand == "+":
        start = int(start) - int(line[start_idx]) + 1
    else:
        start = int(line[end_idx]) - int(end)
    iso = isomir()
    iso.align = line
    iso.set_pos(start, len(sequence))
    logger.debug("READ::From BAM start %s end %s at chrom %s" % (iso.start, iso.end, chrom))
    if len(precursors[chrom]) < start + len(sequence):
        logger.debug("READ::%s start + %s sequence size are bigger than"
                     " size precursor %s" % (
                                             chrom,
                                             len(sequence),
                                             len(precursors[chrom])))
    iso.subs, iso.add, iso.cigar = filter.tune(
        sequence, precursors[chrom],
        start, None)
    logger.debug("READ::After tune start %s end %s" % (iso.start, iso.end))
    logger.debug("READ::iso add %s iso subs %s" % (iso.add, iso.subs))

    idu = make_id(sequence)
    reads[query_name] = hits()
    reads[query_name].set_sequence(sequence)
    reads[query_name].counts = counts
    reads[query_name].sequence = sequence
    reads[query_name].set_precursor(chrom, iso)
    reads = annotate(reads, args.matures, args.precursors, quiet=True)
    gff_line = body.create(reads, args.database, sample, args, quiet=True)
    if start not in gff_line[chrom]:
        return None
    line = gff_line[chrom][start][0][4]
    logger.debug("READ::line:%s" % line)
    if args.add_extra:
        extra = variant_with_nt(line, args.precursors,
                                args.matures)
        line = "%s Changes %s;" % (line, extra)

    line = paste_columns(feature(line), sep=sep)
    return {'chrom': chrom,
            'start': start,
            'name': query_name,
            'mirna': reads[query_name].precursors[chrom].mirna,
            'line': [idu, chrom, counts, sample, line]}

def _bed(handle, bed_fn):
    with open(bed_fn, 'w') as outh:
        for line in handle:
            if line.startswith("@"):
                continue
            cols = line.strip().split()
            if cols[2]=="*":
                logger.debug("READ::Sequence not mapped: %s" % cols[0])
                continue
            query_name = cols[0]
            query_sequence = cols[9]
            counts = cols[14]
            start = int(cols[3])
            strand = cols[1]
            chrom = cols[2]
            # is there no hits
            # is the sequence always matching the read, assuming YES now
            # if not current or query_name!=current:
            query_sequence = query_sequence if not strand=="-" else reverse_complement(query_sequence)
            # logger.debug(("READ::Read name:{0} and Read sequence:{1}").format(line.query_name, sequence))
            if query_sequence and query_sequence.find("N") > -1:
                continue
            end = start + len(query_sequence) - 1
            bed_line = "\t".join(map(str, [chrom, start, end, query_name,
                                           query_sequence, strand, counts]))
            outh.write(bed_line + '\n')
<|endoftext|>


<commit_before>import logging

from django.conf import settings

from django_filters import rest_framework as djfilters

from rest_framework import viewsets, generics, filters, mixins
from rest_framework.decorators import list_route, detail_route
from rest_framework.response import Response
from rest_framework.pagination import PageNumberPagination
from rest_framework.permissions import AllowAny, IsAdminUser

from ..transcript.models import (
    Transcript, TranscriptPhrase, TranscriptPhraseVote,
    TranscriptPhraseCorrection,
    TranscriptPhraseCorrectionVote,
    Source, Topic,
)
from ..accounts.models import (
    Profile, Score, Leaderboard, ContributionStatistics
)
from ..game.models import LoadingScreenData
from .serializers import (
    TranscriptSerializer,
    TranscriptCreateSerializer,
    TranscriptStatsSerializer,
    TranscriptPhraseDetailSerializer,
    TranscriptPhraseVoteSerializer,
    TranscriptPhraseCorrectionSerializer,
    TranscriptPhraseCorrectionVoteSerializer,
    ProfileSerializer, ProfilePatchSerializer,
    ProfilePreferenceClearSerializer, ProfileTranscriptSkipSerializer,
    SourceSerializer, TopicSerializer,
    ScoreSerializer, LeaderboardSerializer,
    LoadingScreenSerializer, ContributionStatisticsSerializer
)
from .permissions import IsOwner, IsOwnerOrStaff, IsOwnerOrReadOnly

django_log = logging.getLogger('django')

phrase_positive_limit = settings.TRANSCRIPT_PHRASE_POSITIVE_CONFIDENCE_LIMIT
phrase_negative_limit = settings.TRANSCRIPT_PHRASE_NEGATIVE_CONFIDENCE_LIMIT
correction_lower_limit = settings.TRANSCRIPT_PHRASE_CORRECTION_LOWER_LIMIT
correction_upper_limit = settings.TRANSCRIPT_PHRASE_CORRECTION_UPPER_LIMIT

class StandardResultsSetPagination(PageNumberPagination):
    page_size = 100
    page_size_query_param = 'page_size'
    max_page_size = 1000

class TranscriptViewSet(mixins.ListModelMixin,
                        viewsets.GenericViewSet):
    permission_classes = (AllowAny,)
    queryset = Transcript.objects.all()
    lookup_field = 'asset_name'
    serializer_class = TranscriptSerializer

    @list_route()
    def random(self, request):
        transcript = Transcript.objects.random_transcript()
        serializer = self.get_serializer(transcript, many=True)
        return Response(serializer.data)

    @list_route()
    def game_one(self, request):
        transcripts, phrases = Transcript.objects.game_one(request.user)
        serializer = self.get_serializer(
            transcripts, many=True,
        )
        if phrases:
            django_log.info(phrases)
            for transcript in serializer.data:
                for phrase in transcript['phrases']:
                    if phrase['pk'] in phrases:
                        phrase['user_can_vote'] = False
                    else:
                        phrase['user_can_vote'] = True
        return Response(serializer.data)

    @list_route()
    def game_two(self, request):
        transcripts, phrases_for_correction = Transcript.objects.game_two(
            request.user
        )
        serializer = self.get_serializer(
            transcripts, many=True,
        )
        for transcript in serializer.data:
            for phrase in transcript['phrases']:
                if phrase['pk'] in phrases_for_correction:
                    phrase['needs_correction'] = True
                else:
                    phrase['needs_correction'] = False
        return Response(serializer.data)

    @list_route()
    def game_three(self, request):
        transcripts, corrected_phrases = Transcript.objects.game_three(
            request.user
        )
        serializer = self.get_serializer(
            transcripts, many=True
        )
        for transcript in serializer.data:
            for phrase in transcript['phrases']:
                for correction in corrected_phrases:
                    if phrase['pk'] in correction:
                        phrase['corrections'] = [
                            {
                                'pk': correction.pk,
                                'corrected_text': correction.correction
                            } for correction in correction[phrase['pk']]
                        ]
                        correction[phrase['pk']]

        return Response(serializer.data)

    @detail_route(methods=['get'])
    def corrected(self, request, pk=None, asset_name=None):
        '''Returns a corrected transcript'''
        transcript = self.get_object()
        highest_rated_corrections = []
        for phrase in transcript.phrases.all():
            if phrase.confidence >= phrase_positive_limit:
                pass
            else:
                corrections = TranscriptPhraseCorrection.objects.filter(
                    transcript_phrase=phrase,
                    confidence__gte=correction_lower_limit
                ).order_by('-confidence')
                if corrections.count() > 0:
                    highest_rated_corrections.append(corrections.first())
        serializer = self.get_serializer(transcript)
        for correction in highest_rated_corrections:
            for phrase in serializer.data['phrases']:
                if phrase['pk'] == correction.transcript_phrase.pk:
                    phrase['text'] = correction.correction
        return Response(serializer.data)

class TranscriptCreateViewset(mixins.CreateModelMixin,
                              viewsets.GenericViewSet):
    permission_classes = (IsAdminUser,)
    queryset = Transcript.objects.all()
    serializer_class = TranscriptCreateSerializer

class TranscriptStatsViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Transcript.objects.all()
    lookup_field = 'asset_name'
    serializer_class = TranscriptStatsSerializer
    filter_backends = (filters.OrderingFilter,
                       djfilters.DjangoFilterBackend)
    ordering_fields = ('complete', 'in_progress')
    filter_fields = ('complete', 'in_progress')

class TranscriptPhraseFilter(djfilters.FilterSet):
    min_confidence = djfilters.NumberFilter(name='confidence', lookup_expr='gte')
    max_confidence = djfilters.NumberFilter(name='confidence', lookup_expr='lte')
    min_votes = djfilters.NumberFilter(name='num_votes', lookup_expr='gte')
    max_votes = djfilters.NumberFilter(name='num_votes', lookup_expr='lte')

    class Meta:
        model = TranscriptPhrase
        fields = ('confidence', 'num_corrections', 'num_votes')

class TranscriptPhraseDetailViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = TranscriptPhrase.objects.all()
    serializer_class = TranscriptPhraseDetailSerializer
    filter_backends = (djfilters.DjangoFilterBackend,)
    filter_class = TranscriptPhraseFilter

class TranscriptPhraseVoteViewSet(mixins.ListModelMixin,
                                  mixins.CreateModelMixin,
                                  viewsets.GenericViewSet):
    permission_classes = (IsOwnerOrReadOnly,)
    queryset = TranscriptPhraseVote.objects.all()
    serializer_class = TranscriptPhraseVoteSerializer

class TranscriptPhraseCorrectionViewSet(mixins.ListModelMixin,
                                        mixins.CreateModelMixin,
                                        viewsets.GenericViewSet):
    def get_queryset(self):
        return TranscriptPhraseCorrection.objects.filter(
            user=self.request.user
        )
    permission_classes = (IsOwnerOrReadOnly,)
    queryset = TranscriptPhraseCorrection.objects.all()
    serializer_class = TranscriptPhraseCorrectionSerializer

class TranscriptPhraseCorrectionVoteViewSet(mixins.ListModelMixin,
                                            mixins.CreateModelMixin,
                                            viewsets.GenericViewSet):
    def get_queryset(self):
        return TranscriptPhraseCorrectionVote.objects.filter(
            user=self.request.user
        )
    permission_classes = (IsOwnerOrReadOnly,)
    queryset = TranscriptPhraseCorrectionVote.objects.all()
    serializer_class = TranscriptPhraseCorrectionVoteSerializer

class SourceViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = (AllowAny,)
    queryset = Source.objects.all()
    serializer_class = SourceSerializer
    pagination_class = StandardResultsSetPagination

class TopicViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = (AllowAny,)
    queryset = Topic.objects.all()
    serializer_class = TopicSerializer

class ProfileViewSet(viewsets.ModelViewSet):
    permission_classes = (IsOwner,)
    queryset = Profile.objects.all()
    serializer_class = ProfileSerializer

    def get_queryset(self):
        return Profile.objects.filter(user=self.request.user)

    @detail_route(methods=['patch'], serializer_class=ProfilePatchSerializer)
    def completed(self, request, *args, **kwargs):
        self.get_object().update_completed(request.data['completed'])
        return self.update(request, *args, **kwargs)

    @detail_route(methods=['patch'], serializer_class=ProfilePreferenceClearSerializer)
    def clear_preferences(self, request, *args, **kwargs):
        self.get_object().clear_preferences(request.data)
        return self.update(request, *args, **kwargs)

    @detail_route(methods=['patch'], serializer_class=ProfileTranscriptSkipSerializer)
    def skip_transcript(self, request, *args, **kwargs):
        self.get_object().skip_transcript(request.data)
        return self.update(request, *args, **kwargs)

class ProfileStatsViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = (IsOwnerOrStaff,)
    queryset = Profile.objects.all()
    serializer_class = ProfileSerializer
    lookup_field = 'username'

class ScoreViewSet(mixins.ListModelMixin,
                   mixins.CreateModelMixin,
                   viewsets.GenericViewSet):
    permission_classes = (IsOwner,)
    queryset = Score.objects.all()
    serializer_class = ScoreSerializer

class LeaderboardView(generics.RetrieveAPIView):
    permission_classes = (AllowAny,)
    queryset = Leaderboard.objects.all()
    serializer_class = LeaderboardSerializer

    def get_object(self):
        obj = Leaderboard.objects.latest('date')
        self.check_object_permissions(self.request, obj)
        return obj

class LoadingScreenView(generics.RetrieveAPIView):
    permission_classes = (AllowAny,)
    queryset = LoadingScreenData.objects.all()
    serializer_class = LoadingScreenSerializer

    def get_object(self):
        obj = LoadingScreenData.objects.latest('date')
        self.check_object_permissions(self.request, obj)
        return obj

class ContributionStatisticsView(generics.RetrieveAPIView):
    permission_classes = (AllowAny,)
    queryset = ContributionStatistics.objects.all()
    serializer_class = ContributionStatisticsSerializer

    def get_object(self):
        obj = ContributionStatistics.objects.latest('date')
        self.check_object_permissions(self.request, obj)
        return obj
<commit_msg>Remove logging<commit_after>import logging

from django.conf import settings

from django_filters import rest_framework as djfilters

from rest_framework import viewsets, generics, filters, mixins
from rest_framework.decorators import list_route, detail_route
from rest_framework.response import Response
from rest_framework.pagination import PageNumberPagination
from rest_framework.permissions import AllowAny, IsAdminUser

from ..transcript.models import (
    Transcript, TranscriptPhrase, TranscriptPhraseVote,
    TranscriptPhraseCorrection,
    TranscriptPhraseCorrectionVote,
    Source, Topic,
)
from ..accounts.models import (
    Profile, Score, Leaderboard, ContributionStatistics
)
from ..game.models import LoadingScreenData
from .serializers import (
    TranscriptSerializer,
    TranscriptCreateSerializer,
    TranscriptStatsSerializer,
    TranscriptPhraseDetailSerializer,
    TranscriptPhraseVoteSerializer,
    TranscriptPhraseCorrectionSerializer,
    TranscriptPhraseCorrectionVoteSerializer,
    ProfileSerializer, ProfilePatchSerializer,
    ProfilePreferenceClearSerializer, ProfileTranscriptSkipSerializer,
    SourceSerializer, TopicSerializer,
    ScoreSerializer, LeaderboardSerializer,
    LoadingScreenSerializer, ContributionStatisticsSerializer
)
from .permissions import IsOwner, IsOwnerOrStaff, IsOwnerOrReadOnly

django_log = logging.getLogger('django')

phrase_positive_limit = settings.TRANSCRIPT_PHRASE_POSITIVE_CONFIDENCE_LIMIT
phrase_negative_limit = settings.TRANSCRIPT_PHRASE_NEGATIVE_CONFIDENCE_LIMIT
correction_lower_limit = settings.TRANSCRIPT_PHRASE_CORRECTION_LOWER_LIMIT
correction_upper_limit = settings.TRANSCRIPT_PHRASE_CORRECTION_UPPER_LIMIT

class StandardResultsSetPagination(PageNumberPagination):
    page_size = 100
    page_size_query_param = 'page_size'
    max_page_size = 1000

class TranscriptViewSet(mixins.ListModelMixin,
                        viewsets.GenericViewSet):
    permission_classes = (AllowAny,)
    queryset = Transcript.objects.all()
    lookup_field = 'asset_name'
    serializer_class = TranscriptSerializer

    @list_route()
    def random(self, request):
        transcript = Transcript.objects.random_transcript()
        serializer = self.get_serializer(transcript, many=True)
        return Response(serializer.data)

    @list_route()
    def game_one(self, request):
        transcripts, phrases = Transcript.objects.game_one(request.user)
        serializer = self.get_serializer(
            transcripts, many=True,
        )
        if phrases:
            for transcript in serializer.data:
                for phrase in transcript['phrases']:
                    if phrase['pk'] in phrases:
                        phrase['user_can_vote'] = False
                    else:
                        phrase['user_can_vote'] = True
        return Response(serializer.data)

    @list_route()
    def game_two(self, request):
        transcripts, phrases_for_correction = Transcript.objects.game_two(
            request.user
        )
        serializer = self.get_serializer(
            transcripts, many=True,
        )
        for transcript in serializer.data:
            for phrase in transcript['phrases']:
                if phrase['pk'] in phrases_for_correction:
                    phrase['needs_correction'] = True
                else:
                    phrase['needs_correction'] = False
        return Response(serializer.data)

    @list_route()
    def game_three(self, request):
        transcripts, corrected_phrases = Transcript.objects.game_three(
            request.user
        )
        serializer = self.get_serializer(
            transcripts, many=True
        )
        for transcript in serializer.data:
            for phrase in transcript['phrases']:
                for correction in corrected_phrases:
                    if phrase['pk'] in correction:
                        phrase['corrections'] = [
                            {
                                'pk': correction.pk,
                                'corrected_text': correction.correction
                            } for correction in correction[phrase['pk']]
                        ]
                        correction[phrase['pk']]

        return Response(serializer.data)

    @detail_route(methods=['get'])
    def corrected(self, request, pk=None, asset_name=None):
        '''Returns a corrected transcript'''
        transcript = self.get_object()
        highest_rated_corrections = []
        for phrase in transcript.phrases.all():
            if phrase.confidence >= phrase_positive_limit:
                pass
            else:
                corrections = TranscriptPhraseCorrection.objects.filter(
                    transcript_phrase=phrase,
                    confidence__gte=correction_lower_limit
                ).order_by('-confidence')
                if corrections.count() > 0:
                    highest_rated_corrections.append(corrections.first())
        serializer = self.get_serializer(transcript)
        for correction in highest_rated_corrections:
            for phrase in serializer.data['phrases']:
                if phrase['pk'] == correction.transcript_phrase.pk:
                    phrase['text'] = correction.correction
        return Response(serializer.data)

class TranscriptCreateViewset(mixins.CreateModelMixin,
                              viewsets.GenericViewSet):
    permission_classes = (IsAdminUser,)
    queryset = Transcript.objects.all()
    serializer_class = TranscriptCreateSerializer

class TranscriptStatsViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Transcript.objects.all()
    lookup_field = 'asset_name'
    serializer_class = TranscriptStatsSerializer
    filter_backends = (filters.OrderingFilter,
                       djfilters.DjangoFilterBackend)
    ordering_fields = ('complete', 'in_progress')
    filter_fields = ('complete', 'in_progress')

class TranscriptPhraseFilter(djfilters.FilterSet):
    min_confidence = djfilters.NumberFilter(name='confidence', lookup_expr='gte')
    max_confidence = djfilters.NumberFilter(name='confidence', lookup_expr='lte')
    min_votes = djfilters.NumberFilter(name='num_votes', lookup_expr='gte')
    max_votes = djfilters.NumberFilter(name='num_votes', lookup_expr='lte')

    class Meta:
        model = TranscriptPhrase
        fields = ('confidence', 'num_corrections', 'num_votes')

class TranscriptPhraseDetailViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = TranscriptPhrase.objects.all()
    serializer_class = TranscriptPhraseDetailSerializer
    filter_backends = (djfilters.DjangoFilterBackend,)
    filter_class = TranscriptPhraseFilter

class TranscriptPhraseVoteViewSet(mixins.ListModelMixin,
                                  mixins.CreateModelMixin,
                                  viewsets.GenericViewSet):
    permission_classes = (IsOwnerOrReadOnly,)
    queryset = TranscriptPhraseVote.objects.all()
    serializer_class = TranscriptPhraseVoteSerializer

class TranscriptPhraseCorrectionViewSet(mixins.ListModelMixin,
                                        mixins.CreateModelMixin,
                                        viewsets.GenericViewSet):
    def get_queryset(self):
        return TranscriptPhraseCorrection.objects.filter(
            user=self.request.user
        )
    permission_classes = (IsOwnerOrReadOnly,)
    queryset = TranscriptPhraseCorrection.objects.all()
    serializer_class = TranscriptPhraseCorrectionSerializer

class TranscriptPhraseCorrectionVoteViewSet(mixins.ListModelMixin,
                                            mixins.CreateModelMixin,
                                            viewsets.GenericViewSet):
    def get_queryset(self):
        return TranscriptPhraseCorrectionVote.objects.filter(
            user=self.request.user
        )
    permission_classes = (IsOwnerOrReadOnly,)
    queryset = TranscriptPhraseCorrectionVote.objects.all()
    serializer_class = TranscriptPhraseCorrectionVoteSerializer

class SourceViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = (AllowAny,)
    queryset = Source.objects.all()
    serializer_class = SourceSerializer
    pagination_class = StandardResultsSetPagination

class TopicViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = (AllowAny,)
    queryset = Topic.objects.all()
    serializer_class = TopicSerializer

class ProfileViewSet(viewsets.ModelViewSet):
    permission_classes = (IsOwner,)
    queryset = Profile.objects.all()
    serializer_class = ProfileSerializer

    def get_queryset(self):
        return Profile.objects.filter(user=self.request.user)

    @detail_route(methods=['patch'], serializer_class=ProfilePatchSerializer)
    def completed(self, request, *args, **kwargs):
        self.get_object().update_completed(request.data['completed'])
        return self.update(request, *args, **kwargs)

    @detail_route(methods=['patch'], serializer_class=ProfilePreferenceClearSerializer)
    def clear_preferences(self, request, *args, **kwargs):
        self.get_object().clear_preferences(request.data)
        return self.update(request, *args, **kwargs)

    @detail_route(methods=['patch'], serializer_class=ProfileTranscriptSkipSerializer)
    def skip_transcript(self, request, *args, **kwargs):
        self.get_object().skip_transcript(request.data)
        return self.update(request, *args, **kwargs)

class ProfileStatsViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = (IsOwnerOrStaff,)
    queryset = Profile.objects.all()
    serializer_class = ProfileSerializer
    lookup_field = 'username'

class ScoreViewSet(mixins.ListModelMixin,
                   mixins.CreateModelMixin,
                   viewsets.GenericViewSet):
    permission_classes = (IsOwner,)
    queryset = Score.objects.all()
    serializer_class = ScoreSerializer

class LeaderboardView(generics.RetrieveAPIView):
    permission_classes = (AllowAny,)
    queryset = Leaderboard.objects.all()
    serializer_class = LeaderboardSerializer

    def get_object(self):
        obj = Leaderboard.objects.latest('date')
        self.check_object_permissions(self.request, obj)
        return obj

class LoadingScreenView(generics.RetrieveAPIView):
    permission_classes = (AllowAny,)
    queryset = LoadingScreenData.objects.all()
    serializer_class = LoadingScreenSerializer

    def get_object(self):
        obj = LoadingScreenData.objects.latest('date')
        self.check_object_permissions(self.request, obj)
        return obj

class ContributionStatisticsView(generics.RetrieveAPIView):
    permission_classes = (AllowAny,)
    queryset = ContributionStatistics.objects.all()
    serializer_class = ContributionStatisticsSerializer

    def get_object(self):
        obj = ContributionStatistics.objects.latest('date')
        self.check_object_permissions(self.request, obj)
        return obj
<|endoftext|>


<commit_before># Authors: Alexandre Barachant <alexandre.barachant@gmail.com>
#
# License: BSD (3-clause)

import numpy as np
from scipy import linalg
import copy as cp

from ..io.proj import Projection
from ..io.base import _BaseRaw
from .. import Covariance, EvokedArray, compute_raw_data_covariance
from ..io.pick import pick_types, pick_info

def _least_square_evoked(data, events, event_id, tmin, tmax, sfreq, decim):
    """Least square estimation of evoked response from data
    return evoked data and toeplitz matrices.
    """
    nmin = int(tmin * sfreq / decim)
    nmax = int(tmax * sfreq / decim)
    data = data[:, ::decim]

    window = nmax - nmin
    ne, ns = data.shape
    to = {}

    for eid in event_id:
        # select events by type
        ix_ev = events[:, -1] == event_id[eid]

        # build toeplitz matrix
        trig = np.zeros((ns, 1))
        ix_trig = (events[ix_ev, 0] / decim) + nmin
        trig[ix_trig] = 1
        toep = linalg.toeplitz(trig[0:window], trig)
        to[eid] = toep

    # Concatenate toeplitz
    to_tot = np.concatenate(to.values())

    # least square estimation
    evo = np.dot(np.dot(linalg.pinv(np.dot(to_tot, to_tot.T)), to_tot), data.T)

    # parse evoked response
    evoked_data = {}
    for i, eid in enumerate(event_id):
        evoked_data[eid] = np.array(evo[(i * window):(i + 1) * window, :]).T

    return evoked_data, to

def least_square_evoked(raw, events, event_id, tmin=0.0, tmax=1.0, decim=1,
                        picks=None, return_toeplitz=False):
    """ Least square estimation of evoked response from a raw instance.

    Parameters
    ----------
    raw : Raw object
        An instance of Raw.
    events : array, shape (n_events, 3)
        The events typically returned by the read_events function.
        If some events don't match the events of interest as specified
        by event_id, they will be ignored.
    event_id : int | list of int | dict | None
        The id of the event to consider. If dict,
        the keys can later be used to acces associated events. Example:
        dict(auditory=1, visual=3). If int, a dict will be created with
        the id as string. If a list, all events with the IDs specified
        in the list are used. If None, all events will be used with
        and a dict is created with string integer names corresponding
        to the event id integers.
    tmin : float
        Start time before event.
    tmax : float
        End time after event.
    picks : array-like of int | None (default)
        Indices of channels to include (if None, all channels are used).
    decim : int
        Factor by which to downsample the data from the raw file upon import.
        Warning: This simply selects every nth sample, data is not filtered
        here. If data is not properly filtered, aliasing artifacts may occur.
    return_toeplitz : bool
        if true, compute the toeplitz matrix

    Returns
    -------
    evokeds : array of evoked instance
        An array of evoked instance for each event type in event_id.
    """
    if not isinstance(raw, _BaseRaw):
        raise ValueError('The raw must be an instance of `mne.io.Raw`')

    if event_id is None:  # convert to int to make typing-checks happy
        event_id = dict((str(e), int(e)) for e in np.unique(events[:, 2]))
    elif isinstance(event_id, int):
        event_id = dict(str(event_id), int(event_id))
    elif isinstance(event_id, list):
        event_id = dict((str(e), int(e)) for e in event_id)
    elif not isinstance(event_id, dict):
        raise ValueError('event_id must be None, int, list of int, or dict')

    for key, val in event_id.items():
        if val not in events[:, 2]:
            raise ValueError('No matching events found for %s '
                             '(event id %i)' % (key, val))
    if picks is None:
        picks = pick_types(raw.info, meg=True, eeg=True)

    evs = events.copy()
    evs[:, 0] -= raw.first_samp
    evo, to = _least_square_evoked(raw._data[picks], evs, event_id,
                                   tmin=tmin, tmax=tmax,
                                   sfreq=raw.info['sfreq'], decim=decim)
    evokeds = []
    info = pick_info(raw.info, picks)
    info['sfreq'] /= decim

    for name, data in evo.items():
        n_events = len(events[events[:, 2] == event_id[name]])
        evoked = EvokedArray(data, info, tmin=tmin,
                             comment=name, nave=n_events)
        evokeds.append(evoked)

    if return_toeplitz:
        return evokeds, to

    return evokeds

class Xdawn():
    """
    Parameters
    ----------
    n_components : int, default 2
        The number of components to decompose M/EEG signals.
    reg : float, str, None
        if not None, allow regularization for covariance estimation
        if float, shrinkage covariance is used (0 <= shrinkage <= 1).
        if str, optimal shrinkage using Ledoit-Wolf Shrinkage ('lws') or
        Oracle Approximating Shrinkage ('oas').

    Attributes
    ----------
    filters_ : dict of ndarray
        If fit, the Xdawn components used to decompose the data for each event
        type, else empty.
    patterns_ : dict of ndarray
        If fit, the Xdawn patterns used to restore M/EEG signals for each event
        type, else empty.
    evokeds_ : dict of evoked instance
        If fit, the evoked response for each event type
    """
    def __init__(self,n_components=2, reg=None):
        self.n_components = n_components
        self.reg = reg
        self.filters_ = {}
        self.patterns_ = {}
        self.evokeds_ = {}
        self.projs_ = {}

    def fit(self, raw, events, event_id, tmin=0.0, tmax=1.0, decim=1,
            picks=None):
        """
        Fit xdawn
        """
        if picks is None:
            picks = pick_types(raw.info, meg=True, eeg=True)

        evokeds, to = least_square_evoked(raw=raw, events=events,
                                          event_id=event_id, tmin=tmin,
                                          tmax=tmax, decim=decim, picks=picks,
                                          return_toeplitz=True)

        # Compute noise cov
        # FIXME : Use mne method
        noise_cov = np.cov(raw._data[picks, ::decim])

        for i, eid in enumerate(event_id):
            self.evokeds_[eid] = evokeds[i]

            if self.reg is None:
                evoked_cov = np.cov(np.dot(evokeds[i].data, to[eid]))
            else:
                # FIXME : use mne reg
                raise NotImplementedError('Regularization not implemented')

            evals, evecs = linalg.eigh(evoked_cov, noise_cov)
            evecs = evecs[:, np.argsort(evals)[::-1]]  # sort eigenvectors
            evecs /= np.apply_along_axis(np.linalg.norm, 0, evecs)

            self.filters_[eid] = evecs
            self.patterns_[eid] = linalg.inv(evecs.T)

            # Create and add projectors
            ch_names = evokeds[i].ch_names
            projs = []
            for j in range(self.n_components):
                proj_data = dict(col_names=ch_names, row_names=None,
                                 data=evecs[:, j].T, nrow=1,
                                 ncol=len(ch_names))
                projs.append(Projection(active=True, data=proj_data,
                             desc="%s Xdawn #%d" % (eid, j)))

            # self.evokeds_[eid].add_proj(projs)
            self.projs_[eid] = projs

        return self
<commit_msg>add apply<commit_after># Authors: Alexandre Barachant <alexandre.barachant@gmail.com>
#
# License: BSD (3-clause)

import numpy as np
from scipy import linalg
import copy as cp

from ..io.proj import Projection
from ..io.base import _BaseRaw
from ..epochs import _BaseEpochs
from .. import Covariance, EvokedArray, compute_raw_data_covariance
from ..io.pick import pick_types, pick_info
from .ica import _get_fast_dot
from ..utils import logger

def _least_square_evoked(data, events, event_id, tmin, tmax, sfreq, decim):
    """Least square estimation of evoked response from data
    return evoked data and toeplitz matrices.
    """
    nmin = int(tmin * sfreq / decim)
    nmax = int(tmax * sfreq / decim)
    data = data[:, ::decim]

    window = nmax - nmin
    ne, ns = data.shape
    to = {}

    for eid in event_id:
        # select events by type
        ix_ev = events[:, -1] == event_id[eid]

        # build toeplitz matrix
        trig = np.zeros((ns, 1))
        ix_trig = (events[ix_ev, 0] / decim) + nmin
        trig[ix_trig] = 1
        toep = linalg.toeplitz(trig[0:window], trig)
        to[eid] = toep

    # Concatenate toeplitz
    to_tot = np.concatenate(to.values())

    # least square estimation
    evo = np.dot(np.dot(linalg.pinv(np.dot(to_tot, to_tot.T)), to_tot), data.T)

    # parse evoked response
    evoked_data = {}
    for i, eid in enumerate(event_id):
        evoked_data[eid] = np.array(evo[(i * window):(i + 1) * window, :]).T

    return evoked_data, to

def least_square_evoked(raw, events, event_id, tmin=0.0, tmax=1.0, decim=1,
                        picks=None, return_toeplitz=False):
    """ Least square estimation of evoked response from a raw instance.

    Parameters
    ----------
    raw : Raw object
        An instance of Raw.
    events : array, shape (n_events, 3)
        The events typically returned by the read_events function.
        If some events don't match the events of interest as specified
        by event_id, they will be ignored.
    event_id : int | list of int | dict | None
        The id of the event to consider. If dict,
        the keys can later be used to acces associated events. Example:
        dict(auditory=1, visual=3). If int, a dict will be created with
        the id as string. If a list, all events with the IDs specified
        in the list are used. If None, all events will be used with
        and a dict is created with string integer names corresponding
        to the event id integers.
    tmin : float
        Start time before event.
    tmax : float
        End time after event.
    picks : array-like of int | None (default)
        Indices of channels to include (if None, all channels are used).
    decim : int
        Factor by which to downsample the data from the raw file upon import.
        Warning: This simply selects every nth sample, data is not filtered
        here. If data is not properly filtered, aliasing artifacts may occur.
    return_toeplitz : bool
        if true, compute the toeplitz matrix

    Returns
    -------
    evokeds : array of evoked instance
        An array of evoked instance for each event type in event_id.
    """
    if not isinstance(raw, _BaseRaw):
        raise ValueError('The raw must be an instance of `mne.io.Raw`')

    if event_id is None:  # convert to int to make typing-checks happy
        event_id = dict((str(e), int(e)) for e in np.unique(events[:, 2]))
    elif isinstance(event_id, int):
        event_id = dict(str(event_id), int(event_id))
    elif isinstance(event_id, list):
        event_id = dict((str(e), int(e)) for e in event_id)
    elif not isinstance(event_id, dict):
        raise ValueError('event_id must be None, int, list of int, or dict')

    for key, val in event_id.items():
        if val not in events[:, 2]:
            raise ValueError('No matching events found for %s '
                             '(event id %i)' % (key, val))
    if picks is None:
        picks = pick_types(raw.info, meg=True, eeg=True)

    evs = events.copy()
    evs[:, 0] -= raw.first_samp
    evo, to = _least_square_evoked(raw._data[picks], evs, event_id,
                                   tmin=tmin, tmax=tmax,
                                   sfreq=raw.info['sfreq'], decim=decim)
    evokeds = []
    info = pick_info(raw.info, picks)
    info['sfreq'] /= decim

    for name, data in evo.items():
        n_events = len(events[events[:, 2] == event_id[name]])
        evoked = EvokedArray(data, info, tmin=tmin,
                             comment=name, nave=n_events)
        evokeds.append(evoked)

    if return_toeplitz:
        return evokeds, to

    return evokeds

class Xdawn():
    """
    Parameters
    ----------
    n_components : int, default 2
        The number of components to decompose M/EEG signals.
    reg : float, str, None
        if not None, allow regularization for covariance estimation
        if float, shrinkage covariance is used (0 <= shrinkage <= 1).
        if str, optimal shrinkage using Ledoit-Wolf Shrinkage ('lws') or
        Oracle Approximating Shrinkage ('oas').

    Attributes
    ----------
    filters_ : dict of ndarray
        If fit, the Xdawn components used to decompose the data for each event
        type, else empty.
    patterns_ : dict of ndarray
        If fit, the Xdawn patterns used to restore M/EEG signals for each event
        type, else empty.
    evokeds_ : dict of evoked instance
        If fit, the evoked response for each event type
    """
    def __init__(self,n_components=2, reg=None):
        self.n_components = n_components
        self.reg = reg
        self.filters_ = {}
        self.patterns_ = {}
        self.evokeds_ = {}
        self.projs_ = {}

    def fit(self, raw, events, event_id, tmin=0.0, tmax=1.0, decim=1,
            picks=None):
        """
        Fit xdawn
        """
        if picks is None:
            picks = pick_types(raw.info, meg=True, eeg=True)

        self.picks = picks
        evokeds, to = least_square_evoked(raw=raw, events=events,
                                          event_id=event_id, tmin=tmin,
                                          tmax=tmax, decim=decim,
                                          picks=self.picks,
                                          return_toeplitz=True)
        self.ch_names = [raw.ch_names[i] for i in self.picks]
        self.exclude = range(self.n_components, len(self.picks))
        # Compute noise cov
        # FIXME : Use mne method
        noise_cov = np.cov(raw._data[self.picks, ::decim])

        for i, eid in enumerate(event_id):
            self.evokeds_[eid] = evokeds[i]

            if self.reg is None:
                evoked_cov = np.cov(np.dot(evokeds[i].data, to[eid]))
            else:
                # FIXME : use mne reg
                raise NotImplementedError('Regularization not implemented')

            evals, evecs = linalg.eigh(evoked_cov, noise_cov)
            evecs = evecs[:, np.argsort(evals)[::-1]]  # sort eigenvectors
            evecs /= np.apply_along_axis(np.linalg.norm, 0, evecs)

            self.filters_[eid] = evecs
            self.patterns_[eid] = linalg.inv(evecs.T)

            # Create and add projectors
            ch_names = evokeds[i].ch_names
            projs = []
            for j in range(self.n_components):
                proj_data = dict(col_names=ch_names, row_names=None,
                                 data=evecs[:, j].T, nrow=1,
                                 ncol=len(ch_names))
                projs.append(Projection(active=True, data=proj_data,
                             desc="%s Xdawn #%d" % (eid, j)))

            # self.evokeds_[eid].add_proj(projs)
            self.projs_[eid] = projs

        return self

    def apply(self, inst, event_id=None, include=None, exclude=None):
        """Remove selected components from the signal.

        Given the unmixing matrix, transform data,
        zero out components, and inverse transform the data.
        This procedure will reconstruct M/EEG signals from which
        the dynamics described by the excluded components is subtracted.

        Parameters
        ----------
        inst : instance of Raw, Epochs or Evoked
            The data to be processed.
        include : array_like of int.
            The indices refering to columns in the ummixing matrix. The
            components to be kept.
        exclude : array_like of int.
            The indices refering to columns in the ummixing matrix. The
            components to be zeroed out.
        event_id : dict or None
            The kind of event to appy. if none, an array of inst will be return
            one for each type of event xdawn has been fitted.
        copy : bool
            Whether to return a copy or whether to apply the solution in place.
            Defaults to False.
        """
        if isinstance(inst, _BaseRaw):
            out = self._apply_raw(raw=inst, include=include,
                                  exclude=exclude, event_id=event_id)
        elif isinstance(inst, _BaseEpochs):
            out = self._apply_epochs(epochs=inst, include=include,
                                     exclude=exclude, event_id=event_id)
        elif isinstance(inst, Evoked):
            out = self._apply_evoked(evoked=inst, include=include,
                                     exclude=exclude, event_id=event_id)
        else:
            raise ValueError('Data input must be of Raw, Epochs or Evoked '
                             'type')
        return out

    def _apply_raw(self, raw, include, exclude, event_id):
        """Aux method"""
        if not raw.preload:
            raise ValueError('Raw data must be preloaded to apply Xdawn')

        if exclude is None:
            exclude = list(set(self.exclude))
        else:
            exclude = list(set(self.exclude + exclude))

        picks = pick_types(raw.info, meg=False, include=self.ch_names,
                           exclude='bads')
        raws = {}
        for eid in event_id:
            data = raw[picks, :][0]

            data = self._pick_sources(data, include, exclude, eid)

            raw_r = raw.copy()

            raw_r[picks, :] = data
            raws[eid] = raw_r
        return raws

    def _apply_epochs(self, epochs, include, exclude, event_id):

        if not epochs.preload:
            raise ValueError('Epochs must be preloaded to apply ICA')

        picks = pick_types(epochs.info, meg=False, ref_meg=False,
                           include=self.ch_names,
                           exclude='bads')

        # special case where epochs come picked but fit was 'unpicked'.
        if len(picks) != len(self.ch_names):
            raise RuntimeError('Epochs don\'t match fitted data: %i channels '
                               'fitted but %i channels supplied. \nPlease '
                               'provide Epochs compatible with '
                               'ica.ch_names' % (len(self.ch_names),
                                                 len(picks)))

        epochs_dict = {}
        data = np.hstack(epochs.get_data()[:, picks])

        for eid in event_id:

            data_r = self._pick_sources(data, include, exclude, eid)

            epochs_r = epochs.copy()
            epochs_r._data[:, picks] = np.array(np.split(data_r,
                                                len(epochs.events), 1))
            epochs_r.preload = True
            epochs_dict[eid] = epochs_r

        return epochs_dict

    def _apply_evoked(self, evoked, include, exclude, event_id):

        picks = pick_types(evoked.info, meg=False, ref_meg=False,
                           include=self.ch_names,
                           exclude='bads')

        # special case where evoked come picked but fit was 'unpicked'.
        if len(picks) != len(self.ch_names):
            raise RuntimeError('Evoked does not match fitted data: %i channels'
                               ' fitted but %i channels supplied. \nPlease '
                               'provide an Evoked object that\'s compatible '
                               'with ica.ch_names' % (len(self.ch_names),
                                                      len(picks)))

        data = evoked.data[picks]
        evokeds = {}

        for eid in event_id:

            data_r = self._pick_sources(data, include, exclude, eid)
            evo = evoked.copy()

            # restore evoked
            evo.data[picks] = data_r
            evokeds[eid] = evo

        return evokeds

    def _pick_sources(self, data, include, exclude, eid):
        """Aux function"""
        fast_dot = _get_fast_dot()
        if exclude is None:
            exclude = self.exclude
        else:
            exclude = list(set(self.exclude + list(exclude)))

        n_components = self.n_components
        logger.info('Transforming to Xdawn space (%i components)' %
                    n_components)

        # Apply unmixing
        sources = fast_dot(self.filters_[eid], data)

        if include not in (None, []):
            mask = np.ones(len(sources), dtype=np.bool)
            mask[np.unique(include)] = False
            sources[mask] = 0.
            logger.info('Zeroing out %i Xdawn components' % mask.sum())
        elif exclude not in (None, []):
            exclude_ = np.unique(exclude)
            sources[exclude_] = 0.
            logger.info('Zeroing out %i Xdawn components' % len(exclude_))
        logger.info('Inverse transforming to PCA space')
        data = fast_dot(self.patterns_[eid], sources)

        return data
<|endoftext|>


<commit_before>#!/usr/bin/python
#    Copyright 2017 Dhvani Patel
#
#    This file is part of UnnaturalCode.
#
#    UnnaturalCode is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    UnnaturalCode is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with UnnaturalCode.  If not, see <http://www.gnu.org/licenses/>.

# Takes in a string of Java code and checks for errors
# NOTE: FOR JAVAC

import os
import subprocess
import sys
import tempfile
from compile_error import CompileError

# Method for finding index of certain characters in a string, n being the n'th occurence of the character/string
def find_nth(haystack, needle, n):
    start = haystack.find(needle)
    while start >= 0 and n > 1:
        start = haystack.find(needle, start+len(needle))
        n -= 1
    return start

# Main method
def checkJavaCSyntax(src):
		#with open (src, "r") as myfile:
   		#	data = myfile.read()
		#print data
		myFile = open("ToCheck.java", "w")
		myFile.write(src)
		myFile.close()
		proc = subprocess.Popen(['javac', 'ToCheck.java'], stderr=subprocess.PIPE)
		streamdata, err = proc.communicate()
		rc = proc.returncode
		if rc == 0:
			# No errors, all good
			os.remove("ToCheck.java")
			return None
		else:
			# Error, disect data for constructor
			#print err
			err = err[:len(err)-1]
			#print err

			lastLine = err.rfind("\n")
			#print lastLine
			#print "split"
			#print len(err)
			lastErrorNum = err[lastLine:]

			lastError = lastErrorNum[:lastErrorNum.index('error')-1]
			numError = int(lastError)

			lineNums = []
			for ind in range(numError):
				fileInd = find_nth(err, "ToCheck.java:", ind+1)
				temp = err[fileInd:]

				cutColInd = find_nth(temp, ":", 2)
				line = err[fileInd+13:cutColInd+fileInd]
				lineNums.append(int(line))

			#print "----OUT----"
			checkInd = err.find("is public, should be declared in a file named")
			#print msgNo
			#print lineNums
			if checkInd != -1:
				check = err[:checkInd]
				lastCheck = err.rfind("ToCheck.java:")
				tempR = err[lastCheck:]
				cutColInd = find_nth(tempR, ":", 2)
				lineRemov = err[lastCheck+13:cutColInd+lastCheck]
				rid = int(lineRemov)
				goOver = lineNums[:]
				for x in goOver:
					if x == rid:
						lineNums.remove(rid)
			msgNo = []
			for x in range(len(lineNums)):
				msgNo.append(x+1)

			#print msgNo
			#print lineNums

			if len(msgNo) == 0 and len(lineNums) == 0:
				os.remove("ToCheck.java")
				return None
			else:
				#errorObj = CompileError(fileName, line, column, None, text, errorname)
				#print err
				#print msgNo
				#print lineNums
				os.remove("ToCheck.java")
				return msgNo, numError
<commit_msg>Update check_javac_syntax.py<commit_after>#!/usr/bin/python
#    Copyright 2017 Dhvani Patel
#
#    This file is part of UnnaturalCode.
#
#    UnnaturalCode is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    UnnaturalCode is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with UnnaturalCode.  If not, see <http://www.gnu.org/licenses/>.

# Takes in a string of Java code and checks for errors
# NOTE: FOR JAVAC

import os
import subprocess
import sys
import tempfile
from compile_error import CompileError

# Method for finding index of certain characters in a string, n being the n'th occurence of the character/string
def find_nth(haystack, needle, n):
    start = haystack.find(needle)
    while start >= 0 and n > 1:
        start = haystack.find(needle, start+len(needle))
        n -= 1
    return start

# Main method
def checkJavaCSyntax(src):
		with open (src, "r") as myfile:
   			data = myfile.read()
		#print data
		myFile = open("ToCheck.java", "w")
		myFile.write(data)
		myFile.close()
		proc = subprocess.Popen(['javac', 'ToCheck.java'], stderr=subprocess.PIPE)
		streamdata, err = proc.communicate()
		rc = proc.returncode
		if rc == 0:
			# No errors, all good
			os.remove("ToCheck.java")
			return None
		else:
			# Error, disect data for constructor
			#print err
			err = err[:len(err)-1]
			#print err

			lastLine = err.rfind("\n")
			#print lastLine
			#print "split"
			#print len(err)
			lastErrorNum = err[lastLine:]

			lastError = lastErrorNum[:lastErrorNum.index('error')-1]
			numError = int(lastError)

			lineNums = []
			insToks = []
			indRepeats = []
			typeErrors = []
			origLineNums = []
			flag = False
			for ind in range(numError):
				fileInd = find_nth(err, "ToCheck.java:", ind+1)
				temp = err[fileInd:]
				#print "ok"
				#print temp
				firstNewLine = find_nth(temp, "\n", 1)
				firstLine = temp[:firstNewLine]
				#print firstLine
				#temp = temp[:firstNewLine]
				errorColInd = find_nth(firstLine, "error:", 1)
				expectInd = find_nth(firstLine, "expected", 1)
				#print expectInd
				if expectInd != -1:
					#print expectInd
					typeErrors.append('i')
					beforeS = firstLine[errorColInd+7:errorColInd+8]
					afterS = firstLine[expectInd-2:expectInd-1]
					if beforeS == '\'' and afterS == '\'':
						insTok = firstLine[errorColInd+8:expectInd-2]
					else:
						insTok = firstLine[errorColInd+7:expectInd-1]
					if insTok == "class, interface, or enum":
						flag = True
						insToks.append("class")
						insToks.append("interface")
						insToks.append("enum")
						typeErrors.append('i')
						typeErrors.append('i')
					else:
						flag = False
						insToks.append(insTok)

				else:
					typeErrors.append('')
					insToks.append('')
				cutColInd = find_nth(temp, ":", 2)
				line = err[fileInd+13:cutColInd+fileInd]
				#print flag
				if flag == True:
					#print int(line)
					lineNums.append(int(line))
					origLineNums.append(int(line))
					indRepeats.append(origLineNums.index(int(line)))
					lineNums.append(int(line))
				else:
					origLineNums.append(int(line))
				lineNums.append(int(line))

			#print lineNums
			#print origLineNums
			#print "RADHA"

			#print "----OUT----"
			checkInd = err.find("is public, should be declared in a file named")
			#print msgNo
			#print lineNums
			if checkInd != -1:
				check = err[:checkInd]
				lastCheck = err.rfind("ToCheck.java:")
				tempR = err[lastCheck:]
				cutColInd = find_nth(tempR, ":", 2)
				lineRemov = err[lastCheck+13:cutColInd+lastCheck]
				rid = int(lineRemov)
				goOver = lineNums[:]
				for x in goOver:
					if x == rid:
						lineNums.remove(rid)
			msgNo = []
			#print indRepeats
			for x in range(len(lineNums) - (len(indRepeats)*2)):
				if x in indRepeats:
					#print "TRUE"
					msgNo.append(x+1)
					msgNo.append(x+1)
				msgNo.append(x+1)

			#print msgNo
			#print lineNums

			if len(msgNo) == 0 and len(lineNums) == 0:
				#os.remove("ToCheck.java")
				return None
			else:
				#errorObj = CompileError(fileName, line, column, None, text, errorname)
				#print err
				#print msgNo
				#print lineNums
				#print insToks
				#print typeErrors
				#print len(msgNo)
				#print len(lineNums)
				#print len(insToks)
				#print len(typeErrors)
				os.remove("ToCheck.java")
				return msgNo, lineNums, insToks, typeErrors
<|endoftext|>


<commit_before>from project.utils import auth
from flask import Blueprint, request, redirect, Response, abort
from project import db, session
from project.models import User, Group
from datetime import datetime, date
import jsonpickle
import project.constants as constants

bathing = Blueprint('bathing', __name__)

@bathing.route('/api/bathing', methods=['POST'])
def add_bathing():
    user_id = request.json['user_id']
    user = User.query.get_or_404(user_id)
    now = datetime.now()
    group = Group.query.get_or_404(user.groupID)
    group.currentBathingID = user_id
    group.currentBathingStart = now
    db.session.commit()
    return 'OK'

@bathing.route('/api/check_bathing', methods=['POST'])
def check_bathing():
    user_id = request.json['user_id']
    user = User.query.filter_by(id=user_id).first()
    group_id = user.groupID
    group = Group.query.get_or_404(group_id)
    if group.currentBathingID == constants.DEFAULT_BATHING_ID:
        result = {"bathing": False}
    else:
        user = User.query.get_or_404(group.currentBathingID)
        result = {"bathing": True,
                  "username": user.username,
                  "start": group.currentBathingStart.strftime('%H:%M')}

    return jsonpickle.encode(result)
<commit_msg>routes for start and stop bathing<commit_after>from project.utils import auth
from flask import Blueprint, request, redirect, Response, abort
from project import db, session
from project.models import User, Group
from datetime import datetime, date
import json
import jsonpickle
import project.constants as constants

bathing = Blueprint('bathing', __name__)

@bathing.route('/api/start_bathing', methods=['POST'])
def start_bathing():
    user_id = request.json['user_id']
    user = User.query.filter_by(id=user_id).first()
    now = datetime.now()
    group = Group.query.get_or_404(user.groupID)
    group.currentBathingID = user_id
    group.currentBathingStart = now
    db.session.commit()
    return 'OK'

@bathing.route('/api/stop_bathing', methods=['POST'])
def stop_bathing():
    user_id = request.json['user_id']
    user = User.query.filter_by(id=user_id).first()
    group = Group.query.get_or_404(user.groupID)
    if group.currentBathingID == constants.DEFAULT_BATHING_ID:
        abort(400)
    elif group.currentBathingID != user_id:
        return json.dumps({'correct_person': False})
    else:
        group.currentBathingID = constants.DEFAULT_BATHING_ID
        group.currentBathingStart = None
        db.session.commit()
        return json.dumps({'correct_person': True})

@bathing.route('/api/check_bathing', methods=['POST'])
def check_bathing():
    user_id = request.json['user_id']

    # TODO: ask user which group if user is in multiple groups
    user = User.query.filter_by(id=user_id).first()
    group_id = user.groupID
    group = Group.query.get_or_404(group_id)
    if group.currentBathingID == constants.DEFAULT_BATHING_ID:
        result = {"bathing": False}
    else:
        user = User.query.filter_by(id=group.currentBathingID, groupID=group_id).first()
        result = {"bathing": True,
                  "username": user.username,
                  "start": group.currentBathingStart.strftime('%H:%M')}

    return jsonpickle.encode(result)
<|endoftext|>