<commit_before>import inspect
from openpnm.core import Workspace
from openpnm.utils.misc import PrintableDict
ws = Workspace()

class ModelsDict(PrintableDict):

    def __str__(self):
        horizontal_rule = '―' * 78
        lines = [horizontal_rule]
        strg = '{0:<3s} {1:<25s} {2:<25s} {3}'
        lines.append(strg.format('#', 'Property Name', 'Parameter', 'Value'))
        lines.append(horizontal_rule)
        for i, item in enumerate(self.keys()):
            temp = self[item].copy()
            regen_mode = temp.pop('regen_mode', None)
            model = str(temp.pop('model')).split(' ')[1]
            lines.append(strg.format(str(i+1), item, 'model:', model))
            for param in temp.keys():
                lines.append(strg.format('', '', param+':', temp[param]))
            lines.append(strg.format('', '', 'regeneration mode:', regen_mode))
            lines.append(horizontal_rule)
        return '\n'.join(lines)

class ModelsMixin():

    def add_model(self, propname, model, regen_mode='deferred', **kwargs):
        # Add model and regen_mode to kwargs dictionary
        kwargs.update({'model': model, 'regen_mode': regen_mode})
        # Insepct model to extract arguments and default values
        if model.__defaults__:
            vals = list(inspect.getargspec(model).defaults)
            keys = inspect.getargspec(model).args[-len(vals):]
            # Put defaults into dict_
            for k, v in zip(keys, vals):
                # Skip if argument was given in kwargs
                if k not in kwargs:
                    kwargs.update({k: v})
        # Store all keyword argumnents in model
        self.models[propname] = kwargs
        # Regenerate model values if necessary
        if regen_mode != 'deferred':
            self._regen(propname)

    def regenerate_models(self, propnames=None, exclude=[]):
        # If no props given, then regenerate them all
        if propnames is None:
            propnames = list(self.models.keys())
        # If only one prop given, as string, put into a list
        elif type(propnames) is str:
            propnames = [propnames]
        [propnames.remove(i) for i in exclude if i in propnames]
        # Scan through list of propnames and regenerate each one
        for item in propnames:
            self._regen(item)

    def _regen(self, prop):
        # Create a temporary dict of all model arguments
        kwargs = self.models[prop].copy()
        # Pop model and regen_mode from temporary dict
        model = kwargs.pop('model')
        regen_mode = kwargs.pop('regen_mode', None)
        # Only regenerate model if regen_mode is correct
        if regen_mode == 'constant':
            if prop not in self.keys():
                self[prop] = model(target=self, **kwargs)
        else:
            self[prop] = model(target=self, **kwargs)

    def _get_models(self):
        if not hasattr(self, '_models_dict'):
            self._models_dict = ModelsDict()
        return self._models_dict

    def _set_models(self, dict_):
        self._models_dict = ModelsDict()
        self._models_dict.update(dict_)
        # Renerate all models in new dict if regen mode says so
        for model in dict_:
            # In case regen mode is not set, do it now
            dict_[model].setdefault('regen_mode', 'normal')
        if self.settings['freeze_models']:
            pass
        else:
            self.regenerate_models()

    models = property(fget=_get_models, fset=_set_models)
<commit_msg>Added remove_model function to ModelsMixin<commit_after>import inspect
from openpnm.core import Workspace
from openpnm.utils.misc import PrintableDict
ws = Workspace()

class ModelsDict(PrintableDict):

    def __str__(self):
        horizontal_rule = '―' * 78
        lines = [horizontal_rule]
        strg = '{0:<3s} {1:<25s} {2:<25s} {3}'
        lines.append(strg.format('#', 'Property Name', 'Parameter', 'Value'))
        lines.append(horizontal_rule)
        for i, item in enumerate(self.keys()):
            temp = self[item].copy()
            regen_mode = temp.pop('regen_mode', None)
            model = str(temp.pop('model')).split(' ')[1]
            lines.append(strg.format(str(i+1), item, 'model:', model))
            for param in temp.keys():
                lines.append(strg.format('', '', param+':', temp[param]))
            lines.append(strg.format('', '', 'regeneration mode:', regen_mode))
            lines.append(horizontal_rule)
        return '\n'.join(lines)

class ModelsMixin():

    def add_model(self, propname, model, regen_mode='deferred', **kwargs):
        r"""
        """
        # Add model and regen_mode to kwargs dictionary
        kwargs.update({'model': model, 'regen_mode': regen_mode})
        # Insepct model to extract arguments and default values
        if model.__defaults__:
            vals = list(inspect.getargspec(model).defaults)
            keys = inspect.getargspec(model).args[-len(vals):]
            # Put defaults into dict_
            for k, v in zip(keys, vals):
                # Skip if argument was given in kwargs
                if k not in kwargs:
                    kwargs.update({k: v})
        # Store all keyword argumnents in model
        self.models[propname] = kwargs
        # Regenerate model values if necessary
        if regen_mode != 'deferred':
            self._regen(propname)

    def remove_model(self, propname=None, mode=['model', 'data']):
        r"""
        """
        if type(propname) is str:
            propname = [propname]
        for item in propname:
            if 'model' in mode:
                del self.models[item]
            if 'data' in mode:
                del self[item]

    def regenerate_models(self, propnames=None, exclude=[]):
        r"""
        """
        # If no props given, then regenerate them all
        if propnames is None:
            propnames = list(self.models.keys())
        # If only one prop given, as string, put into a list
        elif type(propnames) is str:
            propnames = [propnames]
        [propnames.remove(i) for i in exclude if i in propnames]
        # Scan through list of propnames and regenerate each one
        for item in propnames:
            self._regen(item)

    def _regen(self, prop):
        # Create a temporary dict of all model arguments
        kwargs = self.models[prop].copy()
        # Pop model and regen_mode from temporary dict
        model = kwargs.pop('model')
        regen_mode = kwargs.pop('regen_mode', None)
        # Only regenerate model if regen_mode is correct
        if regen_mode == 'constant':
            if prop not in self.keys():
                self[prop] = model(target=self, **kwargs)
        else:
            self[prop] = model(target=self, **kwargs)

    def _get_models(self):
        if not hasattr(self, '_models_dict'):
            self._models_dict = ModelsDict()
        return self._models_dict

    def _set_models(self, dict_):
        self._models_dict = ModelsDict()
        self._models_dict.update(dict_)
        # Renerate all models in new dict if regen mode says so
        for model in dict_:
            # In case regen mode is not set, do it now
            dict_[model].setdefault('regen_mode', 'normal')
        if self.settings['freeze_models']:
            pass
        else:
            self.regenerate_models()

    models = property(fget=_get_models, fset=_set_models)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2017-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
import re
import zmq
import logging

context = zmq.Context()

# from integer socket_type to string
SOCKTYPE = {zmq.REQ: 'REQ', zmq.REP: 'REP',
            zmq.PUSH: 'PUSH', zmq.PULL: 'PULL',
            zmq.ROUTER: 'ROUTER', zmq.DEALER: 'DEALER'}

def bind(end_point, socket_type):
    """
    Bind to a zmq URL; raise a proper error if the URL is invalid; return
    a zmq socket.
    """
    sock = context.socket(socket_type)
    try:
        sock.bind(end_point)
    except zmq.error.ZMQError as exc:
        sock.close()
        raise exc.__class__('%s: %s' % (exc, end_point))
    return sock

def connect(end_point, socket_type):
    """
    Connect to a zmq URL; raise a proper error if the URL is invalid; return
    a zmq socket.
    """
    sock = context.socket(socket_type)
    try:
        sock.connect(end_point)
    except zmq.error.ZMQError as exc:
        sock.close()
        raise exc.__class__('%s: %s' % (exc, end_point))
    return sock

class Socket(object):
    """
    A Socket class to be used with code like the following::

     # server
     with Socket('tcp://127.0.0.1:9000', zmq.REP, 'bind') as sock:
         for tup in sock:
             sock.send(tup)

     # client
     with Socket('tcp://127.0.0.1:9000', zmq.REQ, 'connect') as sock:
        sock.send(tup)

    It also support zmq.PULL/zmq.PUSH sockets, which are asynchronous.

    :param end_point: zmq end point string
    :param socket_type: zmq socket type (integer)
    :param mode: default 'bind', accepts also 'connect'
    :param timeout: default 5000 ms, used when polling the underlying socket
    """
    def __init__(self, end_point, socket_type, mode, timeout=5000):
        assert socket_type in (zmq.REP, zmq.REQ, zmq.PULL, zmq.PUSH)
        assert mode in ('bind', 'connect'), mode
        if mode == 'bind':
            assert 'localhost' not in end_point, 'Use 127.0.0.1 instead'
        self.end_point = end_point
        self.socket_type = socket_type
        self.mode = mode
        self.timeout = timeout
        self.running = False

    def __enter__(self):
        """Instantiate the underlying zmq socket"""
        # first check if the end_point ends in :<min_port>-<max_port>
        port_range = re.search(r':(\d+)-(\d+)$', self.end_point)
        if port_range:
            assert self.mode == 'bind', self.mode
            p1, p2 = map(int, port_range.groups())
            end_point = self.end_point.rsplit(':', 1)[0]  # strip port range
            self.zsocket = context.socket(self.socket_type)
            port = self.zsocket.bind_to_random_port(end_point, p1, p2)
            # NB: will raise a ZMQBindError if no port is available
            self.port = port
        elif self.mode == 'bind':
            self.zsocket = bind(self.end_point, self.socket_type)
        else:  # connect
            self.zsocket = connect(self.end_point, self.socket_type)
        port = re.search(r':(\d+)$', self.end_point)
        if port:
            self.port = int(port.group(1))
        self.zsocket.__enter__()
        self.num_sent = 0
        return self

    def __exit__(self, *args):
        self.zsocket.__exit__(*args)
        del self.zsocket

    def __iter__(self):
        """
        Iterate on the socket and yield the received arguments. Exits if

        1. the flag .running is set to False
        2. SIGTERM is sent
        """
        # works with zmq.REP and zmq.PULL sockets
        self.running = True
        while self.running:
            try:
                if self.zsocket.poll(self.timeout):
                    yield self.zsocket.recv_pyobj()
                elif self.socket_type == zmq.PULL:
                    logging.debug('Timeout in %s', self)
            except zmq.ZMQError:
                # sending SIGTERM raises ZMQError
                break

    def send(self, obj):
        """
        Send an object to the remote server; block and return the reply
        if the socket type is REQ.

        :param obj:
            the Python object to send
        """
        self.zsocket.send_pyobj(obj)
        self.num_sent += 1
        if self.socket_type == zmq.REQ:
            return self.zsocket.recv_pyobj()

    def __repr__(self):
        return '<%s %s %s>' % (self.__class__.__name__,
                               SOCKTYPE[self.socket_type], self.mode)
<commit_msg>Printed the zmq socket port in the timeout message [skip CI]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2017-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
import re
import zmq
import logging

context = zmq.Context()

# from integer socket_type to string
SOCKTYPE = {zmq.REQ: 'REQ', zmq.REP: 'REP',
            zmq.PUSH: 'PUSH', zmq.PULL: 'PULL',
            zmq.ROUTER: 'ROUTER', zmq.DEALER: 'DEALER'}

def bind(end_point, socket_type):
    """
    Bind to a zmq URL; raise a proper error if the URL is invalid; return
    a zmq socket.
    """
    sock = context.socket(socket_type)
    try:
        sock.bind(end_point)
    except zmq.error.ZMQError as exc:
        sock.close()
        raise exc.__class__('%s: %s' % (exc, end_point))
    return sock

def connect(end_point, socket_type):
    """
    Connect to a zmq URL; raise a proper error if the URL is invalid; return
    a zmq socket.
    """
    sock = context.socket(socket_type)
    try:
        sock.connect(end_point)
    except zmq.error.ZMQError as exc:
        sock.close()
        raise exc.__class__('%s: %s' % (exc, end_point))
    return sock

class Socket(object):
    """
    A Socket class to be used with code like the following::

     # server
     with Socket('tcp://127.0.0.1:9000', zmq.REP, 'bind') as sock:
         for tup in sock:
             sock.send(tup)

     # client
     with Socket('tcp://127.0.0.1:9000', zmq.REQ, 'connect') as sock:
        sock.send(tup)

    It also support zmq.PULL/zmq.PUSH sockets, which are asynchronous.

    :param end_point: zmq end point string
    :param socket_type: zmq socket type (integer)
    :param mode: default 'bind', accepts also 'connect'
    :param timeout: default 5000 ms, used when polling the underlying socket
    """
    def __init__(self, end_point, socket_type, mode, timeout=5000):
        assert socket_type in (zmq.REP, zmq.REQ, zmq.PULL, zmq.PUSH)
        assert mode in ('bind', 'connect'), mode
        if mode == 'bind':
            assert 'localhost' not in end_point, 'Use 127.0.0.1 instead'
        self.end_point = end_point
        self.socket_type = socket_type
        self.mode = mode
        self.timeout = timeout
        self.running = False

    def __enter__(self):
        """Instantiate the underlying zmq socket"""
        # first check if the end_point ends in :<min_port>-<max_port>
        port_range = re.search(r':(\d+)-(\d+)$', self.end_point)
        if port_range:
            assert self.mode == 'bind', self.mode
            p1, p2 = map(int, port_range.groups())
            end_point = self.end_point.rsplit(':', 1)[0]  # strip port range
            self.zsocket = context.socket(self.socket_type)
            port = self.zsocket.bind_to_random_port(end_point, p1, p2)
            # NB: will raise a ZMQBindError if no port is available
            self.port = port
        elif self.mode == 'bind':
            self.zsocket = bind(self.end_point, self.socket_type)
        else:  # connect
            self.zsocket = connect(self.end_point, self.socket_type)
        port = re.search(r':(\d+)$', self.end_point)
        if port:
            self.port = int(port.group(1))
        self.zsocket.__enter__()
        self.num_sent = 0
        return self

    def __exit__(self, *args):
        self.zsocket.__exit__(*args)
        del self.zsocket

    def __iter__(self):
        """
        Iterate on the socket and yield the received arguments. Exits if

        1. the flag .running is set to False
        2. SIGTERM is sent
        """
        # works with zmq.REP and zmq.PULL sockets
        self.running = True
        while self.running:
            try:
                if self.zsocket.poll(self.timeout):
                    yield self.zsocket.recv_pyobj()
                elif self.socket_type == zmq.PULL:
                    logging.debug('Timeout in %s:%d', self, self.port)
            except zmq.ZMQError:
                # sending SIGTERM raises ZMQError
                break

    def send(self, obj):
        """
        Send an object to the remote server; block and return the reply
        if the socket type is REQ.

        :param obj:
            the Python object to send
        """
        self.zsocket.send_pyobj(obj)
        self.num_sent += 1
        if self.socket_type == zmq.REQ:
            return self.zsocket.recv_pyobj()

    def __repr__(self):
        return '<%s %s %s>' % (self.__class__.__name__,
                               SOCKTYPE[self.socket_type], self.mode)
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7

from __future__ import print_function
import fileinput
import re
import sys

def userCode(name):
    if name not in userCode.mapping:
        userCode.maxCode = userCode.maxCode + 1
        userCode.mapping[name] = userCode.maxCode
        print(str(userCode.maxCode) + "\t" + name, file=sys.stderr)

    return userCode.mapping[name]

userCode.maxCode = 0
userCode.mapping = {}

def main():
    usernamePattern = re.compile('\W@([a-zA-Z0-9_]+)')

    print("Source\tTarget\tTimestamp")
    print("Id\tLabel", file=sys.stderr)

    for line in fileinput.input():
        parts = line.split("\t", 2)
        timestamp = parts[0]
        user = parts[1].lower()

        for mention in usernamePattern.findall(parts[2]):
            if len(mention) > 15:
                continue
            print(str(userCode(user)) + "\t" + str(userCode(mention.lower())) +  "\t" + timestamp)

if __name__ == "__main__":
    main()
<commit_msg>Verbeter mention regex.<commit_after>#!/usr/bin/env python2.7

from __future__ import print_function
import fileinput
import re
import sys

def userCode(name):
    if name not in userCode.mapping:
        userCode.maxCode = userCode.maxCode + 1
        userCode.mapping[name] = userCode.maxCode
        print(str(userCode.maxCode) + "\t" + name, file=sys.stderr)

    return userCode.mapping[name]

userCode.maxCode = 0
userCode.mapping = {}

def main():
    usernamePattern = re.compile('\W@([a-zA-Z0-9_]{1,15})\W')

    print("Source\tTarget\tTimestamp")
    print("Id\tLabel", file=sys.stderr)

    for line in fileinput.input():
        parts = line.split("\t", 2)
        timestamp = parts[0]
        user = parts[1].lower()

        for mention in usernamePattern.findall(parts[2]):
            if len(mention) > 15:
                continue
            print(str(userCode(user)) + "\t" + str(userCode(mention.lower())) +  "\t" + timestamp)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2016, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
import sys
import subprocess
from openquake.baselib import sap
from openquake.server import dbserver

def rundjango(subcmd, hostport=None):
    args = [sys.executable, '-m', 'openquake.server.manage', subcmd]
    if hostport:
        args.append(hostport)
    subprocess.call(args)

@sap.Script
def webui(cmd, hostport='127.0.0.1:8800'):
    """
    start the webui server in foreground or perform other operation on the
    django application
    """
    dbserver.ensure_on()  # start the dbserver in a subproces
    if cmd == 'start':
        rundjango('runserver', hostport)
    elif cmd == 'migrate':
        rundjango('migrate')
    # For backward compatibility with Django 1.6
    elif cmd == 'syncdb':
        rundjango('syncdb')

webui.arg('cmd', 'webui command', choices='start migrate syncdb'.split())
webui.arg('hostport', 'a string of the form <hostname:port>')
<commit_msg>Run the webui commands only if the user is the DB owner and start the dbserver only when really needed<commit_after>#  -*- coding: utf-8 -*-
#  vim: tabstop=4 shiftwidth=4 softtabstop=4

#  Copyright (c) 2016, GEM Foundation

#  OpenQuake is free software: you can redistribute it and/or modify it
#  under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.

#  OpenQuake is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.

#  You should have received a copy of the GNU Affero General Public License
#  along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import subprocess
from openquake.baselib import sap
from openquake.commonlib import config
from openquake.server import dbserver

def rundjango(subcmd, hostport=None):
    args = [sys.executable, '-m', 'openquake.server.manage', subcmd]
    if hostport:
        args.append(hostport)
    subprocess.call(args)

@sap.Script
def webui(cmd, hostport='127.0.0.1:8800'):
    """
    start the webui server in foreground or perform other operation on the
    django application
    """

    db_owner = os.path.expanduser(config.get('dbserver', 'file'))
    if not os.access(db_owner, os.W_OK):
        sys.exit('This command must be run by the proper user: '
                 'see the documentation for details')
    if cmd == 'start':
        dbserver.ensure_on()  # start the dbserver in a subproces
        rundjango('runserver', hostport)
    elif cmd == 'migrate':
        rundjango('migrate')
    # For backward compatibility with Django 1.6
    elif cmd == 'syncdb':
        rundjango('syncdb')

webui.arg('cmd', 'webui command', choices='start migrate syncdb'.split())
webui.arg('hostport', 'a string of the form <hostname:port>')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""
Set up some system-wide loggers
"""

import os.path
import logging
from datetime import datetime
from contextlib import contextmanager
from openquake.baselib import zeromq, config

LEVELS = {'debug': logging.DEBUG,
          'info': logging.INFO,
          'warn': logging.WARNING,
          'error': logging.ERROR,
          'critical': logging.CRITICAL}

LOG_FORMAT = ('[%(asctime)s job #%(job_id)s %(hostname)s '
              '%(levelname)s %(processName)s/%(process)s] %(message)s')

LOG = logging.getLogger()

DBSERVER_PORT = int(os.environ.get('OQ_DBSERVER_PORT') or config.dbserver.port)

def dbcmd(action, *args):
    """
    A dispatcher to the database server.

    :param action: database action to perform
    :param args: arguments
    """
    sock = zeromq.Socket('tcp://%s:%s' % (config.dbserver.host, DBSERVER_PORT),
                         zeromq.zmq.REQ, 'connect')
    with sock:
        res, etype, _mon = sock.send((action,) + args)
    if etype:
        raise etype(res)
    return res

def touch_log_file(log_file):
    """
    If a log file destination is specified, attempt to open the file in
    'append' mode ('a'). If the specified file is not writable, an
    :exc:`IOError` will be raised.
    """
    open(os.path.abspath(log_file), 'a').close()

def set_level(level):
    """
    Initialize logs to write records with level `level` or above.
    """
    logging.root.setLevel(LEVELS.get(level, logging.WARNING))

def _update_log_record(self, record):
    """
    Massage a log record before emitting it. Intended to be used by the
    custom log handlers defined in this module.
    """
    if not hasattr(record, 'hostname'):
        record.hostname = '-'
    if not hasattr(record, 'job_id'):
        record.job_id = self.job_id

class LogStreamHandler(logging.StreamHandler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogStreamHandler, self).__init__()
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogStreamHandler, self).emit(record)

class LogFileHandler(logging.FileHandler):
    """
    Log file handler
    """
    def __init__(self, job_id, log_file):
        super(LogFileHandler, self).__init__(log_file)
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id
        self.log_file = log_file

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogFileHandler, self).emit(record)

class LogDatabaseHandler(logging.Handler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogDatabaseHandler, self).__init__()
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        if record.levelno >= logging.INFO:
            dbcmd('log', self.job_id, datetime.utcnow(), record.levelname,
                  '%s/%s' % (record.processName, record.process),
                  record.getMessage())

@contextmanager
def handle(job_id, log_level='info', log_file=None):
    """
    Context manager adding and removing log handlers.

    :param job_id:
         ID of the current job
    :param log_level:
         one of debug, info, warn, error, critical
    :param log_file:
         log file path (if None, logs on stdout only)
    """
    handlers = [LogDatabaseHandler(job_id)]  # log on db always
    if log_file is None:
        # add a StreamHandler if not already there
        if not any(h for h in logging.root.handlers
                   if isinstance(h, logging.StreamHandler)):
            handlers.append(LogStreamHandler(job_id))
    else:
        handlers.append(LogFileHandler(job_id, log_file))
    for handler in handlers:
        logging.root.addHandler(handler)
    set_level(log_level)
    try:
        yield
    finally:
        # sanity check to make sure that the logging on file is working
        if (log_file and log_file != os.devnull and
                os.path.getsize(log_file) == 0):
            logging.root.warn('The log file %s is empty!?' % log_file)
        for handler in handlers:
            logging.root.removeHandler(handler)
<commit_msg>Fixed dbcmd [demos]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""
Set up some system-wide loggers
"""

import os.path
import logging
from datetime import datetime
from contextlib import contextmanager
from openquake.baselib import zeromq, config

LEVELS = {'debug': logging.DEBUG,
          'info': logging.INFO,
          'warn': logging.WARNING,
          'error': logging.ERROR,
          'critical': logging.CRITICAL}

LOG_FORMAT = ('[%(asctime)s job #%(job_id)s %(hostname)s '
              '%(levelname)s %(processName)s/%(process)s] %(message)s')

LOG = logging.getLogger()

DBSERVER_PORT = int(os.environ.get('OQ_DBSERVER_PORT') or config.dbserver.port)

def dbcmd(action, *args):
    """
    A dispatcher to the database server.

    :param action: database action to perform
    :param args: arguments
    """
    sock = zeromq.Socket('tcp://%s:%s' % (config.dbserver.host, DBSERVER_PORT),
                         zeromq.zmq.REQ, 'connect')
    with sock:
        pik, etype, _mon = sock.send((action,) + args)
        res = pik.unpickle()
    if etype:
        raise etype(res)
    return res

def touch_log_file(log_file):
    """
    If a log file destination is specified, attempt to open the file in
    'append' mode ('a'). If the specified file is not writable, an
    :exc:`IOError` will be raised.
    """
    open(os.path.abspath(log_file), 'a').close()

def set_level(level):
    """
    Initialize logs to write records with level `level` or above.
    """
    logging.root.setLevel(LEVELS.get(level, logging.WARNING))

def _update_log_record(self, record):
    """
    Massage a log record before emitting it. Intended to be used by the
    custom log handlers defined in this module.
    """
    if not hasattr(record, 'hostname'):
        record.hostname = '-'
    if not hasattr(record, 'job_id'):
        record.job_id = self.job_id

class LogStreamHandler(logging.StreamHandler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogStreamHandler, self).__init__()
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogStreamHandler, self).emit(record)

class LogFileHandler(logging.FileHandler):
    """
    Log file handler
    """
    def __init__(self, job_id, log_file):
        super(LogFileHandler, self).__init__(log_file)
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id
        self.log_file = log_file

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogFileHandler, self).emit(record)

class LogDatabaseHandler(logging.Handler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogDatabaseHandler, self).__init__()
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        if record.levelno >= logging.INFO:
            dbcmd('log', self.job_id, datetime.utcnow(), record.levelname,
                  '%s/%s' % (record.processName, record.process),
                  record.getMessage())

@contextmanager
def handle(job_id, log_level='info', log_file=None):
    """
    Context manager adding and removing log handlers.

    :param job_id:
         ID of the current job
    :param log_level:
         one of debug, info, warn, error, critical
    :param log_file:
         log file path (if None, logs on stdout only)
    """
    handlers = [LogDatabaseHandler(job_id)]  # log on db always
    if log_file is None:
        # add a StreamHandler if not already there
        if not any(h for h in logging.root.handlers
                   if isinstance(h, logging.StreamHandler)):
            handlers.append(LogStreamHandler(job_id))
    else:
        handlers.append(LogFileHandler(job_id, log_file))
    for handler in handlers:
        logging.root.addHandler(handler)
    set_level(log_level)
    try:
        yield
    finally:
        # sanity check to make sure that the logging on file is working
        if (log_file and log_file != os.devnull and
                os.path.getsize(log_file) == 0):
            logging.root.warn('The log file %s is empty!?' % log_file)
        for handler in handlers:
            logging.root.removeHandler(handler)
<|endoftext|>


<commit_before>from Scenery.announcer_chair import *

class Emcee(object):

    def __init__(self, board_object):
        self.table_top = board_object
        self.announcer = Announcer()

    def choose_who_goes_first(self):
        self.announcer.show(self.announcer.start)
        self.announcer.show(self.announcer.select)
        choice = self.get_choice()
        if choice == '2':
           self.table_top.give_computer_the_first_move()

    def get_choice(self):
        choice = self.ask_human()
        if choice == '1' or choice == '2':
            return choice
        else:
            self.announcer.show(self.announcer.bad_move)
            return self.get_choice()

    def end_game(self, winner):
        if winner == 'tie':
            return self.announcer.show(self.announcer.tie)
        elif winner == 'computer':
            return self.announcer.show(self.announcer.computer)
        elif winner == 'human':
            return self.announcer.show(self.announcer.human)

    def ask_human(self):
        return raw_input('> ')
<commit_msg>add 3 strikes rule to Emcee.get_choice()<commit_after>import sys
from Scenery.announcer_chair import *

class Emcee(object):

    def __init__(self, board_object):
        self.table_top = board_object
        self.announcer = Announcer()

    strikes = 0

    def choose_who_goes_first(self):
        self.announcer.show(self.announcer.start)
        self.announcer.show(self.announcer.select)
        choice = self.get_choice()
        if choice == '2':
           self.table_top.give_computer_the_first_move()

    def get_choice(self):
        choice = self.ask_human()
        if choice == '1' or choice == '2':
            return choice
        elif self.strikes == 2:
            self.announcer.show(self.announcer.strike_3)
            sys.exit()
        else:
            self.announcer.show(self.announcer.bad_move)
            self.strikes += 1
            return self.get_choice()

    def end_game(self, winner):
        if winner == 'tie':
            return self.announcer.show(self.announcer.tie)
        elif winner == 'computer':
            return self.announcer.show(self.announcer.computer)
        elif winner == 'human':
            return self.announcer.show(self.announcer.human)

    def ask_human(self):
        return raw_input('> ')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""
Set up some system-wide loggers
"""

import os.path
import logging
from datetime import datetime
from contextlib import contextmanager
from openquake.baselib import zeromq

from openquake.commonlib import config

LEVELS = {'debug': logging.DEBUG,
          'info': logging.INFO,
          'warn': logging.WARNING,
          'error': logging.ERROR,
          'critical': logging.CRITICAL}

LOG_FORMAT = ('[%(asctime)s job #%(job_id)s %(hostname)s '
              '%(levelname)s %(processName)s/%(process)s] %(message)s')

LOG = logging.getLogger()

def dbcmd(action, *args):
    """
    A dispatcher to the database server.

    :param action: database action to perform
    :param args: arguments
    """
    sock = zeromq.Socket('tcp://%s:%s' % config.DBS_ADDRESS, zeromq.zmq.REP)
    res, etype, _mon = sock.req(action, *args)
    if etype:
        raise etype(res)
    return res

def touch_log_file(log_file):
    """
    If a log file destination is specified, attempt to open the file in
    'append' mode ('a'). If the specified file is not writable, an
    :exc:`IOError` will be raised.
    """
    open(os.path.abspath(log_file), 'a').close()

def set_level(level):
    """
    Initialize logs to write records with level `level` or above.
    """
    logging.root.setLevel(LEVELS.get(level, logging.WARNING))

def _update_log_record(self, record):
    """
    Massage a log record before emitting it. Intended to be used by the
    custom log handlers defined in this module.
    """
    if not hasattr(record, 'hostname'):
        record.hostname = '-'
    if not hasattr(record, 'job_id'):
        record.job_id = self.job_id

class LogStreamHandler(logging.StreamHandler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogStreamHandler, self).__init__()
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogStreamHandler, self).emit(record)

class LogFileHandler(logging.FileHandler):
    """
    Log file handler
    """
    def __init__(self, job_id, log_file):
        super(LogFileHandler, self).__init__(log_file)
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id
        self.log_file = log_file

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogFileHandler, self).emit(record)

class LogDatabaseHandler(logging.Handler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogDatabaseHandler, self).__init__()
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        if record.levelno >= logging.INFO:
            dbcmd('log', self.job_id, datetime.utcnow(), record.levelname,
                  '%s/%s' % (record.processName, record.process),
                  record.getMessage())

@contextmanager
def handle(job_id, log_level='info', log_file=None):
    """
    Context manager adding and removing log handlers.

    :param job_id:
         ID of the current job
    :param log_level:
         one of debug, info, warn, error, critical
    :param log_file:
         log file path (if None, logs on stdout only)
    """
    handlers = [LogDatabaseHandler(job_id)]  # log on db always
    if log_file is None:
        # add a StreamHandler if not already there
        if not any(h for h in logging.root.handlers
                   if isinstance(h, logging.StreamHandler)):
            handlers.append(LogStreamHandler(job_id))
    else:
        handlers.append(LogFileHandler(job_id, log_file))
    for handler in handlers:
        logging.root.addHandler(handler)
    set_level(log_level)
    try:
        yield
    finally:
        # sanity check to make sure that the logging on file is working
        if log_file and os.path.getsize(log_file) == 0:
            logging.root.warn('The log file %s is empty!?' % log_file)
        for handler in handlers:
            logging.root.removeHandler(handler)
<commit_msg>Another fix [skip hazardlib] [demos]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2010-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
"""
Set up some system-wide loggers
"""

import os.path
import logging
from datetime import datetime
from contextlib import contextmanager
from openquake.baselib import zeromq

from openquake.commonlib import config

LEVELS = {'debug': logging.DEBUG,
          'info': logging.INFO,
          'warn': logging.WARNING,
          'error': logging.ERROR,
          'critical': logging.CRITICAL}

LOG_FORMAT = ('[%(asctime)s job #%(job_id)s %(hostname)s '
              '%(levelname)s %(processName)s/%(process)s] %(message)s')

LOG = logging.getLogger()

def dbcmd(action, *args):
    """
    A dispatcher to the database server.

    :param action: database action to perform
    :param args: arguments
    """
    sock = zeromq.Socket('tcp://%s:%s' % config.DBS_ADDRESS, zeromq.zmq.REQ)
    res, etype, _mon = sock.req(action, *args)
    if etype:
        raise etype(res)
    return res

def touch_log_file(log_file):
    """
    If a log file destination is specified, attempt to open the file in
    'append' mode ('a'). If the specified file is not writable, an
    :exc:`IOError` will be raised.
    """
    open(os.path.abspath(log_file), 'a').close()

def set_level(level):
    """
    Initialize logs to write records with level `level` or above.
    """
    logging.root.setLevel(LEVELS.get(level, logging.WARNING))

def _update_log_record(self, record):
    """
    Massage a log record before emitting it. Intended to be used by the
    custom log handlers defined in this module.
    """
    if not hasattr(record, 'hostname'):
        record.hostname = '-'
    if not hasattr(record, 'job_id'):
        record.job_id = self.job_id

class LogStreamHandler(logging.StreamHandler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogStreamHandler, self).__init__()
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogStreamHandler, self).emit(record)

class LogFileHandler(logging.FileHandler):
    """
    Log file handler
    """
    def __init__(self, job_id, log_file):
        super(LogFileHandler, self).__init__(log_file)
        self.setFormatter(logging.Formatter(LOG_FORMAT))
        self.job_id = job_id
        self.log_file = log_file

    def emit(self, record):  # pylint: disable=E0202
        _update_log_record(self, record)
        super(LogFileHandler, self).emit(record)

class LogDatabaseHandler(logging.Handler):
    """
    Log stream handler
    """
    def __init__(self, job_id):
        super(LogDatabaseHandler, self).__init__()
        self.job_id = job_id

    def emit(self, record):  # pylint: disable=E0202
        if record.levelno >= logging.INFO:
            dbcmd('log', self.job_id, datetime.utcnow(), record.levelname,
                  '%s/%s' % (record.processName, record.process),
                  record.getMessage())

@contextmanager
def handle(job_id, log_level='info', log_file=None):
    """
    Context manager adding and removing log handlers.

    :param job_id:
         ID of the current job
    :param log_level:
         one of debug, info, warn, error, critical
    :param log_file:
         log file path (if None, logs on stdout only)
    """
    handlers = [LogDatabaseHandler(job_id)]  # log on db always
    if log_file is None:
        # add a StreamHandler if not already there
        if not any(h for h in logging.root.handlers
                   if isinstance(h, logging.StreamHandler)):
            handlers.append(LogStreamHandler(job_id))
    else:
        handlers.append(LogFileHandler(job_id, log_file))
    for handler in handlers:
        logging.root.addHandler(handler)
    set_level(log_level)
    try:
        yield
    finally:
        # sanity check to make sure that the logging on file is working
        if log_file and os.path.getsize(log_file) == 0:
            logging.root.warn('The log file %s is empty!?' % log_file)
        for handler in handlers:
            logging.root.removeHandler(handler)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    openslides.config.models
    ~~~~~~~~~~~~~~~~~~~~~~~~

    Models for the config app.

    :copyright: 2011, 2012 by OpenSlides team, see AUTHORS.
    :license: GNU GPL, see LICENSE for more details.
"""

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models
from django.dispatch import receiver
from django.utils.importlib import import_module
from django.utils.translation import ugettext_lazy as _, ugettext_noop

from openslides.utils.jsonfield import JSONField
from openslides.utils.signals import template_manipulation

from openslides.config.signals import default_config_value

class ConfigStore(models.Model):
    """
    Stores the config values.
    """
    key = models.CharField(max_length=100, primary_key=True)
    value = JSONField()

    def __unicode__(self):
        return self.key

    class Meta:
        verbose_name = 'config'
        permissions = (
            ('can_manage_config', ugettext_noop("Can manage configuration")),
        )

class Config(object):
    """
    Access the config values via config[...]
    """
    def __getitem__(self, key):
        try:
            return ConfigStore.objects.get(key=key).value
        except ConfigStore.DoesNotExist:
            pass

        for receiver, value in default_config_value.send(sender='config',
                                                         key=key):
            if value is not None:
                return value
        if settings.DEBUG:
            print "No default value for: %s" % key
        return None

    def __setitem__(self, key, value):
        try:
            c = ConfigStore.objects.get(pk=key)
        except ConfigStore.DoesNotExist:
            c = ConfigStore(pk=key)
        c.value = value
        c.save()

    def __contains__(self, item):
        return ConfigStore.objects.filter(key=item).exists()

config = Config()

@receiver(default_config_value, dispatch_uid="config_default_config")
def default_config(sender, key, **kwargs):
    """
    Global default values.
    """
    return {
        'event_name': 'OpenSlides',
        'event_description':
        _('Presentation and assembly system'),
        'event_date': '',
        'event_location': '',
        'event_organizer': '',
        'presentation': '',
        'welcome_title': _('Welcome to OpenSlides'),
        'welcome_text': _('[Place for your welcome text.]'),
        'system_enable_anonymous': False,
    }.get(key)

@receiver(template_manipulation, dispatch_uid="config_submenu")
def set_submenu(sender, request, context, **kwargs):
    """
    Submenu for the config tab.
    """
    if not request.path.startswith('/config/'):
        return None
    menu_links = [
        (reverse('config_general'), _('General'),
            request.path == reverse('config_general')),
    ]

    for app in settings.INSTALLED_APPS:
        try:
            mod = import_module(app)
            views = mod.views
            views.Config
        except (ImportError, AttributeError):
            continue

        appname = mod.__name__.split('.')[-1]

        selected = reverse('config_%s' % appname) == request.path
        try:
            title = mod.NAME
        except AttributeError:
            title = appname.title()
        menu_links.append(
            (reverse('config_%s' % appname), _(title), selected)
        )

    menu_links.append((
        reverse('config_version'), _('Version'),
        request.path == reverse('config_version')))

    context.update({
        'menu_links': menu_links})
<commit_msg>Fixed Issue #539 - Default config value in agenda app<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    openslides.config.models
    ~~~~~~~~~~~~~~~~~~~~~~~~

    Models for the config app.

    :copyright: 2011, 2012 by OpenSlides team, see AUTHORS.
    :license: GNU GPL, see LICENSE for more details.
"""

from django.conf import settings
from django.core.urlresolvers import reverse
from django.db import models
from django.dispatch import receiver
from django.utils.importlib import import_module
from django.utils.translation import ugettext_lazy as _, ugettext_noop

from openslides.utils.jsonfield import JSONField
from openslides.utils.signals import template_manipulation

from openslides.config.signals import default_config_value

class ConfigStore(models.Model):
    """
    Stores the config values.
    """
    key = models.CharField(max_length=100, primary_key=True)
    value = JSONField()

    def __unicode__(self):
        return self.key

    class Meta:
        verbose_name = 'config'
        permissions = (
            ('can_manage_config', ugettext_noop("Can manage configuration")),
        )

class Config(object):
    """
    Access the config values via config[...]
    """
    def __getitem__(self, key):
        try:
            return ConfigStore.objects.get(key=key).value
        except ConfigStore.DoesNotExist:
            pass

        for receiver, value in default_config_value.send(sender='config',
                                                         key=key):
            if value is not None:
                return value
        if settings.DEBUG:
            print "No default value for: %s" % key
        return None

    def __setitem__(self, key, value):
        try:
            c = ConfigStore.objects.get(pk=key)
        except ConfigStore.DoesNotExist:
            c = ConfigStore(pk=key)
        c.value = value
        c.save()

    def __contains__(self, item):
        return ConfigStore.objects.filter(key=item).exists()

config = Config()

@receiver(default_config_value, dispatch_uid="config_default_config")
def default_config(sender, key, **kwargs):
    """
    Global default values.
    """
    return {
        'event_name': 'OpenSlides',
        'event_description':
        _('Presentation and assembly system'),
        'event_date': '',
        'event_location': '',
        'event_organizer': '',
        'presentation': '',
        'welcome_title': _('Welcome to OpenSlides'),
        'welcome_text': _('[Place for your welcome text.]'),
        'system_enable_anonymous': False,
        'agenda_start_event_date_time': '',
    }.get(key)

@receiver(template_manipulation, dispatch_uid="config_submenu")
def set_submenu(sender, request, context, **kwargs):
    """
    Submenu for the config tab.
    """
    if not request.path.startswith('/config/'):
        return None
    menu_links = [
        (reverse('config_general'), _('General'),
            request.path == reverse('config_general')),
    ]

    for app in settings.INSTALLED_APPS:
        try:
            mod = import_module(app)
            views = mod.views
            views.Config
        except (ImportError, AttributeError):
            continue

        appname = mod.__name__.split('.')[-1]

        selected = reverse('config_%s' % appname) == request.path
        try:
            title = mod.NAME
        except AttributeError:
            title = appname.title()
        menu_links.append(
            (reverse('config_%s' % appname), _(title), selected)
        )

    menu_links.append((
        reverse('config_version'), _('Version'),
        request.path == reverse('config_version')))

    context.update({
        'menu_links': menu_links})
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    openslides.config.models
    ~~~~~~~~~~~~~~~~~~~~~~~~

    Models for the config app.

    :copyright: 2011 by the OpenSlides team, see AUTHORS.
    :license: GNU GPL, see LICENSE for more details.
"""
from pickle import dumps, loads
import base64

from django.db import models
from django.dispatch import receiver

from utils.translation_ext import ugettext as _

from openslides.config.signals import default_config_value

class ConfigStore(models.Model):
    key = models.CharField(max_length=100, primary_key=True)
    value = models.CharField(max_length=100)

    def __unicode__(self):
        return self.key

    class Meta:
        verbose_name = 'config'
        permissions = (
            ('can_manage_config', _("Can manage configuration", fixstr=True)),
        )

# TODO:
# I used base64 to save pickled Data, there has to be another way see:
# http://stackoverflow.com/questions/2524970/djangounicodedecodeerror-while-storing-pickled-data

class Config(object):
    def load_config(self):
        self.config = {}
        for key, value in ConfigStore.objects.all().values_list():
            self.config[key] = loads(base64.decodestring(str(value)))

    def __getitem__(self, key):
        try:
            self.config
        except AttributeError:
            self.load_config()

        try:
            return self.config[key]
        except KeyError:
            pass

        for receiver, value in default_config_value.send(sender='config', key=key):
            if value is not None:
                return value
        return None

    def __setitem__(self, key, value):
        try:
            c = ConfigStore.objects.get(pk=key)
        except ConfigStore.DoesNotExist:
            c = ConfigStore(pk=key)
        c.value = base64.encodestring(dumps(value))
        c.save()
        try:
            self.config[key] = value
        except AttributeError:
            self.load_config()
            self.config[key] = value

config = Config()

@receiver(default_config_value, dispatch_uid="config_default_config")
def default_config(sender, key, **kwargs):
    return {
        'event_name': 'OpenSlides',
        'event_description': 'Presentation and voting system',
        'frontpage_title': 'Welcome',
        'frontpage_welcometext': 'Welcome to OpenSlides!',
    }.get(key)

from django.dispatch import receiver
from django.core.urlresolvers import reverse
from django.utils.importlib import import_module
import settings

from openslides.utils.signals import template_manipulation

@receiver(template_manipulation, dispatch_uid="config_submenu")
def set_submenu(sender, request, context, **kwargs):
    if not request.path.startswith('/config/'):
        return None
    menu_links = [
        (reverse('config_general'), _('General'), request.path == reverse('config_general') ),
    ]

    for app in settings.INSTALLED_APPS:
        try:
            mod = import_module(app + '.views')
            mod.Config
        except (ImportError, AttributeError):
            continue

        appname = mod.__name__.split('.')[0]
        selected = True if reverse('config_%s' % appname) == request.path else False
        menu_links.append(
            (reverse('config_%s' % appname), _(appname.title()), selected)
        )

    menu_links.append (
        (reverse('config_version'), _('Version'), request.path == reverse('config_version') )
    )

    context.update({
        'menu_links': menu_links,
    })
<commit_msg>use plugin.NAME for submenu-title in config-tab<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
    openslides.config.models
    ~~~~~~~~~~~~~~~~~~~~~~~~

    Models for the config app.

    :copyright: 2011 by the OpenSlides team, see AUTHORS.
    :license: GNU GPL, see LICENSE for more details.
"""
from pickle import dumps, loads
import base64

from django.db import models
from django.dispatch import receiver

from utils.translation_ext import ugettext as _

from openslides.config.signals import default_config_value

class ConfigStore(models.Model):
    key = models.CharField(max_length=100, primary_key=True)
    value = models.CharField(max_length=100)

    def __unicode__(self):
        return self.key

    class Meta:
        verbose_name = 'config'
        permissions = (
            ('can_manage_config', _("Can manage configuration", fixstr=True)),
        )

# TODO:
# I used base64 to save pickled Data, there has to be another way see:
# http://stackoverflow.com/questions/2524970/djangounicodedecodeerror-while-storing-pickled-data

class Config(object):
    def load_config(self):
        self.config = {}
        for key, value in ConfigStore.objects.all().values_list():
            self.config[key] = loads(base64.decodestring(str(value)))

    def __getitem__(self, key):
        try:
            self.config
        except AttributeError:
            self.load_config()

        try:
            return self.config[key]
        except KeyError:
            pass

        for receiver, value in default_config_value.send(sender='config', key=key):
            if value is not None:
                return value
        return None

    def __setitem__(self, key, value):
        try:
            c = ConfigStore.objects.get(pk=key)
        except ConfigStore.DoesNotExist:
            c = ConfigStore(pk=key)
        c.value = base64.encodestring(dumps(value))
        c.save()
        try:
            self.config[key] = value
        except AttributeError:
            self.load_config()
            self.config[key] = value

config = Config()

@receiver(default_config_value, dispatch_uid="config_default_config")
def default_config(sender, key, **kwargs):
    return {
        'event_name': 'OpenSlides',
        'event_description': 'Presentation and voting system',
        'frontpage_title': 'Welcome',
        'frontpage_welcometext': 'Welcome to OpenSlides!',
    }.get(key)

from django.dispatch import receiver
from django.core.urlresolvers import reverse
from django.utils.importlib import import_module
import settings

from openslides.utils.signals import template_manipulation

@receiver(template_manipulation, dispatch_uid="config_submenu")
def set_submenu(sender, request, context, **kwargs):
    if not request.path.startswith('/config/'):
        return None
    menu_links = [
        (reverse('config_general'), _('General'), request.path == reverse('config_general') ),
    ]

    for app in settings.INSTALLED_APPS:
        try:
            mod = import_module(app)
            views = mod.views
            views.Config
        except (ImportError, AttributeError):
            continue

        appname = mod.__name__.split('.')[0]
        selected = reverse('config_%s' % appname) == request.path
        try:
            title = mod.NAME
        except AttributeError:
            title = appname.title()
        menu_links.append(
            (reverse('config_%s' % appname), _(title), selected)
        )

    menu_links.append (
        (reverse('config_version'), _('Version'), request.path == reverse('config_version') )
    )

    context.update({
        'menu_links': menu_links,
    })
<|endoftext|>


<commit_before>"""
this module is actual fix some parameter by using
bayesian optimization. if we run this function in parallel,
we need to satisfied many condition which mentioned in the paper
named 'practical bayesian optimization in machine learning algorithm'
"""

from ..optimizeHyper import run as optimizeHyper
from ..calculateError import run as calculateError
from ..loadParser.loadLabel import run as loadLabel
from ..loadParser.parseLabel import run as parseLabel

def learnMACSparam(args, test_set, validation_set):
    """
    this function actually control learning steps. args is given by
    oppa.py ( main function of this program ) from command line.
    and make wrapper_function to use BayesianOptimization library,
    only wrapper_function`s arguments will be learned by library.

    :param args:

    :return: learned parameter.
    """

    input_file = args.input
    control = args.control
    call_type = args.callType

    def wrapper_function(opt_Qval):
        error = run(input_file, validation_set, str(opt_Qval), call_type,control)
        return -error

    parameter_bound = {'opt_Qval' : (10**-16,0.8)}
    number_of_init_sample = 2

    result = optimizeHyper(wrapper_function, parameter_bound, number_of_init_sample)
    final_error = run(input_file, test_set, str(result['max_params']), call_type, control)

    print " final error about test set is :::" + str(final_error)

def run(input_file, valid_set, Qval, call_type, control = None):
    """
    this function run MACS and calculate error at once.
    each arguments will be given by learnMACSparam that from command line.

    :param input_file:
        input file name.
    :param valid_set:
        python list of labeled data
    :param Qval:
        Q-value of MACS. it will be learned.
    :param control:
        control bam file in MACS. not necessary.

    :return:
        error rate of between MACS_output and labeled Data.
    """
    import MACS

    """it will be runned by protoPFC.py"""
    bam_name = input_file[:-4]  ## delete '.bam'
    reference_char = ".REF_chr"

    p1 = MACS.run(bam_name + reference_char + "1.bam", Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "2.bam", Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "3.bam", Qval ,call_type,control)
    p4 = MACS.run(bam_name + reference_char + "4.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()
    p4.wait()

    p1 = MACS.run(bam_name + reference_char + "5.bam",  Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "6.bam",  Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "7.bam",  Qval ,call_type,control)
    p4 = MACS.run(bam_name + reference_char + "8.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()
    p4.wait()

    p1 = MACS.run(bam_name + reference_char + "9.bam",  Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "10.bam",  Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "11.bam",  Qval ,call_type,control)
    p4 = MACS.run(bam_name + reference_char + "12.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()
    p4.wait()

    p1 = MACS.run(bam_name + reference_char + "13.bam",  Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "14.bam",  Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "15.bam",  Qval ,call_type,control)
    p4 = MACS.run(bam_name + reference_char + "16.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()
    p4.wait()

    p1 = MACS.run(bam_name + reference_char + "17.bam",  Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "18.bam",  Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "19.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()

    p1 = MACS.run(bam_name + reference_char + "20.bam",  Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "21.bam",  Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "22.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()

    p1 = MACS.run(bam_name + reference_char + "M.bam",  Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "X.bam",  Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "Y.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()

    #there must be valid validation set and test set.
    if not valid_set:
        print "there are no matched validation set :p\n"
        exit()

    #actual learning part
    else:
        error_num, label_num = summerize_error(bam_name, valid_set)
       	return error_num/label_num

def summerize_error(bam_name, validation_set):
    """

    :param bam_name:
    :param validation_set:
    :return:
    """
    sum_error_num = 0
    sum_label_num = 0
    reference_char = ".REF_chr"

    for chr_no in range(22):
        input_name = bam_name + reference_char + str(chr_no+1) + ".bam_peaks" + ".broadPeak"
        error_num, label_num = calculateError(input_name, parseLabel(validation_set, input_name))
        sum_error_num += error_num
        sum_label_num += label_num

    # add about sexual chromosome
    input_name = bam_name + reference_char + str('X') + ".bam_peaks" + ".broadPeak"
    error_num, label_num = calculateError(input_name, parseLabel(validation_set, input_name))
    sum_error_num += error_num
    sum_label_num += label_num

    input_name = bam_name + reference_char + str('Y') + ".bam_peaks" + ".broadPeak"
    error_num, label_num = calculateError(input_name, parseLabel(validation_set, input_name))
    sum_error_num += error_num
    sum_label_num += label_num

    return sum_error_num , sum_label_num

def is_exist_chr(valid_set, chr):
    """
    this method to decide either run MACS or not about
    each chromosome in label data. if some chromosome did not
    included by label data, there is no need to run MACS that
    chromosome.

    :param valid_set:
        validation set before parse to python maps. it just list
        of each line in file like a
        ['chrN:0000-1132 peakStart bcell monocyte' , ... ]

    :param chr:
        input chromosome.

    :return:
        T/F about some chromosome exist in label.
    """

    for label in valid_set:
        if chr in label:
            return True
    return False
<commit_msg>Update learnMACSparam.py<commit_after>"""
this module is actual fix some parameter by using
bayesian optimization. if we run this function in parallel,
we need to satisfied many condition which mentioned in the paper
named 'practical bayesian optimization in machine learning algorithm'
"""

from ..optimizeHyper import run as optimizeHyper
from ..calculateError import run as calculateError
from ..loadParser.loadLabel import run as loadLabel
from ..loadParser.parseLabel import run as parseLabel

def learnMACSparam(args, test_set, validation_set):
    """
    this function actually control learning steps. args is given by
    oppa.py ( main function of this program ) from command line.
    and make wrapper_function to use BayesianOptimization library,
    only wrapper_function`s arguments will be learned by library.

    :param args:

    :return: learned parameter.
    """

    input_file = args.input
    control = args.control
    call_type = args.callType

    def wrapper_function(opt_Qval):
        error = run(input_file, validation_set, str(opt_Qval), call_type,control)
        return -error

    parameter_bound = {'opt_Qval' : (10**-16,0.8)}
    number_of_init_sample = 2

    result = optimizeHyper(wrapper_function, parameter_bound, number_of_init_sample)
    final_error = run(input_file, test_set, str(result['max_params']), call_type, control)

    print " final error about test set is :::" + str(final_error)

def run(input_file, valid_set, Qval, call_type, control = None):
    """
    this function run MACS and calculate error at once.
    each arguments will be given by learnMACSparam that from command line.

    :param input_file:
        input file name.
    :param valid_set:
        python list of labeled data
    :param Qval:
        Q-value of MACS. it will be learned.
    :param control:
        control bam file in MACS. not necessary.

    :return:
        error rate of between MACS_output and labeled Data.
    """
    import MACS

    """it will be runned by protoPFC.py"""
    bam_name = input_file[:-4]  ## delete '.bam'
    reference_char = ".REF_chr"

    p1 = MACS.run(bam_name + reference_char + "1.bam", Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "2.bam", Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "3.bam", Qval ,call_type,control)
    p4 = MACS.run(bam_name + reference_char + "4.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()
    p4.wait()

    p1 = MACS.run(bam_name + reference_char + "5.bam",  Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "6.bam",  Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "7.bam",  Qval ,call_type,control)
    p4 = MACS.run(bam_name + reference_char + "8.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()
    p4.wait()

    p1 = MACS.run(bam_name + reference_char + "9.bam",  Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "10.bam",  Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "11.bam",  Qval ,call_type,control)
    p4 = MACS.run(bam_name + reference_char + "12.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()
    p4.wait()

    p1 = MACS.run(bam_name + reference_char + "13.bam",  Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "14.bam",  Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "15.bam",  Qval ,call_type,control)
    p4 = MACS.run(bam_name + reference_char + "16.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()
    p4.wait()

    p1 = MACS.run(bam_name + reference_char + "17.bam",  Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "18.bam",  Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "19.bam",  Qval ,call_type,control)
    p4 = MACS.run(bam_name + reference_char + "X.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()
    p4.wait()

    p1 = MACS.run(bam_name + reference_char + "20.bam",  Qval ,call_type,control)
    p2 = MACS.run(bam_name + reference_char + "21.bam",  Qval ,call_type,control)
    p3 = MACS.run(bam_name + reference_char + "22.bam",  Qval ,call_type,control)
    p4 = MACS.run(bam_name + reference_char + "Y.bam",  Qval ,call_type,control)
    p1.wait()
    p2.wait()
    p3.wait()
    p4.wait()

    #there must be valid validation set and test set.
    if not valid_set:
        print "there are no matched validation set :p\n"
        exit()

    #actual learning part
    else:
        error_num, label_num = summerize_error(bam_name, valid_set)
       	return error_num/label_num

def summerize_error(bam_name, validation_set):
    """

    :param bam_name:
    :param validation_set:
    :return:
    """
    sum_error_num = 0
    sum_label_num = 0
    reference_char = ".REF_chr"

    for chr_no in range(22):
        input_name = bam_name + reference_char + str(chr_no+1) + ".bam_peaks" + ".broadPeak"
        error_num, label_num = calculateError(input_name, parseLabel(validation_set, input_name))
        sum_error_num += error_num
        sum_label_num += label_num

    # add about sexual chromosome
    input_name = bam_name + reference_char + str('X') + ".bam_peaks" + ".broadPeak"
    error_num, label_num = calculateError(input_name, parseLabel(validation_set, input_name))
    sum_error_num += error_num
    sum_label_num += label_num

    input_name = bam_name + reference_char + str('Y') + ".bam_peaks" + ".broadPeak"
    error_num, label_num = calculateError(input_name, parseLabel(validation_set, input_name))
    sum_error_num += error_num
    sum_label_num += label_num

    return sum_error_num , sum_label_num

def is_exist_chr(valid_set, chr):
    """
    this method to decide either run MACS or not about
    each chromosome in label data. if some chromosome did not
    included by label data, there is no need to run MACS that
    chromosome.

    :param valid_set:
        validation set before parse to python maps. it just list
        of each line in file like a
        ['chrN:0000-1132 peakStart bcell monocyte' , ... ]

    :param chr:
        input chromosome.

    :return:
        T/F about some chromosome exist in label.
    """

    for label in valid_set:
        if chr in label:
            return True
    return False
<|endoftext|>


<commit_before># Connect Google Drive
# Reference: https://developers.google.com/api-client-library/python/auth/web-app

# Python
import json
import httplib2
import urllib2
from urllib import urlencode
import re

# Flask
from flask import render_template, flash, redirect, request, url_for
from flask import session, abort
from flask.ext.login import login_user, logout_user, login_required, current_user

# App
from login import app
from login import login_manager

# DB
from models import get_gdrive_token, set_gdrive_token
from login import db

# Google Drive API
from oauth2client.client import OAuth2WebServerFlow, OAuth2Credentials
from apiclient.discovery import build
from oauth2client import client
from oauth2client.client import Credentials

GDRIVE_CLIENT_SECRET = app.config['GDRIVE_CLIENT_SECRET']
GDRIVE_CLIENT_ID = app.config['GDRIVE_CLIENT_ID']

# Full, permissive scope to access all of a user's files
OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive'

# Create JSON element from dict CLIENT_SECRET
# CLIENT_SECRET = json.dumps(app.config['CLIENT_SECRET'])

def get_gdrive_refresh_token():
	""" Get user credentials from database
	"""
	email = session.get('user')
	print "inside get_gdrive_access_token() \nemail = ", email
	if email is None:
		return None

	# Get the refresh token from the DB
	refresh_token = get_gdrive_token(email)

	# print "refresh_token form db = ", refresh_token
	if refresh_token:
		return refresh_token

	print "Error no record of refresh_token found in DB. Not connected to Google Drive"
	return None

def refresh_access_token(cred):
	""" Take credentials (with expired access token) as arg and return
		credentials with renewed access token.

		Get refresh token for this purpose from db. To look in db records,
		use corresponding email (taken from session). After renewing, write
		new credentials back to DB

	Args:
		cred: From DB; contains expired access_token; also contains refresh_token
	Returns:
		cred: Contains renewed access_token
	"""

	try:
		# Get email from session. Email used for querying DB
		email = session.get('user')
		print "inside gdrive-auth-finish. \nEmail = ", email
		if email is None:
			abort(403)

		# Extract out refresh_token from cred
		refresh_token = cred.refresh_token

		# Build the JSON variable for credentials
		renew = {
			'client_id' : GDRIVE_CLIENT_ID,
			'client_secret' : GDRIVE_CLIENT_SECRET,
			'refresh_token' : refresh_token,
			'grant_type' : "refresh_token"
		}

		# Renew access_token using the above refresh_token
		req = urllib2.Request("https://www.googleapis.com/oauth2/v3/token", urlencode(renew))
		resp = urllib2.urlopen(req)
		content = resp.read()
		cont = json.loads(content)

		# Replace old access_token with renewed one
		cred.access_token = cont["access_token"]

		# Update DB records with new credentials.
		set_gdrive_token(email, cred.to_json())

		return cred

	except AttributeError as e:
		print "No refresh_token found in old_credentials. NoneType?", e
		return None

	except Exception as e:
		print "Error: ", e
		return None

def get_user_info(credentials):
	"""Send a request to the UserInfo API to retrieve the user's information.

	Args:
	credentials: oauth2client.client.OAuth2Credentials instance to authorize the
	             request.
	Returns:
	User information as a dict.
	"""

	try:
		# Returns user information as a JSON object
		user_info_service = build(
			serviceName = 'drive', version = 'v2',
			http = credentials.authorize(httplib2.Http()) )
		# user_info is of type dict
		user_info = user_info_service.files().list().execute()
		if user_info:
			return user_info
		else:
			return None
	except Exception as e:
		# logging.error('An error occurred: %s', e)
		print "An error occurred: ", e

def gdrive_connect():
	"""Interfaces with views.py

	Returns:
		User information as a JSON string if it exists otherwise None.
	"""
	try:
		cred = get_gdrive_refresh_token()
		print "gdrive_connect() - cred from db = ", cred
		# Convert JSON represenation to an instance of 'OAuth2Credentials'
		credentials = Credentials.new_from_json(cred)

		# No record found in DB
		if credentials is None:
			print "No records for credentials found in DB = none"
			return None

		# Expired access_token
		if credentials.access_token_expired:
			# Get new access_token
			print "credentials.access_token_expired"
			credentials = refresh_access_token(credentials)

		user_info = get_user_info(credentials)
		# print "\n\nuser_info = ", json.dumps(user_info, indent=4, sort_keys=True)
		return json.dumps(user_info)

	except client.AccessTokenRefreshError as e:
		flash('Drive access revoked!')
		print "client.AccessTokenRefreshError", e
		# User revoked access, so remove gdrive token from DB
		set_gdrive_token(email, None)
		return None

	except AttributeError as e:
		print "no record found in DB. \nArributeError", e
		# No record exists in DB for GDrive
		return None

	except TypeError as e:
		print "no record found in DB. \nTypeError", e
		return None

@app.route('/gdrive-auth-finish')
def gdrive_auth_finish():
	email = session.get('user')
	print "inside gdrive-auth-finish. \nEmail = ", email
	if email is None:
		abort(403)
	flow = get_auth_flow_object()
	if 'code' not in request.args:
		auth_uri = flow.step1_get_authorize_url()
		return redirect(auth_uri)
	else:
		auth_code = request.args.get('code')

		# Credentials is of type class 'oauth2client.client.OAuth2Credentials'
		credentials = flow.step2_exchange(auth_code)

		# Convert credentials to a JSON representation before storing
		if set_gdrive_token(email, credentials.to_json()):
			print "credentials added to DB: type", type(credentials)
			return redirect(url_for('profile'))

		flash('Error in adding Gdrive token to DB')
		print "Error, Could not add credentials to DB"
		return redirect(url_for('profile'))

@app.route('/gdrive-auth-start')
def gdrive_auth_start():
	# Checking for appropriate user session
	if 'user' not in session:
		abort(403)
	email = session.get('user')
	print "inside gdrive_auth_start. \nUser in session"
	print "email = ", email
	if email is None:
		abort(403)

	return redirect(url_for('gdrive_auth_finish'))

def get_auth_flow_object():
	REDIRECT_URI = url_for('gdrive_auth_finish', _external=True)
	flow = OAuth2WebServerFlow(client_id=GDRIVE_CLIENT_ID,
                           client_secret=GDRIVE_CLIENT_SECRET,
                           scope=OAUTH_SCOPE,
                           redirect_uri=REDIRECT_URI)
	flow.params['access_type'] = 'offline'
	flow.params['approval_prompt'] = 'force'
	flow.params['include_granted_scopes'] = 'true'
	return flow

@app.route('/gdrive-disconnect')
def gdrive_disconnect():
	"""Disconnect Google Drive refresh_token from DB records
	"""
	print "inside gdrive-disconnect"
	email = session.get('user')
	if email is None:
		abort(403)
	print "Disconnecting Google Drive for email: ", email
	if set_gdrive_token(email, None):
		print "Disconnected Google Drive. removed from DB"
		return redirect(url_for('profile'))

	flash("Disconnect error, Try again")
	return redirect(url_for('profile'))
<commit_msg>Fixed HTTP 403 Error: bug fixes<commit_after># Connect Google Drive
# Reference: https://developers.google.com/api-client-library/python/auth/web-app

# Python
import json
import httplib2
import urllib2
from urllib import urlencode
import re

# Flask
from flask import render_template, flash, redirect, request, url_for
from flask import session, abort
from flask.ext.login import login_user, logout_user, login_required, current_user

# App
from login import app
from login import login_manager

# DB
from models import get_gdrive_token, set_gdrive_token
from login import db

# Google Drive API
from oauth2client.client import OAuth2WebServerFlow, OAuth2Credentials
from apiclient.discovery import build
from oauth2client import client
from oauth2client.client import Credentials

GDRIVE_CLIENT_SECRET = app.config['GDRIVE_CLIENT_SECRET']
GDRIVE_CLIENT_ID = app.config['GDRIVE_CLIENT_ID']

# Full, permissive scope to access all of a user's files
OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive'

# Create JSON element from dict CLIENT_SECRET
# CLIENT_SECRET = json.dumps(app.config['CLIENT_SECRET'])

def get_gdrive_refresh_token():
	""" Get user credentials from database
	"""
	email = session.get('user')
	print "inside get_gdrive_access_token() \nemail = ", email
	if email is None:
		return None

	# Get the refresh token from the DB
	refresh_token = get_gdrive_token(email)

	# print "refresh_token form db = ", refresh_token
	if refresh_token:
		return refresh_token

	print "Error no record of refresh_token found in DB. Not connected to Google Drive"
	return None

def refresh_access_token(cred):
	""" Take credentials (with expired access token) as arg and return
		credentials with renewed access token.

		Get refresh token for this purpose from db. To look in db records,
		use corresponding email (taken from session). After renewing, write
		new credentials back to DB

	Args:
		cred: From DB; contains expired access_token; also contains refresh_token
	Returns:
		cred: Contains renewed access_token
	"""

	try:
		# Get email from session. Email used for querying DB
		email = session.get('user')
		print "inside gdrive-auth-finish. \nEmail = ", email
		if email is None:
			abort(403)

		# Extract out refresh_token from cred
		refresh_token = cred.refresh_token

		# Build the JSON variable for credentials
		renew = {
			'client_id' : GDRIVE_CLIENT_ID,
			'client_secret' : GDRIVE_CLIENT_SECRET,
			'refresh_token' : refresh_token,
			'grant_type' : "refresh_token"
		}

		# Renew access_token using the above refresh_token
		req = urllib2.Request("https://www.googleapis.com/oauth2/v3/token", urlencode(renew))
		resp = urllib2.urlopen(req)
		content = resp.read()
		cont = json.loads(content)

		# Replace old access_token with renewed one
		cred.access_token = cont["access_token"]

		# Update DB records with new credentials.
		set_gdrive_token(email, cred.to_json())

		return cred

	except AttributeError as e:
		print "No refresh_token found in old_credentials. NoneType?", e
		return None

	except Exception as e:
		print "Error: ", e
		return None

def get_user_info(credentials):
	"""Send a request to the UserInfo API to retrieve the user's information.

	Args:
		credentials: oauth2client.client.OAuth2Credentials instance to authorize the
		             request.
	Returns:
		User information as a dict.
	"""
	try:
		# Returns user information as a JSON object
		user_info_service = build(
			serviceName = 'drive', version = 'v2',
			http = credentials.authorize(httplib2.Http()) )
		# user_info is of type dict
		user_info = user_info_service.files().list().execute()
		if user_info:
			return user_info
		else:
			return None
	except Exception as e:
		# logging.error('An error occurred: %s', e)
		print "An error occurred: ", e

def gdrive_connect():
	"""Interfaces with views.py

	Returns:
		User information as a JSON string if it exists otherwise None.
	"""
	try:
		cred = get_gdrive_refresh_token()
		print "gdrive_connect() - cred from db = ", cred
		# Convert JSON represenation to an instance of 'OAuth2Credentials'
		credentials = Credentials.new_from_json(cred)

		# No record found in DB
		if credentials is None:
			print "No records for credentials found in DB = none"
			return None

		# Expired access_token
		if credentials.access_token_expired:
			# Get new access_token
			print "credentials.access_token_expired"
			credentials = refresh_access_token(credentials)

		user_info = get_user_info(credentials)
		# print "\n\nuser_info = ", json.dumps(user_info, indent=4, sort_keys=True)
		return json.dumps(user_info)

	except client.AccessTokenRefreshError as e:
		flash('Drive access revoked!')
		print "client.AccessTokenRefreshError", e
		# User revoked access, so remove gdrive token from DB
		set_gdrive_token(email, None)
		return None

	except AttributeError as e:
		print "no record found in DB. \nArributeError", e
		# No record exists in DB for GDrive
		return None

	except TypeError as e:
		print "no record found in DB. \nTypeError", e
		return None

@app.route('/gdrive-auth-finish')
def gdrive_auth_finish():
	email = session.get('user')
	print "inside gdrive-auth-finish. \nEmail = ", email
	if email is None:
		abort(403)
	flow = get_auth_flow_object()
	if 'code' not in request.args:
		auth_uri = flow.step1_get_authorize_url()
		return redirect(auth_uri)
	else:
		auth_code = request.args.get('code')

		# Credentials is of type class 'oauth2client.client.OAuth2Credentials'
		credentials = flow.step2_exchange(auth_code)

		# Convert credentials to a JSON representation before storing
		if set_gdrive_token(email, credentials.to_json()):
			print "credentials added to DB: type", type(credentials)
			return redirect(url_for('profile'))

		flash('Error in adding Gdrive token to DB')
		print "Error, Could not add credentials to DB"
		return redirect(url_for('profile'))

@app.route('/gdrive-auth-start')
def gdrive_auth_start():
	# Checking for appropriate user session
	if 'user' not in session:
		abort(403)
	email = session.get('user')
	print "inside gdrive_auth_start. \nUser in session"
	print "email = ", email
	if email is None:
		abort(403)

	return redirect(url_for('gdrive_auth_finish'))

def get_auth_flow_object():
	REDIRECT_URI = url_for('gdrive_auth_finish', _external=True)
	flow = OAuth2WebServerFlow(client_id=GDRIVE_CLIENT_ID,
                           client_secret=GDRIVE_CLIENT_SECRET,
                           scope=OAUTH_SCOPE,
                           redirect_uri=REDIRECT_URI)
	flow.params['access_type'] = 'offline'
	flow.params['approval_prompt'] = 'force'
	flow.params['include_granted_scopes'] = 'true'
	return flow

@app.route('/gdrive-disconnect')
def gdrive_disconnect():
	"""Disconnect Google Drive refresh_token from DB records
	"""
	print "inside gdrive-disconnect"
	email = session.get('user')
	if email is None:
		abort(403)
	print "Disconnecting Google Drive for email: ", email
	if set_gdrive_token(email, None):
		print "Disconnected Google Drive. removed from DB"
		return redirect(url_for('profile'))

	flash("Disconnect error, Try again")
	return redirect(url_for('profile'))
<|endoftext|>