<commit_before>#
# Copyright (c) 2005 rPath, Inc.
#
# All Rights Reserved
#
from mod_python import apache
from mod_python import util

import base64
import os
import xmlrpclib
import zlib
import re
import sys
import time
import traceback

from conary.lib import util as conaryutil
from conary.lib import epdb, log
from conary.repository.netrepos import netserver
from conary.repository.filecontainer import FileContainer
from conary.repository import changeset
from conary.repository import errors

from mint import config
from mint import mint_server
from mint import users
from webhandler import normPath
import app
import cookie_http

profiling = False
BUFFER=1024 * 256

def getHttpAuth(req):
    # special header to pass a session id through
    # instead of a real http authorization token
    if 'X-Session-Id' in req.headers_in:
        return req.headers_in['X-Session-Id']

    if not 'Authorization' in req.headers_in:
        return ('anonymous', 'anonymous')

    info = req.headers_in['Authorization'].split()
    if len(info) != 2 or info[0] != "Basic":
        return apache.HTTP_BAD_REQUEST

    try:
        authString = base64.decodestring(info[1])
    except:
        return apache.HTTP_BAD_REQUEST

    if authString.count(":") != 1:
        return apache.HTTP_BAD_REQUEST

    authToken = authString.split(":")

    return authToken

def checkAuth(req, repos):
    if not req.headers_in.has_key('Authorization'):
        return None
    else:
        authToken = getHttpAuth(req)
        if type(authToken) != tuple:
            return authToken

        if not repos.auth.checkUserPass(authToken):
            return None

    return authToken

def post(port, isSecure, repos, cfg, req):
    if isSecure:
        protocol = "https"
    else:
        protocol = "http"

    if req.headers_in['Content-Type'] == "text/xml":
        encoding = req.headers_in.get('Content-Encoding', None)
        data = req.read()
        if encoding == 'deflate':
            data = zlib.decompress(data)

        authToken = getHttpAuth(req)
        if type(authToken) is int:
            return authToken

        if authToken[0] != "anonymous" and not isSecure and repos.forceSecure:
            return apache.HTTP_FORBIDDEN

        (params, method) = xmlrpclib.loads(data)

        wrapper = repos.callWrapper
        params = [protocol, port, method, authToken, params]
        try:
            result = wrapper(*params)
        except (errors.InsufficientPermission):
            sys.stderr.flush()
            return apache.HTTP_FORBIDDEN

        resp = xmlrpclib.dumps((result,), methodresponse=1)
        req.content_type = "text/xml"
        encoding = req.headers_in.get('Accept-encoding', '')
        if len(resp) > 200 and 'deflate' in encoding:
            req.headers_out['Content-encoding'] = 'deflate'
            resp = zlib.compress(resp, 5)
        req.write(resp)
        return apache.OK
    else:
        webfe = app.MintApp(req, cfg, repServer = repos)
        return webfe._handle(req.uri)

def get(port, isSecure, repos, cfg, req):
    def _writeNestedFile(req, name, tag, size, f, sizeCb):
        if changeset.ChangedFileTypes.refr[4:] == tag[2:]:
            path = f.read()
            size = os.stat(path).st_size
            tag = tag[0:2] + changeset.ChangedFileTypes.file[4:]
            sizeCb(size, tag)
            req.sendfile(path)
        else:
            sizeCb(size, tag)
            req.write(f.read())

    if isSecure:
        protocol = "https"
    else:
        protocol = "http"

    uri = req.uri
    if uri.endswith('/'):
        uri = uri[:-1]
    cmd = os.path.basename(uri)
    fields = util.FieldStorage(req)

    if cmd == "changeset":
        authToken = getHttpAuth(req)
        if type(authToken) is int:
            return authToken
        if authToken[0] != "anonymous" and not isSecure and repos.forceSecure:
            return apache.HTTP_FORBIDDEN

        localName = repos.tmpPath + "/" + req.args + "-out"
        size = os.stat(localName).st_size

        if localName.endswith(".cf-out"):
            try:
                f = open(localName, "r")
            except IOError:
                self.send_error(404, "File not found")
                return None

            os.unlink(localName)

            items = []
            totalSize = 0
            for l in f.readlines():
                (path, size) = l.split()
                size = int(size)
                totalSize += size
                items.append((path, size))
            del f
        else:
            size = os.stat(localName).st_size;
            items = [ (localName, size) ]
            totalSize = size

        req.content_type = "application/x-conary-change-set"
        for (path, size) in items:
            if path.endswith('.ccs-out'):
                cs = FileContainer(open(path))
                cs.dump(req.write,
                        lambda name, tag, size, f, sizeCb:
                            _writeNestedFile(req, name, tag, size, f,
                                             sizeCb))

                del cs
            else:
                req.sendfile(path)

            if path.startswith(repos.tmpPath) and \
                    not(os.path.basename(path)[0:6].startswith('cache-')):
                os.unlink(path)

        return apache.OK
    else:
        webfe = app.MintApp(req, cfg, repServer = repos)
        return webfe._handle(req.uri)

def putFile(port, isSecure, repos, req):
    if not isSecure and repos.forceSecure:
        return apache.HTTP_FORBIDDEN

    path = repos.tmpPath + "/" + req.args + "-in"
    size = os.stat(path).st_size
    if size != 0:
	return apache.HTTP_UNAUTHORIZED

    f = open(path, "w+")
    s = req.read(BUFFER)
    while s:
	f.write(s)
	s = req.read(BUFFER)

    f.close()

    return apache.OK

def conaryHandler(req, cfg, pathInfo):
    paths = normPath(req.uri).split("/")
    if paths[1] == "repos":
        # test suite hook: lop off any port specified in cfg file
        domainName = cfg.projectDomainName.split(":")[0]
        repName = paths[2] + "." + domainName
    else:
        repName = req.hostname

    method = req.method.upper()
    port = req.connection.local_addr[1]
    secure = (req.subprocess_env.get('HTTPS', 'off') == 'on')

    repHash = repName + req.hostname
    if not repositories.has_key(repHash):
        repositoryDir = os.path.join(cfg.reposPath, repName)
        tmpPath = os.path.join(cfg.reposPath, repName, "tmp")
        logFile = os.path.join(repositoryDir, "contents.log")

        if os.path.basename(req.uri) == "changeset":
           rest = os.path.dirname(req.uri) + "/"
        else:
           rest = req.uri

        # pull out any queryargs
        if '?' in rest:
            rest = req.uri.split("?")[0]

        urlBase = "%%(protocol)s://%s:%%(port)d" % \
                        (req.hostname) + rest

        # set up the commitAction
        buildLabel = repName + "@" + cfg.defaultBranch
        projectName = repName.split(".")[0]
        if cfg.SSL:
            protocol = "https"
            host = cfg.secureHost
        else:
            protocol = "http"
            host = req.hostname

        repMapStr = "%s://%s:%s@%s/repos/%s/" % (protocol, cfg.authUser, cfg.authPass, host, projectName)

        repMap = {buildLabel: repMapStr,
                  'conary.rpath.com': 'http://conary-commits.rpath.com/conary/'}
        if cfg.commitAction:
            commitAction = cfg.commitAction % {'repMap': repName + " " + repMapStr,
                                               'buildLabel': buildLabel,
                                               'projectName': projectName,
                                               'commitEmail': cfg.commitEmail,
                                               'basePath' : cfg.basePath}
        else:
            commitAction = None

        # XXX hack to override commitAction for foresight until foresight
        # switches to our mailing lists.
        if req.hostname == "foresight.rpath.org":
            commitAction = '/usr/lib64/python2.4/site-packages/conary/commitaction --module "/usr/lib/python2.4/site-packages/mint/rbuilderaction.py --user %%(user)s --url http://www.rpath.org/xmlrpc-private/" --module "/usr/lib64/python2.4/site-packages/conary/changemail.py --user %(user)s --email desktop-commits@bizrace.com"'

        if os.access(repositoryDir, os.F_OK):
            repositories[repHash] = netserver.NetworkRepositoryServer(
                                        repositoryDir,
                                        tmpPath,
                                        urlBase,
                                        repName,
                                        repMap,
                                        commitAction = commitAction,
                                        cacheChangeSets = True,
                                        logFile = logFile
                                    )
            repositories[repHash].forceSecure = cfg.SSL
            repositories[repHash].cfg = cfg
        else:
            repositories[repHash] = None

    repo = repositories[repHash]

    if method == "POST":
	return post(port, secure, repo, cfg, req)
    elif method == "GET":
	return get(port, secure, repo, cfg, req)
    elif method == "PUT":
	return putFile(port, secure, repo, req)
    else:
	return apache.HTTP_METHOD_NOT_ALLOWED

def xmlrpcHandler(req, cfg, pathInfo):
    if req.method.upper() != "POST":
        return apache.HTTP_METHOD_NOT_ALLOWED
    if req.headers_in['Content-Type'] != "text/xml":
        return apache.HTTP_NOT_FOUND

    authToken = getHttpAuth(req)
    if type(authToken) is int:
        return authToken

    (params, method) = xmlrpclib.loads(req.read())
    params = [method, authToken, params]

    if req.uri.startswith("/xmlrpc-private"):
        server = mint_server.MintServer(cfg, allowPrivate = True)
    elif req.uri.startswith("/xmlrpc"):
        server = mint_server.MintServer(cfg, allowPrivate = False)
    try:
        result = server.callWrapper(*params)
    except (errors.InsufficientPermission, mint_server.PermissionDenied):
        return apache.HTTP_FORBIDDEN

    resp = xmlrpclib.dumps((result,), methodresponse=1)
    req.content_type = "text/xml"
    encoding = req.headers_in.get('Accept-encoding', '')
    useragent = req.headers_in.get('User-Agent', '')
    if len(resp) > 200 and 'deflate' in encoding and 'MSIE' not in useragent:
        req.headers_out['Content-encoding'] = 'deflate'
        resp = zlib.compress(resp, 5)
    req.write(resp)
    return apache.OK

def mintHandler(req, cfg, pathInfo):
    webfe = app.MintApp(req, cfg)
    return webfe._handle(pathInfo)

urls = (
    (r'^/conary/',           conaryHandler),
    (r'^/xmlrpc/',           xmlrpcHandler),
    (r'^/xmlrpc-private/',   xmlrpcHandler),
    (r'^/repos/',            conaryHandler),
    (r'^/',                  mintHandler),
)

def logErrorAndEmail(req, cfg, exception, e, bt):
    c = req.connection
    req.add_common_vars()
    info_dict = {
        'local_addr'     : c.local_ip + ':' + str(c.local_addr[1]),
        'remote_addr'    : c.remote_ip + ':' + str(c.remote_addr[1]),
        'remote_host'    : c.remote_host,
        'remote_logname' : c.remote_logname,
        'aborted'        : c.aborted,
        'keepalive'      : c.keepalive,
        'double_reverse' : c.double_reverse,
        'keepalives'     : c.keepalives,
        'local_host'     : c.local_host,
        'connection_id'  : c.id,
        'notes'          : c.notes,
        'the_request'    : req.the_request,
        'proxyreq'       : req.proxyreq,
        'header_only'    : req.header_only,
        'protocol'       : req.protocol,
        'proto_num'      : req.proto_num,
        'hostname'       : req.hostname,
        'request_time'   : time.ctime(req.request_time),
        'status_line'    : req.status_line,
        'status'         : req.status,
        'method'         : req.method,
        'allowed'        : req.allowed,
        'headers_in'     : req.headers_in,
        'headers_out'    : req.headers_out,
        'uri'            : req.uri,
        'unparsed_uri'   : req.unparsed_uri,
        'args'           : req.args,
        'parsed_uri'     : req.parsed_uri,
        'filename'       : req.filename,
        'subprocess_env' : req.subprocess_env,
        'referer'        : req.headers_in.get('referer', 'N/A')
    }
    info_dict_small = {
        'local_addr'     : c.local_ip + ':' + str(c.local_addr[1]),
        'uri'            : req.uri,
        'request_time'   : time.ctime(req.request_time),
    }

    timeStamp = time.ctime(time.time())
    # log error
    log.error('[%s] Unhandled exception from mint web interface: %s: %s', timeStamp, exception.__name__, e)
    log.error('sending mail to %s and %s' % (cfg.bugsEmail, cfg.smallBugsEmail))
    # send email
    body = 'Unhandled exception from mint web interface:\n\n%s: %s\n\n' %(exception.__name__, e)
    body += 'Time of occurrence: %s\n\n' %timeStamp
    body += ''.join( traceback.format_tb(bt))
    body += '\nConnection Information:\n'
    for key, val in sorted(info_dict.items()):
        body += '\n' + key + ': ' + str(val)

    body_small = 'Mint Exception: %s: %s' % (exception.__name__, e)
    for key, val in sorted(info_dict_small.items()):
        body_small += '\n' + key + ': ' + str(val)

    if cfg.bugsEmail:
        users.sendMailWithChecks(cfg.bugsEmail, cfg.bugsEmailName,
                                 cfg.bugsEmail, cfg.bugsEmailSubject, body)
    if cfg.smallBugsEmail:
        users.sendMailWithChecks(cfg.bugsEmail, cfg.bugsEmailName,
                                 cfg.smallBugsEmail, cfg.bugsEmailSubject, body_small)

cfg = None
def handler(req):
    startTime = time.time()
    if not req.hostname:
        return apache.HTTP_BAD_REQUEST

    global cfg
    if not cfg:
        cfg = config.MintConfig()
        cfg.read(req.filename)

    # normalize req path and base path
    pathInfo = normPath(req.uri)
    basePath = normPath(cfg.basePath)

    # strip off base path and normalize again
    pathInfo = pathInfo[len(basePath):]
    pathInfo = normPath(pathInfo)

    ret = apache.HTTP_NOT_FOUND
    for match, urlHandler in urls:
        if re.match(match, pathInfo):
            newPath = normPath(pathInfo[len(match)-1:])
            try:
                ret = urlHandler(req, cfg, newPath)
            except apache.SERVER_RETURN:
                raise
            except:
                # we only want to handle errors in production mode
                if cfg.debugMode or req.bytes_sent > 0:
                    raise
                # only handle actual mint errors
                if match !='^/':
                    raise
                exception, e, bt = sys.exc_info()
                logErrorAndEmail(req, cfg, exception, e, bt)
                ret = urlHandler(req, cfg, '/unknownError')
            break
    if cfg.profiling:
        print >> sys.stderr, "WEB HIT: %.2fms" % ((time.time() - startTime) * 1000)
        sys.stderr.flush()
    return ret

repositories = {}
<commit_msg>add hooks for code test coverage tracking<commit_after>#
# Copyright (c) 2005 rPath, Inc.
#
# All Rights Reserved
#

try:
    import coverage
    coverage.use_cache(True)
    coverage.start()
except ImportError:
    coverage = None

from mod_python import apache
from mod_python import util

import base64
import os
import xmlrpclib
import zlib
import re
import sys
import time
import traceback

from conary.lib import util as conaryutil
from conary.lib import epdb, log
from conary.repository.netrepos import netserver
from conary.repository.filecontainer import FileContainer
from conary.repository import changeset
from conary.repository import errors

from mint import config
from mint import mint_server
from mint import users
from webhandler import normPath
import app
import cookie_http

profiling = False
BUFFER=1024 * 256

def getHttpAuth(req):
    # special header to pass a session id through
    # instead of a real http authorization token
    if 'X-Session-Id' in req.headers_in:
        return req.headers_in['X-Session-Id']

    if not 'Authorization' in req.headers_in:
        return ('anonymous', 'anonymous')

    info = req.headers_in['Authorization'].split()
    if len(info) != 2 or info[0] != "Basic":
        return apache.HTTP_BAD_REQUEST

    try:
        authString = base64.decodestring(info[1])
    except:
        return apache.HTTP_BAD_REQUEST

    if authString.count(":") != 1:
        return apache.HTTP_BAD_REQUEST

    authToken = authString.split(":")

    return authToken

def checkAuth(req, repos):
    if not req.headers_in.has_key('Authorization'):
        return None
    else:
        authToken = getHttpAuth(req)
        if type(authToken) != tuple:
            return authToken

        if not repos.auth.checkUserPass(authToken):
            return None

    return authToken

def post(port, isSecure, repos, cfg, req):
    if isSecure:
        protocol = "https"
    else:
        protocol = "http"

    if req.headers_in['Content-Type'] == "text/xml":
        encoding = req.headers_in.get('Content-Encoding', None)
        data = req.read()
        if encoding == 'deflate':
            data = zlib.decompress(data)

        authToken = getHttpAuth(req)
        if type(authToken) is int:
            return authToken

        if authToken[0] != "anonymous" and not isSecure and repos.forceSecure:
            return apache.HTTP_FORBIDDEN

        (params, method) = xmlrpclib.loads(data)

        wrapper = repos.callWrapper
        params = [protocol, port, method, authToken, params]
        try:
            result = wrapper(*params)
        except (errors.InsufficientPermission):
            sys.stderr.flush()
            return apache.HTTP_FORBIDDEN

        resp = xmlrpclib.dumps((result,), methodresponse=1)
        req.content_type = "text/xml"
        encoding = req.headers_in.get('Accept-encoding', '')
        if len(resp) > 200 and 'deflate' in encoding:
            req.headers_out['Content-encoding'] = 'deflate'
            resp = zlib.compress(resp, 5)
        req.write(resp)
        return apache.OK
    else:
        webfe = app.MintApp(req, cfg, repServer = repos)
        return webfe._handle(req.uri)

def get(port, isSecure, repos, cfg, req):
    def _writeNestedFile(req, name, tag, size, f, sizeCb):
        if changeset.ChangedFileTypes.refr[4:] == tag[2:]:
            path = f.read()
            size = os.stat(path).st_size
            tag = tag[0:2] + changeset.ChangedFileTypes.file[4:]
            sizeCb(size, tag)
            req.sendfile(path)
        else:
            sizeCb(size, tag)
            req.write(f.read())

    if isSecure:
        protocol = "https"
    else:
        protocol = "http"

    uri = req.uri
    if uri.endswith('/'):
        uri = uri[:-1]
    cmd = os.path.basename(uri)
    fields = util.FieldStorage(req)

    if cmd == "changeset":
        authToken = getHttpAuth(req)
        if type(authToken) is int:
            return authToken
        if authToken[0] != "anonymous" and not isSecure and repos.forceSecure:
            return apache.HTTP_FORBIDDEN

        localName = repos.tmpPath + "/" + req.args + "-out"
        size = os.stat(localName).st_size

        if localName.endswith(".cf-out"):
            try:
                f = open(localName, "r")
            except IOError:
                self.send_error(404, "File not found")
                return None

            os.unlink(localName)

            items = []
            totalSize = 0
            for l in f.readlines():
                (path, size) = l.split()
                size = int(size)
                totalSize += size
                items.append((path, size))
            del f
        else:
            size = os.stat(localName).st_size;
            items = [ (localName, size) ]
            totalSize = size

        req.content_type = "application/x-conary-change-set"
        for (path, size) in items:
            if path.endswith('.ccs-out'):
                cs = FileContainer(open(path))
                cs.dump(req.write,
                        lambda name, tag, size, f, sizeCb:
                            _writeNestedFile(req, name, tag, size, f,
                                             sizeCb))

                del cs
            else:
                req.sendfile(path)

            if path.startswith(repos.tmpPath) and \
                    not(os.path.basename(path)[0:6].startswith('cache-')):
                os.unlink(path)

        return apache.OK
    else:
        webfe = app.MintApp(req, cfg, repServer = repos)
        return webfe._handle(req.uri)

def putFile(port, isSecure, repos, req):
    if not isSecure and repos.forceSecure:
        return apache.HTTP_FORBIDDEN

    path = repos.tmpPath + "/" + req.args + "-in"
    size = os.stat(path).st_size
    if size != 0:
	return apache.HTTP_UNAUTHORIZED

    f = open(path, "w+")
    s = req.read(BUFFER)
    while s:
	f.write(s)
	s = req.read(BUFFER)

    f.close()

    return apache.OK

def conaryHandler(req, cfg, pathInfo):
    paths = normPath(req.uri).split("/")
    if paths[1] == "repos":
        # test suite hook: lop off any port specified in cfg file
        domainName = cfg.projectDomainName.split(":")[0]
        repName = paths[2] + "." + domainName
    else:
        repName = req.hostname

    method = req.method.upper()
    port = req.connection.local_addr[1]
    secure = (req.subprocess_env.get('HTTPS', 'off') == 'on')

    repHash = repName + req.hostname
    if not repositories.has_key(repHash):
        repositoryDir = os.path.join(cfg.reposPath, repName)
        tmpPath = os.path.join(cfg.reposPath, repName, "tmp")
        logFile = os.path.join(repositoryDir, "contents.log")

        if os.path.basename(req.uri) == "changeset":
           rest = os.path.dirname(req.uri) + "/"
        else:
           rest = req.uri

        # pull out any queryargs
        if '?' in rest:
            rest = req.uri.split("?")[0]

        urlBase = "%%(protocol)s://%s:%%(port)d" % \
                        (req.hostname) + rest

        # set up the commitAction
        buildLabel = repName + "@" + cfg.defaultBranch
        projectName = repName.split(".")[0]
        if cfg.SSL:
            protocol = "https"
            host = cfg.secureHost
        else:
            protocol = "http"
            host = req.hostname

        repMapStr = "%s://%s:%s@%s/repos/%s/" % (protocol, cfg.authUser, cfg.authPass, host, projectName)

        repMap = {buildLabel: repMapStr,
                  'conary.rpath.com': 'http://conary-commits.rpath.com/conary/'}
        if cfg.commitAction:
            commitAction = cfg.commitAction % {'repMap': repName + " " + repMapStr,
                                               'buildLabel': buildLabel,
                                               'projectName': projectName,
                                               'commitEmail': cfg.commitEmail,
                                               'basePath' : cfg.basePath}
        else:
            commitAction = None

        # XXX hack to override commitAction for foresight until foresight
        # switches to our mailing lists.
        if req.hostname == "foresight.rpath.org":
            commitAction = '/usr/lib64/python2.4/site-packages/conary/commitaction --module "/usr/lib/python2.4/site-packages/mint/rbuilderaction.py --user %%(user)s --url http://www.rpath.org/xmlrpc-private/" --module "/usr/lib64/python2.4/site-packages/conary/changemail.py --user %(user)s --email desktop-commits@bizrace.com"'

        if os.access(repositoryDir, os.F_OK):
            repositories[repHash] = netserver.NetworkRepositoryServer(
                                        repositoryDir,
                                        tmpPath,
                                        urlBase,
                                        repName,
                                        repMap,
                                        commitAction = commitAction,
                                        cacheChangeSets = True,
                                        logFile = logFile
                                    )
            repositories[repHash].forceSecure = cfg.SSL
            repositories[repHash].cfg = cfg
        else:
            repositories[repHash] = None

    repo = repositories[repHash]

    if method == "POST":
	return post(port, secure, repo, cfg, req)
    elif method == "GET":
	return get(port, secure, repo, cfg, req)
    elif method == "PUT":
	return putFile(port, secure, repo, req)
    else:
	return apache.HTTP_METHOD_NOT_ALLOWED

def xmlrpcHandler(req, cfg, pathInfo):
    if req.method.upper() != "POST":
        return apache.HTTP_METHOD_NOT_ALLOWED
    if req.headers_in['Content-Type'] != "text/xml":
        return apache.HTTP_NOT_FOUND

    authToken = getHttpAuth(req)
    if type(authToken) is int:
        return authToken

    (params, method) = xmlrpclib.loads(req.read())
    params = [method, authToken, params]

    if req.uri.startswith("/xmlrpc-private"):
        server = mint_server.MintServer(cfg, allowPrivate = True)
    elif req.uri.startswith("/xmlrpc"):
        server = mint_server.MintServer(cfg, allowPrivate = False)
    try:
        result = server.callWrapper(*params)
    except (errors.InsufficientPermission, mint_server.PermissionDenied):
        return apache.HTTP_FORBIDDEN

    resp = xmlrpclib.dumps((result,), methodresponse=1)
    req.content_type = "text/xml"
    encoding = req.headers_in.get('Accept-encoding', '')
    useragent = req.headers_in.get('User-Agent', '')
    if len(resp) > 200 and 'deflate' in encoding and 'MSIE' not in useragent:
        req.headers_out['Content-encoding'] = 'deflate'
        resp = zlib.compress(resp, 5)
    req.write(resp)
    return apache.OK

def mintHandler(req, cfg, pathInfo):
    webfe = app.MintApp(req, cfg)
    return webfe._handle(pathInfo)

urls = (
    (r'^/conary/',           conaryHandler),
    (r'^/xmlrpc/',           xmlrpcHandler),
    (r'^/xmlrpc-private/',   xmlrpcHandler),
    (r'^/repos/',            conaryHandler),
    (r'^/',                  mintHandler),
)

def logErrorAndEmail(req, cfg, exception, e, bt):
    c = req.connection
    req.add_common_vars()
    info_dict = {
        'local_addr'     : c.local_ip + ':' + str(c.local_addr[1]),
        'remote_addr'    : c.remote_ip + ':' + str(c.remote_addr[1]),
        'remote_host'    : c.remote_host,
        'remote_logname' : c.remote_logname,
        'aborted'        : c.aborted,
        'keepalive'      : c.keepalive,
        'double_reverse' : c.double_reverse,
        'keepalives'     : c.keepalives,
        'local_host'     : c.local_host,
        'connection_id'  : c.id,
        'notes'          : c.notes,
        'the_request'    : req.the_request,
        'proxyreq'       : req.proxyreq,
        'header_only'    : req.header_only,
        'protocol'       : req.protocol,
        'proto_num'      : req.proto_num,
        'hostname'       : req.hostname,
        'request_time'   : time.ctime(req.request_time),
        'status_line'    : req.status_line,
        'status'         : req.status,
        'method'         : req.method,
        'allowed'        : req.allowed,
        'headers_in'     : req.headers_in,
        'headers_out'    : req.headers_out,
        'uri'            : req.uri,
        'unparsed_uri'   : req.unparsed_uri,
        'args'           : req.args,
        'parsed_uri'     : req.parsed_uri,
        'filename'       : req.filename,
        'subprocess_env' : req.subprocess_env,
        'referer'        : req.headers_in.get('referer', 'N/A')
    }
    info_dict_small = {
        'local_addr'     : c.local_ip + ':' + str(c.local_addr[1]),
        'uri'            : req.uri,
        'request_time'   : time.ctime(req.request_time),
    }

    timeStamp = time.ctime(time.time())
    # log error
    log.error('[%s] Unhandled exception from mint web interface: %s: %s', timeStamp, exception.__name__, e)
    log.error('sending mail to %s and %s' % (cfg.bugsEmail, cfg.smallBugsEmail))
    # send email
    body = 'Unhandled exception from mint web interface:\n\n%s: %s\n\n' %(exception.__name__, e)
    body += 'Time of occurrence: %s\n\n' %timeStamp
    body += ''.join( traceback.format_tb(bt))
    body += '\nConnection Information:\n'
    for key, val in sorted(info_dict.items()):
        body += '\n' + key + ': ' + str(val)

    body_small = 'Mint Exception: %s: %s' % (exception.__name__, e)
    for key, val in sorted(info_dict_small.items()):
        body_small += '\n' + key + ': ' + str(val)

    if cfg.bugsEmail:
        users.sendMailWithChecks(cfg.bugsEmail, cfg.bugsEmailName,
                                 cfg.bugsEmail, cfg.bugsEmailSubject, body)
    if cfg.smallBugsEmail:
        users.sendMailWithChecks(cfg.bugsEmail, cfg.bugsEmailName,
                                 cfg.smallBugsEmail, cfg.bugsEmailSubject, body_small)

cfg = None
def handler(req):
    startTime = time.time()
    if not req.hostname:
        return apache.HTTP_BAD_REQUEST

    global cfg
    if not cfg:
        cfg = config.MintConfig()
        cfg.read(req.filename)

    # normalize req path and base path
    pathInfo = normPath(req.uri)
    basePath = normPath(cfg.basePath)

    # strip off base path and normalize again
    pathInfo = pathInfo[len(basePath):]
    pathInfo = normPath(pathInfo)

    ret = apache.HTTP_NOT_FOUND
    for match, urlHandler in urls:
        if re.match(match, pathInfo):
            newPath = normPath(pathInfo[len(match)-1:])
            try:
                ret = urlHandler(req, cfg, newPath)
            except apache.SERVER_RETURN:
                raise
            except:
                # we only want to handle errors in production mode
                if cfg.debugMode or req.bytes_sent > 0:
                    raise
                # only handle actual mint errors
                if match !='^/':
                    raise
                exception, e, bt = sys.exc_info()
                logErrorAndEmail(req, cfg, exception, e, bt)
                ret = urlHandler(req, cfg, '/unknownError')
            break
    if cfg.profiling:
        print >> sys.stderr, "WEB HIT: %.2fms" % ((time.time() - startTime) * 1000)
        sys.stderr.flush()
    if coverage:
        coverage.the_coverage.save()
    return ret

repositories = {}

if coverage:
    import atexit
    atexit.register(coverage.the_coverage.save)
<|endoftext|>


<commit_before>import time
from urllib.parse import parse_qs, urlencode, urlsplit

import requests
from requests.auth import HTTPBasicAuth

class OAuth:
    AUTHORIZE_URL = 'https://accounts.spotify.com/authorize'
    TOKEN_URL = 'https://accounts.spotify.com/api/token'

    def __init__(self, client_id, client_secret, redirect_uri=None, scopes=(), state=None,
                 auto_refresh=None, requests=requests):
        """
        Handles OAuth authentication for the Spotify web API.
        Supports client credentials and user auth flows.

        Args:
            client_id(str): The spotify app client id
            client_secret(str): The Spotify app secret
            redirect_uri(str(url)): Redirect URI for this app (not needed for client credentials)
            scopes(list|space separated str): A list of user data scopes
                (see: https://developer.spotify.com/web-api/using-scopes/)
            state(str): Optional unique state value to verify requests
            auto_refresh(int): Optional, when set the token is auto refreshed
                as needed on access if it expires in less than the value given in seconds.
            requests_session(requests.Session() or compatible object), defaults to a new requests.Sesssion()
        """
        self.client_id = client_id
        self.client_secret = client_secret
        self.redirect_uri = redirect_uri
        if isinstance(scopes, str):
            self.scopes = scopes.split(' ')
        else:
            self.scopes = list(scopes)
        self.state = None
        self._token = None
        self.auto_refresh = auto_refresh

        self.session = requests or requests.Session()

    @property
    def authorize_url(self):
        """
        Generated OAuth authorization URL that user can be redirected to.

        Returns:
            str (URL)
        """
        params = dict(
            client_id=self.client_id,
            redirect_uri=self.redirect_uri,
            response_type='code',
        )
        if self.scopes:
            params['scopes'] = self.scopes
        if self.state:
            params['state'] = self.state
        params = urlencode(params)
        return '{}?{}'.format(self.base_url, params)

    @property
    def token(self):
        if self.auto_refresh:
            self.refresh_token_if_needed(self.auto_refresh)
        return self._token

    @token.setter
    def token(self, value):
        self._token = value

    def parse_response_code(self, url):
        q = parse_qs(urlsplit(url).query)
        return q['code'][0]

    def request_token(self, url_or_code):
        """
        Requests an access token given a response code or url containing
        response code in a ?code query param.
        The token will be set to this instance's `token` attribute.

        Raises HTTPError on any non 2xx response code
        """
        code = url_or_code if url_or_code.startswith('http') else self.parse_response_code(url_or_code)

        params = {
            'redirect_uri': self.redirect_uri,
            'code': code,
            'grant_type': 'authorization_code'
        }
        if self.scope:
            params['scope'] = self.scopes
        if self.state:
            params['state'] = self.state

        auth = HTTPBasicAuth(self.client_id, self.client_secret)

        response = self.session.post(self.TOKEN_URL, data=params, auth=auth, verify=True)
        response.raise_for_status()
        token = response.json()
        token['expires_at'] = int(time.time()) + token['expires_in']
        self.token = token

    def request_client_credentials(self):
        """
        Fetches a client credentials token using the current app settings.
        resulting token will be saved in `self.token`

        Raises HTTPError on any non 2xx response code
        """
        params = {'grant_type': 'client_credentials'}
        auth = HTTPBasicAuth(self.client_id, self.client_secret)
        response = self.session.post(self.TOKEN_URL, data=params, auth=auth, verify=True)
        response.raise_for_status()
        token = response.json()
        token['expires_at'] = int(time.time()) + token['expires_in']
        self.token = token

    def refresh_token(self):
        """
        Refresh token regardless of when it expires.

        Raises HTTPError on any non 2xx response code
        """
        params = {
            'refresh_token': self.token['refresh_token'],
            'grant_type': 'refresh_token'
        }
        auth = HTTPBasicAuth(self.client_id, self.client_secret)

        response = self.session.post(self.TOKEN_URL, data=params, auth=auth)
        response.raise_for_status()
        token = response.json()
        token['expires_at'] = int(time.time()) + token['expires_in']
        self.token = token

    def refresh_token_if_needed(self, expires_in=300):
        """
        Refresh the current token if it expires in < `expires_in` seconds.

        Raises HTTPError on failed token refresh
        """
        if self.token['expires_at'] <= time.time() + expires_in:
            if 'refresh_token' in self.token:
                self.refresh_token()
            else:
                self.request_client_credentials()
<commit_msg>Use internal _token instead of token to prevent infinite recursion with auto_refresh<commit_after>import time
from urllib.parse import parse_qs, urlencode, urlsplit

import requests
from requests.auth import HTTPBasicAuth

class OAuth:
    AUTHORIZE_URL = 'https://accounts.spotify.com/authorize'
    TOKEN_URL = 'https://accounts.spotify.com/api/token'

    def __init__(self, client_id, client_secret, redirect_uri=None, scopes=(), state=None,
                 auto_refresh=None, requests=requests):
        """
        Handles OAuth authentication for the Spotify web API.
        Supports client credentials and user auth flows.

        Args:
            client_id(str): The spotify app client id
            client_secret(str): The Spotify app secret
            redirect_uri(str(url)): Redirect URI for this app (not needed for client credentials)
            scopes(list|space separated str): A list of user data scopes
                (see: https://developer.spotify.com/web-api/using-scopes/)
            state(str): Optional unique state value to verify requests
            auto_refresh(int): Optional, when set the token is auto refreshed
                as needed on access if it expires in less than the value given in seconds.
            requests_session(requests.Session() or compatible object), defaults to a new requests.Sesssion()
        """
        self.client_id = client_id
        self.client_secret = client_secret
        self.redirect_uri = redirect_uri
        if isinstance(scopes, str):
            self.scopes = scopes.split(' ')
        else:
            self.scopes = list(scopes)
        self.state = None
        self._token = None
        self.auto_refresh = auto_refresh

        self.session = requests or requests.Session()

    @property
    def authorize_url(self):
        """
        Generated OAuth authorization URL that user can be redirected to.

        Returns:
            str (URL)
        """
        params = dict(
            client_id=self.client_id,
            redirect_uri=self.redirect_uri,
            response_type='code',
        )
        if self.scopes:
            params['scopes'] = self.scopes
        if self.state:
            params['state'] = self.state
        params = urlencode(params)
        return '{}?{}'.format(self.base_url, params)

    @property
    def token(self):
        if self.auto_refresh:
            self.refresh_token_if_needed(self.auto_refresh)
        return self._token

    @token.setter
    def token(self, value):
        self._token = value

    def parse_response_code(self, url):
        q = parse_qs(urlsplit(url).query)
        return q['code'][0]

    def request_token(self, url_or_code):
        """
        Requests an access token given a response code or url containing
        response code in a ?code query param.
        The token will be set to this instance's `token` attribute.

        Raises HTTPError on any non 2xx response code
        """
        code = url_or_code if url_or_code.startswith('http') else self.parse_response_code(url_or_code)

        params = {
            'redirect_uri': self.redirect_uri,
            'code': code,
            'grant_type': 'authorization_code'
        }
        if self.scope:
            params['scope'] = self.scopes
        if self.state:
            params['state'] = self.state

        auth = HTTPBasicAuth(self.client_id, self.client_secret)

        response = self.session.post(self.TOKEN_URL, data=params, auth=auth, verify=True)
        response.raise_for_status()
        token = response.json()
        token['expires_at'] = int(time.time()) + token['expires_in']
        self.token = token

    def request_client_credentials(self):
        """
        Fetches a client credentials token using the current app settings.
        resulting token will be saved in `self.token`

        Raises HTTPError on any non 2xx response code
        """
        params = {'grant_type': 'client_credentials'}
        auth = HTTPBasicAuth(self.client_id, self.client_secret)
        response = self.session.post(self.TOKEN_URL, data=params, auth=auth, verify=True)
        response.raise_for_status()
        token = response.json()
        token['expires_at'] = int(time.time()) + token['expires_in']
        self.token = token

    def refresh_token(self):
        """
        Refresh token regardless of when it expires.

        Raises HTTPError on any non 2xx response code
        """
        params = {
            'refresh_token': self._token['refresh_token'],
            'grant_type': 'refresh_token'
        }
        auth = HTTPBasicAuth(self.client_id, self.client_secret)

        response = self.session.post(self.TOKEN_URL, data=params, auth=auth)
        response.raise_for_status()
        token = response.json()
        token['expires_at'] = int(time.time()) + token['expires_in']
        self.token = token

    def refresh_token_if_needed(self, expires_in=300):
        """
        Refresh the current token if it expires in < `expires_in` seconds.

        Raises HTTPError on failed token refresh
        """
        if self._token['expires_at'] <= time.time() + expires_in:
            if 'refresh_token' in self._token:
                self.refresh_token()
            else:
                self.request_client_credentials()
<|endoftext|>


<commit_before>import difflib
import logging

from django.conf import settings
from django.contrib.sites.models import Site
from django.core.urlresolvers import reverse as django_reverse
from django.utils.translation import ugettext as _, ugettext_lazy as _lazy

from bleach import clean
from tidings.events import InstanceEvent, Event, EventUnion
from tidings.utils import hash_to_unsigned
from wikimarkup.parser import ALLOWED_TAGS, ALLOWED_ATTRIBUTES

from kitsune.sumo import email_utils
from kitsune.sumo.templatetags.jinja_helpers import add_utm
from kitsune.sumo.urlresolvers import reverse
from kitsune.wiki.models import Document

log = logging.getLogger('k.wiki.events')

def get_diff_for(doc, old_rev, new_rev):
        fromfile = u'[%s] %s #%s' % (doc.locale, doc.title, old_rev.id)
        tofile = u'[%s] %s #%s' % (doc.locale, doc.title, new_rev.id)
        # difflib expects these to be bytes, not unicode
        fromfile = fromfile.encode('utf8')
        tofile = tofile.encode('utf8')

        # Get diff
        diff_parts = difflib.unified_diff(
            old_rev.content.splitlines(1),
            new_rev.content.splitlines(1),
            fromfile=fromfile, tofile=tofile)

        # Join diff parts
        # XXX this is super goofy
        acc = u''
        for d in diff_parts:
            if isinstance(d, unicode):
                acc = acc + d
            else:
                acc = acc + d.decode('utf8')

        # Clean output
        return clean(acc, ALLOWED_TAGS, ALLOWED_ATTRIBUTES)

def context_dict(revision, ready_for_l10n=False, revision_approved=False):
    """Return a dict that fills in the blanks in KB notification templates."""
    diff = ''
    l10n = revision.document.revisions.filter(is_ready_for_localization=True)
    approved = revision.document.revisions.filter(is_approved=True)

    if ready_for_l10n and l10n.count() > 1:
        old_rev = l10n.order_by('-created')[1]
        diff = get_diff_for(revision.document, old_rev, revision)
    elif revision_approved and approved.count() > 1:
        old_rev = approved.order_by('-created')[1]
        diff = get_diff_for(revision.document, old_rev, revision)
    elif revision.document.current_revision is not None:
        old_rev = revision.document.current_revision
        diff = get_diff_for(revision.document, old_rev, revision)

    return {
        'document_title': revision.document.title,
        'creator': revision.creator,
        'host': Site.objects.get_current().domain,
        'diff': diff,
        'summary': clean(revision.summary, ALLOWED_TAGS, ALLOWED_ATTRIBUTES),
        'fulltext': clean(revision.content, ALLOWED_TAGS, ALLOWED_ATTRIBUTES),
    }

class EditDocumentEvent(InstanceEvent):
    """Event fired when a certain document is edited"""
    event_type = 'wiki edit document'
    content_type = Document

    def __init__(self, revision):
        super(EditDocumentEvent, self).__init__(revision.document)
        self.revision = revision

    def _mails(self, users_and_watches):
        revision = self.revision
        document = revision.document
        log.debug('Sending edited notification email for document (id=%s)' %
                  document.id)

        subject = _lazy(u'{title} was edited by {creator}')
        url = reverse('wiki.document_revisions', locale=document.locale,
                      args=[document.slug])

        context = context_dict(revision)
        context['revisions_url'] = add_utm(url, 'wiki-edit')
        context['locale'] = document.locale
        context['title'] = document.title
        context['creator'] = revision.creator
        context['comment'] = revision.comment

        return email_utils.emails_with_users_and_watches(
            subject=subject,
            text_template='wiki/email/edited.ltxt',
            html_template='wiki/email/edited.html',
            context_vars=context,
            users_and_watches=users_and_watches,
            default_locale=document.locale)

class _RevisionConstructor(object):
    """An event that receives a revision when constructed"""
    def __init__(self, revision):
        super(_RevisionConstructor, self).__init__()
        self.revision = revision

class _BaseProductFilter(object):
    """A base class for product filters.

    It adds a _filter_by_product method that filters down a list of
    (user, watches) to only the users watching the products for the
    revision.
    """
    def _filter_by_product(self, all_watchers):
        products = self.revision.document.get_products()
        product_hashes = [hash_to_unsigned(s.slug) for s in products]

        watchers_and_watches = []

        # Weed out the users that have a product filter that isn't one of the
        # document's products.
        for user, watches in all_watchers:
            for watch in watches:
                # Get the product filters for the watch, if any.
                prods = watch.filters.filter(
                    name='product').values_list('value', flat=True)

                # If there are no product filters, they are watching them all.
                if len(prods) == 0:
                    watchers_and_watches.append((user, watches))
                    break

                # Otherwise, check if they are watching any of the document's
                # products.
                for prod in prods:
                    if prod in product_hashes:
                        watchers_and_watches.append((user, watches))
                        break

        return watchers_and_watches

class _ProductFilter(_BaseProductFilter):
    """An event that receives a revision when constructed and filters according
    to that revision's document's products"""
    filters = set(['product'])

    # notify(), stop_notifying(), and is_notifying() take...
    # (user_or_email, product=optional_product)

    def _users_watching(self, **kwargs):
        # Get the users watching any or all products.
        users = list(self._users_watching_by_filter(**kwargs))

        # Weed out the users that have a product filter that isn't one of the
        # document's products.
        return self._filter_by_product(users)

class _LocaleAndProductFilter(_BaseProductFilter):
    """An event that receives a revision when constructed and filters according
    to that revision's document's locale and products."""
    filters = set(['locale', 'product'])

    # notify(), stop_notifying(), and is_notifying() take...
    # (user_or_email, locale=some_locale, product=optional_product)

    def _users_watching(self, **kwargs):
        locale = self.revision.document.locale

        # Get the users just subscribed to the locale (any and all products).
        users = list(self._users_watching_by_filter(locale=locale, **kwargs))

        # Weed out the users that have a product filter that isn't one of the
        # document's products.
        return self._filter_by_product(users)

class ReviewableRevisionInLocaleEvent(_RevisionConstructor,
                                      _LocaleAndProductFilter,
                                      Event):
    """Event fired when any revision in a certain locale is ready for review"""
    # Our event_type suffices to limit our scope, so we don't bother
    # setting content_type.
    event_type = 'reviewable wiki in locale'

    def _mails(self, users_and_watches):
        revision = self.revision
        document = revision.document
        log.debug('Sending ready for review email for revision (id=%s)' %
                  revision.id)
        subject = _lazy(u'{title} is ready for review ({creator})')
        url = reverse('wiki.review_revision',
                      locale=document.locale,
                      args=[document.slug, revision.id])

        context = context_dict(revision)
        context['revision_url'] = add_utm(url, 'wiki-ready-review')
        context['locale'] = document.locale
        context['title'] = document.title
        context['creator'] = revision.creator
        context['comment'] = revision.comment

        users = []
        for u, w in users_and_watches:
            if document.allows(u, 'review_revision'):
                users.append((u, w))

        return email_utils.emails_with_users_and_watches(
            subject=subject,
            text_template='wiki/email/ready_for_review.ltxt',
            html_template='wiki/email/ready_for_review.html',
            context_vars=context,
            users_and_watches=users,
            default_locale=document.locale)

class ReadyRevisionEvent(_RevisionConstructor, _ProductFilter, Event):
    """Event fired when a revision becomes ready for l10n."""
    event_type = 'ready wiki'

    def _mails(self, users_and_watches):
        """Send readiness mails."""
        revision = self.revision
        document = revision.document
        log.debug('Sending ready notifications for revision (id=%s)' %
                  revision.id)

        subject = _lazy(u'{title} has a revision ready for localization')

        url = django_reverse('wiki.select_locale', args=[document.slug])

        context = context_dict(revision, ready_for_l10n=True)
        context['l10n_url'] = add_utm(url, 'wiki-ready-l10n')
        context['title'] = document.title

        return email_utils.emails_with_users_and_watches(
            subject=subject,
            text_template='wiki/email/ready_for_l10n.ltxt',
            html_template='wiki/email/ready_for_l10n.html',
            context_vars=context,
            users_and_watches=users_and_watches,
            default_locale=document.locale)

class ApproveRevisionInLocaleEvent(_RevisionConstructor,
                                   _LocaleAndProductFilter, Event):
    """Event fed to a union when any revision in a certain locale is approved

    Not intended to be fired individually

    """
    # No other content types have a concept of approval, so we don't bother
    # setting content_type.
    event_type = 'approved wiki in locale'

class ApprovedOrReadyUnion(EventUnion):
    """Event union fired when a revision is approved and also possibly ready

    Unioned events must have a `revision` attr.

    """
    def __init__(self, *args, **kwargs):
        super(ApprovedOrReadyUnion, self).__init__(*args, **kwargs)
        self._revision = self.events[0].revision

    def _mails(self, users_and_watches):
        """Send approval or readiness mails, as appropriate.

        If a given user is watching the Ready event and the revision
        is in fact ready, say so. Otherwise, just send the Approval
        email.

        """
        revision = self._revision
        document = revision.document
        is_ready = revision.is_ready_for_localization
        log.debug('Sending approved/ready notifications for revision (id=%s)' %
                  revision.id)

        # Localize the subject and message with the appropriate
        # context. If there is an error, fall back to English.
        @email_utils.safe_translation
        def _make_mail(locale, user, watches):
            if (is_ready and
                    ReadyRevisionEvent.event_type in
                    (w.event_type for w in watches)):
                c = context_dict(revision, ready_for_l10n=True)
                # TODO: Expose all watches
                c['watch'] = watches[0]

                url = django_reverse(
                    'wiki.select_locale', args=[document.slug])
                c['l10n_url'] = add_utm(url, 'wiki-ready-l10n')

                subject = _(u'{title} has a revision ready for '
                            'localization')
                text_template = 'wiki/email/ready_for_l10n.ltxt'
                html_template = 'wiki/email/ready_for_l10n.html'

            else:
                c = context_dict(revision, revision_approved=True)
                approved_url = reverse('wiki.document',
                                       locale=document.locale,
                                       args=[document.slug])

                c['document_url'] = add_utm(approved_url, 'wiki-approved')
                # TODO: Expose all watches.
                c['watch'] = watches[0]
                c['reviewer'] = revision.reviewer

                subject = _(u'{title} ({locale}) has a new approved '
                            'revision ({reviewer})')
                text_template = 'wiki/email/approved.ltxt'
                html_template = 'wiki/email/approved.html'

            subject = subject.format(
                title=document.title,
                reviewer=revision.reviewer.username,
                locale=document.locale)

            mail = email_utils.make_mail(
                subject=subject,
                text_template=text_template,
                html_template=html_template,
                context_vars=c,
                from_email=settings.TIDINGS_FROM_ADDRESS,
                to_email=user.email)

            return mail

        for user, watches in users_and_watches:
            # Figure out the locale to use for l10n.
            if hasattr(user, 'profile'):
                locale = user.profile.locale
            else:
                locale = document.locale

            yield _make_mail(locale, user, watches)
<commit_msg>[Bug 1095972] Change link in revision ready e-mail (#2896)<commit_after>import difflib
import logging

from django.conf import settings
from django.contrib.sites.models import Site
from django.core.urlresolvers import reverse as django_reverse
from django.utils.translation import ugettext as _, ugettext_lazy as _lazy

from bleach import clean
from tidings.events import InstanceEvent, Event, EventUnion
from tidings.utils import hash_to_unsigned
from wikimarkup.parser import ALLOWED_TAGS, ALLOWED_ATTRIBUTES

from kitsune.sumo import email_utils
from kitsune.sumo.templatetags.jinja_helpers import add_utm
from kitsune.sumo.urlresolvers import reverse
from kitsune.wiki.models import Document

log = logging.getLogger('k.wiki.events')

def get_diff_for(doc, old_rev, new_rev):
        fromfile = u'[%s] %s #%s' % (doc.locale, doc.title, old_rev.id)
        tofile = u'[%s] %s #%s' % (doc.locale, doc.title, new_rev.id)
        # difflib expects these to be bytes, not unicode
        fromfile = fromfile.encode('utf8')
        tofile = tofile.encode('utf8')

        # Get diff
        diff_parts = difflib.unified_diff(
            old_rev.content.splitlines(1),
            new_rev.content.splitlines(1),
            fromfile=fromfile, tofile=tofile)

        # Join diff parts
        # XXX this is super goofy
        acc = u''
        for d in diff_parts:
            if isinstance(d, unicode):
                acc = acc + d
            else:
                acc = acc + d.decode('utf8')

        # Clean output
        return clean(acc, ALLOWED_TAGS, ALLOWED_ATTRIBUTES)

def context_dict(revision, ready_for_l10n=False, revision_approved=False):
    """Return a dict that fills in the blanks in KB notification templates."""
    diff = ''
    l10n = revision.document.revisions.filter(is_ready_for_localization=True)
    approved = revision.document.revisions.filter(is_approved=True)

    if ready_for_l10n and l10n.count() > 1:
        old_rev = l10n.order_by('-created')[1]
        diff = get_diff_for(revision.document, old_rev, revision)
    elif revision_approved and approved.count() > 1:
        old_rev = approved.order_by('-created')[1]
        diff = get_diff_for(revision.document, old_rev, revision)
    elif revision.document.current_revision is not None:
        old_rev = revision.document.current_revision
        diff = get_diff_for(revision.document, old_rev, revision)

    return {
        'document_title': revision.document.title,
        'creator': revision.creator,
        'host': Site.objects.get_current().domain,
        'diff': diff,
        'summary': clean(revision.summary, ALLOWED_TAGS, ALLOWED_ATTRIBUTES),
        'fulltext': clean(revision.content, ALLOWED_TAGS, ALLOWED_ATTRIBUTES),
    }

class EditDocumentEvent(InstanceEvent):
    """Event fired when a certain document is edited"""
    event_type = 'wiki edit document'
    content_type = Document

    def __init__(self, revision):
        super(EditDocumentEvent, self).__init__(revision.document)
        self.revision = revision

    def _mails(self, users_and_watches):
        revision = self.revision
        document = revision.document
        log.debug('Sending edited notification email for document (id=%s)' %
                  document.id)

        subject = _lazy(u'{title} was edited by {creator}')
        url = reverse('wiki.document_revisions', locale=document.locale,
                      args=[document.slug])

        context = context_dict(revision)
        context['revisions_url'] = add_utm(url, 'wiki-edit')
        context['locale'] = document.locale
        context['title'] = document.title
        context['creator'] = revision.creator
        context['comment'] = revision.comment

        return email_utils.emails_with_users_and_watches(
            subject=subject,
            text_template='wiki/email/edited.ltxt',
            html_template='wiki/email/edited.html',
            context_vars=context,
            users_and_watches=users_and_watches,
            default_locale=document.locale)

class _RevisionConstructor(object):
    """An event that receives a revision when constructed"""
    def __init__(self, revision):
        super(_RevisionConstructor, self).__init__()
        self.revision = revision

class _BaseProductFilter(object):
    """A base class for product filters.

    It adds a _filter_by_product method that filters down a list of
    (user, watches) to only the users watching the products for the
    revision.
    """
    def _filter_by_product(self, all_watchers):
        products = self.revision.document.get_products()
        product_hashes = [hash_to_unsigned(s.slug) for s in products]

        watchers_and_watches = []

        # Weed out the users that have a product filter that isn't one of the
        # document's products.
        for user, watches in all_watchers:
            for watch in watches:
                # Get the product filters for the watch, if any.
                prods = watch.filters.filter(
                    name='product').values_list('value', flat=True)

                # If there are no product filters, they are watching them all.
                if len(prods) == 0:
                    watchers_and_watches.append((user, watches))
                    break

                # Otherwise, check if they are watching any of the document's
                # products.
                for prod in prods:
                    if prod in product_hashes:
                        watchers_and_watches.append((user, watches))
                        break

        return watchers_and_watches

class _ProductFilter(_BaseProductFilter):
    """An event that receives a revision when constructed and filters according
    to that revision's document's products"""
    filters = set(['product'])

    # notify(), stop_notifying(), and is_notifying() take...
    # (user_or_email, product=optional_product)

    def _users_watching(self, **kwargs):
        # Get the users watching any or all products.
        users = list(self._users_watching_by_filter(**kwargs))

        # Weed out the users that have a product filter that isn't one of the
        # document's products.
        return self._filter_by_product(users)

class _LocaleAndProductFilter(_BaseProductFilter):
    """An event that receives a revision when constructed and filters according
    to that revision's document's locale and products."""
    filters = set(['locale', 'product'])

    # notify(), stop_notifying(), and is_notifying() take...
    # (user_or_email, locale=some_locale, product=optional_product)

    def _users_watching(self, **kwargs):
        locale = self.revision.document.locale

        # Get the users just subscribed to the locale (any and all products).
        users = list(self._users_watching_by_filter(locale=locale, **kwargs))

        # Weed out the users that have a product filter that isn't one of the
        # document's products.
        return self._filter_by_product(users)

class ReviewableRevisionInLocaleEvent(_RevisionConstructor,
                                      _LocaleAndProductFilter,
                                      Event):
    """Event fired when any revision in a certain locale is ready for review"""
    # Our event_type suffices to limit our scope, so we don't bother
    # setting content_type.
    event_type = 'reviewable wiki in locale'

    def _mails(self, users_and_watches):
        revision = self.revision
        document = revision.document
        log.debug('Sending ready for review email for revision (id=%s)' %
                  revision.id)
        subject = _lazy(u'{title} is ready for review ({creator})')
        url = reverse('wiki.review_revision',
                      locale=document.locale,
                      args=[document.slug, revision.id])

        context = context_dict(revision)
        context['revision_url'] = add_utm(url, 'wiki-ready-review')
        context['locale'] = document.locale
        context['title'] = document.title
        context['creator'] = revision.creator
        context['comment'] = revision.comment

        users = []
        for u, w in users_and_watches:
            if document.allows(u, 'review_revision'):
                users.append((u, w))

        return email_utils.emails_with_users_and_watches(
            subject=subject,
            text_template='wiki/email/ready_for_review.ltxt',
            html_template='wiki/email/ready_for_review.html',
            context_vars=context,
            users_and_watches=users,
            default_locale=document.locale)

class ReadyRevisionEvent(_RevisionConstructor, _ProductFilter, Event):
    """Event fired when a revision becomes ready for l10n."""
    event_type = 'ready wiki'

    def _mails(self, users_and_watches):
        """Send readiness mails."""
        revision = self.revision
        document = revision.document
        log.debug('Sending ready notifications for revision (id=%s)' %
                  revision.id)

        subject = _lazy(u'{title} has a revision ready for localization')

        url = django_reverse('wiki.translate', args=[document.slug])

        context = context_dict(revision, ready_for_l10n=True)
        context['l10n_url'] = add_utm(url, 'wiki-ready-l10n')
        context['title'] = document.title

        return email_utils.emails_with_users_and_watches(
            subject=subject,
            text_template='wiki/email/ready_for_l10n.ltxt',
            html_template='wiki/email/ready_for_l10n.html',
            context_vars=context,
            users_and_watches=users_and_watches,
            default_locale=document.locale)

class ApproveRevisionInLocaleEvent(_RevisionConstructor,
                                   _LocaleAndProductFilter, Event):
    """Event fed to a union when any revision in a certain locale is approved

    Not intended to be fired individually

    """
    # No other content types have a concept of approval, so we don't bother
    # setting content_type.
    event_type = 'approved wiki in locale'

class ApprovedOrReadyUnion(EventUnion):
    """Event union fired when a revision is approved and also possibly ready

    Unioned events must have a `revision` attr.

    """
    def __init__(self, *args, **kwargs):
        super(ApprovedOrReadyUnion, self).__init__(*args, **kwargs)
        self._revision = self.events[0].revision

    def _mails(self, users_and_watches):
        """Send approval or readiness mails, as appropriate.

        If a given user is watching the Ready event and the revision
        is in fact ready, say so. Otherwise, just send the Approval
        email.

        """
        revision = self._revision
        document = revision.document
        is_ready = revision.is_ready_for_localization
        log.debug('Sending approved/ready notifications for revision (id=%s)' %
                  revision.id)

        # Localize the subject and message with the appropriate
        # context. If there is an error, fall back to English.
        @email_utils.safe_translation
        def _make_mail(locale, user, watches):
            if (is_ready and
                    ReadyRevisionEvent.event_type in
                    (w.event_type for w in watches)):
                c = context_dict(revision, ready_for_l10n=True)
                # TODO: Expose all watches
                c['watch'] = watches[0]

                url = reverse('wiki.translate', args=[document.slug], locale=locale)
                c['l10n_url'] = add_utm(url, 'wiki-ready-l10n')

                subject = _(u'{title} has a revision ready for '
                            'localization')
                text_template = 'wiki/email/ready_for_l10n.ltxt'
                html_template = 'wiki/email/ready_for_l10n.html'

            else:
                c = context_dict(revision, revision_approved=True)
                approved_url = reverse('wiki.document',
                                       locale=document.locale,
                                       args=[document.slug])

                c['document_url'] = add_utm(approved_url, 'wiki-approved')
                # TODO: Expose all watches.
                c['watch'] = watches[0]
                c['reviewer'] = revision.reviewer

                subject = _(u'{title} ({locale}) has a new approved '
                            'revision ({reviewer})')
                text_template = 'wiki/email/approved.ltxt'
                html_template = 'wiki/email/approved.html'

            subject = subject.format(
                title=document.title,
                reviewer=revision.reviewer.username,
                locale=document.locale)

            mail = email_utils.make_mail(
                subject=subject,
                text_template=text_template,
                html_template=html_template,
                context_vars=c,
                from_email=settings.TIDINGS_FROM_ADDRESS,
                to_email=user.email)

            return mail

        for user, watches in users_and_watches:
            # Figure out the locale to use for l10n.
            if hasattr(user, 'profile'):
                locale = user.profile.locale
            else:
                locale = document.locale

            yield _make_mail(locale, user, watches)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import logging
logging.getLogger('scss').addHandler(logging.StreamHandler())

import os, sys, shutil, subprocess
import cli, config, scss
from markdown import markdown
from datetime import datetime
from jinja2 import Environment, FileSystemLoader
from os.path import exists, join
from glob import glob

DATE_FORMAT = '%a %b %d %H:%M:%S %Y'
scss.LOAD_PATHS = ['_template', config.compass_path]

def _unslug(slug):
    return slug.replace('-', ' ').title()

def _extract_title(body_md):
    return body_md.split('\n', 1)[0]

def _ask_for_idea():
    slugs = [g.replace('ideas/', '').replace('.md', '')
             for g in glob('ideas/*.md')]
    rng = ['[{}]'.format(x) for x in range(len(slugs))]
    print '\n'.join([' '.join(x) for x in zip(rng, slugs)])
    return slugs[int(raw_input('Which idea? '))]

def _edit_file(filename):
    cmd = config.editor.format(join(os.getcwd(), filename))
    subprocess.call(cmd, shell=True)

@cli.command
def idea(slug):
    _exists = lambda folder: exists('{}/{}.md'.format(folder, slug))

    if _exists('ideas') or _exists('posts'):
        cli.EXIT('That slug already exists!')

    content = open('_template/_new_idea.md').read().format(
        date = datetime.now().strftime(DATE_FORMAT),
        title = _unslug(slug)
    )

    filename = 'ideas/{}.md'.format(slug)
    with open(filename, 'w') as _idea:
        _idea.write(content)
    _edit_file(filename)

@cli.command
def publish(slug=None):
    slug = slug or _ask_for_idea()

    if not exists('ideas/{}.md'.format(slug)):
        cli.EXIT('No idea by that name!')

    if exists('posts/{}.md'.format(slug)):
        cli.EXIT('That post already exists!')

    shutil.move('ideas/{}.md'.format(slug), 'posts/{}.md'.format(slug))
    build()

@cli.command
def build():
    def write_file(slug, html):
        with open(join('_public', slug+'.html'), 'w') as output:
            output.write(html)

    def parse_post(filename):
        head, md = open(filename).read().split('\n\n', 1)
        head = dict([line.split(': ') for line in head.split('\n')])
        head['Title'] = md.split('\n', 1)[0].replace('# ', '')
        head['Date'] = datetime.strptime(head['Date'], DATE_FORMAT)
        return head, md

    def render_single(slug, md):
        ctx = dict(headers[slug],
                   **{'post_html': markdown(md, extensions=['smartypants'])})
        write_file(slug, tpl_single.render(ctx))

    def render_index():
        ctx = {'posts': [dict(headers[x[0]], **{'Slug':x[0]}) for x in chron_order]}
        write_file('index', tpl_index.render(ctx))

    def render_error():
        write_file('error', env.get_template('error.html').render())

    def render_css():
        css = scss.Scss().compile(open(join('_template', 'style.scss')).read())
        with open(join('_public', 'style.css'), 'w') as output:
            output.write(css)

    def copy_assets():
        shutil.rmtree('_public/assets')
        shutil.copytree('assets', '_public/assets')

    posts = glob('posts/*.md')
    env = Environment(loader=FileSystemLoader('_template'))
    tpl_single = env.get_template('single.html')
    tpl_index = env.get_template('index.html')

    headers = {}
    chron_order = []

    for post in posts:
        slug = post.replace('.md', '').replace('posts/', '')
        headers[slug], md = parse_post(post)
        chron_order.append((slug, headers[slug]['Date']))
        render_single(slug, md)

    chron_order = sorted(chron_order, key=lambda tup: tup[1], reverse=True)
    render_index()
    render_error()
    render_css()
    copy_assets()
    print("BUILT!")

@cli.command
def deploy():
    build()
    subprocess.call(config.deploy_command, shell=True)

if __name__ == '__main__':
    cli.run()

<commit_msg>Fix deploy error if _public folder doesn't exist<commit_after>#!/usr/bin/env python

import logging
logging.getLogger('scss').addHandler(logging.StreamHandler())

import os, sys, shutil, subprocess
import cli, config, scss
from markdown import markdown
from datetime import datetime
from jinja2 import Environment, FileSystemLoader
from os.path import exists, join
from glob import glob

DATE_FORMAT = '%a %b %d %H:%M:%S %Y'
scss.LOAD_PATHS = ['_template', config.compass_path]

def _unslug(slug):
    return slug.replace('-', ' ').title()

def _extract_title(body_md):
    return body_md.split('\n', 1)[0]

def _ask_for_idea():
    slugs = [g.replace('ideas/', '').replace('.md', '')
             for g in glob('ideas/*.md')]
    rng = ['[{}]'.format(x) for x in range(len(slugs))]
    print '\n'.join([' '.join(x) for x in zip(rng, slugs)])
    return slugs[int(raw_input('Which idea? '))]

def _edit_file(filename):
    cmd = config.editor.format(join(os.getcwd(), filename))
    subprocess.call(cmd, shell=True)

@cli.command
def idea(slug):
    _exists = lambda folder: exists('{}/{}.md'.format(folder, slug))

    if _exists('ideas') or _exists('posts'):
        cli.EXIT('That slug already exists!')

    content = open('_template/_new_idea.md').read().format(
        date = datetime.now().strftime(DATE_FORMAT),
        title = _unslug(slug)
    )

    filename = 'ideas/{}.md'.format(slug)
    with open(filename, 'w') as _idea:
        _idea.write(content)
    _edit_file(filename)

@cli.command
def publish(slug=None):
    slug = slug or _ask_for_idea()

    if not exists('ideas/{}.md'.format(slug)):
        cli.EXIT('No idea by that name!')

    if exists('posts/{}.md'.format(slug)):
        cli.EXIT('That post already exists!')

    shutil.move('ideas/{}.md'.format(slug), 'posts/{}.md'.format(slug))
    build()

@cli.command
def build():
    def write_file(slug, html):
        with open(join('_public', slug+'.html'), 'w') as output:
            output.write(html)

    def parse_post(filename):
        head, md = open(filename).read().split('\n\n', 1)
        head = dict([line.split(': ') for line in head.split('\n')])
        head['Title'] = md.split('\n', 1)[0].replace('# ', '')
        head['Date'] = datetime.strptime(head['Date'], DATE_FORMAT)
        return head, md

    def render_single(slug, md):
        ctx = dict(headers[slug],
                   **{'post_html': markdown(md, extensions=['smartypants'])})
        write_file(slug, tpl_single.render(ctx))

    def render_index():
        ctx = {'posts': [dict(headers[x[0]], **{'Slug':x[0]}) for x in chron_order]}
        write_file('index', tpl_index.render(ctx))

    def render_error():
        write_file('error', env.get_template('error.html').render())

    def render_css():
        css = scss.Scss().compile(open(join('_template', 'style.scss')).read())
        with open(join('_public', 'style.css'), 'w') as output:
            output.write(css)

    def copy_assets():
        shutil.rmtree('_public/assets')
        shutil.copytree('assets', '_public/assets')

    if not os.path.exists('_public/assets'):
        os.makedirs('_public/assets')

    posts = glob('posts/*.md')
    env = Environment(loader=FileSystemLoader('_template'))
    tpl_single = env.get_template('single.html')
    tpl_index = env.get_template('index.html')

    headers = {}
    chron_order = []

    for post in posts:
        slug = post.replace('.md', '').replace('posts/', '')
        headers[slug], md = parse_post(post)
        chron_order.append((slug, headers[slug]['Date']))
        render_single(slug, md)

    chron_order = sorted(chron_order, key=lambda tup: tup[1], reverse=True)
    render_index()
    render_error()
    render_css()
    copy_assets()
    print("BUILT!")

@cli.command
def deploy():
    build()
    subprocess.call(config.deploy_command, shell=True)

if __name__ == '__main__':
    cli.run()

<|endoftext|>


<commit_before># Copyright (c) 2015, 2014 Computational Molecular Biology Group, Free University
# Berlin, 14195 Berlin, Germany.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import numpy as np

from pyemma.coordinates.data.interface import ReaderInterface

__author__ = 'noe, marscher'

class DataInMemory(ReaderInterface):

    r"""
    multi-dimensional data fully stored in memory.

    Used to pass arbitrary coordinates to pipeline. Data is being flattened to
    two dimensions to ensure it is compatible.

    Parameters
    ----------
    data : ndarray (nframe, ndim) or list of ndarrays (nframe, ndim)
        Data has to be either one 2d array which stores amount of frames in first
        dimension and coordinates/features in second dimension or a list of this
        arrays.
    """

    def __init__(self, data, chunksize=5000):
        super(DataInMemory, self).__init__(chunksize=chunksize)

        # storage
        self._data = []

        if not isinstance(data, (list, tuple)):
            data = [data]

        # everything is an array
        if all(isinstance(d, np.ndarray) for d in data):
            for d in data:
                self._add_array_to_storage(d)
        else:
            raise ValueError("Please supply numpy.ndarray, or list/tuple of ndarray."
                             " Your input was %s" % str(data))

        self.__set_dimensions_and_lenghts()
        self._parametrized = True

    @classmethod
    def load_from_files(cls, files):
        """ construct this by loading all files into memory

        Parameters
        ----------
        files: str or list of str
            filenames to read from
        """
        # import here to avoid cyclic import
        from pyemma.coordinates.data.numpy_filereader import NumPyFileReader

        reader = NumPyFileReader(files)
        data = reader.get_output()
        return cls(data)

    def describe(self):
        return "[DataInMemory array shapes: %s]" % [np.shape(x) for x in self._data]

    def __set_dimensions_and_lenghts(self):
        # number of trajectories/data sets
        self._ntraj = len(self._data)
        if self._ntraj == 0:
            raise ValueError("no valid data")

        # this works since everything is flattened to 2d
        self._lengths = [np.shape(d)[0] for d in self._data]

        # ensure all trajs have same dim
        ndims = [np.shape(x)[1] for x in self._data]
        if not np.unique(ndims).size == 1:
            raise ValueError("input data has different dimensions!"
                             "Dimensions are = %s" % ndims)

        self._ndim = ndims[0]

    def _reset(self, stride=1):
        """Resets the data producer
        """
        self._itraj = 0
        self._t = 0

    def _next_chunk(self, lag=0, stride=1):
        # finished once with all trajectories? so _reset the pointer to allow
        # multi-pass
        if self._itraj >= self._ntraj:
            self._reset()

        traj_len = self._lengths[self._itraj]
        traj = self._data[self._itraj]

        # complete trajectory mode
        if self._chunksize == 0:
            X = traj[::stride]
            self._itraj += 1

            if lag == 0:
                return X
            else:
                Y = traj[lag * stride:traj_len:stride]
                return (X, Y)
        # chunked mode
        else:
            upper_bound = min(
                self._t + self._chunksize * stride, traj_len)
            slice_x = slice(self._t, upper_bound, stride)

            X = traj[slice_x]

            if lag == 0:
                self._t = upper_bound

                if upper_bound >= traj_len:
                    self._itraj += 1
                    self._t = 0
                return X
            else:
                # its okay to return empty chunks
                upper_bound = min(
                    self._t + stride * (lag + self._chunksize), traj_len)
                slice_y = slice(self._t + lag, upper_bound, stride)
                self._t = upper_bound

                if self._t >= traj_len:
                    self._itraj += 1
                    self._t = 0
                Y = traj[slice_y]
                return X, Y
<commit_msg>[coordinates] fix data in memory with lag and stride; implemented old convention to be compatible with feature reader<commit_after># Copyright (c) 2015, 2014 Computational Molecular Biology Group, Free University
# Berlin, 14195 Berlin, Germany.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation and/or
# other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import numpy as np

from pyemma.coordinates.data.interface import ReaderInterface

__author__ = 'noe, marscher'

class DataInMemory(ReaderInterface):

    r"""
    multi-dimensional data fully stored in memory.

    Used to pass arbitrary coordinates to pipeline. Data is being flattened to
    two dimensions to ensure it is compatible.

    Parameters
    ----------
    data : ndarray (nframe, ndim) or list of ndarrays (nframe, ndim)
        Data has to be either one 2d array which stores amount of frames in first
        dimension and coordinates/features in second dimension or a list of this
        arrays.
    """

    def __init__(self, data, chunksize=5000):
        super(DataInMemory, self).__init__(chunksize=chunksize)

        # storage
        self._data = []

        if not isinstance(data, (list, tuple)):
            data = [data]

        # everything is an array
        if all(isinstance(d, np.ndarray) for d in data):
            for d in data:
                self._add_array_to_storage(d)
        else:
            raise ValueError("Please supply numpy.ndarray, or list/tuple of ndarray."
                             " Your input was %s" % str(data))

        self.__set_dimensions_and_lenghts()
        self._parametrized = True

    @classmethod
    def load_from_files(cls, files):
        """ construct this by loading all files into memory

        Parameters
        ----------
        files: str or list of str
            filenames to read from
        """
        # import here to avoid cyclic import
        from pyemma.coordinates.data.numpy_filereader import NumPyFileReader

        reader = NumPyFileReader(files)
        data = reader.get_output()
        return cls(data)

    def describe(self):
        return "[DataInMemory array shapes: %s]" % [np.shape(x) for x in self._data]

    def __set_dimensions_and_lenghts(self):
        # number of trajectories/data sets
        self._ntraj = len(self._data)
        if self._ntraj == 0:
            raise ValueError("no valid data")

        # this works since everything is flattened to 2d
        self._lengths = [np.shape(d)[0] for d in self._data]

        # ensure all trajs have same dim
        ndims = [np.shape(x)[1] for x in self._data]
        if not np.unique(ndims).size == 1:
            raise ValueError("input data has different dimensions!"
                             "Dimensions are = %s" % ndims)

        self._ndim = ndims[0]

    def _reset(self, stride=1):
        """Resets the data producer
        """
        self._itraj = 0
        self._t = 0

    def _next_chunk(self, lag=0, stride=1):
        # finished once with all trajectories? so _reset the pointer to allow
        # multi-pass
        if self._itraj >= self._ntraj:
            self._reset()

        traj_len = self._lengths[self._itraj]
        traj = self._data[self._itraj]

        # complete trajectory mode
        if self._chunksize == 0:
            X = traj[::stride]
            self._itraj += 1

            if lag == 0:
                return X
            else:
                Y = traj[lag * stride:traj_len:stride]
                return (X, Y)
        # chunked mode
        else:
            upper_bound = min(
                self._t + self._chunksize * stride, traj_len)
            slice_x = slice(self._t, upper_bound, stride)

            X = traj[slice_x]

            if lag!=0:
                 upper_bound_Y = min(
                     self._t + (lag + self._chunksize) * stride, traj_len)
                 slice_y = slice(self._t + lag*stride, upper_bound_Y, stride)
                 Y = traj[slice_y]

            self._t = upper_bound

            if upper_bound >= traj_len:
                 self._itraj += 1
                 self._t = 0

            if lag==0:
                return X
            else:
                return (X, Y)
<|endoftext|>