<commit_before># -*- coding: utf-8 -*-
#
# Copyright P. Christeas <p_christ@hol.gr> 2008,2009
# A part of the code comes from the ganeti project:  http://www.mail-archive.com/ganeti-devel@googlegroups.com/msg00713.html#
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
###############################################################################

""" Framework for generic http servers

"""

import socket
import base64
import SocketServer
from BaseHTTPServer import *
from SimpleHTTPServer import SimpleHTTPRequestHandler

class AuthRequiredExc(Exception):
    def __init__(self,atype,realm):
        Exception.__init__(self)
        self.atype = atype
        self.realm = realm

class AuthRejectedExc(Exception):
    pass

class AuthProvider:
    def __init__(self,realm):
        self.realm = realm

    def setupAuth(self, multi,handler):
        """ Attach an AuthProxy object to handler
        """
        pass

    def authenticate(self, user, passwd, client_address):
        return False

    def log(self, msg):
        print msg

class BasicAuthProvider(AuthProvider):
    def setupAuth(self, multi, handler):
        if not multi.sec_realms.has_key(self.realm):
            multi.sec_realms[self.realm] = BasicAuthProxy(self)

class AuthProxy:
    """ This class will hold authentication information for a handler,
        i.e. a connection
    """
    def __init__(self, provider):
        self.provider = provider

    def checkRequest(self,handler,path = '/'):
        """ Check if we are allowed to process that request
        """
        pass

class BasicAuthProxy(AuthProxy):
    """ Require basic authentication..
    """
    def __init__(self,provider):
        AuthProxy.__init__(self,provider)
        self.auth_creds = None
        self.auth_tries = 0

    def checkRequest(self,handler,path = '/'):
        if self.auth_creds:
            return True
        auth_str = handler.headers.get('Authorization',False)
        if auth_str and auth_str.startswith('Basic '):
            auth_str=auth_str[len('Basic '):]
            (user,passwd) = base64.decodestring(auth_str).split(':')
            self.provider.log("Found user=\"%s\", passwd=\"%s\"" %(user,passwd))
            self.auth_creds = self.provider.authenticate(user,passwd,handler.client_address)
            if self.auth_creds:
                return True
        if self.auth_tries > 5:
            self.provider.log("Failing authorization after 5 requests w/o password")
            raise AuthRejectedExc("Authorization failed.")
        self.auth_tries += 1
        raise AuthRequiredExc(atype = 'Basic', realm=self.provider.realm)

class HTTPHandler(SimpleHTTPRequestHandler):
    def __init__(self,request, client_address, server):
        SimpleHTTPRequestHandler.__init__(self,request,client_address,server)
        # print "Handler for %s inited" % str(client_address)
        self.protocol_version = 'HTTP/1.1'
        self.connection = dummyconn()

    def handle(self):
        """ Classes here should NOT handle inside their constructor
        """
        pass

    def finish(self):
        pass

    def setup(self):
        pass

class HTTPDir:
    """ A dispatcher class, like a virtual folder in httpd
    """
    def __init__(self,path,handler, auth_provider = None):
        self.path = path
        self.handler = handler
        self.auth_provider = auth_provider

    def matches(self, request):
        """ Test if some request matches us. If so, return
            the matched path. """
        if request.startswith(self.path):
            return self.path
        return False

class noconnection:
    """ a class to use instead of the real connection
    """
    def makefile(self, mode, bufsize):
        return None

class dummyconn:
    def shutdown(self, tru):
        pass

def _quote_html(html):
    return html.replace("&", "&").replace("<", "<").replace(">", ">")

class FixSendError:
    #error_message_format = """ """
    def send_error(self, code, message=None):
        #overriden from BaseHTTPRequestHandler, we also send the content-length
        try:
            short, long = self.responses[code]
        except KeyError:
            short, long = '???', '???'
        if message is None:
            message = short
        explain = long
        self.log_error("code %d, message %s", code, message)
        # using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201)
        content = (self.error_message_format %
                   {'code': code, 'message': _quote_html(message), 'explain': explain})
        self.send_response(code, message)
        self.send_header("Content-Type", self.error_content_type)
        self.send_header('Connection', 'close')
        self.send_header('Content-Length', len(content) or 0)
        self.end_headers()
        if self.command != 'HEAD' and code >= 200 and code not in (204, 304):
            self.wfile.write(content)

class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler):
    """ this is a multiple handler, that will dispatch each request
        to a nested handler, iff it matches

        The handler will also have *one* dict of authentication proxies,
        groupped by their realm.
    """

    protocol_version = "HTTP/1.1"
    default_request_version = "HTTP/0.9"    # compatibility with py2.5

    auth_required_msg = """ <title>Authorization required</title>
    You must authenticate to use this service\r\r"""

    def __init__(self, request, client_address, server):
        self.in_handlers = {}
        self.sec_realms = {}
        SocketServer.StreamRequestHandler.__init__(self,request,client_address,server)
        self.log_message("MultiHttpHandler init for %s" %(str(client_address)))

    def _handle_one_foreign(self,fore, path, auth_provider):
        """ This method overrides the handle_one_request for *children*
            handlers. It is required, since the first line should not be
            read again..

        """
        fore.raw_requestline = "%s %s %s\n" % (self.command, path, self.version)
        if not fore.parse_request(): # An error code has been sent, just exit
            return
        self.request_version = fore.request_version
        if auth_provider and auth_provider.realm:
            try:
                self.sec_realms[auth_provider.realm].checkRequest(fore,path)
            except AuthRequiredExc,ae:
                if self.request_version != 'HTTP/1.1':
                    self.log_error("Cannot require auth at %s",self.request_version)
                    self.send_error(401)
                    return
                self._get_ignore_body(fore) # consume any body that came, not loose sync with input
                self.send_response(401,'Authorization required')
                self.send_header('WWW-Authenticate','%s realm="%s"' % (ae.atype,ae.realm))
                self.send_header('Connection', 'keep-alive')
                self.send_header('Content-Type','text/html')
                self.send_header('Content-Length',len(self.auth_required_msg))
                self.end_headers()
                self.wfile.write(self.auth_required_msg)
                return
            except AuthRejectedExc,e:
                self.log_error("Rejected auth: %s" % e.args[0])
                self.send_error(401,e.args[0])
                self.close_connection = 1
                return
        mname = 'do_' + fore.command
        if not hasattr(fore, mname):
            fore.send_error(501, "Unsupported method (%r)" % fore.command)
            return
        fore.close_connection = 0
        method = getattr(fore, mname)
        method()
        if fore.close_connection:
            # print "Closing connection because of handler"
            self.close_connection = fore.close_connection
        if hasattr(fore, '_flush'):
            fore._flush()

    def parse_rawline(self):
        """Parse a request (internal).

        The request should be stored in self.raw_requestline; the results
        are in self.command, self.path, self.request_version and
        self.headers.

        Return True for success, False for failure; on failure, an
        error is sent back.

        """
        self.command = None  # set in case of error on the first line
        self.request_version = version = self.default_request_version
        self.close_connection = 1
        requestline = self.raw_requestline
        if requestline[-2:] == '\r\n':
            requestline = requestline[:-2]
        elif requestline[-1:] == '\n':
            requestline = requestline[:-1]
        self.requestline = requestline
        words = requestline.split()
        if len(words) == 3:
            [command, path, version] = words
            if version[:5] != 'HTTP/':
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            try:
                base_version_number = version.split('/', 1)[1]
                version_number = base_version_number.split(".")
                # RFC 2145 section 3.1 says there can be only one "." and
                #   - major and minor numbers MUST be treated as
                #      separate integers;
                #   - HTTP/2.4 is a lower version than HTTP/2.13, which in
                #      turn is lower than HTTP/12.3;
                #   - Leading zeros MUST be ignored by recipients.
                if len(version_number) != 2:
                    raise ValueError
                version_number = int(version_number[0]), int(version_number[1])
            except (ValueError, IndexError):
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            if version_number >= (1, 1):
                self.close_connection = 0
            if version_number >= (2, 0):
                self.send_error(505,
                          "Invalid HTTP Version (%s)" % base_version_number)
                return False
        elif len(words) == 2:
            [command, path] = words
            self.close_connection = 1
            if command != 'GET':
                self.send_error(400,
                                "Bad HTTP/0.9 request type (%r)" % command)
                return False
        elif not words:
            return False
        else:
            #self.send_error(400, "Bad request syntax (%r)" % requestline)
            return False
        self.request_version = version
        self.command, self.path, self.version = command, path, version
        return True

    def handle_one_request(self):
        """Handle a single HTTP request.
           Dispatch to the correct handler.
        """
        self.request.setblocking(True)
        self.raw_requestline = self.rfile.readline()
        if not self.raw_requestline:
            self.close_connection = 1
            # self.log_message("no requestline, connection closed?")
            return
        if not self.parse_rawline():
            self.log_message("Could not parse rawline.")
            return
        # self.parse_request(): # Do NOT parse here. the first line should be the only
        for vdir in self.server.vdirs:
            p = vdir.matches(self.path)
            if p == False:
                continue
            npath = self.path[len(p):]
            if not npath.startswith('/'):
                npath = '/' + npath

            if not self.in_handlers.has_key(p):
                self.in_handlers[p] = vdir.handler(noconnection(),self.client_address,self.server)
                if vdir.auth_provider:
                    vdir.auth_provider.setupAuth(self, self.in_handlers[p])
            hnd = self.in_handlers[p]
            hnd.rfile = self.rfile
            hnd.wfile = self.wfile
            self.rlpath = self.raw_requestline
            self._handle_one_foreign(hnd,npath, vdir.auth_provider)
            # print "Handled, closing = ", self.close_connection
            return
        # if no match:
        self.send_error(404, "Path not found: %s" % self.path)
        return

    def _get_ignore_body(self,fore):
        if not fore.headers.has_key("content-length"):
            return
        max_chunk_size = 10*1024*1024
        size_remaining = int(fore.headers["content-length"])
        got = ''
        while size_remaining:
            chunk_size = min(size_remaining, max_chunk_size)
            got = fore.rfile.read(chunk_size)
            size_remaining -= len(got)

class SecureMultiHTTPHandler(MultiHTTPHandler):
    def getcert_fnames(self):
        """ Return a pair with the filenames of ssl cert,key

            Override this to direct to other filenames
        """
        return ('server.cert','server.key')

    def setup(self):
        import ssl
        certfile, keyfile = self.getcert_fnames()
        try:
            self.connection = ssl.wrap_socket(self.request,
                                server_side=True,
                                certfile=certfile,
                                keyfile=keyfile,
                                ssl_version=ssl.PROTOCOL_SSLv23)
            self.rfile = self.connection.makefile('rb', self.rbufsize)
            self.wfile = self.connection.makefile('wb', self.wbufsize)
            self.log_message("Secure %s connection from %s",self.connection.cipher(),self.client_address)
        except Exception:
            self.request.shutdown(socket.SHUT_RDWR)
            raise

    def finish(self):
        # With ssl connections, closing the filehandlers alone may not
        # work because of ref counting. We explicitly tell the socket
        # to shutdown.
        MultiHTTPHandler.finish(self)
        try:
            self.connection.shutdown(socket.SHUT_RDWR)
        except Exception:
            pass

import threading
class ConnThreadingMixIn:
    """Mix-in class to handle each _connection_ in a new thread.

       This is necessary for persistent connections, where multiple
       requests should be handled synchronously at each connection, but
       multiple connections can run in parallel.
    """

    # Decides how threads will act upon termination of the
    # main process
    daemon_threads = False

    def _handle_request_noblock(self):
        """Start a new thread to process the request."""
        t = threading.Thread(target = self._handle_request2)
        if self.daemon_threads:
            t.setDaemon (1)
        t.start()

    def _handle_request2(self):
        """Handle one request, without blocking.

        I assume that select.select has returned that the socket is
        readable before this function was called, so there should be
        no risk of blocking in get_request().
        """
        try:
            request, client_address = self.get_request()
        except socket.error:
            return
        if self.verify_request(request, client_address):
            try:
                self.process_request(request, client_address)
            except Exception:
                self.handle_error(request, client_address)
                self.close_request(request)

#eof
<commit_msg>[MERGE] merged xrg's fixes for HTTP auth error codes: access denied is 403<commit_after># -*- coding: utf-8 -*-
#
# Copyright P. Christeas <p_christ@hol.gr> 2008,2009
# A part of the code comes from the ganeti project:  http://www.mail-archive.com/ganeti-devel@googlegroups.com/msg00713.html#
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
###############################################################################

""" Framework for generic http servers

"""

import socket
import base64
import SocketServer
from BaseHTTPServer import *
from SimpleHTTPServer import SimpleHTTPRequestHandler

class AuthRequiredExc(Exception):
    def __init__(self,atype,realm):
        Exception.__init__(self)
        self.atype = atype
        self.realm = realm

class AuthRejectedExc(Exception):
    pass

class AuthProvider:
    def __init__(self,realm):
        self.realm = realm

    def setupAuth(self, multi,handler):
        """ Attach an AuthProxy object to handler
        """
        pass

    def authenticate(self, user, passwd, client_address):
        return False

    def log(self, msg):
        print msg

class BasicAuthProvider(AuthProvider):
    def setupAuth(self, multi, handler):
        if not multi.sec_realms.has_key(self.realm):
            multi.sec_realms[self.realm] = BasicAuthProxy(self)

class AuthProxy:
    """ This class will hold authentication information for a handler,
        i.e. a connection
    """
    def __init__(self, provider):
        self.provider = provider

    def checkRequest(self,handler,path = '/'):
        """ Check if we are allowed to process that request
        """
        pass

class BasicAuthProxy(AuthProxy):
    """ Require basic authentication..
    """
    def __init__(self,provider):
        AuthProxy.__init__(self,provider)
        self.auth_creds = None
        self.auth_tries = 0

    def checkRequest(self,handler,path = '/'):
        if self.auth_creds:
            return True
        auth_str = handler.headers.get('Authorization',False)
        if auth_str and auth_str.startswith('Basic '):
            auth_str=auth_str[len('Basic '):]
            (user,passwd) = base64.decodestring(auth_str).split(':')
            self.provider.log("Found user=\"%s\", passwd=\"%s\"" %(user,passwd))
            self.auth_creds = self.provider.authenticate(user,passwd,handler.client_address)
            if self.auth_creds:
                return True
        if self.auth_tries > 5:
            self.provider.log("Failing authorization after 5 requests w/o password")
            raise AuthRejectedExc("Authorization failed.")
        self.auth_tries += 1
        raise AuthRequiredExc(atype = 'Basic', realm=self.provider.realm)

class HTTPHandler(SimpleHTTPRequestHandler):
    def __init__(self,request, client_address, server):
        SimpleHTTPRequestHandler.__init__(self,request,client_address,server)
        # print "Handler for %s inited" % str(client_address)
        self.protocol_version = 'HTTP/1.1'
        self.connection = dummyconn()

    def handle(self):
        """ Classes here should NOT handle inside their constructor
        """
        pass

    def finish(self):
        pass

    def setup(self):
        pass

class HTTPDir:
    """ A dispatcher class, like a virtual folder in httpd
    """
    def __init__(self,path,handler, auth_provider = None):
        self.path = path
        self.handler = handler
        self.auth_provider = auth_provider

    def matches(self, request):
        """ Test if some request matches us. If so, return
            the matched path. """
        if request.startswith(self.path):
            return self.path
        return False

class noconnection:
    """ a class to use instead of the real connection
    """
    def makefile(self, mode, bufsize):
        return None

class dummyconn:
    def shutdown(self, tru):
        pass

def _quote_html(html):
    return html.replace("&", "&").replace("<", "<").replace(">", ">")

class FixSendError:
    #error_message_format = """ """
    def send_error(self, code, message=None):
        #overriden from BaseHTTPRequestHandler, we also send the content-length
        try:
            short, long = self.responses[code]
        except KeyError:
            short, long = '???', '???'
        if message is None:
            message = short
        explain = long
        self.log_error("code %d, message %s", code, message)
        # using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201)
        content = (self.error_message_format %
                   {'code': code, 'message': _quote_html(message), 'explain': explain})
        self.send_response(code, message)
        self.send_header("Content-Type", self.error_content_type)
        self.send_header('Connection', 'close')
        self.send_header('Content-Length', len(content) or 0)
        self.end_headers()
        if self.command != 'HEAD' and code >= 200 and code not in (204, 304):
            self.wfile.write(content)

class MultiHTTPHandler(FixSendError,BaseHTTPRequestHandler):
    """ this is a multiple handler, that will dispatch each request
        to a nested handler, iff it matches

        The handler will also have *one* dict of authentication proxies,
        groupped by their realm.
    """

    protocol_version = "HTTP/1.1"
    default_request_version = "HTTP/0.9"    # compatibility with py2.5

    auth_required_msg = """ <title>Authorization required</title>
    You must authenticate to use this service\r\r"""

    def __init__(self, request, client_address, server):
        self.in_handlers = {}
        self.sec_realms = {}
        SocketServer.StreamRequestHandler.__init__(self,request,client_address,server)
        self.log_message("MultiHttpHandler init for %s" %(str(client_address)))

    def _handle_one_foreign(self,fore, path, auth_provider):
        """ This method overrides the handle_one_request for *children*
            handlers. It is required, since the first line should not be
            read again..

        """
        fore.raw_requestline = "%s %s %s\n" % (self.command, path, self.version)
        if not fore.parse_request(): # An error code has been sent, just exit
            return
        self.request_version = fore.request_version
        if auth_provider and auth_provider.realm:
            try:
                self.sec_realms[auth_provider.realm].checkRequest(fore,path)
            except AuthRequiredExc,ae:
                if self.request_version != 'HTTP/1.1':
                    self.log_error("Cannot require auth at %s",self.request_version)
                    self.send_error(403)
                    return
                self._get_ignore_body(fore) # consume any body that came, not loose sync with input
                self.send_response(401,'Authorization required')
                self.send_header('WWW-Authenticate','%s realm="%s"' % (ae.atype,ae.realm))
                self.send_header('Connection', 'keep-alive')
                self.send_header('Content-Type','text/html')
                self.send_header('Content-Length',len(self.auth_required_msg))
                self.end_headers()
                self.wfile.write(self.auth_required_msg)
                return
            except AuthRejectedExc,e:
                self.log_error("Rejected auth: %s" % e.args[0])
                self.send_error(403,e.args[0])
                self.close_connection = 1
                return
        mname = 'do_' + fore.command
        if not hasattr(fore, mname):
            fore.send_error(501, "Unsupported method (%r)" % fore.command)
            return
        fore.close_connection = 0
        method = getattr(fore, mname)
        method()
        if fore.close_connection:
            # print "Closing connection because of handler"
            self.close_connection = fore.close_connection
        if hasattr(fore, '_flush'):
            fore._flush()

    def parse_rawline(self):
        """Parse a request (internal).

        The request should be stored in self.raw_requestline; the results
        are in self.command, self.path, self.request_version and
        self.headers.

        Return True for success, False for failure; on failure, an
        error is sent back.

        """
        self.command = None  # set in case of error on the first line
        self.request_version = version = self.default_request_version
        self.close_connection = 1
        requestline = self.raw_requestline
        if requestline[-2:] == '\r\n':
            requestline = requestline[:-2]
        elif requestline[-1:] == '\n':
            requestline = requestline[:-1]
        self.requestline = requestline
        words = requestline.split()
        if len(words) == 3:
            [command, path, version] = words
            if version[:5] != 'HTTP/':
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            try:
                base_version_number = version.split('/', 1)[1]
                version_number = base_version_number.split(".")
                # RFC 2145 section 3.1 says there can be only one "." and
                #   - major and minor numbers MUST be treated as
                #      separate integers;
                #   - HTTP/2.4 is a lower version than HTTP/2.13, which in
                #      turn is lower than HTTP/12.3;
                #   - Leading zeros MUST be ignored by recipients.
                if len(version_number) != 2:
                    raise ValueError
                version_number = int(version_number[0]), int(version_number[1])
            except (ValueError, IndexError):
                self.send_error(400, "Bad request version (%r)" % version)
                return False
            if version_number >= (1, 1):
                self.close_connection = 0
            if version_number >= (2, 0):
                self.send_error(505,
                          "Invalid HTTP Version (%s)" % base_version_number)
                return False
        elif len(words) == 2:
            [command, path] = words
            self.close_connection = 1
            if command != 'GET':
                self.send_error(400,
                                "Bad HTTP/0.9 request type (%r)" % command)
                return False
        elif not words:
            return False
        else:
            #self.send_error(400, "Bad request syntax (%r)" % requestline)
            return False
        self.request_version = version
        self.command, self.path, self.version = command, path, version
        return True

    def handle_one_request(self):
        """Handle a single HTTP request.
           Dispatch to the correct handler.
        """
        self.request.setblocking(True)
        self.raw_requestline = self.rfile.readline()
        if not self.raw_requestline:
            self.close_connection = 1
            # self.log_message("no requestline, connection closed?")
            return
        if not self.parse_rawline():
            self.log_message("Could not parse rawline.")
            return
        # self.parse_request(): # Do NOT parse here. the first line should be the only
        for vdir in self.server.vdirs:
            p = vdir.matches(self.path)
            if p == False:
                continue
            npath = self.path[len(p):]
            if not npath.startswith('/'):
                npath = '/' + npath

            if not self.in_handlers.has_key(p):
                self.in_handlers[p] = vdir.handler(noconnection(),self.client_address,self.server)
                if vdir.auth_provider:
                    vdir.auth_provider.setupAuth(self, self.in_handlers[p])
            hnd = self.in_handlers[p]
            hnd.rfile = self.rfile
            hnd.wfile = self.wfile
            self.rlpath = self.raw_requestline
            self._handle_one_foreign(hnd,npath, vdir.auth_provider)
            # print "Handled, closing = ", self.close_connection
            return
        # if no match:
        self.send_error(404, "Path not found: %s" % self.path)
        return

    def _get_ignore_body(self,fore):
        if not fore.headers.has_key("content-length"):
            return
        max_chunk_size = 10*1024*1024
        size_remaining = int(fore.headers["content-length"])
        got = ''
        while size_remaining:
            chunk_size = min(size_remaining, max_chunk_size)
            got = fore.rfile.read(chunk_size)
            size_remaining -= len(got)

class SecureMultiHTTPHandler(MultiHTTPHandler):
    def getcert_fnames(self):
        """ Return a pair with the filenames of ssl cert,key

            Override this to direct to other filenames
        """
        return ('server.cert','server.key')

    def setup(self):
        import ssl
        certfile, keyfile = self.getcert_fnames()
        try:
            self.connection = ssl.wrap_socket(self.request,
                                server_side=True,
                                certfile=certfile,
                                keyfile=keyfile,
                                ssl_version=ssl.PROTOCOL_SSLv23)
            self.rfile = self.connection.makefile('rb', self.rbufsize)
            self.wfile = self.connection.makefile('wb', self.wbufsize)
            self.log_message("Secure %s connection from %s",self.connection.cipher(),self.client_address)
        except Exception:
            self.request.shutdown(socket.SHUT_RDWR)
            raise

    def finish(self):
        # With ssl connections, closing the filehandlers alone may not
        # work because of ref counting. We explicitly tell the socket
        # to shutdown.
        MultiHTTPHandler.finish(self)
        try:
            self.connection.shutdown(socket.SHUT_RDWR)
        except Exception:
            pass

import threading
class ConnThreadingMixIn:
    """Mix-in class to handle each _connection_ in a new thread.

       This is necessary for persistent connections, where multiple
       requests should be handled synchronously at each connection, but
       multiple connections can run in parallel.
    """

    # Decides how threads will act upon termination of the
    # main process
    daemon_threads = False

    def _handle_request_noblock(self):
        """Start a new thread to process the request."""
        t = threading.Thread(target = self._handle_request2)
        if self.daemon_threads:
            t.setDaemon (1)
        t.start()

    def _handle_request2(self):
        """Handle one request, without blocking.

        I assume that select.select has returned that the socket is
        readable before this function was called, so there should be
        no risk of blocking in get_request().
        """
        try:
            request, client_address = self.get_request()
        except socket.error:
            return
        if self.verify_request(request, client_address):
            try:
                self.process_request(request, client_address)
            except Exception:
                self.handle_error(request, client_address)
                self.close_request(request)

#eof
<|endoftext|>


<commit_before>#!/usr/bin/python3

import requests
import argparse
import sys

def err(msg, *args):
    print('error:', msg % args, file = sys.stderr)

token = open('token').read().strip()

URL = 'https://api.github.com/repos/{owner}/{repo}/hooks?access_token=' + token

events = [
    '*', 'commit_comment', 'create', 'delete', 'deployment', 'deployment_status', 'fork',
    'gollum', 'issue_comment', 'issues', 'member', 'membership', 'page_build', 'public',
    'pull_request_review_comment', 'pull_request', 'push', 'repository', 'release',
    'status', 'team_add', 'watch'
]

parser = argparse.ArgumentParser(description = 'add a webhook to many repos')
parser.add_argument('-u', '--url', metavar = 'URL', type=str, required=True,
                    help='URL for the webhook')
parser.add_argument('-t', '--content-type', choices=['json', 'form'], default='json',
                    help='Content-type for the webhook')
parser.add_argument('-e', '--event', choices=events, action='append', default=['push'],
                    help='Events to support')
parser.add_argument('-s', '--secret', default='', help='Secret for the webhook')
parser.add_argument('-o', '--owner', type=str, help='Owner of repos without a /')
parser.add_argument('repo', nargs='*', type=str,
                    help='Repositories to add to')

args = parser.parse_args()

hook = {
    'name': "web",
    'active': True,
    'events': args.event,
    'config': {
        'url': args.url,
        'content_type': args.content_type,
        'secret': args.secretg
    }
}

for rawrepo in args.repo:
    splits = rawrepo.split('/')
    if len(splits) == 1:
        if args.owner is None:
            err('`%s` has no owner specified and there is no --owner argument', rawrepo)
            continue
        owner, repo = args.owner, rawrepo
    elif len(splits) == 2:
        owner, repo = splits
    else:
        err('`%s` is not a valid GitHub repo', rawrepo)
        continue

    print('Adding to %s/%s' % (owner, repo))
    url = URL.format(owner = owner, repo = repo)
    requests.post(url, json = hook)
<commit_msg>Pressing C in C-g is hard sometimes.<commit_after>#!/usr/bin/python3

import requests
import argparse
import sys

def err(msg, *args):
    print('error:', msg % args, file = sys.stderr)

token = open('token').read().strip()

URL = 'https://api.github.com/repos/{owner}/{repo}/hooks?access_token=' + token

events = [
    '*', 'commit_comment', 'create', 'delete', 'deployment', 'deployment_status', 'fork',
    'gollum', 'issue_comment', 'issues', 'member', 'membership', 'page_build', 'public',
    'pull_request_review_comment', 'pull_request', 'push', 'repository', 'release',
    'status', 'team_add', 'watch'
]

parser = argparse.ArgumentParser(description = 'add a webhook to many repos')
parser.add_argument('-u', '--url', metavar = 'URL', type=str, required=True,
                    help='URL for the webhook')
parser.add_argument('-t', '--content-type', choices=['json', 'form'], default='json',
                    help='Content-type for the webhook')
parser.add_argument('-e', '--event', choices=events, action='append', default=['push'],
                    help='Events to support')
parser.add_argument('-s', '--secret', default='', help='Secret for the webhook')
parser.add_argument('-o', '--owner', type=str, help='Owner of repos without a /')
parser.add_argument('repo', nargs='*', type=str,
                    help='Repositories to add to')

args = parser.parse_args()

hook = {
    'name': "web",
    'active': True,
    'events': args.event,
    'config': {
        'url': args.url,
        'content_type': args.content_type,
        'secret': args.secret
    }
}

for rawrepo in args.repo:
    splits = rawrepo.split('/')
    if len(splits) == 1:
        if args.owner is None:
            err('`%s` has no owner specified and there is no --owner argument', rawrepo)
            continue
        owner, repo = args.owner, rawrepo
    elif len(splits) == 2:
        owner, repo = splits
    else:
        err('`%s` is not a valid GitHub repo', rawrepo)
        continue

    print('Adding to %s/%s' % (owner, repo))
    url = URL.format(owner = owner, repo = repo)
    requests.post(url, json = hook)
<|endoftext|>


<commit_before>"""
This app is intended to provide the core functionality for tracking user
engagement with content and Kolibri in general. As such, it is intended
to store details of user interactions with content, a summary of those
interactions, interactions with the software in general, as well as user
feedback on the content and the software.
"""

from __future__ import unicode_literals

from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models
from kolibri.auth.constants import role_kinds
from kolibri.auth.models import AbstractFacilityDataModel, Facility, FacilityUser
from kolibri.auth.permissions.base import RoleBasedPermissions
from kolibri.auth.permissions.general import IsOwn
from kolibri.content.models import UUIDField

from .permissions import AnyoneCanWriteAnonymousLogs

class BaseLogModel(AbstractFacilityDataModel):

    permissions = (
        AnyoneCanWriteAnonymousLogs() |
        IsOwn() |
        RoleBasedPermissions(
            target_field="user",
            can_be_created_by=(role_kinds.ADMIN,),
            can_be_read_by=(role_kinds.ADMIN, role_kinds.COACH),
            can_be_updated_by=(role_kinds.ADMIN,),
            can_be_deleted_by=(role_kinds.ADMIN,),
        )
    )

    class Meta:
        abstract = True

    def infer_dataset(self):
        if self.user:
            return self.user.dataset
        else:
            facility = Facility.get_default_facility()
            assert facility, "Before you can save logs, you must have a facility"
            return facility.dataset

class ContentSessionLog(BaseLogModel):
    """
    This model provides a record of interactions with a content item within a single visit to that content page.
    """
    user = models.ForeignKey(FacilityUser, blank=True, null=True)
    content_id = UUIDField(db_index=True)
    channel_id = UUIDField()
    start_timestamp = models.DateTimeField()
    end_timestamp = models.DateTimeField(blank=True, null=True)
    time_spent = models.FloatField(help_text="(in seconds)", default=0.0, validators=[MinValueValidator(0)])
    progress = models.FloatField(default=0, validators=[MinValueValidator(0)])
    kind = models.CharField(max_length=200)
    extra_fields = models.TextField(default="{}")

class ContentSummaryLog(BaseLogModel):
    """
    This model provides a summary of all interactions a user has had with a content item.
    """
    user = models.ForeignKey(FacilityUser)
    content_id = UUIDField(db_index=True)
    channel_id = UUIDField()
    start_timestamp = models.DateTimeField()
    end_timestamp = models.DateTimeField(blank=True, null=True)
    completion_timestamp = models.DateTimeField(blank=True, null=True)
    time_spent = models.FloatField(help_text="(in seconds)", default=0.0, validators=[MinValueValidator(0)])
    progress = models.FloatField(default=0, validators=[MinValueValidator(0), MaxValueValidator(1)])
    kind = models.CharField(max_length=200)
    extra_fields = models.TextField(default="{}")

class ContentRatingLog(BaseLogModel):
    """
    This model provides a record of user feedback on a content item.
    """
    user = models.ForeignKey(FacilityUser, blank=True, null=True)
    content_id = UUIDField(db_index=True)
    channel_id = UUIDField()
    quality = models.IntegerField(blank=True, null=True, validators=[MinValueValidator(1), MaxValueValidator(5)])
    ease = models.IntegerField(blank=True, null=True, validators=[MinValueValidator(1), MaxValueValidator(5)])
    learning = models.IntegerField(blank=True, null=True, validators=[MinValueValidator(1), MaxValueValidator(5)])
    feedback = models.TextField(blank=True)

class UserSessionLog(BaseLogModel):
    """
    This model provides a record of a user session in Kolibri.
    """
    user = models.ForeignKey(FacilityUser)
    channels = models.TextField(blank=True)
    start_timestamp = models.DateTimeField(auto_now_add=True)
    completion_timestamp = models.DateTimeField(blank=True, null=True)
    pages = models.TextField(blank=True)
<commit_msg>Add BaseLogQuerySet with filter_by_topic method.<commit_after>"""
This app is intended to provide the core functionality for tracking user
engagement with content and Kolibri in general. As such, it is intended
to store details of user interactions with content, a summary of those
interactions, interactions with the software in general, as well as user
feedback on the content and the software.
"""

from __future__ import unicode_literals

from django.core.validators import MaxValueValidator, MinValueValidator
from django.db import models
from kolibri.auth.constants import role_kinds
from kolibri.auth.models import AbstractFacilityDataModel, Facility, FacilityUser
from kolibri.auth.permissions.base import RoleBasedPermissions
from kolibri.auth.permissions.general import IsOwn
from kolibri.content.content_db_router import default_database_is_attached, get_active_content_database
from kolibri.content.models import UUIDField

from .permissions import AnyoneCanWriteAnonymousLogs

class BaseLogQuerySet(models.QuerySet):

    def filter_by_topic(self, topic, content_id_lookup="content_id"):
        """
        Filter a set of logs by content_id, using content_ids from all descendants of specified topic.
        """

        content_ids = topic.get_descendant_content_ids()

        if default_database_is_attached():
            # perform the query using an efficient cross-database join, if possible
            return self.using(get_active_content_database()).filter(**{content_id_lookup + "__in": content_ids})
        else:
            # if the databases can't be joined, convert the content_ids into a list and pass in
            return self.filter(**{content_id_lookup + "__in": list(content_ids)})

class BaseLogModel(AbstractFacilityDataModel):

    permissions = (
        AnyoneCanWriteAnonymousLogs() |
        IsOwn() |
        RoleBasedPermissions(
            target_field="user",
            can_be_created_by=(role_kinds.ADMIN,),
            can_be_read_by=(role_kinds.ADMIN, role_kinds.COACH),
            can_be_updated_by=(role_kinds.ADMIN,),
            can_be_deleted_by=(role_kinds.ADMIN,),
        )
    )

    objects = BaseLogQuerySet.as_manager()

    class Meta:
        abstract = True

    def infer_dataset(self):
        if self.user:
            return self.user.dataset
        else:
            facility = Facility.get_default_facility()
            assert facility, "Before you can save logs, you must have a facility"
            return facility.dataset

class ContentSessionLog(BaseLogModel):
    """
    This model provides a record of interactions with a content item within a single visit to that content page.
    """
    user = models.ForeignKey(FacilityUser, blank=True, null=True)
    content_id = UUIDField(db_index=True)
    channel_id = UUIDField()
    start_timestamp = models.DateTimeField()
    end_timestamp = models.DateTimeField(blank=True, null=True)
    time_spent = models.FloatField(help_text="(in seconds)", default=0.0, validators=[MinValueValidator(0)])
    progress = models.FloatField(default=0, validators=[MinValueValidator(0)])
    kind = models.CharField(max_length=200)
    extra_fields = models.TextField(default="{}")

class ContentSummaryLog(BaseLogModel):
    """
    This model provides a summary of all interactions a user has had with a content item.
    """
    user = models.ForeignKey(FacilityUser)
    content_id = UUIDField(db_index=True)
    channel_id = UUIDField()
    start_timestamp = models.DateTimeField()
    end_timestamp = models.DateTimeField(blank=True, null=True)
    completion_timestamp = models.DateTimeField(blank=True, null=True)
    time_spent = models.FloatField(help_text="(in seconds)", default=0.0, validators=[MinValueValidator(0)])
    progress = models.FloatField(default=0, validators=[MinValueValidator(0), MaxValueValidator(1)])
    kind = models.CharField(max_length=200)
    extra_fields = models.TextField(default="{}")

class ContentRatingLog(BaseLogModel):
    """
    This model provides a record of user feedback on a content item.
    """
    user = models.ForeignKey(FacilityUser, blank=True, null=True)
    content_id = UUIDField(db_index=True)
    channel_id = UUIDField()
    quality = models.IntegerField(blank=True, null=True, validators=[MinValueValidator(1), MaxValueValidator(5)])
    ease = models.IntegerField(blank=True, null=True, validators=[MinValueValidator(1), MaxValueValidator(5)])
    learning = models.IntegerField(blank=True, null=True, validators=[MinValueValidator(1), MaxValueValidator(5)])
    feedback = models.TextField(blank=True)

class UserSessionLog(BaseLogModel):
    """
    This model provides a record of a user session in Kolibri.
    """
    user = models.ForeignKey(FacilityUser)
    channels = models.TextField(blank=True)
    start_timestamp = models.DateTimeField(auto_now_add=True)
    completion_timestamp = models.DateTimeField(blank=True, null=True)
    pages = models.TextField(blank=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys, sqlite3, time as _time, os.path

if len(sys.argv) != 2:
    print('Usage: adjust.py /path/to/fossil/repo')
    exit(10)

repo_path = sys.argv[1]
conn = sqlite3.connect(repo_path)
c = conn.cursor()

now = int(_time.time())

ticket_options = {
    'type_choices': ['Bug', 'Feature', 'Task', 'Question'],
    'priority_choices': ['High', 'Normal', 'Low'],
    'severity_choices': ['Severe', 'Important', 'Minor'],
    'resolution_choices': ['Open', 'Fixed', 'Rejected', 'Obsolete', 'Duplicate'],
    'status_choices': ['Open', 'Closed', 'Postponed'],
    'subsystem_choices': [],
}

configs = [
    # show links without javascript
    ['auto-hyperlink', 0],
    # no need for captcha
    ['require-captcha', 0],
    # show timeline in local time
    ['timeline-utc', 0],
]

pages = [
    'ticket-editpage',
    'ticket-newpage',
    'ticket-viewpage',
]

reports = {
    'Open Tickets': dict(sqlcode='''
SELECT
case priority
when 'High' then '#f2dcdc'
when 'Normal' then '#cfe8bd'
when 'Low' then '#cacae5'
else '#ffffff' end as bgcolor,
  substr(tkt_uuid,1,10) AS '#',
  date(tkt_ctime) as created,
  date(tkt_mtime) AS updated,
  type,
  title
FROM ticket
where status in ('Open', 'Verified')
order by priority != 'High', priority != 'Normal', tkt_ctime
    ''', cols='''
#ffffff Priority:
#f2dcdc High
#cfe8bd Normal
#cacae5 Low
    '''),
}

for name in reports:
    reports[name]['sqlcode'] = reports[name]['sqlcode'].strip()
    reports[name]['cols'] = reports[name]['cols'].strip()

# ---

def build_ticket_common(ticket_options):
    result = ''
    for key in ticket_options:
        result += 'set %s {\n%s\n}\n' % (key, "\n".join('  ' + key for key in ticket_options[key]))
    return result.strip()

configs.append(['ticket-common', build_ticket_common(ticket_options)])

for page in pages:
    with open(os.path.join(os.path.dirname(__file__), 'templates', '%s.html' % page)) as f:
        content = f.read()
    configs.append([page, content])

for name, value in configs:
    c.execute('''replace into config (name, value, mtime) values (?, ?, ?)''',
        (name, value, now))

def fetchfirst(c):
    row = c.fetchone()
    if row is None:
        return None
    else:
        return row[0]

for name in reports:
    c.execute('select rn from reportfmt where title=?', (name,))
    rn = fetchfirst(c)
    if rn:
        c.execute('update reportfmt set sqlcode=?, cols=?, mtime=?, owner=?',
            (reports[name]['sqlcode'], reports[name]['cols'], 'now', 'predefined'))
    else:
        c.execute('insert into reportfmt (owner, title, mtime, sqlcode, cols) values (?, ?, ?, ?, ?)',
            ('predefined', name, 'now', reports[name]['sqlcode'], reports[name]['cols']))

conn.commit()
c.close()
<commit_msg>Recently closed tickets report<commit_after>#!/usr/bin/env python

import sys, sqlite3, time as _time, os.path

if len(sys.argv) != 2:
    print('Usage: adjust.py /path/to/fossil/repo')
    exit(10)

repo_path = sys.argv[1]
conn = sqlite3.connect(repo_path)
c = conn.cursor()

now = int(_time.time())

ticket_options = {
    'type_choices': ['Bug', 'Feature', 'Task', 'Question'],
    'priority_choices': ['High', 'Normal', 'Low'],
    'severity_choices': ['Severe', 'Important', 'Minor'],
    'resolution_choices': ['Open', 'Fixed', 'Rejected', 'Obsolete', 'Duplicate'],
    'status_choices': ['Open', 'Closed', 'Postponed'],
    'subsystem_choices': [],
}

configs = [
    # show links without javascript
    ['auto-hyperlink', 0],
    # no need for captcha
    ['require-captcha', 0],
    # show timeline in local time
    ['timeline-utc', 0],
]

pages = [
    'ticket-editpage',
    'ticket-newpage',
    'ticket-viewpage',
]

reports = {
    'Open Tickets': dict(sqlcode='''
SELECT
case priority
when 'High' then '#f2dcdc'
when 'Normal' then '#cfe8bd'
when 'Low' then '#cacae5'
else '#ffffff' end as bgcolor,
  substr(tkt_uuid,1,10) AS '#',
  date(tkt_ctime) as created,
  date(tkt_mtime) AS updated,
  type,
  title
FROM ticket
where status in ('Open', 'Verified')
order by priority != 'High', priority != 'Normal', tkt_ctime
    ''', cols='''
#ffffff Priority:
#f2dcdc High
#cfe8bd Normal
#cacae5 Low
    '''),
    'Recently Closed Tickets': dict(sqlcode='''
SELECT
case date(tkt_mtime)
when date('now') then '#fcf8cf'
when date('now', '-1 day') then '#cfe7fc'
when date('now', '-2 days') then '#fccffa'
when date('now', '-3 days') then '#cffcd5'
when date('now', '-4 days') then '#e7cffc'
else '#fcdecf' end bgcolor,
  substr(tkt_uuid,1,10) AS '#',
  date(tkt_ctime) as created,
  date(tkt_mtime) AS updated,
  type,
  title
FROM ticket
where status in ('Closed')
and date(tkt_mtime) > date('now', '-7 days')
order by tkt_mtime desc
    ''', cols='''
#ffffff Closed:
#fcf8cf Today
#cfe7fc Yesterday
#fccffa 2 days ago
#cffcd5 3 days ago
#e7cffc 4 days ago
#fcdecf 5+ days ago
    '''),
}

for name in reports:
    reports[name]['sqlcode'] = reports[name]['sqlcode'].strip()
    reports[name]['cols'] = reports[name]['cols'].strip()

# ---

def build_ticket_common(ticket_options):
    result = ''
    for key in ticket_options:
        result += 'set %s {\n%s\n}\n' % (key, "\n".join('  ' + key for key in ticket_options[key]))
    return result.strip()

configs.append(['ticket-common', build_ticket_common(ticket_options)])

for page in pages:
    with open(os.path.join(os.path.dirname(__file__), 'templates', '%s.html' % page)) as f:
        content = f.read()
    configs.append([page, content])

for name, value in configs:
    c.execute('''replace into config (name, value, mtime) values (?, ?, ?)''',
        (name, value, now))

def fetchfirst(c):
    row = c.fetchone()
    if row is None:
        return None
    else:
        return row[0]

for name in reports:
    c.execute('select rn from reportfmt where title=?', (name,))
    rn = fetchfirst(c)
    if rn:
        c.execute('update reportfmt set sqlcode=?, cols=?, mtime=?, owner=?',
            (reports[name]['sqlcode'], reports[name]['cols'], 'now', 'predefined'))
    else:
        c.execute('insert into reportfmt (owner, title, mtime, sqlcode, cols) values (?, ?, ?, ?, ?)',
            ('predefined', name, 'now', reports[name]['sqlcode'], reports[name]['cols']))

conn.commit()
c.close()
<|endoftext|>


<commit_before>from experiment_config_lib import ControllerConfig
from sts.topology import MeshTopology
from sts.control_flow import Fuzzer, Interactive
from sts.input_traces.input_logger import InputLogger
from sts.invariant_checker import InvariantChecker
from sts.simulation_state import SimulationConfig

# Use POX as our controller
command_line = ('''./pox.py --verbose openflow.debug ''' #sts.syncproto.pox_syncer '''
                '''forwarding.l2_multi '''
                #'''sts.util.socket_mux.pox_monkeypatcher '''
                '''openflow.of_01 --address=__address__ --port=__port__''')
controllers = [ControllerConfig(command_line, cwd="betta")]

topology_class = MeshTopology
topology_params = "num_switches=2"
dataplane_trace = "dataplane_traces/ping_pong_same_subnet.trace"

simulation_config = SimulationConfig(controller_configs=controllers,
                                     topology_class=topology_class,
                                     topology_params=topology_params,
                                     dataplane_trace=dataplane_trace,
                                     multiplex_sockets=False)

control_flow = Fuzzer(simulation_config, check_interval=80,
                      halt_on_violation=False,
                      input_logger=InputLogger(),
                      invariant_check=InvariantChecker.check_connectivity)
#control_flow = Interactive(simulation_config, input_logger=InputLogger())
<commit_msg>Switch to normal pox instead of betta<commit_after>from experiment_config_lib import ControllerConfig
from sts.topology import MeshTopology
from sts.control_flow import Fuzzer, Interactive
from sts.input_traces.input_logger import InputLogger
from sts.invariant_checker import InvariantChecker
from sts.simulation_state import SimulationConfig

# Use POX as our controller
command_line = ('''./pox.py --verbose sts.syncproto.pox_syncer '''
                '''openflow.discovery forwarding.l2_multi '''
                '''sts.util.socket_mux.pox_monkeypatcher '''
                '''openflow.of_01 --address=__address__ --port=__port__''')
controllers = [ControllerConfig(command_line, cwd="pox", sync="tcp:localhost:18899")]
topology_class = MeshTopology
topology_params = "num_switches=2"
dataplane_trace = "dataplane_traces/ping_pong_same_subnet.trace"

simulation_config = SimulationConfig(controller_configs=controllers,
                                     topology_class=topology_class,
                                     topology_params=topology_params,
                                     dataplane_trace=dataplane_trace,
                                     multiplex_sockets=True)

control_flow = Fuzzer(simulation_config, check_interval=80,
                      halt_on_violation=False,
                      input_logger=InputLogger(),
                      invariant_check=InvariantChecker.check_loops)
#control_flow = Interactive(simulation_config, input_logger=InputLogger())
<|endoftext|>


<commit_before>from lab.with_config import WithConfig
from lab.with_log import WithLogMixIn
from lab.laboratory import Laboratory

class FakeCloud(object):
    pod = Laboratory()

def starter(worker):
    return worker.start_worker_parallel()

class RunnerHA(WithConfig, WithLogMixIn):
    def __init__(self):
        from prettytable import PrettyTable

        self.table = PrettyTable()
        self.table.field_names = ['path', 'time', 'status', 'result', 'tims']
        self.cloud = None

    def __repr__(self):
        return u'RunnerHA'

    def execute_single_test(self, test_case):
        import multiprocessing
        import fabric.network
        import time

        manager = multiprocessing.Manager()
        status_dict = manager.dict()

        workers = test_case.workers
        for worker in workers:
            worker.status_dict = status_dict
            worker.set_status(worker.STATUS_CREATED)
            if not test_case.is_debug:
                worker.setup_worker()  # run all setup_worker in non-parallel
            else:
                worker.log('Setup...')

        fabric.network.disconnect_all()  # we do that since URL: http://stackoverflow.com/questions/29480850/paramiko-hangs-at-get-channel-while-using-multiprocessing
        time.sleep(2)

        pool = multiprocessing.Pool(len(workers))
        self.log('\n\n***AFTER THIS LINE PARALLEL EXECUTION STARTS***\n\n')
        results = pool.map(starter, workers)
        self.log('\n\n***PARALLEL EXECUTION FINISHED***\n\n')

        test_case.after_run(results=results)
        self.table.add_row([test_case.path, test_case.time, test_case.tcr.status, test_case.tcr.text, test_case.tcr.tims_url])

        map(lambda x: x.teardown_worker(), workers)  # run all teardown_workers

    def run(self, pod_name, test_regex, is_noclean, is_debug):
        import time
        from lab.elk import Elk

        available_tc = self.ls_configs(directory='ha')
        test_paths = sorted(filter(lambda x: test_regex in x, available_tc))

        if not test_paths:
            raise ValueError('Provided regexp "{}" does not match any tests'.format(test_regex))

        self.cloud = self.init_cloud(pod_name=pod_name, is_debug=is_debug, test_paths=test_paths)

        tests = self.create_tests(test_paths=test_paths, is_noclean=is_noclean, is_debug=is_debug)

        start_time = time.time()
        map(lambda x: self.execute_single_test(test_case=x), tests)

        with self.open_artifact('main_results.txt', 'w') as f:
            f.write(self.table.get_string())

        if type(self.cloud) is not FakeCloud:
            elk = Elk(proxy=self.cloud.mediator)
            elk.filter_error_warning_in_last_seconds(seconds=time.time() - start_time)
            self.cloud.pod.r_collect_info(regex='error', comment=test_regex)
        print self.table

    @staticmethod
    def init_cloud(pod_name, is_debug, test_paths):
        from lab.deployers.deployer_existing import DeployerExisting

        if len(test_paths) == 1 and test_paths[0] == 'dev01-test-parallel.yaml':  # special test case to test infrastructure itself, does not require cloud
            return FakeCloud()

        if not is_debug:
            deployer = DeployerExisting(lab_name=pod_name)
            cloud = deployer.execute({'clouds': [], 'servers': []})
            if len(cloud.computes) < 2:
                raise RuntimeError('{}: not possible to run on this cloud, number of compute hosts less then 2'.format(cloud))
            return cloud
        else:
            return FakeCloud()  # debug mode just to show the time sequence of parrallel workers and there interactions

    def create_tests(self, test_paths, is_noclean, is_debug):
        from lab.test_case import TestCase

        unique_id_seen = []
        test_cases = []
        for test_path in test_paths:
            test = TestCase(path=test_path, is_noclean=is_noclean, is_debug=is_debug, cloud=self.cloud)
            assert test.unique_id not in unique_id_seen
            unique_id_seen.append(test.unique_id)
            test_cases.append(test)

        return test_cases
<commit_msg>ha: don't run teardown is is_debug == True<commit_after>from lab.with_config import WithConfig
from lab.with_log import WithLogMixIn
from lab.laboratory import Laboratory

class FakeCloud(object):
    pod = Laboratory()

def starter(worker):
    return worker.start_worker_parallel()

class RunnerHA(WithConfig, WithLogMixIn):
    def __init__(self):
        from prettytable import PrettyTable

        self.table = PrettyTable()
        self.table.field_names = ['path', 'time', 'status', 'result', 'tims']
        self.cloud = None

    def __repr__(self):
        return u'RunnerHA'

    def execute_single_test(self, test_case):
        import multiprocessing
        import fabric.network
        import time

        manager = multiprocessing.Manager()
        status_dict = manager.dict()

        workers = test_case.workers
        for worker in workers:
            worker.status_dict = status_dict
            worker.set_status(worker.STATUS_CREATED)
            if not test_case.is_debug:
                worker.setup_worker()  # run all setup_worker in non-parallel
            else:
                worker.log('Setup...')

        fabric.network.disconnect_all()  # we do that since URL: http://stackoverflow.com/questions/29480850/paramiko-hangs-at-get-channel-while-using-multiprocessing
        time.sleep(2)

        pool = multiprocessing.Pool(len(workers))
        self.log('\n\n***AFTER THIS LINE PARALLEL EXECUTION STARTS***\n\n')
        results = pool.map(starter, workers)
        self.log('\n\n***PARALLEL EXECUTION FINISHED***\n\n')

        test_case.after_run(results=results)
        self.table.add_row([test_case.path, test_case.time, test_case.tcr.status, test_case.tcr.text, test_case.tcr.tims_url])

        if not test_case.is_debug:
            map(lambda x: x.teardown_worker(), workers)  # run all teardown_workers

    def run(self, pod_name, test_regex, is_noclean, is_debug):
        import time
        from lab.elk import Elk

        available_tc = self.ls_configs(directory='ha')
        test_paths = sorted(filter(lambda x: test_regex in x, available_tc))

        if not test_paths:
            raise ValueError('Provided regexp "{}" does not match any tests'.format(test_regex))

        self.cloud = self.init_cloud(pod_name=pod_name, is_debug=is_debug, test_paths=test_paths)

        tests = self.create_tests(test_paths=test_paths, is_noclean=is_noclean, is_debug=is_debug)

        start_time = time.time()
        map(lambda x: self.execute_single_test(test_case=x), tests)

        with self.open_artifact('main_results.txt', 'w') as f:
            f.write(self.table.get_string())

        if type(self.cloud) is not FakeCloud:
            elk = Elk(proxy=self.cloud.mediator)
            elk.filter_error_warning_in_last_seconds(seconds=time.time() - start_time)
            self.cloud.pod.r_collect_info(regex='error', comment=test_regex)
        print self.table

    @staticmethod
    def init_cloud(pod_name, is_debug, test_paths):
        from lab.deployers.deployer_existing import DeployerExisting

        if len(test_paths) == 1 and test_paths[0] == 'dev01-test-parallel.yaml':  # special test case to test infrastructure itself, does not require cloud
            return FakeCloud()

        if not is_debug:
            deployer = DeployerExisting(lab_name=pod_name)
            cloud = deployer.execute({'clouds': [], 'servers': []})
            if len(cloud.computes) < 2:
                raise RuntimeError('{}: not possible to run on this cloud, number of compute hosts less then 2'.format(cloud))
            return cloud
        else:
            return FakeCloud()  # debug mode just to show the time sequence of parrallel workers and there interactions

    def create_tests(self, test_paths, is_noclean, is_debug):
        from lab.test_case import TestCase

        unique_id_seen = []
        test_cases = []
        for test_path in test_paths:
            test = TestCase(path=test_path, is_noclean=is_noclean, is_debug=is_debug, cloud=self.cloud)
            assert test.unique_id not in unique_id_seen
            unique_id_seen.append(test.unique_id)
            test_cases.append(test)

        return test_cases
<|endoftext|>


<commit_before># HTTP-Server
serverPort = 8000

# PRINTER
printerIp = '172.22.27.26'
printerPort = 9100<commit_msg>added timeout<commit_after># HTTP-Server
SERVER_PORT = 8000
SERVER_DEFAULT_TEMPLATE = '/choose'

# PRINTER
PRINTER_TIMEOUT = 10  # in seconds
PRINTER_HOST = '172.22.27.26'
PRINTER_PORT = 9100
<|endoftext|>


<commit_before>import os
import logging

# HTTP-Server
SERVER_ADDRESS = '::'
SERVER_PORT = 8000
SERVER_DEFAULT_TEMPLATE = '/choose'

# PRINTER
PRINTER_TIMEOUT = 10  # in seconds
PRINTER_HOST = '172.22.26.67'
PRINTER_PORT = 9100

# error logging
SENTRY_DSN = None
DEBUG_LOG_LEVEL = logging.ERROR

# try to overwrite default vars with the local config file
try:
    from labelprinterServeConf_local import *
except ImportError:
    pass

# loop over all local vars and overwrite with found environ vars
for name in list(vars().keys()):
    if name.isupper() and name in os.environ:
        try:
            locals()[name] = int(os.environ[name])
        except ValueError:
            locals()[name] = os.environ[name]

# get SENTRY_DSN from a secret (if it exists)
if os.path.exists("/run/secrets/SENTRY_DSN"):
    SENTRY_DSN = open("/run/secrets/SENTRY_DSN").read().strip()

logging.basicConfig(level=DEBUG_LOG_LEVEL)
<commit_msg>Update default printer IP<commit_after>import os
import logging

# HTTP-Server
SERVER_ADDRESS = '::'
SERVER_PORT = 8000
SERVER_DEFAULT_TEMPLATE = '/choose'

# PRINTER
PRINTER_TIMEOUT = 10  # in seconds
PRINTER_HOST = '10.1.255.228'
PRINTER_PORT = 9100

# error logging
SENTRY_DSN = None
DEBUG_LOG_LEVEL = logging.ERROR

# try to overwrite default vars with the local config file
try:
    from labelprinterServeConf_local import *
except ImportError:
    pass

# loop over all local vars and overwrite with found environ vars
for name in list(vars().keys()):
    if name.isupper() and name in os.environ:
        try:
            locals()[name] = int(os.environ[name])
        except ValueError:
            locals()[name] = os.environ[name]

# get SENTRY_DSN from a secret (if it exists)
if os.path.exists("/run/secrets/SENTRY_DSN"):
    SENTRY_DSN = open("/run/secrets/SENTRY_DSN").read().strip()

logging.basicConfig(level=DEBUG_LOG_LEVEL)
<|endoftext|>


<commit_before>"""
Provide the head of a (potentially gzipped) file in S3. Stream to limit
disk and RAM pressure.

Lambda functions can have up to 3GB of RAM and only 512MB of disk.
"""
import io
import json
from urllib.parse import urlparse
import zlib

import requests

from t4_lambda_shared.decorator import api, validate
from t4_lambda_shared.utils import get_default_origins, make_json_response

# Number of bytes for read routines like decompress() and
# response.content.iter_content()
CHUNK = 1024*8
# MAX_BYTES is bytes scanned, so functions as an upper bound on bytes returned
# in practice we will hit MAX_LINES first
# we need a largish number for things like VCF where we will discard many bytes
# Only applied to _from_stream() types. _to_memory types are size limited either
# by pandas or by exclude_output='true'
MAX_BYTES = 1024*1024
MAX_LINES = 512 # must be positive int
MIN_VCF_COLS = 8 # per 4.2 spec on header and data lines

S3_DOMAIN_SUFFIX = '.s3.amazonaws.com'

FILE_EXTENSIONS = ["csv", "excel", "ipynb", "parquet", "vcf"]
# BED https://genome.ucsc.edu/FAQ/FAQformat.html#format1
TEXT_TYPES = ["bed", "txt"]
FILE_EXTENSIONS.extend(TEXT_TYPES)

SCHEMA = {
    'type': 'object',
    'properties': {
        'url': {
            'type': 'string'
        },
        # separator for CSV files
        'sep': {
            'minLength': 1,
            'maxLength': 1
        },
        'max_bytes' : {
            'type': 'string',
        },
        # line_count used to be an integer with a max and min, which is more correct
        # nevertheless, request.args has it as a string, even if
        # the request specifies it as an integer
        'line_count' : {
            'type': 'string',
        },
        'input': {
            'enum': FILE_EXTENSIONS
        },
        'exclude_output': {
            'enum': ['true', 'false']
        },
        'compression': {
            'enum': ['gz']
        }
    },
    'required': ['url', 'input'],
    'additionalProperties': False
}

@api(cors_origins=get_default_origins())
@validate(SCHEMA)
def lambda_handler(request):
    """
    dynamically handle preview requests for bytes in S3
    caller must specify input_type (since there may be no file extension)

    Returns:
        JSON response
    """
    url = request.args['url']
    input_type = request.args.get('input')
    compression = request.args.get('compression')
    separator = request.args.get('sep') or ','
    exclude_output = request.args.get('exclude_output') == 'true'
    max_bytes = request.args.get('max_bytes', MAX_BYTES)

    parsed_url = urlparse(url, allow_fragments=False)
    if not (parsed_url.scheme == 'https' and
            parsed_url.netloc.endswith(S3_DOMAIN_SUFFIX) and
            parsed_url.username is None and
            parsed_url.password is None):
        return make_json_response(400, {
            'title': 'Invalid url=. Expected S3 virtual-host URL.'
        })

    try:
        line_count = _str_to_line_count(request.args.get('line_count', str(MAX_LINES)))
    except ValueError as error:
        # format https://jsonapi.org/format/1.1/#error-objects
        return make_json_response(400, {
            'title': f'Unexpected line_count= value',
            'detail': str(error)
        })

    # stream=True saves memory almost equal to file size
    resp = requests.get(url, stream=True)
    if resp.ok:
        if input_type == 'csv':
            html, info = extract_csv(
                _from_stream(resp, compression, line_count, max_bytes),
                separator
            )
        elif input_type == 'excel':
            html, info = extract_excel(_to_memory(resp, compression))
        elif input_type == 'ipynb':
            html, info = extract_ipynb(_to_memory(resp, compression), exclude_output)
        elif input_type == 'parquet':
            html, info = extract_parquet(_to_memory(resp, compression))
        elif input_type == 'vcf':
            html, info = extract_vcf(_from_stream(resp, compression, line_count, max_bytes))
        elif input_type in TEXT_TYPES:
            html, info = extract_txt(_from_stream(resp, compression, line_count, max_bytes))
        else:
            assert False, f'unexpected input_type: {input_type}'

        assert isinstance(html, str), 'expected html parameter as string'
        assert isinstance(info, dict), 'expected info metadata to be a dict'

        ret_val = {
            'info': info,
            'html': html,
        }

    else:
        ret_val = {
            'error': resp.reason
        }

    return make_json_response(200, ret_val)

def extract_csv(head, separator):
    """
    csv file => data frame => html
    Args:
        file_ - file-like object opened in binary mode, pointing to .csv
    Returns:
        html - html version of *first sheet only* in workbook
        info - metadata
    """
    # doing this locally because it might be slow
    import pandas
    import re
    # this shouldn't balloon memory because head is limited in size by _from_stream
    try:
        data = pandas.read_csv(
            io.StringIO('\n'.join(head)),
            sep=separator
        )
    # ParserError happens when TSVs are labeled CSVs and/or columns have mixed types
    except pandas.errors.ParserError:
        data = pandas.read_csv(
            io.StringIO('\n'.join(head)),
            # this slower (doesn't use C) but deduces the separator
            sep=None
        )

    html = data._repr_html_() # pylint: disable=protected-access
    html = re.sub(
        r'(
\n
)\d+ rows × \d+ columns(
\n
)$',
        r'\1\2',
        html
    )
    return html, {
        'note': (
            'Object truncated for preview. Row count may be smaller than expected. '
            'S3 data remain intact, full length.'
        )
    }

def extract_excel(file_):
    """
    excel file => data frame => html
    Args:
        file_ - file-like object opened in binary mode, pointing to XLS or XLSX
    Returns:
        html - html version of *first sheet only* in workbook
        info - metadata
    """
    # doing this locally because it might be slow
    import pandas

    first_sheet = pandas.read_excel(file_, sheet_name=0)
    html = first_sheet._repr_html_() # pylint: disable=protected-access
    return html, {}

def extract_ipynb(file_, exclude_output):
    """
    parse and extract ipynb files

    Args:
        file_ - file-like object opened in binary mode (+b)

    Returns:
        html - html version of notebook
        info - unmodified (is also passed in)
    """
    # local import reduces amortized latency, saves memory
    from nbconvert import HTMLExporter
    import nbformat

    html_exporter = HTMLExporter()
    html_exporter.template_file = 'basic'
    html_exporter.exclude_output = exclude_output

    notebook = nbformat.read(file_, 4)
    html, _ = html_exporter.from_notebook_node(notebook)

    return html, {}

def extract_parquet(file_):
    """
    parse and extract key metadata from parquet files

    Args:
        file_ - file-like object opened in binary mode (+b)

    Returns:
        dict
            html - html summary of main contents (if applicable)
            info - metdata for user consumption
    """
    # TODO: generalize to datasets, multipart files
    # As written, only works for single files, and metadata
    # is slanted towards the first row_group

    # local import reduces amortized latency, saves memory
    import pyarrow.parquet as pq

    meta = pq.read_metadata(file_)

    info = {}
    info['created_by'] = meta.created_by
    info['format_version'] = meta.format_version
    info['metadata'] = {
        # seems silly but sets up a simple json.dumps(info) below
        k.decode():json.loads(meta.metadata[k])
        for k in meta.metadata
    } if meta.metadata is not None else {}
    info['num_row_groups'] = meta.num_row_groups
    info['schema'] = {
        name: {
            'logical_type': meta.schema.column(i).logical_type,
            'max_definition_level': meta.schema.column(i).max_definition_level,
            'max_repetition_level': meta.schema.column(i).max_repetition_level,
            'path': meta.schema.column(i).path,
            'physical_type': meta.schema.column(i).physical_type,
        }
        for i, name in enumerate(meta.schema.names)
    }
    info['serialized_size'] = meta.serialized_size
    info['shape'] = [meta.num_rows, meta.num_columns]

    file_.seek(0)
    # TODO: make this faster with n_threads > 1?
    row_group = pq.ParquetFile(file_).read_row_group(0)
    # convert to str since FileMetaData is not JSON.dumps'able (below)
    html = row_group.to_pandas()._repr_html_() # pylint: disable=protected-access

    return html, info

def extract_vcf(head):
    """
    Pull summary info from VCF: meta-information, header line, and data lines
    VCF file format: https://github.com/samtools/hts-specs/blob/master/VCFv4.3.pdf

    Args:
        array of first few lines of file
    Returns:
        dict
    """
    meta = []
    header = None
    data = []
    variants = []
    limit = MIN_VCF_COLS + 1 # +1 to get the FORMAT column
    for line in head:
        if line.startswith('##'):
            meta.append(line)
        elif line.startswith('#'):
            if header:
                print('Unexpected multiple headers:', header)
            header = line
            columns = header.split() # VCF is tab-delimited
            # only grab first "limit"-many rows
            header = columns[:limit]
            variants = columns[limit:]
        elif line:
            columns = line.split()[:limit]
            data.append(columns)
    info = {
        'data': {
            'meta': meta,
            'header': header,
            'data': data
        },
        'metadata': {
            'variants': variants,
            'variant_count': len(variants)
        }
    }

    return '', info

def extract_txt(head):
    """
    Display first N lines of a potentially large file.

    Args:
        file_ - file-like object opened in binary mode (+b)
    Returns:
        dict - head and tail. tail may be empty. returns at most MAX_LINES
        lines that occupy a total of MAX_BYTES bytes.
    """
    info = {
        'data': {
            'head': head,
            # retain tail for backwards compatibility with client
            'tail': []
        }
    }

    return '', info

def _from_stream(response, compression, max_lines, max_bytes):
    """
    for files we can read in a streaming manner
    """
    buffer = []
    size = 0
    line_count = 0

    if compression:
        assert compression == 'gz', 'only gzip compression is supported'
        dec = zlib.decompressobj(zlib.MAX_WBITS + 32)
    else:
        dec = None

    for chunk in response.iter_content(chunk_size=CHUNK):
        if dec:
            chunk = dec.decompress(chunk)

        buffer.append(chunk)
        size += len(chunk)
        line_count += chunk.count(b'\n')

        # TODO why are we hitting dec.eof after ~65kb of large gz files?
        # not >= since we might get lucky and complete a line if we wait
        if size > max_bytes or line_count > max_lines or (dec and dec.eof):
            break

    lines = b''.join(buffer).splitlines()

    # If we stopped because of max_bytes, then drop the last, possibly incomplete line -
    # as long as we have more than one line.
    if size > max_bytes and len(lines) > 1:
        lines.pop()

    # Drop any lines over the max.
    del lines[max_lines:]

    # We may still be over max_bytes at this point, up to max_bytes + CHUNK,
    # but we don't really care.

    return [l.decode('utf-8', 'ignore') for l in lines]

def _str_to_line_count(int_string, lower=1, upper=MAX_LINES):
    """
    validates an integer string

    Raises: ValueError
    """
    integer = int(int_string)
    if integer < lower or integer > upper:
        raise ValueError(f'{integer} out of range: [{lower}, {upper}]')

    return integer

def _to_memory(response, compression):
    """
    for file-types where we don't support streaming read;
    drop the entire file into memory
    """
    if compression:
        assert compression == 'gz', 'only gzip compression is supported'
        # +32 => automatically accepts zlib or gzip
        # https://docs.python.org/2/library/zlib.html#zlib.decompress
        return io.BytesIO(zlib.decompress(response.content, zlib.MAX_WBITS + 32))
    return io.BytesIO(response.content)
<commit_msg>Relax S3 url syntax (#1066)<commit_after>"""
Provide the head of a (potentially gzipped) file in S3. Stream to limit
disk and RAM pressure.

Lambda functions can have up to 3GB of RAM and only 512MB of disk.
"""
import io
import json
from urllib.parse import urlparse
import zlib

import requests

from t4_lambda_shared.decorator import api, validate
from t4_lambda_shared.utils import get_default_origins, make_json_response

# Number of bytes for read routines like decompress() and
# response.content.iter_content()
CHUNK = 1024*8
# MAX_BYTES is bytes scanned, so functions as an upper bound on bytes returned
# in practice we will hit MAX_LINES first
# we need a largish number for things like VCF where we will discard many bytes
# Only applied to _from_stream() types. _to_memory types are size limited either
# by pandas or by exclude_output='true'
MAX_BYTES = 1024*1024
MAX_LINES = 512 # must be positive int
MIN_VCF_COLS = 8 # per 4.2 spec on header and data lines

S3_DOMAIN_SUFFIX = '.amazonaws.com'

FILE_EXTENSIONS = ["csv", "excel", "ipynb", "parquet", "vcf"]
# BED https://genome.ucsc.edu/FAQ/FAQformat.html#format1
TEXT_TYPES = ["bed", "txt"]
FILE_EXTENSIONS.extend(TEXT_TYPES)

SCHEMA = {
    'type': 'object',
    'properties': {
        'url': {
            'type': 'string'
        },
        # separator for CSV files
        'sep': {
            'minLength': 1,
            'maxLength': 1
        },
        'max_bytes' : {
            'type': 'string',
        },
        # line_count used to be an integer with a max and min, which is more correct
        # nevertheless, request.args has it as a string, even if
        # the request specifies it as an integer
        'line_count' : {
            'type': 'string',
        },
        'input': {
            'enum': FILE_EXTENSIONS
        },
        'exclude_output': {
            'enum': ['true', 'false']
        },
        'compression': {
            'enum': ['gz']
        }
    },
    'required': ['url', 'input'],
    'additionalProperties': False
}

@api(cors_origins=get_default_origins())
@validate(SCHEMA)
def lambda_handler(request):
    """
    dynamically handle preview requests for bytes in S3
    caller must specify input_type (since there may be no file extension)

    Returns:
        JSON response
    """
    url = request.args['url']
    input_type = request.args.get('input')
    compression = request.args.get('compression')
    separator = request.args.get('sep') or ','
    exclude_output = request.args.get('exclude_output') == 'true'
    max_bytes = request.args.get('max_bytes', MAX_BYTES)

    parsed_url = urlparse(url, allow_fragments=False)
    if not (parsed_url.scheme == 'https' and
            parsed_url.netloc.endswith(S3_DOMAIN_SUFFIX) and
            parsed_url.username is None and
            parsed_url.password is None):
        return make_json_response(400, {
            'title': 'Invalid url=. Expected S3 virtual-host URL.'
        })

    try:
        line_count = _str_to_line_count(request.args.get('line_count', str(MAX_LINES)))
    except ValueError as error:
        # format https://jsonapi.org/format/1.1/#error-objects
        return make_json_response(400, {
            'title': f'Unexpected line_count= value',
            'detail': str(error)
        })

    # stream=True saves memory almost equal to file size
    resp = requests.get(url, stream=True)
    if resp.ok:
        if input_type == 'csv':
            html, info = extract_csv(
                _from_stream(resp, compression, line_count, max_bytes),
                separator
            )
        elif input_type == 'excel':
            html, info = extract_excel(_to_memory(resp, compression))
        elif input_type == 'ipynb':
            html, info = extract_ipynb(_to_memory(resp, compression), exclude_output)
        elif input_type == 'parquet':
            html, info = extract_parquet(_to_memory(resp, compression))
        elif input_type == 'vcf':
            html, info = extract_vcf(_from_stream(resp, compression, line_count, max_bytes))
        elif input_type in TEXT_TYPES:
            html, info = extract_txt(_from_stream(resp, compression, line_count, max_bytes))
        else:
            assert False, f'unexpected input_type: {input_type}'

        assert isinstance(html, str), 'expected html parameter as string'
        assert isinstance(info, dict), 'expected info metadata to be a dict'

        ret_val = {
            'info': info,
            'html': html,
        }

    else:
        ret_val = {
            'error': resp.reason
        }

    return make_json_response(200, ret_val)

def extract_csv(head, separator):
    """
    csv file => data frame => html
    Args:
        file_ - file-like object opened in binary mode, pointing to .csv
    Returns:
        html - html version of *first sheet only* in workbook
        info - metadata
    """
    # doing this locally because it might be slow
    import pandas
    import re
    # this shouldn't balloon memory because head is limited in size by _from_stream
    try:
        data = pandas.read_csv(
            io.StringIO('\n'.join(head)),
            sep=separator
        )
    # ParserError happens when TSVs are labeled CSVs and/or columns have mixed types
    except pandas.errors.ParserError:
        data = pandas.read_csv(
            io.StringIO('\n'.join(head)),
            # this slower (doesn't use C) but deduces the separator
            sep=None
        )

    html = data._repr_html_() # pylint: disable=protected-access
    html = re.sub(
        r'(
\n
)\d+ rows × \d+ columns(
\n
)$',
        r'\1\2',
        html
    )
    return html, {
        'note': (
            'Object truncated for preview. Row count may be smaller than expected. '
            'S3 data remain intact, full length.'
        )
    }

def extract_excel(file_):
    """
    excel file => data frame => html
    Args:
        file_ - file-like object opened in binary mode, pointing to XLS or XLSX
    Returns:
        html - html version of *first sheet only* in workbook
        info - metadata
    """
    # doing this locally because it might be slow
    import pandas

    first_sheet = pandas.read_excel(file_, sheet_name=0)
    html = first_sheet._repr_html_() # pylint: disable=protected-access
    return html, {}

def extract_ipynb(file_, exclude_output):
    """
    parse and extract ipynb files

    Args:
        file_ - file-like object opened in binary mode (+b)

    Returns:
        html - html version of notebook
        info - unmodified (is also passed in)
    """
    # local import reduces amortized latency, saves memory
    from nbconvert import HTMLExporter
    import nbformat

    html_exporter = HTMLExporter()
    html_exporter.template_file = 'basic'
    html_exporter.exclude_output = exclude_output

    notebook = nbformat.read(file_, 4)
    html, _ = html_exporter.from_notebook_node(notebook)

    return html, {}

def extract_parquet(file_):
    """
    parse and extract key metadata from parquet files

    Args:
        file_ - file-like object opened in binary mode (+b)

    Returns:
        dict
            html - html summary of main contents (if applicable)
            info - metdata for user consumption
    """
    # TODO: generalize to datasets, multipart files
    # As written, only works for single files, and metadata
    # is slanted towards the first row_group

    # local import reduces amortized latency, saves memory
    import pyarrow.parquet as pq

    meta = pq.read_metadata(file_)

    info = {}
    info['created_by'] = meta.created_by
    info['format_version'] = meta.format_version
    info['metadata'] = {
        # seems silly but sets up a simple json.dumps(info) below
        k.decode():json.loads(meta.metadata[k])
        for k in meta.metadata
    } if meta.metadata is not None else {}
    info['num_row_groups'] = meta.num_row_groups
    info['schema'] = {
        name: {
            'logical_type': meta.schema.column(i).logical_type,
            'max_definition_level': meta.schema.column(i).max_definition_level,
            'max_repetition_level': meta.schema.column(i).max_repetition_level,
            'path': meta.schema.column(i).path,
            'physical_type': meta.schema.column(i).physical_type,
        }
        for i, name in enumerate(meta.schema.names)
    }
    info['serialized_size'] = meta.serialized_size
    info['shape'] = [meta.num_rows, meta.num_columns]

    file_.seek(0)
    # TODO: make this faster with n_threads > 1?
    row_group = pq.ParquetFile(file_).read_row_group(0)
    # convert to str since FileMetaData is not JSON.dumps'able (below)
    html = row_group.to_pandas()._repr_html_() # pylint: disable=protected-access

    return html, info

def extract_vcf(head):
    """
    Pull summary info from VCF: meta-information, header line, and data lines
    VCF file format: https://github.com/samtools/hts-specs/blob/master/VCFv4.3.pdf

    Args:
        array of first few lines of file
    Returns:
        dict
    """
    meta = []
    header = None
    data = []
    variants = []
    limit = MIN_VCF_COLS + 1 # +1 to get the FORMAT column
    for line in head:
        if line.startswith('##'):
            meta.append(line)
        elif line.startswith('#'):
            if header:
                print('Unexpected multiple headers:', header)
            header = line
            columns = header.split() # VCF is tab-delimited
            # only grab first "limit"-many rows
            header = columns[:limit]
            variants = columns[limit:]
        elif line:
            columns = line.split()[:limit]
            data.append(columns)
    info = {
        'data': {
            'meta': meta,
            'header': header,
            'data': data
        },
        'metadata': {
            'variants': variants,
            'variant_count': len(variants)
        }
    }

    return '', info

def extract_txt(head):
    """
    Display first N lines of a potentially large file.

    Args:
        file_ - file-like object opened in binary mode (+b)
    Returns:
        dict - head and tail. tail may be empty. returns at most MAX_LINES
        lines that occupy a total of MAX_BYTES bytes.
    """
    info = {
        'data': {
            'head': head,
            # retain tail for backwards compatibility with client
            'tail': []
        }
    }

    return '', info

def _from_stream(response, compression, max_lines, max_bytes):
    """
    for files we can read in a streaming manner
    """
    buffer = []
    size = 0
    line_count = 0

    if compression:
        assert compression == 'gz', 'only gzip compression is supported'
        dec = zlib.decompressobj(zlib.MAX_WBITS + 32)
    else:
        dec = None

    for chunk in response.iter_content(chunk_size=CHUNK):
        if dec:
            chunk = dec.decompress(chunk)

        buffer.append(chunk)
        size += len(chunk)
        line_count += chunk.count(b'\n')

        # TODO why are we hitting dec.eof after ~65kb of large gz files?
        # not >= since we might get lucky and complete a line if we wait
        if size > max_bytes or line_count > max_lines or (dec and dec.eof):
            break

    lines = b''.join(buffer).splitlines()

    # If we stopped because of max_bytes, then drop the last, possibly incomplete line -
    # as long as we have more than one line.
    if size > max_bytes and len(lines) > 1:
        lines.pop()

    # Drop any lines over the max.
    del lines[max_lines:]

    # We may still be over max_bytes at this point, up to max_bytes + CHUNK,
    # but we don't really care.

    return [l.decode('utf-8', 'ignore') for l in lines]

def _str_to_line_count(int_string, lower=1, upper=MAX_LINES):
    """
    validates an integer string

    Raises: ValueError
    """
    integer = int(int_string)
    if integer < lower or integer > upper:
        raise ValueError(f'{integer} out of range: [{lower}, {upper}]')

    return integer

def _to_memory(response, compression):
    """
    for file-types where we don't support streaming read;
    drop the entire file into memory
    """
    if compression:
        assert compression == 'gz', 'only gzip compression is supported'
        # +32 => automatically accepts zlib or gzip
        # https://docs.python.org/2/library/zlib.html#zlib.decompress
        return io.BytesIO(zlib.decompress(response.content, zlib.MAX_WBITS + 32))
    return io.BytesIO(response.content)
<|endoftext|>