<commit_before>from Axon.Component import component
from Axon.Ipc import producerFinished, shutdownMicroprocess, shutdown
from Kamaelia.Internet.TCPClient import TCPClient
import string, time
from Lagger import Lagger

from HTTPParser import HTTPParser, splitUri
def intval(mystring):
    try:
        retval = int(mystring)
    except ValueError:
        retval = None
    return retval

def removeTrailingCr(line):
    if len(line) == 0:
        return ""
    elif line[-1] == "\r":
        return line[0:-1]
    else:
        return line

class HTTPClient(component):
    Inboxes =  { "inbox"          : "URL request",
                 "tcp-outbox"     : "TCP connection/outbox",
                 "tcp-signal"     : "TCP connection/signal",
                 "parser-outbox"  : "HTTP parser/outbox",
                 "parser-signal"  : "HTTP parser/signal",
                 "control"        : "UNUSED" }

    Outboxes = { "outbox"         : "Requested file",
                 "tcp-inbox"      : "TCP connection/inbox",
                 "tcp-control"    : "TCP connection/control",
                 "parser-inbox"   : "HTTP parser/inbox",
                 "parser-control" : "HTTP parser/control",
                 "signal"         : "UNUSED" }

    def __init__(self):
        super(HTTPClient, self).__init__()
        self.readbuffer = ""
        self.tcpclient = None
        self.httpparser = None
        self.lines = ""
        self.requestqueue = []

    def formRequest(self, url):
        splituri = splitUri(url)

        host = splituri["uri-server"]
        if splituri.has_key("uri-port"):
            host += ":" + splituri["uri-port"]

        splituri["request"] = "GET " + splituri["raw-uri"] + " HTTP/1.1\r\nHost: " + host + "\r\nUser-agent: Kamaelia HTTP Client 0.1 (RJL)\r\nConnection: Keep-Alive\r\n\r\n" #keep-alive is a work around for lack of shutdown notification in TCPClient
        return splituri

    def makeRequest(self, request):
        self.tcpclient = None
        self.httpparser = None
        port = intval(request.get("uri-port", ""))
        if port == None:
            port = 80

        #print "Connecting to <" + request["uri-server"] + ">:" + str(port)

        self.tcpclient = TCPClient(request["uri-server"], port)
        self.httpparser = HTTPParser(mode="response")

        self.link( (self, "tcp-inbox"),        (self.tcpclient, "inbox") )
        self.link( (self, "tcp-control"),      (self.tcpclient, "control") )
        self.link( (self.tcpclient, "outbox"), (self, "tcp-outbox") )
        self.link( (self.tcpclient, "signal"), (self, "tcp-signal") )

        self.link( (self, "parser-inbox"),   (self.httpparser, "inbox") )
        self.link( (self, "parser-control"), (self.httpparser, "control") )
        self.link( (self.httpparser, "outbox"), (self, "parser-outbox") )
        self.link( (self.httpparser, "signal"), (self, "parser-signal") )

        self.addChildren( self.tcpclient, self.httpparser )
        self.tcpclient.activate()
        self.httpparser.activate()
        self.send(request["request"], "tcp-inbox")

    def shutdownKids(self):
        if self.tcpclient != None and self.httpparser != None:
            self.removeChild(self.tcpclient)
            self.removeChild(self.httpparser)
            self.send(shutdown(), "tcp-control")
            self.send(shutdown(), "parser-control")
            self.tcpclient = None
            self.httpparser = None

    def main(self):
        waitforrequests = True
        while 1:
            yield 1
            #print "HTTPClient::main"
            while self.dataReady("inbox"):
                url = self.recv("inbox")
                self.requestqueue.append(self.formRequest(url))

            while self.dataReady("parser-outbox"):
                response = self.recv("parser-outbox")
                #print "Parsed"
                self.send(response["body"], "outbox")
                self.shutdownKids()

            while self.dataReady("parser-signal"):
                temp = self.recv("parser-signal")
                print "parser-signal"
                print type(temp)

            while self.dataReady("tcp-outbox"):
                temp = self.recv("tcp-outbox")
                self.send(temp, "parser-inbox")
                #print "Received: " + temp

            while self.dataReady("tcp-signal"):
                temp = self.recv("tcp-signal")
                self.send(temp, "parser-control")
                print "tcp-signal"

                print type(temp)

            while self.dataReady("control"):
                temp = self.recv("control")
                if isinstance(temp, producerFinished):
                    waitforrequests = False
                elif isinstance(temp, shutdownMicroprocess) or isinstance(temp, shutdown):
                    self.shutdownKids()
                    return

            if self.tcpclient == None:
                if len(self.requestqueue) > 0:
                    self.makeRequest(self.requestqueue.pop(0))
                elif waitforrequests == False:
                    return #producer has finished so quit
            self.pause()

if __name__ == '__main__':
    from Kamaelia.Util.PipelineComponent import pipeline
    from Kamaelia.Util.Console import ConsoleReader, ConsoleEchoer
    from Kamaelia.File.Writing import SimpleFileWriter

    # type in a URL e.g. http://www.google.co.uk and have it saved to disk
    pipeline(
        ConsoleReader(">>> ", ""),
        HTTPClient(),
        SimpleFileWriter("downloadedfile.txt"),
    ).run()
<commit_msg>Major restructuring - now uses Carousel.<commit_after>#!/usr/bin/env python
#
# FIXME: Uses the selector service, but has no way of indicating to the
#        selector service that its services are no longer required.
#        This needs resolving.
#
# (C) 2004 British Broadcasting Corporation and Kamaelia Contributors(1)
#     All Rights Reserved.
#
# You may only modify and redistribute this under the terms of any of the
# following licenses(2): Mozilla Public License, V1.1, GNU General
# Public License, V2.0, GNU Lesser General Public License, V2.1
#
# (1) Kamaelia Contributors are listed in the AUTHORS file and at
#     http://kamaelia.sourceforge.net/AUTHORS - please extend this file,
#     not this notice.
# (2) Reproduced in the COPYING file, and at:
#     http://kamaelia.sourceforge.net/COPYING
# Under section 3.5 of the MPL, we are using this text since we deem the MPL
# notice inappropriate for this file. As per MPL/GPL/LGPL removal of this
# notice is prohibited.
#
# Please contact us via: kamaelia-list-owner@lists.sourceforge.net
# to discuss alternative licensing.
# -------------------------------------------------------------------------
"""\
=================
SingleShotHTTPClient
=================

This component is for downloading a file from an HTTP server.
Send to its "inbox"
inbox to send data to the server. Pick up data received from the server on its
"outbox" outbox.

Example Usage
-------------
TO DO

How does it work?
-----------------
Barely
"""

from Axon.Component import component
from Kamaelia.Chassis.Carousel import Carousel
from Axon.Ipc import producerFinished, shutdownMicroprocess, shutdown
from Kamaelia.Internet.TCPClient import TCPClient
import string, time

from HTTPParser import *

def intval(mystring):
    try:
        retval = int(mystring)
    except ValueError:
        retval = None
    return retval

def removeTrailingCr(line):
    if len(line) == 0:
        return ""
    elif line[-1] == "\r":
        return line[0:-1]
    else:
        return line

class HTTPRequest(object):
    def __init__(self, requestobject, redirectcount):
        super(HTTPRequest, self).__init__()
        self.requestobject = requestobject
        self.redirectcount = redirectcount

class SingleShotHTTPClient(component):
    """\
    SingleShotHTTPClient() -> component that can download a file using HTTP by URL

    Arguments:
    - url     -- the URL of the file to download
    """

    Inboxes =  {
        "inbox"          : "UNUSED",
        "control"        : "UNUSED",

        "_parserinbox"   : "Data from HTTP parser",
        "_parsercontrol" : "Signals from HTTP parser",
        "_tcpcontrol"    : "Signals from TCP client",
    }

    Outboxes = {
        "outbox"         : "Requested file",

        "_parsersignal"  : "Signals for HTTP parser",

        "_tcpoutbox"     : "Send over TCP connection",
        "_tcpsignal"     : "Signals shutdown of TCP connection",

        "signal"         : "UNUSED"
    }

    def __init__(self, starturl):
        print "SingleShotHTTPClient.__init__()"
        super(SingleShotHTTPClient, self).__init__()
        self.tcpclient = None
        self.httpparser = None
        self.requestqueue = []
        self.starturl = starturl
        print "Start url: " + starturl

    def formRequest(self, url):
        """Craft a HTTP request string for the supplied url"""
        splituri = splitUri(url)

        host = splituri["uri-server"]
        if splituri.has_key("uri-port"):
            host += ":" + splituri["uri-port"]

        splituri["request"] = "GET " + splituri["raw-uri"] + " HTTP/1.1\r\nHost: " + host + "\r\nUser-agent: Kamaelia HTTP Client 0.3 (RJL)\r\nConnection: Keep-Alive\r\n\r\n" #keep-alive is a work around for lack of shutdown notification in TCPClient
        return splituri

    def makeRequest(self, request):
        """Connect to the remote HTTP server and send request"""
        self.tcpclient = None
        self.httpparser = None
        port = intval(request.requestobject.get("uri-port", ""))
        if port == None:
            port = 80

        self.tcpclient = TCPClient(request.requestobject["uri-server"], port)
        self.httpparser = HTTPParser(mode="response")

        self.link( (self, "_tcpoutbox"),       (self.tcpclient, "inbox") )
        self.link( (self, "_tcpsignal"),       (self.tcpclient, "control") )
        self.link( (self.tcpclient, "signal"), (self, "_tcpcontrol") )

        self.link( (self.tcpclient, "outbox"), (self.httpparser, "inbox") ) #incoming TCP data -> HTTPParser directly

        self.link( (self, "_parsersignal"), (self.httpparser, "control") )
        self.link( (self.httpparser, "outbox"), (self, "_parserinbox") )
        self.link( (self.httpparser, "signal"), (self, "_parsercontrol") )

        self.addChildren( self.tcpclient, self.httpparser )
        self.tcpclient.activate()
        self.httpparser.activate()
        self.response = ""
        self.send(request.requestobject["request"], "_tcpoutbox")

    def shutdownKids(self):
        """Close TCP connection and HTTP parser"""
        if self.tcpclient != None and self.httpparser != None:
            self.removeChild(self.tcpclient)
            self.removeChild(self.httpparser)
            self.send(shutdown(), "_tcpsignal")
            self.send(shutdown(), "_parsersignal")
            self.tcpclient = None
            self.httpparser = None

    def handleRedirect(self, header):
        if header["responsecode"] == "302" or header["responsecode"] == "303" or header["responsecode"] == "307":
            # location header gives the redirect URL
            newurl = header["headers"].get("location", "")
            if newurl != "":
                redirectedrequest = HTTPRequest(self.formRequest(newurl), self.currentrequest.redirectcount + 1)
                self.requestqueue.append(redirectedrequest)
                return True
            else:
                return False
                # do something equivalent to what we'd do for 404
        else:
            return False

    def main(self):
        self.requestqueue.append(HTTPRequest(self.formRequest(self.starturl), 0))
        while self.mainBody():
            yield 1
        self.send(producerFinished(), "signal")
        yield 1
        return

    def mainBody(self):
        print "SingleShotHTTPClient.mainBody()"
        while self.dataReady("_parserinbox"):
            msg = self.recv("_parserinbox")
            print "SingleShotHTTPClient received on _parserinbox"
            print msg

            if isinstance(msg, ParsedHTTPHeader):
                # if the page is a redirect page
                if not self.handleRedirect(msg.header):
                    if msg.header["responsecode"] == "200":
                        self.send(msg, "outbox") # if not redirecting then send the response on
                    else:  #treat as not found
                        pass

            elif isinstance(msg, ParsedHTTPBodyChunk):
                if len(self.requestqueue) == 0: # if not redirecting then send the response on
                    self.send(msg, "outbox")

            elif isinstance(msg, ParsedHTTPEnd):
                if len(self.requestqueue) == 0: # if not redirecting then send the response on
                    self.send(msg, "outbox")
                self.shutdownKids()

        while self.dataReady("_parsercontrol"):
            temp = self.recv("_parsercontrol")
            print "parser-signal"
            print type(temp)

        while self.dataReady("_tcpcontrol"):
            msg = self.recv("_tcpcontrol")
            self.send(msg, "_parsersignal")

        while self.dataReady("control"):
            msg = self.recv("control")
            if isinstance(msg, shutdownMicroprocess) or isinstance(msg, shutdown):
                self.shutdownKids()
                return 0

        # if we're not currently downloading a page
        if self.tcpclient == None:
            # then either we've finished or we should download the next URL (if we've been redirected)
            if len(self.requestqueue) > 0:
                self.currentrequest = self.requestqueue.pop(0)
                if self.currentrequest.redirectcount == 3: # 3 redirects is excessive, give up, we're probably in a loop anyway
                    return 0
                else:
                    self.makeRequest(self.currentrequest)
            else:
                return 0

        self.pause()
        return 1

def makeSSHTTPClient(url):
    return SingleShotHTTPClient(url)

class SimpleHTTPClient(component):
    Inboxes = {
        "inbox"           : "URLs to download",
        "control"         : "Shut me down",
        "_carouselready"  : "Receive NEXT when carousel has completed a request",
        "_carouselinbox"  : "Data from SingleShotHTTPClient via Carousel"
    }
    Outboxes = {
        "outbox"          : "Requested file's data string",
        "signal"          : "Signal I have shutdown",
        "_carouselnext"   : "Create a new SingleShotHTTPClient",
        "_carouselsignal" : "Shutdown the carousel"
    }

    def __init__(self):
        print "SimpleHTTPClient.__init__()"
        super(SimpleHTTPClient, self).__init__()

        self.carousel = Carousel(componentFactory=makeSSHTTPClient)
        self.addChildren(self.carousel)
        self.link((self, "_carouselnext"),        (self.carousel, "next"))
        self.link((self, "_carouselsignal"),      (self.carousel, "control"))
        self.link((self.carousel, "outbox"),      (self, "_carouselinbox"))
        self.link((self.carousel, "requestNext"), (self, "_carouselready"))
        self.carousel.activate()

    def cleanup(self):
        print "SimpleHTTPClient.cleanup()"
        self.send(shutdown(), "_carouselsignal")
        self.removeChild(self.carousel)
        self.send(producerFinished(), "signal")

    def main(self):
        print "SimpleHTTPClient.main()"
        finished = False
        while not finished:
            yield 1
            while self.dataReady("inbox"):
                url = self.recv("inbox")
                print "SimpleHTTPClient received url " + url
                self.send(url, "_carouselnext")

                filebody = ""
                carouselbusy = True

                while carouselbusy:
                    yield 1
                    while self.dataReady("_carouselinbox"):
                        msg = self.recv("_carouselinbox")
                        if isinstance(msg, ParsedHTTPBodyChunk):
                            filebody += msg.bodychunk

                    while self.dataReady("control"):
                        msg = self.recv("control")
                        if isinstance(msg, producerFinished):
                            producerfinished = True
                        elif isinstance(msg, shutdown):
                            self.cleanup()

                    while self.dataReady("_carouselready"):
                        msg = self.recv("_carouselready")
                        carouselbusy = False

                    self.pause()
                self.send(filebody, "outbox")
                filebody = ""

            self.pause()

        self.cleanup()

if __name__ == '__main__':
    from Kamaelia.Util.PipelineComponent import pipeline
    from Kamaelia.Util.Console import ConsoleReader, ConsoleEchoer
    from Kamaelia.File.Writing import SimpleFileWriter

    # type in a URL e.g. http://www.google.co.uk and have it saved to disk
    pipeline(
        ConsoleReader(">>> ", ""),
        SimpleHTTPClient(),
        SimpleFileWriter("downloadedfile.txt"),
    ).run()
<|endoftext|>


<commit_before>import os
import sys
try:
    import termios
except ImportError:
    # Not available on Windows
    termios = None
from contextlib import contextmanager

from invoke.vendor.six import BytesIO, b, iteritems, wraps

from mock import patch, Mock
from spec import trap, Spec, eq_, ok_, skip

from invoke import Program, Runner
from invoke.platform import WINDOWS

support = os.path.join(os.path.dirname(__file__), '_support')

def skip_if_windows(fn):
    @wraps(fn)
    def wrapper(*args, **kwargs):
        if WINDOWS:
            skip()
        return fn(*args, **kwargs)
    return wrapper

@contextmanager
def support_path():
    sys.path.insert(0, support)
    try:
        yield
    finally:
        sys.path.pop(0)

def load(name):
    with support_path():
        return __import__(name)

class IntegrationSpec(Spec):
    def setup(self):
        # Preserve environment for later restore
        self.old_environ = os.environ.copy()
        # Always do things relative to tests/_support
        os.chdir(support)

    def teardown(self):
        # Chdir back to project root to avoid problems
        os.chdir(os.path.join(os.path.dirname(__file__), '..'))
        # Nuke changes to environ
        os.environ.clear()
        os.environ.update(self.old_environ)
        # Strip any test-support task collections from sys.modules to prevent
        # state bleed between tests; otherwise tests can incorrectly pass
        # despite not explicitly loading/cd'ing to get the tasks they call
        # loaded.
        for name, module in iteritems(sys.modules.copy()):
            if module and support in getattr(module, '__file__', ''):
                del sys.modules[name]

@trap
def expect(invocation, out=None, err=None, program=None, invoke=True,
    test=None):
    """
    Run ``invocation`` via ``program`` and expect resulting output to match.

    May give one or both of ``out``/``err`` (but not neither).

    ``program`` defaults to ``Program()``.

    To skip automatically assuming the argv under test starts with ``"invoke
    "``, say ``invoke=False``.

    To customize the operator used for testing (default: equality), use
    ``test`` (which should be an assertion wrapper of some kind).
    """
    if program is None:
        program = Program()
    if invoke:
        invocation = "invoke {0}".format(invocation)
    program.run(invocation, exit=False)
    # Perform tests
    if out is not None:
        (test or eq_)(sys.stdout.getvalue(), out)
    if err is not None:
        (test or eq_)(sys.stderr.getvalue(), err)

def mock_subprocess(out='', err='', exit=0, isatty=None, insert_Popen=False):
    def decorator(f):
        @wraps(f)
        @patch('invoke.runners.Popen')
        @patch('os.read')
        @patch('sys.stdin', new_callable=BytesIO)
        def wrapper(*args, **kwargs):
            args = list(args)
            Popen, read, sys_stdin = args.pop(), args.pop(), args.pop()
            process = Popen.return_value
            process.returncode = exit
            process.stdout.fileno.return_value = 1
            process.stderr.fileno.return_value = 2
            # If requested, mock isatty to fake out pty detection
            if isatty is not None:
                sys_stdin.isatty = Mock(return_value=isatty)
            out_file = BytesIO(b(out))
            err_file = BytesIO(b(err))
            def fakeread(fileno, count):
                fd = {1: out_file, 2: err_file}[fileno]
                return fd.read(count)
            read.side_effect = fakeread
            if insert_Popen:
                args.append(Popen)
            f(*args, **kwargs)
        return wrapper
    return decorator

def mock_pty(out='', err='', exit=0, isatty=None, trailing_error=None,
    skip_asserts=False, insert_os=False):
    # Windows doesn't have ptys, so all the pty tests should be
    # skipped anyway...
    if WINDOWS:
        return skip_if_windows

    def decorator(f):
        import fcntl
        ioctl_patch = patch('invoke.runners.fcntl.ioctl', wraps=fcntl.ioctl)

        @wraps(f)
        @patch('invoke.runners.pty')
        @patch('invoke.runners.os')
        @ioctl_patch
        def wrapper(*args, **kwargs):
            args = list(args)
            pty, os, ioctl = args.pop(), args.pop(), args.pop()
            # Don't actually fork, but pretend we did & that main thread is
            # also the child (pid 0) to trigger execve call; & give 'parent fd'
            # of 1 (stdout).
            pty.fork.return_value = 0, 1
            # We don't really need to care about waiting since not truly
            # forking/etc, so here we just return a nonzero "pid" + sentinel
            # wait-status value (used in some tests about WIFEXITED etc)
            os.waitpid.return_value = None, Mock(name='exitstatus')
            # Either or both of these may get called, depending...
            os.WEXITSTATUS.return_value = exit
            os.WTERMSIG.return_value = exit
            # If requested, mock isatty to fake out pty detection
            if isatty is not None:
                os.isatty.return_value = isatty
            out_file = BytesIO(b(out))
            err_file = BytesIO(b(err))
            def fakeread(fileno, count):
                fd = {1: out_file, 2: err_file}[fileno]
                ret = fd.read(count)
                # If asked, fake a Linux-platform trailing I/O error.
                if not ret and trailing_error:
                    raise trailing_error
                return ret
            os.read.side_effect = fakeread
            if insert_os:
                args.append(os)
            f(*args, **kwargs)
            # Short-circuit if we raised an error in fakeread()
            if trailing_error:
                return
            # Sanity checks to make sure the stuff we mocked, actually got ran!
            # TODO: inject our mocks back into the tests so they can make their
            # own assertions if desired
            pty.fork.assert_called_with()
            # Expect a get, and then later set, of terminal window size
            eq_(ioctl.call_args_list[0][0][1], termios.TIOCGWINSZ)
            eq_(ioctl.call_args_list[1][0][1], termios.TIOCSWINSZ)
            if not skip_asserts:
                for name in ('execve', 'waitpid'):
                    ok_(getattr(os, name).called)
                # Ensure at least one of the exit status getters was called
                ok_(os.WEXITSTATUS.called or os.WTERMSIG.called)
        return wrapper
    return decorator

class _Dummy(Runner):
    """
    Dummy runner subclass that does minimum work required to execute run().

    It also serves as a convenient basic API checker; failure to update it to
    match the current Runner API will cause TypeErrors, NotImplementedErrors,
    and similar.
    """
    # Neuter the input loop sleep, so tests aren't slow (at the expense of CPU,
    # which isn't a problem for testing).
    input_sleep = 0

    def start(self, command, shell, env):
        pass

    def read_proc_stdout(self, num_bytes):
        return ""

    def read_proc_stderr(self, num_bytes):
        return ""

    def _write_proc_stdin(self, data):
        pass

    @property
    def process_is_finished(self):
        return True

    def returncode(self):
        return 0

    def stop(self):
        pass

# Dummy command that will blow up if it ever truly hits a real shell.
_ = "nope"

# Runner that fakes ^C during subprocess exec
class _KeyboardInterruptingRunner(_Dummy):
    def __init__(self, *args, **kwargs):
        super(_KeyboardInterruptingRunner, self).__init__(*args, **kwargs)
        self._interrupted = False

    # Trigger KeyboardInterrupt during wait()
    def wait(self):
        if not self._interrupted:
            self._interrupted = True
            raise KeyboardInterrupt

    # But also, after that has been done, pretend subprocess shutdown happened
    # (or we will loop forever).
    def process_is_finished(self):
        return self._interrupted

class OhNoz(Exception):
    pass
<commit_msg>Recast mock_subprocess as doctest-friendly mock object<commit_after>import os
import sys
try:
    import termios
except ImportError:
    # Not available on Windows
    termios = None
from contextlib import contextmanager

from invoke.vendor.six import BytesIO, b, iteritems, wraps

from mock import patch, Mock
from spec import trap, Spec, eq_, ok_, skip

from invoke import Program, Runner
from invoke.platform import WINDOWS

support = os.path.join(os.path.dirname(__file__), '_support')

def skip_if_windows(fn):
    @wraps(fn)
    def wrapper(*args, **kwargs):
        if WINDOWS:
            skip()
        return fn(*args, **kwargs)
    return wrapper

@contextmanager
def support_path():
    sys.path.insert(0, support)
    try:
        yield
    finally:
        sys.path.pop(0)

def load(name):
    with support_path():
        return __import__(name)

class IntegrationSpec(Spec):
    def setup(self):
        # Preserve environment for later restore
        self.old_environ = os.environ.copy()
        # Always do things relative to tests/_support
        os.chdir(support)

    def teardown(self):
        # Chdir back to project root to avoid problems
        os.chdir(os.path.join(os.path.dirname(__file__), '..'))
        # Nuke changes to environ
        os.environ.clear()
        os.environ.update(self.old_environ)
        # Strip any test-support task collections from sys.modules to prevent
        # state bleed between tests; otherwise tests can incorrectly pass
        # despite not explicitly loading/cd'ing to get the tasks they call
        # loaded.
        for name, module in iteritems(sys.modules.copy()):
            if module and support in getattr(module, '__file__', ''):
                del sys.modules[name]

@trap
def expect(invocation, out=None, err=None, program=None, invoke=True,
    test=None):
    """
    Run ``invocation`` via ``program`` and expect resulting output to match.

    May give one or both of ``out``/``err`` (but not neither).

    ``program`` defaults to ``Program()``.

    To skip automatically assuming the argv under test starts with ``"invoke
    "``, say ``invoke=False``.

    To customize the operator used for testing (default: equality), use
    ``test`` (which should be an assertion wrapper of some kind).
    """
    if program is None:
        program = Program()
    if invoke:
        invocation = "invoke {0}".format(invocation)
    program.run(invocation, exit=False)
    # Perform tests
    if out is not None:
        (test or eq_)(sys.stdout.getvalue(), out)
    if err is not None:
        (test or eq_)(sys.stderr.getvalue(), err)

class MockSubprocess(object):
    def __init__(self, out='', err='', exit=0, isatty=None, autostart=True):
        self.out_file = BytesIO(b(out))
        self.err_file = BytesIO(b(err))
        self.exit = exit
        self.isatty = isatty
        if autostart:
            self.start()

    def start(self):
        # Start patchin'
        self.popen = patch('invoke.runners.Popen')
        Popen = self.popen.start()
        self.read = patch('os.read')
        read = self.read.start()
        self.sys_stdin = patch('sys.stdin', new_callable=BytesIO)
        sys_stdin = self.sys_stdin.start()
        # Setup mocks
        process = Popen.return_value
        process.returncode = self.exit
        process.stdout.fileno.return_value = 1
        process.stderr.fileno.return_value = 2
        # If requested, mock isatty to fake out pty detection
        if self.isatty is not None:
            sys_stdin.isatty = Mock(return_value=self.isatty)
        def fakeread(fileno, count):
            fd = {1: self.out_file, 2: self.err_file}[fileno]
            return fd.read(count)
        read.side_effect = fakeread
        # Return the Popen mock as it's sometimes wanted inside tests
        return Popen

    def stop(self):
        self.popen.stop()
        self.read.stop()
        self.sys_stdin.stop()

def mock_subprocess(out='', err='', exit=0, isatty=None, insert_Popen=False):
    def decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            proc = MockSubprocess(
                out=out, err=err, exit=exit, isatty=isatty, autostart=False,
            )
            Popen = proc.start()
            args = list(args)
            if insert_Popen:
                args.append(Popen)
            try:
                f(*args, **kwargs)
            finally:
                proc.stop()
        return wrapper
    return decorator

def mock_pty(out='', err='', exit=0, isatty=None, trailing_error=None,
    skip_asserts=False, insert_os=False):
    # Windows doesn't have ptys, so all the pty tests should be
    # skipped anyway...
    if WINDOWS:
        return skip_if_windows

    def decorator(f):
        import fcntl
        ioctl_patch = patch('invoke.runners.fcntl.ioctl', wraps=fcntl.ioctl)

        @wraps(f)
        @patch('invoke.runners.pty')
        @patch('invoke.runners.os')
        @ioctl_patch
        def wrapper(*args, **kwargs):
            args = list(args)
            pty, os, ioctl = args.pop(), args.pop(), args.pop()
            # Don't actually fork, but pretend we did & that main thread is
            # also the child (pid 0) to trigger execve call; & give 'parent fd'
            # of 1 (stdout).
            pty.fork.return_value = 0, 1
            # We don't really need to care about waiting since not truly
            # forking/etc, so here we just return a nonzero "pid" + sentinel
            # wait-status value (used in some tests about WIFEXITED etc)
            os.waitpid.return_value = None, Mock(name='exitstatus')
            # Either or both of these may get called, depending...
            os.WEXITSTATUS.return_value = exit
            os.WTERMSIG.return_value = exit
            # If requested, mock isatty to fake out pty detection
            if isatty is not None:
                os.isatty.return_value = isatty
            out_file = BytesIO(b(out))
            err_file = BytesIO(b(err))
            def fakeread(fileno, count):
                fd = {1: out_file, 2: err_file}[fileno]
                ret = fd.read(count)
                # If asked, fake a Linux-platform trailing I/O error.
                if not ret and trailing_error:
                    raise trailing_error
                return ret
            os.read.side_effect = fakeread
            if insert_os:
                args.append(os)
            f(*args, **kwargs)
            # Short-circuit if we raised an error in fakeread()
            if trailing_error:
                return
            # Sanity checks to make sure the stuff we mocked, actually got ran!
            # TODO: inject our mocks back into the tests so they can make their
            # own assertions if desired
            pty.fork.assert_called_with()
            # Expect a get, and then later set, of terminal window size
            eq_(ioctl.call_args_list[0][0][1], termios.TIOCGWINSZ)
            eq_(ioctl.call_args_list[1][0][1], termios.TIOCSWINSZ)
            if not skip_asserts:
                for name in ('execve', 'waitpid'):
                    ok_(getattr(os, name).called)
                # Ensure at least one of the exit status getters was called
                ok_(os.WEXITSTATUS.called or os.WTERMSIG.called)
        return wrapper
    return decorator

class _Dummy(Runner):
    """
    Dummy runner subclass that does minimum work required to execute run().

    It also serves as a convenient basic API checker; failure to update it to
    match the current Runner API will cause TypeErrors, NotImplementedErrors,
    and similar.
    """
    # Neuter the input loop sleep, so tests aren't slow (at the expense of CPU,
    # which isn't a problem for testing).
    input_sleep = 0

    def start(self, command, shell, env):
        pass

    def read_proc_stdout(self, num_bytes):
        return ""

    def read_proc_stderr(self, num_bytes):
        return ""

    def _write_proc_stdin(self, data):
        pass

    @property
    def process_is_finished(self):
        return True

    def returncode(self):
        return 0

    def stop(self):
        pass

# Dummy command that will blow up if it ever truly hits a real shell.
_ = "nope"

# Runner that fakes ^C during subprocess exec
class _KeyboardInterruptingRunner(_Dummy):
    def __init__(self, *args, **kwargs):
        super(_KeyboardInterruptingRunner, self).__init__(*args, **kwargs)
        self._interrupted = False

    # Trigger KeyboardInterrupt during wait()
    def wait(self):
        if not self._interrupted:
            self._interrupted = True
            raise KeyboardInterrupt

    # But also, after that has been done, pretend subprocess shutdown happened
    # (or we will loop forever).
    def process_is_finished(self):
        return self._interrupted

class OhNoz(Exception):
    pass
<|endoftext|>


<commit_before>import os
import sys

try:
    import termios
except ImportError:
    # Not available on Windows
    termios = None
from contextlib import contextmanager

from invoke.vendor.six import BytesIO, b, wraps

from mock import patch, Mock
from pytest import skip
from pytest_relaxed import trap

from invoke import Program, Runner
from invoke.terminals import WINDOWS

support = os.path.join(os.path.dirname(__file__), "_support")

def skip_if_windows(fn):
    @wraps(fn)
    def wrapper(*args, **kwargs):
        if WINDOWS:
            skip()
        return fn(*args, **kwargs)

    return wrapper

@contextmanager
def support_path():
    sys.path.insert(0, support)
    try:
        yield
    finally:
        sys.path.pop(0)

def load(name):
    with support_path():
        imported = __import__(name)
        return imported

def support_file(subpath):
    with open(os.path.join(support, subpath)) as fd:
        return fd.read()

@trap
def run(invocation, program=None, invoke=True):
    """
    Run ``invocation`` via ``program``, returning output stream captures.

    ``program`` defaults to ``Program()``.

    To skip automatically assuming the argv under test starts with ``"invoke
    "``, say ``invoke=False``.

    :returns: Two-tuple of ``stdout, stderr`` strings.
    """
    if program is None:
        program = Program()
    if invoke:
        invocation = "invoke {}".format(invocation)
    program.run(invocation, exit=False)
    return sys.stdout.getvalue(), sys.stderr.getvalue()

def expect(
    invocation, out=None, err=None, program=None, invoke=True, test=None
):
    """
    Run ``invocation`` via ``program`` and expect resulting output to match.

    May give one or both of ``out``/``err`` (but not neither).

    ``program`` defaults to ``Program()``.

    To skip automatically assuming the argv under test starts with ``"invoke
    "``, say ``invoke=False``.

    To customize the operator used for testing (default: equality), use
    ``test`` (which should be an assertion wrapper of some kind).
    """
    stdout, stderr = run(invocation, program, invoke)
    # Perform tests
    if out is not None:
        if test:
            test(stdout, out)
        else:
            assert out == stdout
    if err is not None:
        if test:
            test(stderr, err)
        else:
            assert err == stderr
    # Guard against silent failures; since we say exit=False this is the only
    # real way to tell if stuff died in a manner we didn't expect.
    elif stderr:
        assert False, "Unexpected stderr: {}".format(stderr)
    return stdout, stderr

class MockSubprocess(object):
    def __init__(self, out="", err="", exit=0, isatty=None, autostart=True):
        self.out_file = BytesIO(b(out))
        self.err_file = BytesIO(b(err))
        self.exit = exit
        self.isatty = isatty
        if autostart:
            self.start()

    def start(self):
        # Start patchin'
        self.popen = patch("invoke.runners.Popen")
        Popen = self.popen.start()
        self.read = patch("os.read")
        read = self.read.start()
        self.sys_stdin = patch("sys.stdin", new_callable=BytesIO)
        sys_stdin = self.sys_stdin.start()
        # Setup mocks
        process = Popen.return_value
        process.returncode = self.exit
        process.stdout.fileno.return_value = 1
        process.stderr.fileno.return_value = 2
        # If requested, mock isatty to fake out pty detection
        if self.isatty is not None:
            sys_stdin.isatty = Mock(return_value=self.isatty)

        def fakeread(fileno, count):
            fd = {1: self.out_file, 2: self.err_file}[fileno]
            return fd.read(count)

        read.side_effect = fakeread
        # Return the Popen mock as it's sometimes wanted inside tests
        return Popen

    def stop(self):
        self.popen.stop()
        self.read.stop()
        self.sys_stdin.stop()

def mock_subprocess(out="", err="", exit=0, isatty=None, insert_Popen=False):
    def decorator(f):
        @wraps(f)
        # We have to include a @patch here to trick pytest into ignoring
        # the wrapped test's sometimes-there, sometimes-not mock_Popen arg. (It
        # explicitly "skips ahead" past what it perceives as patch args, even
        # though in our case those are not applying to the test function!)
        # Doesn't matter what we patch as long as it doesn't
        # actually get in our way.
        @patch("invoke.runners.pty")
        def wrapper(*args, **kwargs):
            proc = MockSubprocess(
                out=out, err=err, exit=exit, isatty=isatty, autostart=False
            )
            Popen = proc.start()
            args = list(args)
            args.pop()  # Pop the dummy patch
            if insert_Popen:
                args.append(Popen)
            try:
                f(*args, **kwargs)
            finally:
                proc.stop()

        return wrapper

    return decorator

def mock_pty(
    out="",
    err="",
    exit=0,
    isatty=None,
    trailing_error=None,
    skip_asserts=False,
    insert_os=False,
):
    # Windows doesn't have ptys, so all the pty tests should be
    # skipped anyway...
    if WINDOWS:
        return skip_if_windows

    def decorator(f):
        import fcntl

        ioctl_patch = patch("invoke.runners.fcntl.ioctl", wraps=fcntl.ioctl)

        @wraps(f)
        @patch("invoke.runners.pty")
        @patch("invoke.runners.os")
        @ioctl_patch
        def wrapper(*args, **kwargs):
            args = list(args)
            pty, os, ioctl = args.pop(), args.pop(), args.pop()
            # Don't actually fork, but pretend we did & that main thread is
            # also the child (pid 0) to trigger execve call; & give 'parent fd'
            # of 1 (stdout).
            pty.fork.return_value = 0, 1
            # We don't really need to care about waiting since not truly
            # forking/etc, so here we just return a nonzero "pid" + sentinel
            # wait-status value (used in some tests about WIFEXITED etc)
            os.waitpid.return_value = None, Mock(name="exitstatus")
            # Either or both of these may get called, depending...
            os.WEXITSTATUS.return_value = exit
            os.WTERMSIG.return_value = exit
            # If requested, mock isatty to fake out pty detection
            if isatty is not None:
                os.isatty.return_value = isatty
            out_file = BytesIO(b(out))
            err_file = BytesIO(b(err))

            def fakeread(fileno, count):
                fd = {1: out_file, 2: err_file}[fileno]
                ret = fd.read(count)
                # If asked, fake a Linux-platform trailing I/O error.
                if not ret and trailing_error:
                    raise trailing_error
                return ret

            os.read.side_effect = fakeread
            if insert_os:
                args.append(os)
            f(*args, **kwargs)
            # Short-circuit if we raised an error in fakeread()
            if trailing_error:
                return
            # Sanity checks to make sure the stuff we mocked, actually got ran!
            # TODO: inject our mocks back into the tests so they can make their
            # own assertions if desired
            pty.fork.assert_called_with()
            # Expect a get, and then later set, of terminal window size
            assert ioctl.call_args_list[0][0][1] == termios.TIOCGWINSZ
            assert ioctl.call_args_list[1][0][1] == termios.TIOCSWINSZ
            if not skip_asserts:
                for name in ("execve", "waitpid"):
                    assert getattr(os, name).called
                # Ensure at least one of the exit status getters was called
                assert os.WEXITSTATUS.called or os.WTERMSIG.called

        return wrapper

    return decorator

class _Dummy(Runner):
    """
    Dummy runner subclass that does minimum work required to execute run().

    It also serves as a convenient basic API checker; failure to update it to
    match the current Runner API will cause TypeErrors, NotImplementedErrors,
    and similar.
    """

    # Neuter the input loop sleep, so tests aren't slow (at the expense of CPU,
    # which isn't a problem for testing).
    input_sleep = 0

    def start(self, command, shell, env):
        pass

    def read_proc_stdout(self, num_bytes):
        return ""

    def read_proc_stderr(self, num_bytes):
        return ""

    def _write_proc_stdin(self, data):
        pass

    @property
    def process_is_finished(self):
        return True

    def returncode(self):
        return 0

    def stop(self):
        pass

# Dummy command that will blow up if it ever truly hits a real shell.
_ = "nope"

# Runner that fakes ^C during subprocess exec
class _KeyboardInterruptingRunner(_Dummy):
    def __init__(self, *args, **kwargs):
        super(_KeyboardInterruptingRunner, self).__init__(*args, **kwargs)
        self._interrupted = False

    # Trigger KeyboardInterrupt during wait()
    def wait(self):
        if not self._interrupted:
            self._interrupted = True
            raise KeyboardInterrupt

    # But also, after that has been done, pretend subprocess shutdown happened
    # (or we will loop forever).
    def process_is_finished(self):
        return self._interrupted

class OhNoz(Exception):
    pass
<commit_msg>Change to marginally more realistic mock child pty FD<commit_after>import os
import sys

try:
    import termios
except ImportError:
    # Not available on Windows
    termios = None
from contextlib import contextmanager

from invoke.vendor.six import BytesIO, b, wraps

from mock import patch, Mock
from pytest import skip
from pytest_relaxed import trap

from invoke import Program, Runner
from invoke.terminals import WINDOWS

support = os.path.join(os.path.dirname(__file__), "_support")

def skip_if_windows(fn):
    @wraps(fn)
    def wrapper(*args, **kwargs):
        if WINDOWS:
            skip()
        return fn(*args, **kwargs)

    return wrapper

@contextmanager
def support_path():
    sys.path.insert(0, support)
    try:
        yield
    finally:
        sys.path.pop(0)

def load(name):
    with support_path():
        imported = __import__(name)
        return imported

def support_file(subpath):
    with open(os.path.join(support, subpath)) as fd:
        return fd.read()

@trap
def run(invocation, program=None, invoke=True):
    """
    Run ``invocation`` via ``program``, returning output stream captures.

    ``program`` defaults to ``Program()``.

    To skip automatically assuming the argv under test starts with ``"invoke
    "``, say ``invoke=False``.

    :returns: Two-tuple of ``stdout, stderr`` strings.
    """
    if program is None:
        program = Program()
    if invoke:
        invocation = "invoke {}".format(invocation)
    program.run(invocation, exit=False)
    return sys.stdout.getvalue(), sys.stderr.getvalue()

def expect(
    invocation, out=None, err=None, program=None, invoke=True, test=None
):
    """
    Run ``invocation`` via ``program`` and expect resulting output to match.

    May give one or both of ``out``/``err`` (but not neither).

    ``program`` defaults to ``Program()``.

    To skip automatically assuming the argv under test starts with ``"invoke
    "``, say ``invoke=False``.

    To customize the operator used for testing (default: equality), use
    ``test`` (which should be an assertion wrapper of some kind).
    """
    stdout, stderr = run(invocation, program, invoke)
    # Perform tests
    if out is not None:
        if test:
            test(stdout, out)
        else:
            assert out == stdout
    if err is not None:
        if test:
            test(stderr, err)
        else:
            assert err == stderr
    # Guard against silent failures; since we say exit=False this is the only
    # real way to tell if stuff died in a manner we didn't expect.
    elif stderr:
        assert False, "Unexpected stderr: {}".format(stderr)
    return stdout, stderr

class MockSubprocess(object):
    def __init__(self, out="", err="", exit=0, isatty=None, autostart=True):
        self.out_file = BytesIO(b(out))
        self.err_file = BytesIO(b(err))
        self.exit = exit
        self.isatty = isatty
        if autostart:
            self.start()

    def start(self):
        # Start patchin'
        self.popen = patch("invoke.runners.Popen")
        Popen = self.popen.start()
        self.read = patch("os.read")
        read = self.read.start()
        self.sys_stdin = patch("sys.stdin", new_callable=BytesIO)
        sys_stdin = self.sys_stdin.start()
        # Setup mocks
        process = Popen.return_value
        process.returncode = self.exit
        process.stdout.fileno.return_value = 1
        process.stderr.fileno.return_value = 2
        # If requested, mock isatty to fake out pty detection
        if self.isatty is not None:
            sys_stdin.isatty = Mock(return_value=self.isatty)

        def fakeread(fileno, count):
            fd = {1: self.out_file, 2: self.err_file}[fileno]
            return fd.read(count)

        read.side_effect = fakeread
        # Return the Popen mock as it's sometimes wanted inside tests
        return Popen

    def stop(self):
        self.popen.stop()
        self.read.stop()
        self.sys_stdin.stop()

def mock_subprocess(out="", err="", exit=0, isatty=None, insert_Popen=False):
    def decorator(f):
        @wraps(f)
        # We have to include a @patch here to trick pytest into ignoring
        # the wrapped test's sometimes-there, sometimes-not mock_Popen arg. (It
        # explicitly "skips ahead" past what it perceives as patch args, even
        # though in our case those are not applying to the test function!)
        # Doesn't matter what we patch as long as it doesn't
        # actually get in our way.
        @patch("invoke.runners.pty")
        def wrapper(*args, **kwargs):
            proc = MockSubprocess(
                out=out, err=err, exit=exit, isatty=isatty, autostart=False
            )
            Popen = proc.start()
            args = list(args)
            args.pop()  # Pop the dummy patch
            if insert_Popen:
                args.append(Popen)
            try:
                f(*args, **kwargs)
            finally:
                proc.stop()

        return wrapper

    return decorator

def mock_pty(
    out="",
    err="",
    exit=0,
    isatty=None,
    trailing_error=None,
    skip_asserts=False,
    insert_os=False,
):
    # Windows doesn't have ptys, so all the pty tests should be
    # skipped anyway...
    if WINDOWS:
        return skip_if_windows

    def decorator(f):
        import fcntl

        ioctl_patch = patch("invoke.runners.fcntl.ioctl", wraps=fcntl.ioctl)

        @wraps(f)
        @patch("invoke.runners.pty")
        @patch("invoke.runners.os")
        @ioctl_patch
        def wrapper(*args, **kwargs):
            args = list(args)
            pty, os, ioctl = args.pop(), args.pop(), args.pop()
            # Don't actually fork, but pretend we did & that main thread is
            # also the child (pid 0) to trigger execve call; & give 'parent fd'
            # of 3 (typically, first allocated non-stdin/out/err FD)
            pty.fork.return_value = 0, 3
            # We don't really need to care about waiting since not truly
            # forking/etc, so here we just return a nonzero "pid" + sentinel
            # wait-status value (used in some tests about WIFEXITED etc)
            os.waitpid.return_value = None, Mock(name="exitstatus")
            # Either or both of these may get called, depending...
            os.WEXITSTATUS.return_value = exit
            os.WTERMSIG.return_value = exit
            # If requested, mock isatty to fake out pty detection
            if isatty is not None:
                os.isatty.return_value = isatty
            out_file = BytesIO(b(out))
            err_file = BytesIO(b(err))

            def fakeread(fileno, count):
                fd = {3: out_file, 2: err_file}[fileno]
                ret = fd.read(count)
                # If asked, fake a Linux-platform trailing I/O error.
                if not ret and trailing_error:
                    raise trailing_error
                return ret

            os.read.side_effect = fakeread
            if insert_os:
                args.append(os)
            f(*args, **kwargs)
            # Short-circuit if we raised an error in fakeread()
            if trailing_error:
                return
            # Sanity checks to make sure the stuff we mocked, actually got ran!
            # TODO: inject our mocks back into the tests so they can make their
            # own assertions if desired
            pty.fork.assert_called_with()
            # Expect a get, and then later set, of terminal window size
            assert ioctl.call_args_list[0][0][1] == termios.TIOCGWINSZ
            assert ioctl.call_args_list[1][0][1] == termios.TIOCSWINSZ
            if not skip_asserts:
                for name in ("execve", "waitpid"):
                    assert getattr(os, name).called
                # Ensure at least one of the exit status getters was called
                assert os.WEXITSTATUS.called or os.WTERMSIG.called

        return wrapper

    return decorator

class _Dummy(Runner):
    """
    Dummy runner subclass that does minimum work required to execute run().

    It also serves as a convenient basic API checker; failure to update it to
    match the current Runner API will cause TypeErrors, NotImplementedErrors,
    and similar.
    """

    # Neuter the input loop sleep, so tests aren't slow (at the expense of CPU,
    # which isn't a problem for testing).
    input_sleep = 0

    def start(self, command, shell, env):
        pass

    def read_proc_stdout(self, num_bytes):
        return ""

    def read_proc_stderr(self, num_bytes):
        return ""

    def _write_proc_stdin(self, data):
        pass

    @property
    def process_is_finished(self):
        return True

    def returncode(self):
        return 0

    def stop(self):
        pass

# Dummy command that will blow up if it ever truly hits a real shell.
_ = "nope"

# Runner that fakes ^C during subprocess exec
class _KeyboardInterruptingRunner(_Dummy):
    def __init__(self, *args, **kwargs):
        super(_KeyboardInterruptingRunner, self).__init__(*args, **kwargs)
        self._interrupted = False

    # Trigger KeyboardInterrupt during wait()
    def wait(self):
        if not self._interrupted:
            self._interrupted = True
            raise KeyboardInterrupt

    # But also, after that has been done, pretend subprocess shutdown happened
    # (or we will loop forever).
    def process_is_finished(self):
        return self._interrupted

class OhNoz(Exception):
    pass
<|endoftext|>


<commit_before>#!/usr/bin/python -u
#
# Tests global error handlers at the python level.
#
import libvirt
import sys
import os

errno = None

def handler(ctxt, err):
    global errno

    #print "handler(%s, %s)" % (ctxt, err)
    errno = err

libvirt.registerErrorHandler(handler, 'context')

conn = libvirt.openReadOnly(None)
if conn == None:
    print 'Failed to open connection to the hypervisor'
    sys.exit(1)

try:
    dom0 = conn.lookupByName("Does_not_exist")
    print 'strange found a Does_not_exist domain'
    sys.exit(1)
except:
    pass

del conn

if errno == None:
    print 'failed to get an error'
elif errno[0] == libvirt.VIR_ERR_NO_CONNECT or \
     errno[0] == libvirt.VIR_ERR_INVALID_DOMAIN:
    print "OK"
else:
    print 'got unexpected error %s' % (errno)

sys.exit(0)
<commit_msg>Fixed the test, daniel<commit_after>#!/usr/bin/python -u
#
# Tests global error handlers at the python level.
#
import libvirt
import sys
import os

errno = None

def handler(ctxt, err):
    global errno

    #print "handler(%s, %s)" % (ctxt, err)
    errno = err

libvirt.registerErrorHandler(handler, 'context')

conn = libvirt.openReadOnly(None)
if conn == None:
    print 'Failed to open connection to the hypervisor'
    sys.exit(1)

try:
    dom0 = conn.lookupByName("Does_not_exist")
    print 'strange found a Does_not_exist domain'
    sys.exit(1)
except:
    pass

del conn

if errno == None:
    print 'failed to get an error'
elif errno[0] == libvirt.VIR_ERR_NO_CONNECT or \
     errno[0] == libvirt.VIR_ERR_INVALID_DOMAIN or \
     errno[0] == libvirt.VIR_ERR_GET_FAILED:
    print "OK"
else:
    print 'got unexpected error:', errno

sys.exit(0)
<|endoftext|>


<commit_before>from hbpush.pubsub.publisher import Publisher
from hbpush.pubsub.subscriber import Subscriber, LongPollingSubscriber
from tornado.httpserver import HTTPHeaders, HTTPRequest
from tornado.web import HTTPError
from tornado.ioloop import IOLoop

def decorate_method(method):
    def _new_method(self, channel_id, expect, next):
        self.expect = expect
        self.next = next
        getattr(super(MockHandler, self), method)(channel_id)
    return _new_method

class MockHandler(object):
    def __init__(self, *args, **kwargs):
        super(MockHandler, self).__init__(*args, **kwargs)
        self._write_buf = ''

    delete = decorate_method('delete')
    get = decorate_method('get')
    post = decorate_method('post')
    put = decorate_method('put')

    def write(self, data):
        assert not self._finished
        self._write_buf += data

    def finish(self, *args, **kwargs):
        self._finished = True

        # Here we test expectations
        for callback in self.expect:
            callback(self._status_code, self._headers, self._write_buf)
        self.next()

    def _handle_request_exception(self, e):
        if isinstance(e, HTTPError):
            super(MockHandler, self)._handle_request_exception(e)
        else:
            raise e

class MockPublisher(MockHandler, Publisher):
    pass

class MockSubscriber(MockHandler, Subscriber):
    pass

class MockLongPollingSubscriber(MockHandler, LongPollingSubscriber):
    pass

class MockApplication(object):
    ui_modules = {}
    ui_methods = {}
    _wsgi = False

class MockRequest(object):
    def __init__(self, method='GET', channel=None, headers=None, body='', cb=(), handler=None, **kwargs):
        self.method = method
        self.channel = channel
        self.callbacks = hasattr(cb, '__iter__') and tuple(cb) or (cb,)
        self.handler = handler
        self.body = body
        self.kwargs = kwargs
        self.headers = headers

    def __call__(self, next):
        headers = HTTPHeaders()
        if self.headers:
            for name, val in [h.split(': ') for h in self.headers]:
                headers[name] = val

        req = HTTPRequest(self.method, '', body=self.body, headers=headers)
        del req.connection
        app = MockApplication()
        getattr(self.handler(app, req, **self.kwargs), self.method.lower())(self.channel, self.callbacks, next)

class MockIOLoop(IOLoop):
    def handle_callback_exception(self, callback):
        import sys
        (type, value, traceback) = sys.exc_info()
        raise type, value, traceback
<commit_msg>tests: process all handler exceptions in _handle_request_exception<commit_after>from hbpush.pubsub.publisher import Publisher
from hbpush.pubsub.subscriber import Subscriber, LongPollingSubscriber
from tornado.httpserver import HTTPHeaders, HTTPRequest
from tornado.web import HTTPError
from tornado.ioloop import IOLoop

def decorate_method(method):
    def _new_method(self, channel_id, expect, next):
        self.expect = expect
        self.next = next
        try:
            getattr(super(MockHandler, self), method)(channel_id)
        except Exception, e:
            self._handle_request_exception(e)
    return _new_method

class MockHandler(object):
    def __init__(self, *args, **kwargs):
        super(MockHandler, self).__init__(*args, **kwargs)
        self._write_buf = ''

    delete = decorate_method('delete')
    get = decorate_method('get')
    post = decorate_method('post')
    put = decorate_method('put')

    def write(self, data):
        assert not self._finished
        self._write_buf += data

    def finish(self, *args, **kwargs):
        self._finished = True

        # Here we test expectations
        for callback in self.expect:
            callback(self._status_code, self._headers, self._write_buf)
        self.next()

    def _handle_request_exception(self, e):
        if isinstance(e, HTTPError):
            super(MockHandler, self)._handle_request_exception(e)
        else:
            raise e

class MockPublisher(MockHandler, Publisher):
    pass

class MockSubscriber(MockHandler, Subscriber):
    pass

class MockLongPollingSubscriber(MockHandler, LongPollingSubscriber):
    pass

class MockApplication(object):
    ui_modules = {}
    ui_methods = {}
    _wsgi = False

class MockRequest(object):
    def __init__(self, method='GET', channel=None, headers=None, body='', cb=(), handler=None, **kwargs):
        self.method = method
        self.channel = channel
        self.callbacks = hasattr(cb, '__iter__') and tuple(cb) or (cb,)
        self.handler = handler
        self.body = body
        self.kwargs = kwargs
        self.headers = headers

    def __call__(self, next):
        headers = HTTPHeaders()
        if self.headers:
            for name, val in [h.split(': ') for h in self.headers]:
                headers[name] = val

        req = HTTPRequest(self.method, '', body=self.body, headers=headers)
        del req.connection
        app = MockApplication()
        getattr(self.handler(app, req, **self.kwargs), self.method.lower())(self.channel, self.callbacks, next)

class MockIOLoop(IOLoop):
    def handle_callback_exception(self, callback):
        import sys
        (type, value, traceback) = sys.exc_info()
        raise type, value, traceback
<|endoftext|>


<commit_before>from nutils import model, mesh, function
from . import register, unittest
import numpy

@register
def laplace():
  domain, geom = mesh.rectilinear( [8,8] )
  basis = domain.basis( 'std', degree=1 )
  cons = domain.boundary['left'].project( 0, onto=basis, geometry=geom, ischeme='gauss2' )
  dofs = function.Argument( 'dofs', [len(basis)] )
  u = basis.dot( dofs )
  residual = model.Integral( ( basis.grad(geom) * u.grad(geom) ).sum(-1), domain=domain, geometry=geom, degree=2 ) \
           + model.Integral( basis, domain=domain.boundary['top'], geometry=geom, degree=2 )

  for name in 'direct', 'newton':
    @unittest( name=name )
    def res():
      if name == 'direct':
        lhs = model.solve_linear( 'dofs', residual=residual, constrain=cons )
      else:
        lhs = model.newton( 'dofs', residual=residual, lhs0=cons|0, freezedofs=cons.where ).solve( tol=1e-10, maxiter=0 )
      res = residual.eval(arguments=dict(dofs=lhs))
      resnorm = numpy.linalg.norm( res[~cons.where] )
      assert resnorm < 1e-13

@register
def navierstokes():
  domain, geom = mesh.rectilinear( [numpy.linspace(0,1,9)] * 2 )
  ubasis, pbasis = function.chain([
    domain.basis( 'std', degree=2 ).vector(2),
    domain.basis( 'std', degree=1 ),
  ])
  dofs = function.Argument( 'dofs', [len(ubasis)] )
  u = ubasis.dot( dofs )
  p = pbasis.dot( dofs )
  viscosity = 1
  inertia = model.Integral( (ubasis * u).sum(-1), domain=domain, geometry=geom, degree=5 )
  stokesres = model.Integral( viscosity * ubasis['ni,j'] * (u['i,j']+u['j,i']) - ubasis['nk,k'] * p + pbasis['n'] * u['k,k'], domain=domain, geometry=geom, degree=5 )
  residual = stokesres + model.Integral( ubasis['ni'] * u['i,j'] * u['j'], domain=domain, geometry=geom, degree=5 )
  cons = domain.boundary['top,bottom'].project( [0,0], onto=ubasis, geometry=geom, ischeme='gauss2' ) \
       | domain.boundary['left'].project( [geom[1]*(1-geom[1]),0], onto=ubasis, geometry=geom, ischeme='gauss2' )
  lhs0 = model.solve_linear( 'dofs', residual=stokesres, constrain=cons )

  for name in 'direct', 'newton', 'pseudotime':
    @unittest( name=name, raises=name=='direct' and model.ModelError)
    def res():
      tol = 1e-10
      if name == 'direct':
        lhs = model.solve_linear( 'dofs', residual=residual, constrain=cons )
      elif name == 'newton':
        lhs = model.newton( 'dofs', residual=residual, lhs0=lhs0, freezedofs=cons.where ).solve( tol=tol, maxiter=2 )
      else:
        lhs = model.pseudotime( 'dofs', residual=residual, lhs0=lhs0, freezedofs=cons.where, inertia=inertia, timestep=1 ).solve( tol=tol, maxiter=3 )
      res = residual.eval(arguments=dict(dofs=lhs))
      resnorm = numpy.linalg.norm( res[~cons.where] )
      assert resnorm < tol
<commit_msg>use domain.integral in model tests<commit_after>from nutils import model, mesh, function
from . import register, unittest
import numpy

@register
def laplace():
  domain, geom = mesh.rectilinear( [8,8] )
  basis = domain.basis( 'std', degree=1 )
  cons = domain.boundary['left'].project( 0, onto=basis, geometry=geom, ischeme='gauss2' )
  dofs = function.Argument( 'dofs', [len(basis)] )
  u = basis.dot( dofs )
  residual = domain.integral( ( basis.grad(geom) * u.grad(geom) ).sum(-1), geometry=geom, degree=2 ) \
           + domain.boundary['top'].integral( basis, geometry=geom, degree=2 )

  for name in 'direct', 'newton':
    @unittest( name=name )
    def res():
      if name == 'direct':
        lhs = model.solve_linear( 'dofs', residual=residual, constrain=cons )
      else:
        lhs = model.newton( 'dofs', residual=residual, lhs0=cons|0, freezedofs=cons.where ).solve( tol=1e-10, maxiter=0 )
      res = residual.eval(arguments=dict(dofs=lhs))
      resnorm = numpy.linalg.norm( res[~cons.where] )
      assert resnorm < 1e-13

@register
def navierstokes():
  domain, geom = mesh.rectilinear( [numpy.linspace(0,1,9)] * 2 )
  ubasis, pbasis = function.chain([
    domain.basis( 'std', degree=2 ).vector(2),
    domain.basis( 'std', degree=1 ),
  ])
  dofs = function.Argument( 'dofs', [len(ubasis)] )
  u = ubasis.dot( dofs )
  p = pbasis.dot( dofs )
  viscosity = 1
  inertia = domain.integral( (ubasis * u).sum(-1), geometry=geom, degree=5 )
  stokesres = domain.integral( viscosity * ubasis['ni,j'] * (u['i,j']+u['j,i']) - ubasis['nk,k'] * p + pbasis['n'] * u['k,k'], geometry=geom, degree=5 )
  residual = stokesres + domain.integral( ubasis['ni'] * u['i,j'] * u['j'], geometry=geom, degree=5 )
  cons = domain.boundary['top,bottom'].project( [0,0], onto=ubasis, geometry=geom, ischeme='gauss2' ) \
       | domain.boundary['left'].project( [geom[1]*(1-geom[1]),0], onto=ubasis, geometry=geom, ischeme='gauss2' )
  lhs0 = model.solve_linear( 'dofs', residual=stokesres, constrain=cons )

  for name in 'direct', 'newton', 'pseudotime':
    @unittest( name=name, raises=name=='direct' and model.ModelError)
    def res():
      tol = 1e-10
      if name == 'direct':
        lhs = model.solve_linear( 'dofs', residual=residual, constrain=cons )
      elif name == 'newton':
        lhs = model.newton( 'dofs', residual=residual, lhs0=lhs0, freezedofs=cons.where ).solve( tol=tol, maxiter=2 )
      else:
        lhs = model.pseudotime( 'dofs', residual=residual, lhs0=lhs0, freezedofs=cons.where, inertia=inertia, timestep=1 ).solve( tol=tol, maxiter=3 )
      res = residual.eval(arguments=dict(dofs=lhs))
      resnorm = numpy.linalg.norm( res[~cons.where] )
      assert resnorm < tol
<|endoftext|>


<commit_before>import asyncio
import discord
from   discord.ext import commands
from   Cogs import Settings
from   Cogs import CheckRoles

# This is the message xp module.  It's also likely to be retarded.

class MessageXp:

	# Init with the bot reference, and a reference to the settings var
	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings

	async def message(self, message):
		# Check the message and see if we should allow it - always yes.
		# This module doesn't need to cancel messages.

		server = message.guild
		xpAmount   = int(self.settings.getServerStat(server, "XPPerMessage"))
		xpRAmount  = int(self.settings.getServerStat(server, "XPRPerMessage"))

		if xpRAmount > 0:
			# Bump xp reserve
			self.settings.incrementStat(message.author, server, "XPReserve", xpRAmount)

		if xpAmount > 0:
			# Bump xp
			self.settings.incrementStat(message.author, server, "XP", xpAmount)
			# Check for promotion/demotion
			await CheckRoles.checkroles(message.author, message.channel, self.settings, self.bot)

		return { 'Ignore' : False, 'Delete' : False}
<commit_msg>Message block fix<commit_after>import asyncio
import discord
from   discord.ext import commands
from   Cogs import Settings
from   Cogs import CheckRoles

# This is the message xp module.  It's also likely to be retarded.

class MessageXp:

	# Init with the bot reference, and a reference to the settings var
	def __init__(self, bot, settings):
		self.bot = bot
		self.settings = settings

	async def message(self, message):
		# Check the message and see if we should allow it - always yes.
		# This module doesn't need to cancel messages.

		server = message.guild

		# Check if we're blocked
		xpblock = self.settings.getServerStat(server, "XpBlockArray")
		if message.author.id in xpblock:
			# No xp for you
			return { 'Ignore' : False, 'Delete' : False}
		for role in message.author.roles:
			if role.id in xpblock:
				return { 'Ignore' : False, 'Delete' : False}

		xpAmount   = int(self.settings.getServerStat(server, "XPPerMessage"))
		xpRAmount  = int(self.settings.getServerStat(server, "XPRPerMessage"))

		if xpRAmount > 0:
			# Bump xp reserve
			self.settings.incrementStat(message.author, server, "XPReserve", xpRAmount)

		if xpAmount > 0:
			# Bump xp
			self.settings.incrementStat(message.author, server, "XP", xpAmount)
			# Check for promotion/demotion
			await CheckRoles.checkroles(message.author, message.channel, self.settings, self.bot)

		return { 'Ignore' : False, 'Delete' : False}
<|endoftext|>


<commit_before>from LSP.plugin.core.documents import DocumentSyncListener
from LSP.plugin.core.logging import debug
from LSP.plugin.core.protocol import Notification, Request, WorkspaceFolder
from LSP.plugin.core.registry import windows
from LSP.plugin.core.sessions import Session
from LSP.plugin.core.settings import client_configs
from LSP.plugin.core.types import ClientConfig, LanguageConfig, ClientStates
from os import environ
from os.path import dirname
from os.path import join
from sublime_plugin import view_event_listeners
from test_mocks import basic_responses
from unittesting import DeferrableTestCase
import sublime

CI = any(key in environ for key in ("TRAVIS", "CI", "GITHUB_ACTIONS"))

project_path = dirname(__file__)
test_file_path = join(project_path, "testfile.txt")
workspace_folders = [WorkspaceFolder.from_path(project_path)]
TIMEOUT_TIME = 10000 if CI else 2000
SUPPORTED_SCOPE = "text.plain"
SUPPORTED_SYNTAX = "Packages/Text/Plain text.tmLanguage"
text_language = LanguageConfig("text", [SUPPORTED_SCOPE], [SUPPORTED_SYNTAX])
text_config = ClientConfig("textls", [], None, languages=[text_language])

try:
    from typing import Dict, List, Callable, Any, Optional, Generator
    assert Dict and Callable and List and Any and Optional and Generator
except ImportError:
    pass

class YieldPromise:

    __slots__ = ('__done')

    def __init__(self) -> None:
        self.__done = False

    def __call__(self) -> bool:
        return self.__done

    def fulfill(self) -> None:
        assert not self.__done
        self.__done = True

def make_stdio_test_config() -> ClientConfig:
    return ClientConfig(
        name="TEST",
        binary_args=["python3", join("$packages", "LSP", "tests", "server.py")],
        tcp_port=None,
        languages=[LanguageConfig(
            language_id="txt",
            scopes=[SUPPORTED_SCOPE],
            syntaxes=[SUPPORTED_SYNTAX])],
        enabled=True)

def add_config(config):
    client_configs.all.append(config)

def remove_config(config):
    client_configs.all.remove(config)

def close_test_view(view: sublime.View):
    if view:
        view.set_scratch(True)
        view.close()

def expand(s: str, w: sublime.Window) -> str:
    return sublime.expand_variables(s, w.extract_variables())

class SessionType:
    Stdio = 1
    TcpCreate = 2
    TcpConnectExisting = 3

class TextDocumentTestCase(DeferrableTestCase):

    def __init__(self, *args: 'Any', **kwargs: 'Any') -> None:
        super().__init__(*args, **kwargs)
        self.session = None  # type: Optional[Session]
        self.config = make_stdio_test_config()

    def setUp(self) -> 'Generator':
        super().setUp()
        test_name = self.get_test_name()
        server_capabilities = self.get_test_server_capabilities()
        session_type = self.get_test_session_type()
        self.assertTrue(test_name)
        self.assertTrue(server_capabilities)
        if session_type == SessionType.Stdio:
            pass
        elif session_type == SessionType.TcpCreate:
            # TODO: make the test server do TCP
            pass
        elif session_type == SessionType.TcpConnectExisting:
            # TODO
            pass
        else:
            self.assertFalse(True)
        window = sublime.active_window()
        self.assertTrue(window)
        filename = expand(join("$packages", "LSP", "tests", "{}.txt".format(test_name)), window)
        self.config.init_options["serverResponse"] = server_capabilities
        add_config(self.config)
        self.wm = windows.lookup(window)
        self.wm._configs.all.append(self.config)
        self.view = window.open_file(filename)
        yield {"condition": lambda: not self.view.is_loading(), "timeout": TIMEOUT_TIME}
        self.assertTrue(self.wm._configs.syntax_supported(self.view))
        self.init_view_settings()
        found_document_sync_listener = False
        for listener in view_event_listeners[self.view.id()]:
            if isinstance(listener, DocumentSyncListener):
                # Bug in ST3? Either that, or CI runs with ST window not in focus and that makes ST3 not trigger some
                # events like on_load_async, on_activated, on_deactivated. That makes things not properly initialize on
                # opening file (manager missing in DocumentSyncListener)
                # Revisit this once we're on ST4.
                sublime.set_timeout_async(listener.on_activated_async)
                found_document_sync_listener = True
                break
        self.assertTrue(found_document_sync_listener)
        yield {
            "condition": lambda: self.wm.get_session(self.config.name, self.view.file_name()) is not None,
            "timeout": TIMEOUT_TIME}
        self.session = self.wm.get_session(self.config.name, self.view.file_name())
        self.assertIsNotNone(self.session)
        self.assertEqual(self.session.config.name, self.config.name)
        yield {"condition": lambda: self.session.state == ClientStates.READY, "timeout": TIMEOUT_TIME}
        yield from self.await_boilerplate_begin()

    def get_test_name(self) -> str:
        return "testfile"

    def get_test_server_capabilities(self) -> dict:
        return basic_responses["initialize"]

    def get_test_session_type(self) -> int:
        return SessionType.Stdio

    def init_view_settings(self) -> None:
        s = self.view.settings().set
        s("auto_complete_selector", SUPPORTED_SCOPE)
        s("ensure_newline_at_eof_on_save", False)
        s("rulers", [])
        s("tab_size", 4)
        s("translate_tabs_to_spaces", False)
        s("word_wrap", False)
        s("lsp_format_on_save", False)

    def await_message(self, method: str) -> 'Generator':
        self.assertIsNotNone(self.session)
        assert self.session  # mypy
        promise = YieldPromise()

        def handler(params: 'Any') -> None:
            assert params is None
            promise.fulfill()

        def error_handler(params: 'Any') -> None:
            debug("Got error:", params, "awaiting timeout :(")

        self.session.client.send_request(Request("$test/getReceived", {"method": method}), handler, error_handler)
        yield {"condition": promise, "timeout": TIMEOUT_TIME}

    def set_response(self, method: str, response: 'Any') -> None:
        self.assertIsNotNone(self.session)
        assert self.session  # mypy
        self.session.client.send_notification(
            Notification("$test/setResponse", {"method": method, "response": response}))

    def await_boilerplate_begin(self) -> 'Generator':
        yield from self.await_message("initialize")
        yield from self.await_message("initialized")
        yield from self.await_message("textDocument/didOpen")

    def await_boilerplate_end(self) -> 'Generator':
        self.wm.end_config_sessions(self.config.name)  # TODO: Shouldn't this be automatic once the last view closes?
        if self.session:
            yield lambda: self.session.client is None

    def await_clear_view_and_save(self) -> 'Generator':
        assert self.view  # type: Optional[sublime.View]
        self.view.run_command("select_all")
        self.view.run_command("left_delete")
        self.view.run_command("save")
        yield from self.await_message("textDocument/didChange")
        yield from self.await_message("textDocument/didSave")

    def await_view_change(self, expected_change_count: int) -> 'Generator':
        assert self.view  # type: Optional[sublime.View]

        def condition() -> bool:
            nonlocal self
            nonlocal expected_change_count
            assert self.view
            v = self.view
            return v.change_count() == expected_change_count

        yield {"condition": condition, "timeout": TIMEOUT_TIME}

    def insert_characters(self, characters: str) -> int:
        assert self.view  # type: Optional[sublime.View]
        self.view.run_command("insert", {"characters": characters})
        return self.view.change_count()

    def tearDown(self) -> 'Generator':
        yield from self.await_boilerplate_end()
        super().tearDown()

    def doCleanups(self) -> 'Generator':
        # restore the user's configs
        close_test_view(self.view)
        remove_config(self.config)
        self.wm._configs.all.remove(self.config)
        yield from super().doCleanups()
<commit_msg>Wait for DocumentSyncListener to be created<commit_after>from LSP.plugin.core.documents import DocumentSyncListener
from LSP.plugin.core.logging import debug
from LSP.plugin.core.protocol import Notification, Request, WorkspaceFolder
from LSP.plugin.core.registry import windows
from LSP.plugin.core.sessions import Session
from LSP.plugin.core.settings import client_configs
from LSP.plugin.core.types import ClientConfig, LanguageConfig, ClientStates
from os import environ
from os.path import dirname
from os.path import join
from sublime_plugin import view_event_listeners
from test_mocks import basic_responses
from unittesting import DeferrableTestCase
import sublime

CI = any(key in environ for key in ("TRAVIS", "CI", "GITHUB_ACTIONS"))

project_path = dirname(__file__)
test_file_path = join(project_path, "testfile.txt")
workspace_folders = [WorkspaceFolder.from_path(project_path)]
TIMEOUT_TIME = 10000 if CI else 2000
SUPPORTED_SCOPE = "text.plain"
SUPPORTED_SYNTAX = "Packages/Text/Plain text.tmLanguage"
text_language = LanguageConfig("text", [SUPPORTED_SCOPE], [SUPPORTED_SYNTAX])
text_config = ClientConfig("textls", [], None, languages=[text_language])

try:
    from typing import Dict, List, Callable, Any, Optional, Generator
    assert Dict and Callable and List and Any and Optional and Generator
except ImportError:
    pass

class YieldPromise:

    __slots__ = ('__done')

    def __init__(self) -> None:
        self.__done = False

    def __call__(self) -> bool:
        return self.__done

    def fulfill(self) -> None:
        assert not self.__done
        self.__done = True

def make_stdio_test_config() -> ClientConfig:
    return ClientConfig(
        name="TEST",
        binary_args=["python3", join("$packages", "LSP", "tests", "server.py")],
        tcp_port=None,
        languages=[LanguageConfig(
            language_id="txt",
            scopes=[SUPPORTED_SCOPE],
            syntaxes=[SUPPORTED_SYNTAX])],
        enabled=True)

def add_config(config):
    client_configs.all.append(config)

def remove_config(config):
    client_configs.all.remove(config)

def close_test_view(view: sublime.View):
    if view:
        view.set_scratch(True)
        view.close()

def expand(s: str, w: sublime.Window) -> str:
    return sublime.expand_variables(s, w.extract_variables())

class SessionType:
    Stdio = 1
    TcpCreate = 2
    TcpConnectExisting = 3

class TextDocumentTestCase(DeferrableTestCase):

    def __init__(self, *args: 'Any', **kwargs: 'Any') -> None:
        super().__init__(*args, **kwargs)
        self.session = None  # type: Optional[Session]
        self.config = make_stdio_test_config()

    def setUp(self) -> 'Generator':
        super().setUp()
        test_name = self.get_test_name()
        server_capabilities = self.get_test_server_capabilities()
        session_type = self.get_test_session_type()
        self.assertTrue(test_name)
        self.assertTrue(server_capabilities)
        if session_type == SessionType.Stdio:
            pass
        elif session_type == SessionType.TcpCreate:
            # TODO: make the test server do TCP
            pass
        elif session_type == SessionType.TcpConnectExisting:
            # TODO
            pass
        else:
            self.assertFalse(True)
        window = sublime.active_window()
        self.assertTrue(window)
        filename = expand(join("$packages", "LSP", "tests", "{}.txt".format(test_name)), window)
        self.config.init_options["serverResponse"] = server_capabilities
        add_config(self.config)
        self.wm = windows.lookup(window)
        self.wm._configs.all.append(self.config)
        self.view = window.open_file(filename)
        yield {"condition": lambda: not self.view.is_loading(), "timeout": TIMEOUT_TIME}
        self.assertTrue(self.wm._configs.syntax_supported(self.view))
        self.init_view_settings()
        yield {"condition": self.ensure_document_listener_created, "timeout": TIMEOUT_TIME}
        yield {
            "condition": lambda: self.wm.get_session(self.config.name, self.view.file_name()) is not None,
            "timeout": TIMEOUT_TIME}
        self.session = self.wm.get_session(self.config.name, self.view.file_name())
        self.assertIsNotNone(self.session)
        self.assertEqual(self.session.config.name, self.config.name)
        yield {"condition": lambda: self.session.state == ClientStates.READY, "timeout": TIMEOUT_TIME}
        yield from self.await_boilerplate_begin()

    def get_test_name(self) -> str:
        return "testfile"

    def get_test_server_capabilities(self) -> dict:
        return basic_responses["initialize"]

    def get_test_session_type(self) -> int:
        return SessionType.Stdio

    def init_view_settings(self) -> None:
        s = self.view.settings().set
        s("auto_complete_selector", SUPPORTED_SCOPE)
        s("ensure_newline_at_eof_on_save", False)
        s("rulers", [])
        s("tab_size", 4)
        s("translate_tabs_to_spaces", False)
        s("word_wrap", False)
        s("lsp_format_on_save", False)

    def ensure_document_listener_created(self) -> bool:
        assert self.view
        # Bug in ST3? Either that, or CI runs with ST window not in focus and that makes ST3 not trigger some
        # events like on_load_async, on_activated, on_deactivated. That makes things not properly initialize on
        # opening file (manager missing in DocumentSyncListener)
        # Revisit this once we're on ST4.
        for listener in view_event_listeners[self.view.id()]:
            if isinstance(listener, DocumentSyncListener):
                sublime.set_timeout_async(listener.on_activated_async)
                return True
        return False

    def await_message(self, method: str) -> 'Generator':
        self.assertIsNotNone(self.session)
        assert self.session  # mypy
        promise = YieldPromise()

        def handler(params: 'Any') -> None:
            assert params is None
            promise.fulfill()

        def error_handler(params: 'Any') -> None:
            debug("Got error:", params, "awaiting timeout :(")

        self.session.client.send_request(Request("$test/getReceived", {"method": method}), handler, error_handler)
        yield {"condition": promise, "timeout": TIMEOUT_TIME}

    def set_response(self, method: str, response: 'Any') -> None:
        self.assertIsNotNone(self.session)
        assert self.session  # mypy
        self.session.client.send_notification(
            Notification("$test/setResponse", {"method": method, "response": response}))

    def await_boilerplate_begin(self) -> 'Generator':
        yield from self.await_message("initialize")
        yield from self.await_message("initialized")
        yield from self.await_message("textDocument/didOpen")

    def await_boilerplate_end(self) -> 'Generator':
        self.wm.end_config_sessions(self.config.name)  # TODO: Shouldn't this be automatic once the last view closes?
        if self.session:
            yield lambda: self.session.client is None

    def await_clear_view_and_save(self) -> 'Generator':
        assert self.view  # type: Optional[sublime.View]
        self.view.run_command("select_all")
        self.view.run_command("left_delete")
        self.view.run_command("save")
        yield from self.await_message("textDocument/didChange")
        yield from self.await_message("textDocument/didSave")

    def await_view_change(self, expected_change_count: int) -> 'Generator':
        assert self.view  # type: Optional[sublime.View]

        def condition() -> bool:
            nonlocal self
            nonlocal expected_change_count
            assert self.view
            v = self.view
            return v.change_count() == expected_change_count

        yield {"condition": condition, "timeout": TIMEOUT_TIME}

    def insert_characters(self, characters: str) -> int:
        assert self.view  # type: Optional[sublime.View]
        self.view.run_command("insert", {"characters": characters})
        return self.view.change_count()

    def tearDown(self) -> 'Generator':
        yield from self.await_boilerplate_end()
        super().tearDown()

    def doCleanups(self) -> 'Generator':
        # restore the user's configs
        close_test_view(self.view)
        remove_config(self.config)
        self.wm._configs.all.remove(self.config)
        yield from super().doCleanups()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
"""smoke
    Runs a smoke test of the jahmon installation on mini-mon by ensuring metrics are flowing and creating a new
    notification, alarm and that the Threshold Engine changes the state of the alarm.
    This requires the mon CLI and must be run on one of the mini-mon VMs. Tested running on kafka VM.
    Get it by following the instructions on https://wiki.hpcloud.net/display/iaas/Monitoring+CLI.
    If you want to see the notification, you must install postfix on the kakfa VM, configure it to be local, and
    modify /etc/mon/notification.yaml to use localhost for the email server, then restart

    TODO:
        1. Add check of alarm history when that works
        2. Add check of notification history when that is implemented
        3. Add check of mail getting to root when postfix is added mini-mon. This script will have to run on the kafka VM
"""

from __future__ import print_function
import json
import sys
import os
import subprocess
import time

# export OS_AUTH_TOKEN=82510970543135
# export OS_NO_CLIENT_AUTH=1
# export MON_API_URL=http://192.168.10.4:8080/v2.0/

os.environ["OS_AUTH_TOKEN"] = "82510970543135"
os.environ["OS_NO_CLIENT_AUTH"] = "1"
os.environ["MON_API_URL"] = "http://192.168.10.4:8080/v2.0/"

def get_alarm_state(alarm_id):
    stdout = run_mon_cli(["mon", "--json", "alarm-show", alarm_id])
    response_json = json.loads(stdout)
    return response_json['state']

def check_alarm_history(alarm_id):
    print('Checking Alarm History')
    # Make take a little bit of time for Alarm history to flow all the way through
    for x in range(0, 10):
        stdout = run_mon_cli(["mon", "--json", "alarm-history", alarm_id])
        response_json = json.loads(stdout)
        if len(response_json) > 0:
            break
        time.sleep(4)

    result = True
    if not check_expected(1, len(response_json), 'number of history entries'):
        return False
    alarm_json = response_json[0]
    if not check_expected('UNDETERMINED', alarm_json['old_state'], 'old_state'):
        result = False
    if not check_expected('ALARM', alarm_json['new_state'], 'new_state'):
        result = False
    if not check_expected(alarm_id, alarm_json['alarm_id'], 'alarm_id'):
        result = False
    if result:
        print("Alarm History is OK")
    return result

def check_expected(expected, actual, what):
    if (expected == actual):
        return True
    print("Incorrect value for alarm history " + what + " expected '" + str(expected) + "' but was '" + str(actual) + "'", file=sys.stderr)
    return False

def create_alarm(name, expression, notification_method_id, description=None):
    args = ["mon", "alarm-create"]
    if (description):
            args.append("--description")
            args.append(description)
    args.append("--alarm-actions")
    args.append(notification_method_id)
    args.append("--ok-actions")
    args.append(notification_method_id)
    args.append("--undetermined-actions")
    args.append(notification_method_id)
    args.append(name)
    args.append(expression)
    print("Creating alarm")
    stdout = run_mon_cli(args)
    response_json = json.loads(stdout)

    # Parse out id
    alarm_id = response_json['id']
    return alarm_id

def get_metrics(name, dimensions):
    print("Getting metrics for " + name + str(dimensions))
    dimensions_arg = ""
    for key, value in dimensions.iteritems():
        if dimensions_arg != "":
            dimensions_arg = dimensions_arg + ","
        dimensions_arg = dimensions_arg + key + "=" + value
    stdout = run_mon_cli(["mon", "--json", "measurement-list", "--dimensions", dimensions_arg, name, "00"])
    return json.loads(stdout)

def run_mon_cli(args):
    try:
        stdout = subprocess.check_output(args)
        return stdout
    except subprocess.CalledProcessError as e:
        print(e, file=sys.stderr)
        sys.exit(1)

def create_notification(notification_name, notification_email_addr):
    print("Creating notification")
    stdout = run_mon_cli(["mon", "notification-create", notification_name, "EMAIL", notification_email_addr])
    response_json = json.loads(stdout)

    # Parse out id
    notification_method_id = response_json['id']
    return notification_method_id

def find_id_for_name(object_json, name):
    for obj in object_json:
        this_name = obj['name']
        if name == this_name:
            return obj['id']
    return None

def cleanup(notification_name, alarm_name):
    # Delete our alarm if it already exists
    alarm_json = json.loads(run_mon_cli(["mon", "--json", "alarm-list"]))
    alarm_id = find_id_for_name(alarm_json, alarm_name)
    if alarm_id:
        run_mon_cli(["mon", "alarm-delete", alarm_id])
    # Delete our notification if it already exists
    notification_json = json.loads(run_mon_cli(["mon", "--json", "notification-list"]))
    notification_id = find_id_for_name(notification_json, notification_name)
    if notification_id:
        run_mon_cli(["mon", "notification-delete", notification_id])

def main():
    notification_name = "Jahmon Smoke Test"
    notification_email_addr = "root@kafka"
    alarm_name = "high cpu and load"
    metric_name = "cpu_system_perc"
    metric_dimensions = {"hostname":"thresh"}
    cleanup(notification_name, alarm_name)

    # Query how many metrics there are for the Alarm
    metric_json = get_metrics(metric_name, metric_dimensions)
    if len(metric_json) == 0:
        print("No measurements received for metric " + metric_name + str(metric_dimensions), file=sys.stderr)
        sys.exit(1)

    initial_num_metrics = len(metric_json[0]['measurements'])

    # Create Notification through CLI
    notification_method_id = create_notification(notification_name, notification_email_addr)
    # Create Alarm through CLI
    alarm_id = create_alarm(alarm_name, "max(cpu_system_perc) > 1 and max(load_avg_1_min{hostname=thresh}) > 3", notification_method_id, "System CPU Utilization exceeds 1% and Load exeeds 3 per measurement period")
    state = get_alarm_state(alarm_id)
    # Ensure it is created in the right state
    if state != 'UNDETERMINED':
        print("Wrong initial alarm state, expected UNDETERMINED but was " + state)
        sys.exit(1)
    # Wait for it to
    print("Waiting for alarm to change state")
    change_time = 0
    for x in range(0, 250):
        time.sleep(1)
        state = get_alarm_state(alarm_id)
        if state != 'UNDETERMINED':
            print("Alarm state changed in " + str(x) + " seconds")
            change_time = x
            break

    if state != 'ALARM':
        print("Wrong final state, expected ALARM but was " + state, file=sys.stderr)
        sys.exit(1)
    print("Final state of alarm was " + state)
    # If the alarm changes state too fast, then there isn't time for the new metric to arrive.
    # Unlikely, but it has been seen
    if change_time < 30:
        time.sleep(30 - change_time)
        change_time = 30
    metric_json = get_metrics(metric_name, metric_dimensions)
    final_num_metrics = len(metric_json[0]['measurements'])
    if final_num_metrics <= initial_num_metrics:
        print("No new metrics received", file=sys.stderr)
        sys.exit(1)
    print("Received " + str(final_num_metrics - initial_num_metrics) + " metrics in " + str(change_time) + " seconds")
    if not check_alarm_history(alarm_id):
        sys.exit(1)

    return 0

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>Updated TODO list<commit_after>#!/usr/bin/env python
#
"""smoke
    Runs a smoke test of the jahmon installation on mini-mon by ensuring metrics are flowing and creating a new
    notification, alarm and that the Threshold Engine changes the state of the alarm.
    This requires the mon CLI and must be run on one of the mini-mon VMs. Tested running on kafka VM.
    Get it by following the instructions on https://wiki.hpcloud.net/display/iaas/Monitoring+CLI.
    If you want to see the notification, you must install postfix on the kakfa VM, configure it to be local, and
    modify /etc/mon/notification.yaml to use localhost for the email server, then restart

    TODO:
        1. Add check of notification history when that is implemented
        2. Add check of mail getting to root when postfix is added mini-mon. This script will have to run on the kafka VM
"""

from __future__ import print_function
import json
import sys
import os
import subprocess
import time

# export OS_AUTH_TOKEN=82510970543135
# export OS_NO_CLIENT_AUTH=1
# export MON_API_URL=http://192.168.10.4:8080/v2.0/

os.environ["OS_AUTH_TOKEN"] = "82510970543135"
os.environ["OS_NO_CLIENT_AUTH"] = "1"
os.environ["MON_API_URL"] = "http://192.168.10.4:8080/v2.0/"

def get_alarm_state(alarm_id):
    stdout = run_mon_cli(["mon", "--json", "alarm-show", alarm_id])
    response_json = json.loads(stdout)
    return response_json['state']

def check_alarm_history(alarm_id):
    print('Checking Alarm History')
    # Make take a little bit of time for Alarm history to flow all the way through
    for x in range(0, 10):
        stdout = run_mon_cli(["mon", "--json", "alarm-history", alarm_id])
        response_json = json.loads(stdout)
        if len(response_json) > 0:
            break
        time.sleep(4)

    result = True
    if not check_expected(1, len(response_json), 'number of history entries'):
        return False
    alarm_json = response_json[0]
    if not check_expected('UNDETERMINED', alarm_json['old_state'], 'old_state'):
        result = False
    if not check_expected('ALARM', alarm_json['new_state'], 'new_state'):
        result = False
    if not check_expected(alarm_id, alarm_json['alarm_id'], 'alarm_id'):
        result = False
    if result:
        print("Alarm History is OK")
    return result

def check_expected(expected, actual, what):
    if (expected == actual):
        return True
    print("Incorrect value for alarm history " + what + " expected '" + str(expected) + "' but was '" + str(actual) + "'", file=sys.stderr)
    return False

def create_alarm(name, expression, notification_method_id, description=None):
    args = ["mon", "alarm-create"]
    if (description):
            args.append("--description")
            args.append(description)
    args.append("--alarm-actions")
    args.append(notification_method_id)
    args.append("--ok-actions")
    args.append(notification_method_id)
    args.append("--undetermined-actions")
    args.append(notification_method_id)
    args.append(name)
    args.append(expression)
    print("Creating alarm")
    stdout = run_mon_cli(args)
    response_json = json.loads(stdout)

    # Parse out id
    alarm_id = response_json['id']
    return alarm_id

def get_metrics(name, dimensions):
    print("Getting metrics for " + name + str(dimensions))
    dimensions_arg = ""
    for key, value in dimensions.iteritems():
        if dimensions_arg != "":
            dimensions_arg = dimensions_arg + ","
        dimensions_arg = dimensions_arg + key + "=" + value
    stdout = run_mon_cli(["mon", "--json", "measurement-list", "--dimensions", dimensions_arg, name, "00"])
    return json.loads(stdout)

def run_mon_cli(args):
    try:
        stdout = subprocess.check_output(args)
        return stdout
    except subprocess.CalledProcessError as e:
        print(e, file=sys.stderr)
        sys.exit(1)

def create_notification(notification_name, notification_email_addr):
    print("Creating notification")
    stdout = run_mon_cli(["mon", "notification-create", notification_name, "EMAIL", notification_email_addr])
    response_json = json.loads(stdout)

    # Parse out id
    notification_method_id = response_json['id']
    return notification_method_id

def find_id_for_name(object_json, name):
    for obj in object_json:
        this_name = obj['name']
        if name == this_name:
            return obj['id']
    return None

def cleanup(notification_name, alarm_name):
    # Delete our alarm if it already exists
    alarm_json = json.loads(run_mon_cli(["mon", "--json", "alarm-list"]))
    alarm_id = find_id_for_name(alarm_json, alarm_name)
    if alarm_id:
        run_mon_cli(["mon", "alarm-delete", alarm_id])
    # Delete our notification if it already exists
    notification_json = json.loads(run_mon_cli(["mon", "--json", "notification-list"]))
    notification_id = find_id_for_name(notification_json, notification_name)
    if notification_id:
        run_mon_cli(["mon", "notification-delete", notification_id])

def main():
    notification_name = "Jahmon Smoke Test"
    notification_email_addr = "root@kafka"
    alarm_name = "high cpu and load"
    metric_name = "cpu_system_perc"
    metric_dimensions = {"hostname":"thresh"}
    cleanup(notification_name, alarm_name)

    # Query how many metrics there are for the Alarm
    metric_json = get_metrics(metric_name, metric_dimensions)
    if len(metric_json) == 0:
        print("No measurements received for metric " + metric_name + str(metric_dimensions), file=sys.stderr)
        sys.exit(1)

    initial_num_metrics = len(metric_json[0]['measurements'])

    # Create Notification through CLI
    notification_method_id = create_notification(notification_name, notification_email_addr)
    # Create Alarm through CLI
    alarm_id = create_alarm(alarm_name, "max(cpu_system_perc) > 1 and max(load_avg_1_min{hostname=thresh}) > 3", notification_method_id, "System CPU Utilization exceeds 1% and Load exeeds 3 per measurement period")
    state = get_alarm_state(alarm_id)
    # Ensure it is created in the right state
    if state != 'UNDETERMINED':
        print("Wrong initial alarm state, expected UNDETERMINED but was " + state)
        sys.exit(1)
    # Wait for it to
    print("Waiting for alarm to change state")
    change_time = 0
    for x in range(0, 250):
        time.sleep(1)
        state = get_alarm_state(alarm_id)
        if state != 'UNDETERMINED':
            print("Alarm state changed in " + str(x) + " seconds")
            change_time = x
            break

    if state != 'ALARM':
        print("Wrong final state, expected ALARM but was " + state, file=sys.stderr)
        sys.exit(1)
    print("Final state of alarm was " + state)
    # If the alarm changes state too fast, then there isn't time for the new metric to arrive.
    # Unlikely, but it has been seen
    if change_time < 30:
        time.sleep(30 - change_time)
        change_time = 30
    metric_json = get_metrics(metric_name, metric_dimensions)
    final_num_metrics = len(metric_json[0]['measurements'])
    if final_num_metrics <= initial_num_metrics:
        print("No new metrics received", file=sys.stderr)
        sys.exit(1)
    print("Received " + str(final_num_metrics - initial_num_metrics) + " metrics in " + str(change_time) + " seconds")
    if not check_alarm_history(alarm_id):
        sys.exit(1)

    return 0

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>