<commit_before>#!python
from __future__ import print_function
import sys
import re
import inspect
import optparse
import shutil
from StringIO import StringIO

def get_options():
	parser = optparse.OptionParser()
	options, args = parser.parse_args()
	try:
		options.filename = args.pop()
	except IndexError:
		parser.error("Filename required")
	return options

# each of the replacement functions should have a docstring
#  which is a regular expression to be matched.

def replace_external_link(matcher):
	r"\[(?P<href>(?P<scheme>\w+)\://.+?) (?P<name>.+?)\]"
	return '`{name} <{href}>`_'.format(**matcher.groupdict())

def replace_wiki_link(matcher):
	r"\[wiki\:(?P<ref>.+?) (?P<name>.+?)\]"
	return '`{name} <TODO-fix wiki target {ref}>`_'.format(**matcher.groupdict())

# character array indexed by level for characters
heading_characters = [None, '*', '=', '-', '^']

def replace_headings(matcher):
	r"^(?P<level>=+) (?P<name>.*) (?P=level)$"
	level = len(matcher.groupdict()['level'])
	char = heading_characters[level]
	name = matcher.groupdict()['name']
	return '\n'.join([name, char*len(name)])

def indent(block):
	add_indent = lambda s: '    ' + s
	lines = StringIO(block)
	i_lines = map(add_indent, lines)
	return ''.join(i_lines)

def replace_inline_code(matcher):
	r"\{\{\{(?P[^\n]*?)\}\}\}"
	return '``{code}``'.format(**matcher.groupdict())

def replace_code_block(matcher):
	r"\{\{\{\n(?P(.|\n)*?)^\}\}\}"
	return '::\n\n' + indent(matcher.groupdict()['code'])

replacements = [func for name, func in globals().items() if name.startswith('replace_')]

def convert_file(filename):
	shutil.copy(filename, filename+'.bak')
	text = open(filename).read()
	new_text = text
	for repl in replacements:
		pattern = re.compile(inspect.getdoc(repl), re.MULTILINE)
		new_text = pattern.sub(repl, new_text)

	open(filename, 'w').write(new_text)
	print("done")

def handle_command_line():
	options = get_options()
	convert_file(options.filename)

if __name__ == '__main__':
	handle_command_line()
<commit_msg>Added support for grabbing the content directly from a trac URL. Added substitutions for content encountered in CherryPy trac docs<commit_after>#!python
from __future__ import print_function
import sys
import re
import inspect
import optparse
import shutil
import urllib2
from StringIO import StringIO

def get_options():
	global options
	parser = optparse.OptionParser()
	parser.add_option('-U', '--url', help="Trac URL from which to retrieve source")
	options, args = parser.parse_args()
	try:
		options.filename = args.pop()
	except IndexError:
		parser.error("Filename required")

# each of the replacement functions should have a docstring
#  which is a regular expression to be matched.

def replace_external_link(matcher):
	r"\[(?P<href>(?P<scheme>\w+)\://.+?) (?P<name>.+?)\]"
	return '`{name} <{href}>`_'.format(**matcher.groupdict())

def replace_wiki_link(matcher):
	r"\[wiki\:(?P<ref>.+?) (?P<name>.+?)\]"
	return '`{name} <TODO-fix wiki target {ref}>`_'.format(**matcher.groupdict())

# character array indexed by level for characters
heading_characters = [None, '*', '=', '-', '^']

def replace_headings(matcher):
	r"^(?P<level>=+) (?P<name>.*) (?P=level)$"
	level = len(matcher.groupdict()['level'])
	char = heading_characters[level]
	name = matcher.groupdict()['name']
	return '\n'.join([name, char*len(name)])

def indent(block):
	add_indent = lambda s: '    ' + s
	lines = StringIO(block)
	i_lines = map(add_indent, lines)
	return ''.join(i_lines)

def replace_inline_code(matcher):
	r"\{\{\{(?P[^\n]*?)\}\}\}"
	return '``{code}``'.format(**matcher.groupdict())

def replace_code_block(matcher):
	r"\{\{\{\n(?P(.|\n)*?)^\}\}\}"
	return '::\n\n' + indent(matcher.groupdict()['code'])

def replace_page_outline(matcher):
	r"\[\[PageOutline\]\]\n"
	return ''

# a number of the files end in
"""{{{
#!html

Older versions

}}}""" # and everything after is garbage, so just remove it.
def remove_2x_compat_notes(matcher):
	r"\{\{\{\n#!html\n<h2(.|\n)*"
	return ''

replacements = [remove_2x_compat_notes] + \
	[func for name, func in globals().items() if name.startswith('replace_')]

def normalize_linebreaks(text):
	return text.replace('\r\n', '\n')

def convert_file():
	filename = options.filename
	if options.url:
		text = urllib2.urlopen(options.url).read()
		text = normalize_linebreaks(text)
	else:
		shutil.copy(filename, filename+'.bak')
		text = open(filename).read()
	# iterate over each of the replacements and execute it
	new_text = text
	for repl in replacements:
		pattern = re.compile(inspect.getdoc(repl), re.MULTILINE)
		new_text = pattern.sub(repl, new_text)

	open(filename, 'w').write(new_text)
	print("done")

def handle_command_line():
	get_options()
	convert_file()

if __name__ == '__main__':
	handle_command_line()
<|endoftext|>


<commit_before>import os

from moviepy.video.VideoClip import VideoClip
from moviepy.audio.io.AudioFileClip import AudioFileClip
from moviepy.Clip import Clip
from moviepy.video.io.ffmpeg_reader import FFMPEG_VideoReader

class VideoFileClip(VideoClip):

    """

    A video clip originating from a movie file. For instance: ::

        >>> clip = VideofileClip("myHolidays.mp4")
        >>> clip2 = VideofileClip("myMaskVideo.avi")

    Parameters
    ------------

    filename:
      The name of the video file. It can have any extension supported
      by ffmpeg: .ogv, .mp4, .mpeg, .avi, .mov etc.

    has_mask:
      Set this to 'True' if there is a mask included in the videofile.
      Video files rarely contain masks, but some video codecs enable
      that. For istance if you have a MoviePy VideoClip with a mask you
      can save it to a videofile with a mask. (see also
      ``VideoClip.write_videofile`` for more details).

    audio:
      Set to `False` if the clip doesn't have any audio or if you do not
      wish to read the audio.

    Attributes
    -----------

    filename:
      Name of the original video file.

    fps:
      Frames per second in the original file.

    Read docstrings for Clip() and VideoClip() for other, more generic, attributes.

    """

    def __init__(self, filename, has_mask=False,
                 audio=True, audio_buffersize = 200000,
                 audio_fps=44100, audio_nbytes=2, verbose=False):

        VideoClip.__init__(self)

        # Make a reader
        pix_fmt= "rgba" if has_mask else "rgb24"
        reader = FFMPEG_VideoReader(filename, pix_fmt=pix_fmt)
        self.reader = reader
        # Make some of the reader's attributes accessible from the clip
        self.duration = self.reader.duration
        self.end = self.reader.duration

        self.fps = self.reader.fps
        self.size = self.reader.size

        self.filename = self.reader.filename

        if has_mask:

            self.make_frame = lambda t: reader.get_frame(t)[:,:,:3]
            mask_mf =  lambda t: reader.get_frame(t)[:,:,3]/255.0
            self.mask = (VideoClip(ismask = True, make_frame = mask_mf)
                       .set_duration(self.duration))
            self.mask.fps = self.fps

        else:

            self.make_frame = lambda t: reader.get_frame(t)

        # Make a reader for the audio, if any.
        if audio and self.reader.infos['audio_found']:

            self.audio = AudioFileClip(filename,
                                       buffersize= audio_buffersize,
                                       fps = audio_fps,
                                       nbytes = audio_nbytes)

    def __del__(self):
      """ Close/delete the internal reader. """
      del self.reader
<commit_msg>Fixed typo #375<commit_after>import os

from moviepy.video.VideoClip import VideoClip
from moviepy.audio.io.AudioFileClip import AudioFileClip
from moviepy.Clip import Clip
from moviepy.video.io.ffmpeg_reader import FFMPEG_VideoReader

class VideoFileClip(VideoClip):

    """

    A video clip originating from a movie file. For instance: ::

        >>> clip = VideoFileClip("myHolidays.mp4")
        >>> clip2 = VideoFileClip("myMaskVideo.avi")

    Parameters
    ------------

    filename:
      The name of the video file. It can have any extension supported
      by ffmpeg: .ogv, .mp4, .mpeg, .avi, .mov etc.

    has_mask:
      Set this to 'True' if there is a mask included in the videofile.
      Video files rarely contain masks, but some video codecs enable
      that. For istance if you have a MoviePy VideoClip with a mask you
      can save it to a videofile with a mask. (see also
      ``VideoClip.write_videofile`` for more details).

    audio:
      Set to `False` if the clip doesn't have any audio or if you do not
      wish to read the audio.

    Attributes
    -----------

    filename:
      Name of the original video file.

    fps:
      Frames per second in the original file.

    Read docstrings for Clip() and VideoClip() for other, more generic, attributes.

    """

    def __init__(self, filename, has_mask=False,
                 audio=True, audio_buffersize = 200000,
                 audio_fps=44100, audio_nbytes=2, verbose=False):

        VideoClip.__init__(self)

        # Make a reader
        pix_fmt= "rgba" if has_mask else "rgb24"
        reader = FFMPEG_VideoReader(filename, pix_fmt=pix_fmt)
        self.reader = reader
        # Make some of the reader's attributes accessible from the clip
        self.duration = self.reader.duration
        self.end = self.reader.duration

        self.fps = self.reader.fps
        self.size = self.reader.size

        self.filename = self.reader.filename

        if has_mask:

            self.make_frame = lambda t: reader.get_frame(t)[:,:,:3]
            mask_mf =  lambda t: reader.get_frame(t)[:,:,3]/255.0
            self.mask = (VideoClip(ismask = True, make_frame = mask_mf)
                       .set_duration(self.duration))
            self.mask.fps = self.fps

        else:

            self.make_frame = lambda t: reader.get_frame(t)

        # Make a reader for the audio, if any.
        if audio and self.reader.infos['audio_found']:

            self.audio = AudioFileClip(filename,
                                       buffersize= audio_buffersize,
                                       fps = audio_fps,
                                       nbytes = audio_nbytes)

    def __del__(self):
      """ Close/delete the internal reader. """
      del self.reader
<|endoftext|>


<commit_before>import warnings
from .base import Renderer
from ..exporter import Exporter

class VincentRenderer(Renderer):
    def open_figure(self, fig):
        self.chart = None
        self.figwidth = int(fig.get_figwidth() * fig.dpi)
        self.figheight = int(fig.get_figheight() * fig.dpi)

    def draw_line(self, data, coordinates, style):
        import vincent
        if coordinates != 'data':
            warnings.warn("Only data coordinates supported. Skipping this")
        linedata = {'x': data[:, 0],
                    'y': data[:, 1]}
        line = vincent.Line(linedata, iter_idx='x',
                            width=self.figwidth, height=self.figheight)
        line.scales['color'].range = [style['color']]

        # TODO: set the

        if self.chart is None:
            self.chart = line
        else:
            warnings.warn("Multiple plot elements not yet supported")

    def draw_markers(self, data, coordinates, style):
        import vincent
        if coordinates != 'data':
            warnings.warn("Only data coordinates supported. Skipping this")
        markerdata = {'x': data[:, 0],
                      'y': data[:, 1]}
        markers = vincent.Scatter(markerdata, iter_idx='x',
                                  width=self.figwidth, height=self.figheight)
        markers.scales['color'].range = [style['facecolor']]
        if self.chart is None:
            self.chart = markers
        else:
            warnings.warn("Multiple plot elements not yet supported")

def fig_to_vincent(fig):
    """Convert a matplotlib figure to a vincent object"""
    renderer = VincentRenderer()
    exporter = Exporter(renderer)
    exporter.run(fig)
    return renderer.chart
<commit_msg>add TODO notes<commit_after>import warnings
from .base import Renderer
from ..exporter import Exporter

class VincentRenderer(Renderer):
    def open_figure(self, fig):
        self.chart = None
        self.figwidth = int(fig.get_figwidth() * fig.dpi)
        self.figheight = int(fig.get_figheight() * fig.dpi)

    def draw_line(self, data, coordinates, style):
        import vincent  # only import if VincentRenderer is used
        if coordinates != 'data':
            warnings.warn("Only data coordinates supported. Skipping this")
        linedata = {'x': data[:, 0],
                    'y': data[:, 1]}
        line = vincent.Line(linedata, iter_idx='x',
                            width=self.figwidth, height=self.figheight)

        # TODO: respect the other style settings
        line.scales['color'].range = [style['color']]

        if self.chart is None:
            self.chart = line
        else:
            warnings.warn("Multiple plot elements not yet supported")

    def draw_markers(self, data, coordinates, style):
        import vincent  # only import if VincentRenderer is used
        if coordinates != 'data':
            warnings.warn("Only data coordinates supported. Skipping this")
        markerdata = {'x': data[:, 0],
                      'y': data[:, 1]}
        markers = vincent.Scatter(markerdata, iter_idx='x',
                                  width=self.figwidth, height=self.figheight)

        # TODO: respect the other style settings
        markers.scales['color'].range = [style['facecolor']]

        if self.chart is None:
            self.chart = markers
        else:
            warnings.warn("Multiple plot elements not yet supported")

def fig_to_vincent(fig):
    """Convert a matplotlib figure to a vincent object"""
    renderer = VincentRenderer()
    exporter = Exporter(renderer)
    exporter.run(fig)
    return renderer.chart
<|endoftext|>


<commit_before>from __future__ import print_function

import random
import warnings

from twisted.internet.defer import QueueUnderflow, Deferred, succeed

from spinoff.actor import Actor, actor, baseactor, ActorStopped, ActorNotRunning, ActorAlreadyStopped, ActorAlreadyRunning
from spinoff.util.pattern_matching import ANY, IS_INSTANCE
from spinoff.util.async import CancelledError
from spinoff.util.testing import deferred_result, assert_raises, assert_not_raises, assert_one_warning, MockActor, run, RootActor

warnings.simplefilter('always')

def test_base_actor_not_started():
    parent = MockActor()

    with assert_raises(ActorNotRunning):
        parent.send('whatev')

def test_base_actor():
    root, x = run(MockActor)

    msg = random.random()
    x.send(msg)
    assert x.clear() == [msg]

    x.pause()
    msg = random.random()
    x.send(msg)
    assert x.clear() == []

    x.resume()
    assert x.clear() == [msg]

    assert not root.messages

    x.stop()
    assert [('stopped', x)] == root.messages

def test_base_actor_error():
    @baseactor
    def B(self, message):
        raise MockException()
    _, b = run(B)
    with assert_not_raises(MockException):
        b.send('whatev')
    assert not b.d.called

    @actor
    def P(self):
        b = self.spawn(B)
        b.send('whatev')
        msg_d = self.get(('error', b, (IS_INSTANCE(MockException), ANY), True))
        assert msg_d.called
    run(P)

def test_failure_with_children():
    child_stopped = [False]

    @actor
    def Child(self):
        try:
            yield Deferred()
        except ActorStopped:
            child_stopped[0] = True

    @actor
    def A(self):
        self.spawn(Child)
        raise MockException()

    run(A, raise_only_asserts=True)

    assert child_stopped[0]

def test_actor_refuses_to_stop():
    mock_d = Deferred()

    @actor
    def A(self):
        try:
            yield mock_d
        except ActorStopped:
            pass
        yield succeed(None)
        assert False

    root, a = run(A)

    a.pause()
    a.stop()
    assert [('stopped', a, 'refused')] == root.messages, root.messages

def test_failure_while_stopping():
    mock_d = Deferred()
    print(mock_d)

    @actor
    def A(self):
        try:
            yield mock_d
        except ActorStopped:
            raise MockException()

    r, a = run(A)
    assert a.is_running
    with assert_not_raises(MockException):
        a.stop()
    assert len(r.messages) == 1 and r.messages[0][0:3] == ('stopped', a, 'unclean')

def test_connect_and_put():
    received_msg = []

    @actor
    def Mock(self):
        received_msg.append((yield self.get()))

    root, mock = run(Mock)

    c = actor(lambda self: self.put('msg-1'))()
    c._parent = root
    c.connect(mock)
    c.start()

    assert received_msg == ['msg-1'], received_msg

def test_flow():
    called = [0]

    mock_d = Deferred()

    @actor
    def Proc(self):
        called[0] += 1
        yield mock_d
        called[0] += 1

    proc = Proc()
    root = RootActor.spawn()
    proc._parent = root
    assert not called[0], "creating an actor should not automatically start the coroutine in it"

    proc.start()

    with assert_raises(ActorAlreadyRunning):
        proc.start()

    mock_d.callback(None)
    assert called[0] == 2, "the coroutine in an actor should complete as normal"
    assert not proc.is_alive
    assert root.messages == [('stopped', proc)]
    assert not proc.is_alive

def test_exception():
    mock_d = Deferred()
    exception_caught = [False]

    @actor
    def Y(self):
        try:
            yield mock_d
        except MockException:
            exception_caught[0] = True

    run(Y)
    mock_d.errback(MockException())
    assert exception_caught[0]

def test_failure():
    exc = MockException()

    @actor
    def A(self):
        raise exc

    root, a = run(A, raise_only_asserts=True)
    assert not a.is_alive

    assert [('error', a, (exc, ANY), False)] == root.messages

def test_yielding_a_non_deferred():
    @actor
    def Actor1(self):
        tmp = random.random()
        ret = yield tmp
        assert ret == tmp
    run(Actor1)

    @actor
    def Actor2(self):
        ret = yield
        assert ret is None
    run(Actor2)

def test_pending_failures_are_discarded_with_a_warning():
    mock_d = Deferred()

    @actor
    def X(self):
        yield mock_d

    root, p = run(X)
    p.pause()

    mock_d.errback(Exception())
    with assert_one_warning():
        p.stop()

def test_pausing_resuming_and_stopping():
    stopped = [False]

    mock_d = Deferred()
    retval = random.random()

    @actor
    def X(self):
        try:
            ret = yield mock_d
            assert ret == retval
        except ActorStopped:
            stopped[0] = True

    ### resuming when the async called has been fired
    root, proc = run(X)

    proc.pause()
    assert not proc.is_running
    assert proc.is_alive
    assert proc.is_paused

    with assert_raises(ActorNotRunning):
        proc.pause()

    mock_d.callback(retval)

    assert not proc.d.called, "a paused actor should not be resumed when the call it's waiting on completes"

    proc.resume()

    assert proc.d.called
    assert root.messages == [('stopped', proc)]

    ### resuming when the async call has NOT been fired
    mock_d = Deferred()
    root, proc2 = run(X)

    proc2.pause()
    proc2.resume()

    assert not stopped[0]
    assert not root.messages

    ### resuming when the async call has failed
    mock_d = Deferred()
    exception_caught = [False]

    @actor
    def Y(self):
        try:
            yield mock_d
        except MockException:
            exception_caught[0] = True

    root, x = run(Y)
    x.pause()
    mock_d.errback(MockException())
    x.resume()
    assert exception_caught[0]

    ### can't resume twice
    with assert_raises(ActorAlreadyRunning, "it should not be possible to resume an actor twice"):
        proc2.resume()

    ### stopping
    mock_d = Deferred()
    root, proc3 = run(X)

    proc3.stop()
    with assert_raises(ActorAlreadyStopped):
        proc3.stop()

    assert stopped[0]
    assert [('stopped', proc3)] == root.messages

    with assert_raises(ActorAlreadyStopped):
        proc3.start()
    with assert_raises(ActorAlreadyStopped):
        proc3.resume()

    ### stopping a paused actor
    mock_d = Deferred()
    root, proc4 = run(X)

    proc4.pause()
    proc4.stop()

    assert stopped[0]
    assert [('stopped', proc4)] == root.messages, root.messages

def test_stopping_cancels_the_deferred_on_hold():
    cancelled = [False]
    mock_d = Deferred(lambda _: cancelled.__setitem__(0, True))

    @actor
    def X(self):
        yield mock_d

    root, x = run(X)
    x.stop()

    assert cancelled[0]

def test_actor_does_not_have_to_catch_actorstopped():
    @actor
    def X(self):
        yield Deferred()
    root, proc = run(X)
    with assert_not_raises(ActorStopped):
        proc.stop()
    assert [('stopped', proc)] == root.messages

def test_actor_must_exit_after_being_stopped():
    # actor that violates the rule
    @actor
    def X(self):
        while True:
            try:
                yield Deferred()
            except ActorStopped:
                pass
    root, proc = run(X)
    proc.stop()
    assert [('stopped', proc, 'refused')] == root.messages, \
        "actor should not be allowed to continue working when stopped"

    # actor that complies with the rule
    @actor
    def Proc2(self):
        while True:
            try:
                yield Deferred()
            except ActorStopped:
                break
    root, proc2 = run(Proc2)
    proc2.stop()
    assert [('stopped', proc2)] == root.messages, root.messages

def test_actor_with_args():
    passed_values = [None, None]

    @actor
    def Proc(self, a, b):
        yield
        passed_values[:] = [a, b]

    run(Proc(1, b=2))
    assert passed_values == [1, 2]

def test_actor_doesnt_require_generator():
    @actor
    def Proc(self):
        pass

    root, proc = run(Proc)
    assert [('stopped', proc)] == root.messages

    @actor
    def Proc2(self):
        raise MockException()
    root, proc = run(Proc2, raise_only_asserts=True)

def test_get():
    def _make_getter(filter=None):
        @actor
        def ret(self):
            received_msg.append((yield self.get(filter=filter)))
        return ret

    ###
    received_msg = []
    root, x = run(_make_getter())
    x.send('foo')
    assert ['foo'] == received_msg

    ###
    received_msg = []
    tmp = random.random()
    root, x = run(_make_getter(('foo', ANY)))
    x.send(('foo', tmp))
    root.raise_errors(only_asserts=False)
    assert [tmp] == received_msg, received_msg

    ###
    received_msg = []
    tmp = random.random()
    root, c = run(_make_getter(('baz', ANY)))
    c.send(('foo', tmp))
    root.raise_errors(only_asserts=False)
    assert received_msg == []

    c.send(('baz', tmp))
    root.raise_errors(only_asserts=False)
    assert received_msg == [tmp]

def test_get_removes_message_from_inbox():
    @actor
    def X(self):
        yield self.get()
        msg_d = self.get()
        assert not msg_d.called

    root, x = run(X)
    x.send('whatev')

def test_inbox_underflow():
    @actor
    def GetTwice(self):
        self.get()
        with assert_raises(QueueUnderflow):
            self.get()
    run(GetTwice)

    @actor
    def GetThenCancelAndGetAgain(self):
        msg_d = self.get()
        msg_d.addErrback(lambda f: f.trap(CancelledError))
        msg_d.cancel()
        self.get()
    run(GetThenCancelAndGetAgain)

def test_spawn_child_actor():
    @actor
    def Child(self):
        yield Deferred()

    @actor
    def Parent(self):
        c = self.spawn(Child)
        assert c.parent == self

        c.stop()
        msg = deferred_result(self.get())
        assert msg == ('stopped', c), "child actor forced exit should be sent to its parent"

        c = self.spawn(run_with_error)
        msg = deferred_result(self.get())
        assert msg[:2] == ('error', c) and msg[3] is False and isinstance(msg[-2][0], MockException), \
            "child actor errors should be sent to its parent"

        for retval in [random.random(), None]:
            c = self.spawn(lambda self: retval)
            msg = deferred_result(self.get())
            assert ('stopped', c) == msg, "child actor return value is ignored %s" % repr(msg)

    run(Parent)

    chld_stopped = [False]
    arg = random.random()

    @actor
    def ChildWithArgs(self, foo):
        assert foo == arg
        try:
            yield Deferred()
        except ActorStopped:
            chld_stopped[0] = True

    @actor
    def Parent2(self):
        self.spawn(ChildWithArgs(arg))

    root, parent2 = run(Parent2, raise_only_asserts=False)

    assert chld_stopped[0]

    class Parent3(Actor):
        def __init__(self):
            super(Parent3, self).__init__()
            self.spawn(Child)

    root = RootActor.spawn()
    parent3 = Parent3()
    parent3._parent = root
    with assert_not_raises(ActorAlreadyRunning, "it should be possible for actors to spawn children in the constructor"):
        parent3.start()

def test_pausing_resuming_and_stopping_actor_with_children_does_the_same_with_children():
    children = []
    child_stopped = [False]

    @actor
    def Child(self):
        try:
            yield Deferred()
        except ActorStopped:
            child_stopped[0] = True

    @actor
    def Parent(self):
        children.append(self.spawn(Child))
        yield Deferred()
    root, a = run(Parent)

    a.pause()
    assert all(x.is_paused for x in children)

    a.resume()
    assert all(not x.is_paused for x in children)

    a.stop()
    assert all(not x.is_alive for x in children)
    assert child_stopped[0]

def test_pausing_and_stoping_actor_with_some_finished_children():
    @actor
    def Stillborn(self):
        yield

    child_stopped = [False]

    @actor
    def LongLivingChild(self):
        try:
            yield Deferred()
        except ActorStopped:
            child_stopped[0] = True

    mock_d = Deferred()

    @actor
    def Parent(self):
        self.spawn(Stillborn)
        self.spawn(LongLivingChild)
        yield mock_d

    root, a = run(Parent)

    with assert_not_raises(ActorNotRunning):
        a.pause()

    with assert_not_raises(ActorAlreadyStopped):
        a.resume()

    a.stop()
    assert child_stopped[0]

def test_actor_finishing_before_child_stops_its_children():
    child_stopped = [False]

    @actor
    def Child(self):
        try:
            yield Deferred()
        except ActorStopped:
            child_stopped[0] = True

    @actor
    def Parent(self):
        self.spawn(Child)

    root, p = run(Parent)
    assert child_stopped[0]
    assert not p.is_running
    assert [('stopped', p)] == root.messages

class MockException(Exception):
    pass

@actor
def run_with_error(self):
    raise MockException()
<commit_msg>Added actor_test.test_actor_failinig_stops_its_children<commit_after>from __future__ import print_function

import random
import warnings

from twisted.internet.defer import QueueUnderflow, Deferred, succeed

from spinoff.actor import Actor, actor, baseactor, ActorStopped, ActorNotRunning, ActorAlreadyStopped, ActorAlreadyRunning
from spinoff.util.pattern_matching import ANY, IS_INSTANCE
from spinoff.util.async import CancelledError
from spinoff.util.testing import deferred_result, assert_raises, assert_not_raises, assert_one_warning, MockActor, run, RootActor

warnings.simplefilter('always')

def test_base_actor_not_started():
    parent = MockActor()

    with assert_raises(ActorNotRunning):
        parent.send('whatev')

def test_base_actor():
    root, x = run(MockActor)

    msg = random.random()
    x.send(msg)
    assert x.clear() == [msg]

    x.pause()
    msg = random.random()
    x.send(msg)
    assert x.clear() == []

    x.resume()
    assert x.clear() == [msg]

    assert not root.messages

    x.stop()
    assert [('stopped', x)] == root.messages

def test_base_actor_error():
    @baseactor
    def B(self, message):
        raise MockException()
    _, b = run(B)
    with assert_not_raises(MockException):
        b.send('whatev')
    assert not b.d.called

    @actor
    def P(self):
        b = self.spawn(B)
        b.send('whatev')
        msg_d = self.get(('error', b, (IS_INSTANCE(MockException), ANY), True))
        assert msg_d.called
    run(P)

def test_failure_with_children():
    child_stopped = [False]

    @actor
    def Child(self):
        try:
            yield Deferred()
        except ActorStopped:
            child_stopped[0] = True

    @actor
    def A(self):
        self.spawn(Child)
        raise MockException()

    run(A, raise_only_asserts=True)

    assert child_stopped[0]

def test_actor_refuses_to_stop():
    mock_d = Deferred()

    @actor
    def A(self):
        try:
            yield mock_d
        except ActorStopped:
            pass
        yield succeed(None)
        assert False

    root, a = run(A)

    a.pause()
    a.stop()
    assert [('stopped', a, 'refused')] == root.messages, root.messages

def test_failure_while_stopping():
    mock_d = Deferred()
    print(mock_d)

    @actor
    def A(self):
        try:
            yield mock_d
        except ActorStopped:
            raise MockException()

    r, a = run(A)
    assert a.is_running
    with assert_not_raises(MockException):
        a.stop()
    assert len(r.messages) == 1 and r.messages[0][0:3] == ('stopped', a, 'unclean')

def test_connect_and_put():
    received_msg = []

    @actor
    def Mock(self):
        received_msg.append((yield self.get()))

    root, mock = run(Mock)

    c = actor(lambda self: self.put('msg-1'))()
    c._parent = root
    c.connect(mock)
    c.start()

    assert received_msg == ['msg-1'], received_msg

def test_flow():
    called = [0]

    mock_d = Deferred()

    @actor
    def Proc(self):
        called[0] += 1
        yield mock_d
        called[0] += 1

    proc = Proc()
    root = RootActor.spawn()
    proc._parent = root
    assert not called[0], "creating an actor should not automatically start the coroutine in it"

    proc.start()

    with assert_raises(ActorAlreadyRunning):
        proc.start()

    mock_d.callback(None)
    assert called[0] == 2, "the coroutine in an actor should complete as normal"
    assert not proc.is_alive
    assert root.messages == [('stopped', proc)]
    assert not proc.is_alive

def test_exception():
    mock_d = Deferred()
    exception_caught = [False]

    @actor
    def Y(self):
        try:
            yield mock_d
        except MockException:
            exception_caught[0] = True

    run(Y)
    mock_d.errback(MockException())
    assert exception_caught[0]

def test_failure():
    exc = MockException()

    @actor
    def A(self):
        raise exc

    root, a = run(A, raise_only_asserts=True)
    assert not a.is_alive

    assert [('error', a, (exc, ANY), False)] == root.messages

def test_yielding_a_non_deferred():
    @actor
    def Actor1(self):
        tmp = random.random()
        ret = yield tmp
        assert ret == tmp
    run(Actor1)

    @actor
    def Actor2(self):
        ret = yield
        assert ret is None
    run(Actor2)

def test_pending_failures_are_discarded_with_a_warning():
    mock_d = Deferred()

    @actor
    def X(self):
        yield mock_d

    root, p = run(X)
    p.pause()

    mock_d.errback(Exception())
    with assert_one_warning():
        p.stop()

def test_pausing_resuming_and_stopping():
    stopped = [False]

    mock_d = Deferred()
    retval = random.random()

    @actor
    def X(self):
        try:
            ret = yield mock_d
            assert ret == retval
        except ActorStopped:
            stopped[0] = True

    ### resuming when the async called has been fired
    root, proc = run(X)

    proc.pause()
    assert not proc.is_running
    assert proc.is_alive
    assert proc.is_paused

    with assert_raises(ActorNotRunning):
        proc.pause()

    mock_d.callback(retval)

    assert not proc.d.called, "a paused actor should not be resumed when the call it's waiting on completes"

    proc.resume()

    assert proc.d.called
    assert root.messages == [('stopped', proc)]

    ### resuming when the async call has NOT been fired
    mock_d = Deferred()
    root, proc2 = run(X)

    proc2.pause()
    proc2.resume()

    assert not stopped[0]
    assert not root.messages

    ### resuming when the async call has failed
    mock_d = Deferred()
    exception_caught = [False]

    @actor
    def Y(self):
        try:
            yield mock_d
        except MockException:
            exception_caught[0] = True

    root, x = run(Y)
    x.pause()
    mock_d.errback(MockException())
    x.resume()
    assert exception_caught[0]

    ### can't resume twice
    with assert_raises(ActorAlreadyRunning, "it should not be possible to resume an actor twice"):
        proc2.resume()

    ### stopping
    mock_d = Deferred()
    root, proc3 = run(X)

    proc3.stop()
    with assert_raises(ActorAlreadyStopped):
        proc3.stop()

    assert stopped[0]
    assert [('stopped', proc3)] == root.messages

    with assert_raises(ActorAlreadyStopped):
        proc3.start()
    with assert_raises(ActorAlreadyStopped):
        proc3.resume()

    ### stopping a paused actor
    mock_d = Deferred()
    root, proc4 = run(X)

    proc4.pause()
    proc4.stop()

    assert stopped[0]
    assert [('stopped', proc4)] == root.messages, root.messages

def test_stopping_cancels_the_deferred_on_hold():
    cancelled = [False]
    mock_d = Deferred(lambda _: cancelled.__setitem__(0, True))

    @actor
    def X(self):
        yield mock_d

    root, x = run(X)
    x.stop()

    assert cancelled[0]

def test_actor_does_not_have_to_catch_actorstopped():
    @actor
    def X(self):
        yield Deferred()
    root, proc = run(X)
    with assert_not_raises(ActorStopped):
        proc.stop()
    assert [('stopped', proc)] == root.messages

def test_actor_must_exit_after_being_stopped():
    # actor that violates the rule
    @actor
    def X(self):
        while True:
            try:
                yield Deferred()
            except ActorStopped:
                pass
    root, proc = run(X)
    proc.stop()
    assert [('stopped', proc, 'refused')] == root.messages, \
        "actor should not be allowed to continue working when stopped"

    # actor that complies with the rule
    @actor
    def Proc2(self):
        while True:
            try:
                yield Deferred()
            except ActorStopped:
                break
    root, proc2 = run(Proc2)
    proc2.stop()
    assert [('stopped', proc2)] == root.messages, root.messages

def test_actor_with_args():
    passed_values = [None, None]

    @actor
    def Proc(self, a, b):
        yield
        passed_values[:] = [a, b]

    run(Proc(1, b=2))
    assert passed_values == [1, 2]

def test_actor_doesnt_require_generator():
    @actor
    def Proc(self):
        pass

    root, proc = run(Proc)
    assert [('stopped', proc)] == root.messages

    @actor
    def Proc2(self):
        raise MockException()
    root, proc = run(Proc2, raise_only_asserts=True)

def test_get():
    def _make_getter(filter=None):
        @actor
        def ret(self):
            received_msg.append((yield self.get(filter=filter)))
        return ret

    ###
    received_msg = []
    root, x = run(_make_getter())
    x.send('foo')
    assert ['foo'] == received_msg

    ###
    received_msg = []
    tmp = random.random()
    root, x = run(_make_getter(('foo', ANY)))
    x.send(('foo', tmp))
    root.raise_errors(only_asserts=False)
    assert [tmp] == received_msg, received_msg

    ###
    received_msg = []
    tmp = random.random()
    root, c = run(_make_getter(('baz', ANY)))
    c.send(('foo', tmp))
    root.raise_errors(only_asserts=False)
    assert received_msg == []

    c.send(('baz', tmp))
    root.raise_errors(only_asserts=False)
    assert received_msg == [tmp]

def test_get_removes_message_from_inbox():
    @actor
    def X(self):
        yield self.get()
        msg_d = self.get()
        assert not msg_d.called

    root, x = run(X)
    x.send('whatev')

def test_inbox_underflow():
    @actor
    def GetTwice(self):
        self.get()
        with assert_raises(QueueUnderflow):
            self.get()
    run(GetTwice)

    @actor
    def GetThenCancelAndGetAgain(self):
        msg_d = self.get()
        msg_d.addErrback(lambda f: f.trap(CancelledError))
        msg_d.cancel()
        self.get()
    run(GetThenCancelAndGetAgain)

def test_spawn_child_actor():
    @actor
    def Child(self):
        yield Deferred()

    @actor
    def Parent(self):
        c = self.spawn(Child)
        assert c.parent == self

        c.stop()
        msg = deferred_result(self.get())
        assert msg == ('stopped', c), "child actor forced exit should be sent to its parent"

        c = self.spawn(run_with_error)
        msg = deferred_result(self.get())
        assert msg[:2] == ('error', c) and msg[3] is False and isinstance(msg[-2][0], MockException), \
            "child actor errors should be sent to its parent"

        for retval in [random.random(), None]:
            c = self.spawn(lambda self: retval)
            msg = deferred_result(self.get())
            assert ('stopped', c) == msg, "child actor return value is ignored %s" % repr(msg)

    run(Parent)

    chld_stopped = [False]
    arg = random.random()

    @actor
    def ChildWithArgs(self, foo):
        assert foo == arg
        try:
            yield Deferred()
        except ActorStopped:
            chld_stopped[0] = True

    @actor
    def Parent2(self):
        self.spawn(ChildWithArgs(arg))

    root, parent2 = run(Parent2, raise_only_asserts=False)

    assert chld_stopped[0]

    class Parent3(Actor):
        def __init__(self):
            super(Parent3, self).__init__()
            self.spawn(Child)

    root = RootActor.spawn()
    parent3 = Parent3()
    parent3._parent = root
    with assert_not_raises(ActorAlreadyRunning, "it should be possible for actors to spawn children in the constructor"):
        parent3.start()

def test_pausing_resuming_and_stopping_actor_with_children_does_the_same_with_children():
    children = []
    child_stopped = [False]

    @actor
    def Child(self):
        try:
            yield Deferred()
        except ActorStopped:
            child_stopped[0] = True

    @actor
    def Parent(self):
        children.append(self.spawn(Child))
        yield Deferred()
    root, a = run(Parent)

    a.pause()
    assert all(x.is_paused for x in children)

    a.resume()
    assert all(not x.is_paused for x in children)

    a.stop()
    assert all(not x.is_alive for x in children)
    assert child_stopped[0]

def test_pausing_and_stoping_actor_with_some_finished_children():
    @actor
    def Stillborn(self):
        yield

    child_stopped = [False]

    @actor
    def LongLivingChild(self):
        try:
            yield Deferred()
        except ActorStopped:
            child_stopped[0] = True

    mock_d = Deferred()

    @actor
    def Parent(self):
        self.spawn(Stillborn)
        self.spawn(LongLivingChild)
        yield mock_d

    root, a = run(Parent)

    with assert_not_raises(ActorNotRunning):
        a.pause()

    with assert_not_raises(ActorAlreadyStopped):
        a.resume()

    a.stop()
    assert child_stopped[0]

def test_actor_finishing_before_child_stops_its_children():
    child_stopped = [False]

    @actor
    def Child(self):
        try:
            yield Deferred()
        except ActorStopped:
            child_stopped[0] = True

    @actor
    def Parent(self):
        self.spawn(Child)

    root, p = run(Parent)
    assert child_stopped[0]
    assert not p.is_running
    assert [('stopped', p)] == root.messages

def test_actor_failinig_stops_its_children():
    child_stopped = [False]

    @actor
    def Child(self):
        try:
            yield Deferred()
        except ActorStopped:
            child_stopped[0] = True

    @actor
    def Parent(self):
        self.spawn(Child)
        raise Exception()

    root, p = run(Parent, raise_only_asserts=True)
    root.messages.remove(('error', p, (IS_INSTANCE(Exception), ANY), ANY))
    assert child_stopped[0]

class MockException(Exception):
    pass

@actor
def run_with_error(self):
    raise MockException()
<|endoftext|>


<commit_before>import unittest, requests, json, base64, urllib
from cStringIO import StringIO
from PIL import Image
from splash.tests.utils import TestServers
from splash import defaults

class _BaseRenderTest(unittest.TestCase):

    host = "localhost:%s" % defaults.SPLASH_PORT
    render_format = "html"

    def request(self, query, render_format=None):
        render_format = render_format or self.render_format
        if isinstance(query, dict):
            url= "http://%s/render.%s" % (self.host, render_format)
            return requests.get(url, params=query)
        else:
            url = "http://%s/render.%s?%s" % (self.host, render_format, query)
            return requests.get(url)

class _RenderTest(_BaseRenderTest):

    def test_render_error(self):
        r = self.request("url=http://non-existent-host/")
        self.assertEqual(r.status_code, 502)

    def test_timeout(self):
        r = self.request("url=http://localhost:8998/delay?n=10&timeout=0.5")
        self.assertEqual(r.status_code, 504)

    def test_timeout_out_of_range(self):
        r = self.request("url=http://localhost:8998/delay?n=10&timeout=999")
        self.assertEqual(r.status_code, 400)

    def test_missing_url(self):
        r = self.request("")
        self.assertEqual(r.status_code, 400)
        self.assertTrue("url" in r.text)

    def test_jsalert(self):
        r = self.request("url=http://localhost:8998/jsalert&timeout=3")
        self.assertEqual(r.status_code, 200)

    def test_jsconfirm(self):
        r = self.request("url=http://localhost:8998/jsconfirm&timeout=3")
        self.assertEqual(r.status_code, 200)

    def test_iframes(self):
        r = self.request("url=http://localhost:8998/iframes&timeout=3")
        self.assertEqual(r.status_code, 200)

    def test_wait(self):
        r1 = self.request({"url": "http://localhost:8998/jsinterval"})
        r2 = self.request({"url": "http://localhost:8998/jsinterval&wait=0.2"})
        self.assertEqual(r1.status_code, 200)
        self.assertEqual(r2.status_code, 200)
        self.assertNotEqual(r1.text, r2.text)

class RenderHtmlTest(_RenderTest):

    render_format = "html"

    def test_ok(self):
        self._test_ok("http://localhost:8998/jsrender")

    def test_ok_https(self):
        self._test_ok("https://localhost:8999/jsrender")

    def _test_ok(self, url):
        r = self.request("url=%s" % url)
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers["content-type"].lower(), "text/html; charset=utf-8")
        self.assertTrue("Before" not in r.text)
        self.assertTrue("After" in r.text)

    def test_baseurl(self):
        # first make sure that script.js is served under the right url
        self.assertEqual(404, requests.get("http://localhost:8998/script.js").status_code)
        self.assertEqual(200, requests.get("http://localhost:8998/baseurl/script.js").status_code)

        r = self.request("url=http://localhost:8998/baseurl&baseurl=http://localhost:8998/baseurl/")
        self.assertEqual(r.status_code, 200)
        self.assertTrue("Before" not in r.text)
        self.assertTrue("After" in r.text)

class RenderPngTest(_RenderTest):

    render_format = "png"

    def test_ok(self):
        self._test_ok("http://localhost:8998/jsrender")

    def test_ok_https(self):
        self._test_ok("https://localhost:8999/jsrender")

    def _test_ok(self, url):
        r = self.request("url=%s" % url)
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers["content-type"], "image/png")
        img = Image.open(StringIO(r.content))
        self.assertEqual(img.format, "PNG")
        self.assertEqual(img.size, (1024, 768))

    def test_width(self):
        r = self.request("url=http://localhost:8998/jsrender&width=300")
        self.assertEqual(r.headers["content-type"], "image/png")
        img = Image.open(StringIO(r.content))
        self.assertEqual(img.format, "PNG")
        self.assertEqual(img.size[0], 300)

    def test_width_height(self):
        r = self.request("url=http://localhost:8998/jsrender&width=300&height=100")
        self.assertEqual(r.headers["content-type"], "image/png")
        img = Image.open(StringIO(r.content))
        self.assertEqual(img.format, "PNG")
        self.assertEqual(img.size, (300, 100))

    def test_range_checks(self):
        for arg in ('width', 'height', 'vwidth', 'vheight'):
            for val in (-1, 99999):
                r = self.request("url=http://localhost:8998/jsrender&%s=%d" % (arg, val))
                self.assertEqual(r.status_code, 400)

class RenderJsonTest(_RenderTest):
    render_format = 'json'

    def test_jsrender_html(self):
        self.assertSameHtml('http://localhost:8998/jsrender')

    def test_jsrender_https_html(self):
        self.assertSameHtml('https://localhost:8999/jsrender')

    def test_jsalert_html(self):
        self.assertSameHtml('http://localhost:8998/jsalert', {'timeout': 3})

    def test_jsconfirm_html(self):
        self.assertSameHtml("http://localhost:8998/jsconfirm", {'timeout': 3})

    def test_iframes_html(self):
        self.assertSameHtml("http://localhost:8998/iframes", {'timeout': 3})

    def test_jsrender_png(self):
        self.assertSamePng('http://localhost:8998/jsrender')

    def test_jsrender_https_png(self):
        self.assertSamePng('https://localhost:8999/jsrender')

    def test_jsalert_png(self):
        self.assertSamePng('http://localhost:8998/jsalert', {'timeout': 3})

    def test_jsconfirm_png(self):
        self.assertSamePng("http://localhost:8998/jsconfirm", {'timeout': 3})

    def test_iframes_png(self):
        self.assertSamePng("http://localhost:8998/iframes", {'timeout': 3})

    def test_png_size(self):
        self.assertSamePng('http://localhost:8998/jsrender', {'width': 100})
        self.assertSamePng('http://localhost:8998/jsrender', {'width': 100, 'height': 200})
        self.assertSamePng('http://localhost:8998/jsrender',
                           {'width': 100, 'height': 200, 'vwidth': 100, 'vheight': 200})
        self.assertSamePng('http://localhost:8998/jsrender',
                           {'vwidth': 100})

    def test_fields_all(self):
        query = {'url': "https://localhost:8999/iframes",
                 "html": 1, "png": 1, "iframes": 1}

        res = self.request(query).json()
        self.assertFieldsInResponse(res, ["html", "png", "url", "requestedUrl",
                                          "childFrames", "geometry", "title"])
        frames = res['childFrames']
        self.assertTrue(frames)
        for frame in frames:
            self.assertFieldsInResponse(frame, ["html", "url", "requestedUrl",
                                               "childFrames", "geometry", "title"])
            # no screenshots for individual frames
            self.assertFieldsNotInResponse(frame, ['png'])

    def test_fields_no_html(self):
        # turn off returning HTML
        query = {'url': "https://localhost:8999/iframes",
                 'html': 0, 'png': 1, 'iframes': 1}

        res = self.request(query).json()
        self.assertFieldsInResponse(res, ["png", "url", "requestedUrl",
                                          "childFrames", "geometry", "title"])
        self.assertFieldsNotInResponse(res, ['html'])

        # html=0 also turns off html for iframes
        frames = res['childFrames']
        self.assertTrue(frames)
        for frame in frames:
            self.assertFieldsInResponse(frame, ["url", "requestedUrl",
                                                "childFrames", "geometry", "title"])
            self.assertFieldsNotInResponse(frame, ['html', 'png'])

    def test_fields_no_screenshots(self):
        # turn off screenshots
        query = {'url': "https://localhost:8999/iframes",
                 'html': 1, 'png': 0, 'iframes': 1}
        res = self.request(query).json()
        self.assertFieldsInResponse(res, ["url", "requestedUrl", "childFrames",
                                          "geometry", "title", "html"])
        self.assertFieldsNotInResponse(res, ["png"])

    def test_fields_no_iframes(self):
        query = {'url': "https://localhost:8999/iframes",
                 'html': 1, 'png': 1, 'iframes': 0}
        res = self.request(query).json()
        self.assertFieldsInResponse(res, ["url", "requestedUrl", "geometry",
                                          "title", "html", "png"])
        self.assertFieldsNotInResponse(res, ["childFrames"])

    def test_fields_default(self):
        query = {'url': "https://localhost:8999/iframes"}
        res = self.request(query).json()
        self.assertFieldsInResponse(res, ["url", "requestedUrl", "geometry",
                                          "title"])
        self.assertFieldsNotInResponse(res, ["childFrames", "html", "png"])

    def assertFieldsInResponse(self, res, fields):
        for key in fields:
            self.assertTrue(key in res, "%s is not in response" % key)

    def assertFieldsNotInResponse(self, res, fields):
        for key in fields:
            self.assertTrue(key not in res, "%s is in response" % key)

    def assertSameHtml(self, url, params=None):
        defaults = {'html': 1}
        defaults.update(params or {})
        r1, r2 = self._do_same_requests(url, defaults, 'html')
        html1 = r1.json()['html']
        html2 = r2.text
        self.assertEqual(html1, html2)

    def assertSamePng(self, url, params=None):
        defaults = {'png': 1}
        defaults.update(params or {})
        r1, r2 = self._do_same_requests(url, defaults, 'png')
        png1 = base64.decodestring(r1.json()['png'])
        png2 = r2.content
        self.assertEqual(png1, png2)

    def _do_same_requests(self, url, params, other_format):
        query = {'url': url}
        query.update(params or {})
        r1 = self.request(query, render_format='json')
        r2 = self.request(query, render_format=other_format)
        self.assertEqual(r1.status_code, 200)
        self.assertEqual(r2.status_code, 200)
        return r1, r2

class IframesRenderTest(_BaseRenderTest):
    render_format = 'json'

    def test_basic(self):
        self.assertIframesText('IFRAME_1_OK')

    def test_js_iframes(self):
        self.assertIframesText('IFRAME_2_OK')

    def test_delayed_js_iframes(self):
        self.assertNoIframesText('IFRAME_3_OK', {'wait': 0.0})
        self.assertIframesText('IFRAME_3_OK', {'wait': 0.5})

    def test_onload_iframes(self):
        self.assertIframesText('IFRAME_4_OK')

    def test_document_write_iframes(self):
        self.assertIframesText('IFRAME_5_OK')

    def test_nested_iframes(self):
        self.assertIframesText('IFRAME_6_OK')

    def assertIframesText(self, text, params=None):
        data = self._iframes_request(params)
        self.assertTrue(self._text_is_somewhere(data, text))

    def assertNoIframesText(self, text, params=None):
        data = self._iframes_request(params)
        self.assertFalse(self._text_is_somewhere(data, text))

    def _text_is_somewhere(self, result, text):
        if text in result['html']:
            return True
        return any(self._text_is_somewhere(child, text)
                   for child in result['childFrames'])

    def _iframes_request(self, params):
        query = {'url': 'https://localhost:8999/iframes',
                 'iframes': 1, 'html': 1}
        query.update(params or {})
        return self.request(query).json()

class TestTestSetup(unittest.TestCase):

    def test_mockserver_works(self):
        r = requests.get('http://localhost:8998/jsrender')
        self.assertEqual(r.status_code, 200)

    def test_splashserver_works(self):
        r = requests.get('http://localhost:%s/debug' % defaults.SPLASH_PORT)
        self.assertEqual(r.status_code, 200)

ts = TestServers()

def setup():
    ts.__enter__()

def teardown():
    #leaks = requests.get("http://localhost:8050/debug").json()['leaks']
    #assert not leaks, "Leaks detected:\n%s" % leaks
    ts.__exit__(None, None, None)
<commit_msg>improve and fix test_wait (it used to check wrong things)<commit_after>import unittest, requests, json, base64, urllib
from cStringIO import StringIO
from PIL import Image
from splash.tests.utils import TestServers
from splash import defaults

class _BaseRenderTest(unittest.TestCase):

    host = "localhost:%s" % defaults.SPLASH_PORT
    render_format = "html"

    def request(self, query, render_format=None):
        render_format = render_format or self.render_format
        if isinstance(query, dict):
            url= "http://%s/render.%s" % (self.host, render_format)
            return requests.get(url, params=query)
        else:
            url = "http://%s/render.%s?%s" % (self.host, render_format, query)
            return requests.get(url)

class _RenderTest(_BaseRenderTest):

    def test_render_error(self):
        r = self.request("url=http://non-existent-host/")
        self.assertEqual(r.status_code, 502)

    def test_timeout(self):
        r = self.request("url=http://localhost:8998/delay?n=10&timeout=0.5")
        self.assertEqual(r.status_code, 504)

    def test_timeout_out_of_range(self):
        r = self.request("url=http://localhost:8998/delay?n=10&timeout=999")
        self.assertEqual(r.status_code, 400)

    def test_missing_url(self):
        r = self.request("")
        self.assertEqual(r.status_code, 400)
        self.assertTrue("url" in r.text)

    def test_jsalert(self):
        r = self.request("url=http://localhost:8998/jsalert&timeout=3")
        self.assertEqual(r.status_code, 200)

    def test_jsconfirm(self):
        r = self.request("url=http://localhost:8998/jsconfirm&timeout=3")
        self.assertEqual(r.status_code, 200)

    def test_iframes(self):
        r = self.request("url=http://localhost:8998/iframes&timeout=3")
        self.assertEqual(r.status_code, 200)

    def test_wait(self):
        r1 = self.request("url=http://localhost:8998/jsinterval")
        r2 = self.request("url=http://localhost:8998/jsinterval")
        r3 = self.request("url=http://localhost:8998/jsinterval&wait=0.2")
        self.assertEqual(r1.status_code, 200)
        self.assertEqual(r2.status_code, 200)
        self.assertEqual(r3.status_code, 200)
        self.assertEqual(r1.content, r2.content)
        self.assertNotEqual(r1.content, r3.content)

class RenderHtmlTest(_RenderTest):

    render_format = "html"

    def test_ok(self):
        self._test_ok("http://localhost:8998/jsrender")

    def test_ok_https(self):
        self._test_ok("https://localhost:8999/jsrender")

    def _test_ok(self, url):
        r = self.request("url=%s" % url)
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers["content-type"].lower(), "text/html; charset=utf-8")
        self.assertTrue("Before" not in r.text)
        self.assertTrue("After" in r.text)

    def test_baseurl(self):
        # first make sure that script.js is served under the right url
        self.assertEqual(404, requests.get("http://localhost:8998/script.js").status_code)
        self.assertEqual(200, requests.get("http://localhost:8998/baseurl/script.js").status_code)

        r = self.request("url=http://localhost:8998/baseurl&baseurl=http://localhost:8998/baseurl/")
        self.assertEqual(r.status_code, 200)
        self.assertTrue("Before" not in r.text)
        self.assertTrue("After" in r.text)

class RenderPngTest(_RenderTest):

    render_format = "png"

    def test_ok(self):
        self._test_ok("http://localhost:8998/jsrender")

    def test_ok_https(self):
        self._test_ok("https://localhost:8999/jsrender")

    def _test_ok(self, url):
        r = self.request("url=%s" % url)
        self.assertEqual(r.status_code, 200)
        self.assertEqual(r.headers["content-type"], "image/png")
        img = Image.open(StringIO(r.content))
        self.assertEqual(img.format, "PNG")
        self.assertEqual(img.size, (1024, 768))

    def test_width(self):
        r = self.request("url=http://localhost:8998/jsrender&width=300")
        self.assertEqual(r.headers["content-type"], "image/png")
        img = Image.open(StringIO(r.content))
        self.assertEqual(img.format, "PNG")
        self.assertEqual(img.size[0], 300)

    def test_width_height(self):
        r = self.request("url=http://localhost:8998/jsrender&width=300&height=100")
        self.assertEqual(r.headers["content-type"], "image/png")
        img = Image.open(StringIO(r.content))
        self.assertEqual(img.format, "PNG")
        self.assertEqual(img.size, (300, 100))

    def test_range_checks(self):
        for arg in ('width', 'height', 'vwidth', 'vheight'):
            for val in (-1, 99999):
                r = self.request("url=http://localhost:8998/jsrender&%s=%d" % (arg, val))
                self.assertEqual(r.status_code, 400)

class RenderJsonTest(_RenderTest):
    render_format = 'json'

    def test_jsrender_html(self):
        self.assertSameHtml('http://localhost:8998/jsrender')

    def test_jsrender_https_html(self):
        self.assertSameHtml('https://localhost:8999/jsrender')

    def test_jsalert_html(self):
        self.assertSameHtml('http://localhost:8998/jsalert', {'timeout': 3})

    def test_jsconfirm_html(self):
        self.assertSameHtml("http://localhost:8998/jsconfirm", {'timeout': 3})

    def test_iframes_html(self):
        self.assertSameHtml("http://localhost:8998/iframes", {'timeout': 3})

    def test_jsrender_png(self):
        self.assertSamePng('http://localhost:8998/jsrender')

    def test_jsrender_https_png(self):
        self.assertSamePng('https://localhost:8999/jsrender')

    def test_jsalert_png(self):
        self.assertSamePng('http://localhost:8998/jsalert', {'timeout': 3})

    def test_jsconfirm_png(self):
        self.assertSamePng("http://localhost:8998/jsconfirm", {'timeout': 3})

    def test_iframes_png(self):
        self.assertSamePng("http://localhost:8998/iframes", {'timeout': 3})

    def test_png_size(self):
        self.assertSamePng('http://localhost:8998/jsrender', {'width': 100})
        self.assertSamePng('http://localhost:8998/jsrender', {'width': 100, 'height': 200})
        self.assertSamePng('http://localhost:8998/jsrender',
                           {'width': 100, 'height': 200, 'vwidth': 100, 'vheight': 200})
        self.assertSamePng('http://localhost:8998/jsrender',
                           {'vwidth': 100})

    def test_fields_all(self):
        query = {'url': "https://localhost:8999/iframes",
                 "html": 1, "png": 1, "iframes": 1}

        res = self.request(query).json()
        self.assertFieldsInResponse(res, ["html", "png", "url", "requestedUrl",
                                          "childFrames", "geometry", "title"])
        frames = res['childFrames']
        self.assertTrue(frames)
        for frame in frames:
            self.assertFieldsInResponse(frame, ["html", "url", "requestedUrl",
                                               "childFrames", "geometry", "title"])
            # no screenshots for individual frames
            self.assertFieldsNotInResponse(frame, ['png'])

    def test_fields_no_html(self):
        # turn off returning HTML
        query = {'url': "https://localhost:8999/iframes",
                 'html': 0, 'png': 1, 'iframes': 1}

        res = self.request(query).json()
        self.assertFieldsInResponse(res, ["png", "url", "requestedUrl",
                                          "childFrames", "geometry", "title"])
        self.assertFieldsNotInResponse(res, ['html'])

        # html=0 also turns off html for iframes
        frames = res['childFrames']
        self.assertTrue(frames)
        for frame in frames:
            self.assertFieldsInResponse(frame, ["url", "requestedUrl",
                                                "childFrames", "geometry", "title"])
            self.assertFieldsNotInResponse(frame, ['html', 'png'])

    def test_fields_no_screenshots(self):
        # turn off screenshots
        query = {'url': "https://localhost:8999/iframes",
                 'html': 1, 'png': 0, 'iframes': 1}
        res = self.request(query).json()
        self.assertFieldsInResponse(res, ["url", "requestedUrl", "childFrames",
                                          "geometry", "title", "html"])
        self.assertFieldsNotInResponse(res, ["png"])

    def test_fields_no_iframes(self):
        query = {'url': "https://localhost:8999/iframes",
                 'html': 1, 'png': 1, 'iframes': 0}
        res = self.request(query).json()
        self.assertFieldsInResponse(res, ["url", "requestedUrl", "geometry",
                                          "title", "html", "png"])
        self.assertFieldsNotInResponse(res, ["childFrames"])

    def test_fields_default(self):
        query = {'url': "https://localhost:8999/iframes"}
        res = self.request(query).json()
        self.assertFieldsInResponse(res, ["url", "requestedUrl", "geometry",
                                          "title"])
        self.assertFieldsNotInResponse(res, ["childFrames", "html", "png"])

    def test_wait(self):
        # override parent's test to make it aware of render.json endpoint
        r1 = self.request({"url": "http://localhost:8998/jsinterval", 'html': 1})
        r2 = self.request({"url": "http://localhost:8998/jsinterval", 'html': 1})
        r3 = self.request({"url": "http://localhost:8998/jsinterval", 'wait': 0.2, 'html': 1})
        self.assertEqual(r1.status_code, 200)
        self.assertEqual(r2.status_code, 200)
        self.assertEqual(r3.status_code, 200)

        html1 = r1.json()['html']
        html2 = r2.json()['html']
        html3 = r3.json()['html']
        self.assertEqual(html1, html2)
        self.assertNotEqual(html1, html3)

    def assertFieldsInResponse(self, res, fields):
        for key in fields:
            self.assertTrue(key in res, "%s is not in response" % key)

    def assertFieldsNotInResponse(self, res, fields):
        for key in fields:
            self.assertTrue(key not in res, "%s is in response" % key)

    def assertSameHtml(self, url, params=None):
        defaults = {'html': 1}
        defaults.update(params or {})
        r1, r2 = self._do_same_requests(url, defaults, 'html')
        html1 = r1.json()['html']
        html2 = r2.text
        self.assertEqual(html1, html2)

    def assertSamePng(self, url, params=None):
        defaults = {'png': 1}
        defaults.update(params or {})
        r1, r2 = self._do_same_requests(url, defaults, 'png')
        png1 = base64.decodestring(r1.json()['png'])
        png2 = r2.content
        self.assertEqual(png1, png2)

    def _do_same_requests(self, url, params, other_format):
        query = {'url': url}
        query.update(params or {})
        r1 = self.request(query, render_format='json')
        r2 = self.request(query, render_format=other_format)
        self.assertEqual(r1.status_code, 200)
        self.assertEqual(r2.status_code, 200)
        return r1, r2

class IframesRenderTest(_BaseRenderTest):
    render_format = 'json'

    def test_basic(self):
        self.assertIframesText('IFRAME_1_OK')

    def test_js_iframes(self):
        self.assertIframesText('IFRAME_2_OK')

    def test_delayed_js_iframes(self):
        self.assertNoIframesText('IFRAME_3_OK', {'wait': 0.0})
        self.assertIframesText('IFRAME_3_OK', {'wait': 0.5})

    def test_onload_iframes(self):
        self.assertIframesText('IFRAME_4_OK')

    def test_document_write_iframes(self):
        self.assertIframesText('IFRAME_5_OK')

    def test_nested_iframes(self):
        self.assertIframesText('IFRAME_6_OK')

    def assertIframesText(self, text, params=None):
        data = self._iframes_request(params)
        self.assertTrue(self._text_is_somewhere(data, text))

    def assertNoIframesText(self, text, params=None):
        data = self._iframes_request(params)
        self.assertFalse(self._text_is_somewhere(data, text))

    def _text_is_somewhere(self, result, text):
        if text in result['html']:
            return True
        return any(self._text_is_somewhere(child, text)
                   for child in result['childFrames'])

    def _iframes_request(self, params):
        query = {'url': 'https://localhost:8999/iframes',
                 'iframes': 1, 'html': 1}
        query.update(params or {})
        return self.request(query).json()

class TestTestSetup(unittest.TestCase):

    def test_mockserver_works(self):
        r = requests.get('http://localhost:8998/jsrender')
        self.assertEqual(r.status_code, 200)

    def test_splashserver_works(self):
        r = requests.get('http://localhost:%s/debug' % defaults.SPLASH_PORT)
        self.assertEqual(r.status_code, 200)

ts = TestServers()

def setup():
    ts.__enter__()

def teardown():
    #leaks = requests.get("http://localhost:8050/debug").json()['leaks']
    #assert not leaks, "Leaks detected:\n%s" % leaks
    ts.__exit__(None, None, None)
<|endoftext|>


<commit_before>import os
import asyncio  # noqa: F401
import discord
import logging
from discord.ext import commands
from cogs.utils.dataIO import dataIO
from cogs.utils import checks

class LinkedChannels:
    """links 2 or more channels across discords that the bot can see.
    heavily influenced by Twentysix26's Rift
    https://github.com/Twentysix26/26-Cogs/blob/master/rift/rift.py
    This version is persistent until closed and shows
    who said what on both sides"""

    __author__ = "mikeshardmind"
    __version__ = "1.1"

    def __init__(self, bot):
        self.bot = bot
        self.settings = dataIO.load_json('data/linkedchannels/settings.json')
        self.links = []
        self.initialized = False

    def save_json(self):
        dataIO.save_json("data/linkedchannels/settings.json", self.settings)

    @checks.is_owner()
    @commands.command(name="makelink", pass_context=True)
    async def makelink(self, ctx, name: str, *chan_ids):
        """links two or more channels by id and names the link"""
        if name in self.settings:
            return await self.bot.say("there is an existing link of that name")
        chans = []
        for chan in chan_ids:
            if chan not in chans:
                chans.append(chan)
        if len(chans) > 1:
            self.settings['name'] = {'chans' = chans
                                     'active': False
                                     }

            self.save_json()
            await self.bot.say("Link formed; be sure to activate it")
        else:
            await self.bot.say("I did not get two or more unique channel IDs")

    @checks.is_owner()
    @commands.command(name="unlink", pass_context=True)
    async def unlink(self, ctx, name: str):
        """unlinks two channels by link name"""

        if name in self.links:
            self.links = [l for l in self.links if l['name'] != name]
        self.settings = [l for l in self.settings if l['name'] != name]
        self.save_json()
        await self.bot.say("If there was a link by that name, it has been "
                           "removed.")

    @checks.is_owner()
    @commands.command(name="listlinks", pass_context=True)
    async def list_links(self, ctx):
        """lists the channel links by name"""

        names = [link['name'] for link in self.settings]
        self.bot.say("{}".format(names))

    @checks.is_owner()
    @commands.command(name="togglelink", pass_context=True)
    async def togglelink(self, ctx, name: str):
        """toggles the link status"""
        if name not in self.settings:
            return await self.bot.say("No link by that name")
        self.settings[name]['active'] = not self.settings[name'active]
        self.validate()
        if self.links[name]['active']:
            await self.bot.say("Link activated")
        else:
            await self.bot.say("Link deactivated")

    async def validate(self, name=None):
        channels = self.bot.get_all_channels()
        channels = [c for c in channels if c.type == discord.ChannelType.text]

        for l in self.settings:
            link = {'name': self.settings[l]['name'],
                    'chans': [],
                    'active': self.settings[l]['active']
                    }
            for channel in channels:
                if self.settings[name]['A'] == channel.id:
                    link['chans'].append(channel)
                if self.settings[name]['B'] == channel.id:
                    link['chans'].append(channel)

            if len(link[chans]) >= 2:
                if self.settings[name]['active']:
                    self.links.append(link)

    async def on_message(self, message):
        """Do stuff based on settings"""
        if not self.initialized:
            await self.validate()
            self.initialized = True

        if message.author == self.bot.user:
            return

        channel = message.channel
        destination = None
        for link in self.links:
            if channel in link['chans']:
                destinations = [c for c in link['chans'] if c != channel]

        if destinations is not None:
            for d in destinations:
                await self.sender(d, message)

    async def sender(self, where, message=None):
        """sends the thing"""

        if message:
            channel = message.channel
            server = channel.server
            content = message.clean_content
            author = message.author
            sname = server.name
            cname = channel.name
            timestamp = message.timestamp.strftime('%Y-%m-%d %H:%M')
            avatar = author.avatar_url if author.avatar \
                else author.default_avatar_url
            footer = 'Said in {} #{} at {} UTC'.format(sname, cname, timestamp)
            em = discord.Embed(description=content,
                               color=discord.Color.purple())
            em.set_author(name='{} ({})'.format(author.name, author.id),
                          icon_url=avatar)
            em.set_footer(text=footer)
            await self.bot.send_message(where, embed=em)

def check_folder():
    f = 'data/linkedchannels'
    if not os.path.exists(f):
        os.makedirs(f)

def check_file():
    f = 'data/linkedchannels/settings.json'
    if dataIO.is_valid_json(f) is False:
        dataIO.save_json(f, {})

def setup(bot):
    check_folder()
    check_file()
    n = LinkedChannels(bot)
    bot.add_cog(n)
<commit_msg>After much cofee was had, the links persist properly and do not allow for looping hells.<commit_after>import os
import asyncio  # noqa: F401
import discord
from discord.ext import commands
from cogs.utils.dataIO import dataIO
from cogs.utils import checks

class LinkedChannels:
    """links 2 channels across discord serversthat the bot can see.
    heavily influenced by Twentysix26's Rift
    https://github.com/Twentysix26/26-Cogs/blob/master/rift/rift.py
    This version is persistent across restarts and shows
    who said what on both sides.\n supports multiple active links"""

    __author__ = "mikeshardmind"
    __version__ = "2.0"

    def __init__(self, bot):
        self.bot = bot
        self.settings = dataIO.load_json('data/linkedchannels/settings.json')
        self.links = {}
        self.activechans = []
        self.initialized = False

    def save_json(self):
        dataIO.save_json("data/linkedchannels/settings.json", self.settings)

    @checks.is_owner()
    @commands.command(name="makelink", pass_context=True)
    async def makelink(self, ctx, name: str, chan1: str, chan2: str):
        """links two channels by id and names the link"""
        name = name.lower()
        if name in self.settings:
            return await self.bot.say("there is an existing link of that name")

        channels = self.bot.get_all_channels()
        channels = [c for c in channels if c.type == discord.ChannelType.text]
        channels = [c.id for c in channels if c.id == chan1 or c.id == chan2]

        if bool(set(channels) & set(self.activechans)):
            return await self.bot.say("One or more of these channels is "
                                      "already linked elsewhere")

        if len(channels) == 2:
            self.settings[name] = {'chans': channels}
            self.save_json()
            await self.validate()
            if name in self.links:
                await self.bot.say("Link formed.")
        else:
            await self.bot.say("I did not get two or more unique channel IDs")

    @checks.is_owner()
    @commands.command(name="unlink", pass_context=True)
    async def unlink(self, ctx, name: str):
        """unlinks two channels by link name"""
        name = name.lower()
        if name in self.links:
            chans = self.links[name]
            self.links.pop(name, None)
            self.settings.pop(name, None)
            self.activechans = [c for c in self.activechans if c not in chans]
            self.save_json()
            await self.bot.say("Link removed")
        else:
            await self.bot.say("No such link")

    @checks.is_owner()
    @commands.command(name="listlinks", pass_context=True)
    async def list_links(self, ctx):
        """lists the channel links by name"""

        links = list(self.settings.keys())
        await self.bot.say("Active link names:\n {}".format(links))

    async def validate(self, name=None):
        channels = self.bot.get_all_channels()
        channels = [c for c in channels if c.type == discord.ChannelType.text]

        for name in self.settings:
            chan_ids = list(*self.settings[name].values())
            chans = [c for c in channels if c.id in chan_ids]
            self.links[name] = chans
            self.activechans += chan_ids

    async def on_message(self, message):
        """Do stuff based on settings"""
        if not self.initialized:
            await self.validate()
            self.initialized = True

        if message.author == self.bot.user:
            return

        channel = message.channel
        destination = None
        for link in self.links:
            if channel in self.links[link]:
                destination = [c for c in self.links[link] if c != channel][0]

        if destination is not None:
            await self.sender(destination, message)

    async def sender(self, where, message=None):
        """sends the thing"""

        if message:
            channel = message.channel
            server = channel.server
            content = message.clean_content
            author = message.author
            sname = server.name
            cname = channel.name
            timestamp = message.timestamp.strftime('%Y-%m-%d %H:%M')
            avatar = author.avatar_url if author.avatar \
                else author.default_avatar_url
            footer = 'Said in {} #{} at {} UTC'.format(sname, cname, timestamp)
            em = discord.Embed(description=content,
                               color=discord.Color.purple())
            em.set_author(name='{} ({})'.format(author.name, author.id),
                          icon_url=avatar)
            em.set_footer(text=footer)
            await self.bot.send_message(where, embed=em)

def check_folder():
    f = 'data/linkedchannels'
    if not os.path.exists(f):
        os.makedirs(f)

def check_file():
    f = 'data/linkedchannels/settings.json'
    if dataIO.is_valid_json(f) is False:
        dataIO.save_json(f, {})

def setup(bot):
    check_folder()
    check_file()
    n = LinkedChannels(bot)
    bot.add_cog(n)
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
#
# Module CLI
#
# Part of Nutils: open source numerical utilities for Python. Jointly developed
# by HvZ Computational Engineering, TU/e Multiscale Engineering Fluid Dynamics,
# and others. More info at http://nutils.org <info@nutils.org>. (c) 2014

"""
The cli (command line interface) module provides the `cli.run` function that
can be used set up properties, initiate an output environment, and execute a
python function based arguments specified on the command line.
"""

from . import log, core, version
import sys, inspect, os, datetime, argparse, pdb, signal, subprocess

def _version():
  try:
    githash = subprocess.check_output( ['git','rev-parse','--short','HEAD'], universal_newlines=True, cwd=os.path.dirname(__file__) ).strip()
    if subprocess.check_output( ['git','status','--untracked-files=no','--porcelain'], cwd=os.path.dirname(__file__) ):
      githash += '+'
  except:
    return version
  else:
    return '{} (git:{})'.format( version, githash )

def _bool( s ):
  if s in ('true','True'):
    return True
  if s in ('false','False'):
    return False
  raise argparse.ArgumentTypeError( 'invalid boolean value: {!r}'.format(s) )

def _relative_paths( basepath, path ):
  baseparts = basepath.rstrip( os.path.sep ).split( os.path.sep )
  parts = path.rstrip( os.path.sep ).split( os.path.sep )
  if parts[:len(baseparts)] == baseparts:
    for i in range( len(baseparts), len(parts) ):
      yield os.path.sep.join(parts[:i]), os.path.sep.join(parts[i:])

def _mkbox( *lines ):
  width = max( len(line) for line in lines )
  ul, ur, ll, lr, hh, vv = '┌┐└┘─│' if core.getprop('richoutput') else '++++-|'
  return '\n'.join( [ ul + hh * (width+2) + ur ]
                  + [ vv + (' '+line).ljust(width+2) + vv for line in lines ]
                  + [ ll + hh * (width+2) + lr ] )

def _sigint_handler( mysignal, frame ):
  _handler = signal.signal( mysignal, signal.SIG_IGN ) # temporarily disable handler
  try:
    while True:
      answer = input( 'interrupted. quit, continue or start debugger? [q/c/d]' )
      if answer == 'q':
        raise KeyboardInterrupt
      if answer == 'c' or answer == 'd':
        break
    if answer == 'd': # after break, to minimize code after set_trace
      print( _mkbox(
        'TRACING ACTIVATED. Use the Python debugger',
        'to step through the code at source line',
        'level, list source code, set breakpoints,',
        'and evaluate arbitrary Python code in the',
        'context of any stack frame. Type "h" for',
        'an overview of commands to get going, or',
        '"c" to continue uninterrupted execution.' ) )
      pdb.set_trace()
  finally:
    signal.signal( mysignal, _handler )

def run( *functions ):
  '''parse command line arguments and call function'''

  assert functions

  # parse command line arguments
  parser = argparse.ArgumentParser()
  parser.add_argument( '--nprocs', type=int, metavar='INT', default=core.globalproperties['nprocs'], help='number of processors' )
  parser.add_argument( '--outrootdir', type=str, metavar='PATH', default=core.globalproperties['outrootdir'], help='root directory for output' )
  parser.add_argument( '--outdir', type=str, metavar='PATH', default=None, help='custom directory for output' )
  parser.add_argument( '--verbose', type=int, metavar='INT', default=core.globalproperties['verbose'], help='verbosity level' )
  parser.add_argument( '--richoutput', type=_bool, nargs='?', const=True, metavar='BOOL', default=core.globalproperties['richoutput'], help='use rich output (colors, unicode)' )
  parser.add_argument( '--htmloutput', type=_bool, nargs='?', const=True, metavar='BOOL', default=core.globalproperties['htmloutput'], help='generate a HTML log' )
  parser.add_argument( '--pdb', type=_bool, nargs='?', const=True, metavar='BOOL', default=core.globalproperties['pdb'], help='start python debugger on error' )
  parser.add_argument( '--imagetype', type=str, metavar='STR', default=core.globalproperties['imagetype'], help='default image type' )
  parser.add_argument( '--symlink', type=str, metavar='STR', default=core.globalproperties['symlink'], help='create symlink to latest results' )
  parser.add_argument( '--recache', type=_bool, nargs='?', const=True, metavar='BOOL', default=core.globalproperties['recache'], help='overwrite existing cache' )
  parser.add_argument( '--dot', type=str, metavar='STR', default=core.globalproperties['dot'], help='graphviz executable' )
  parser.add_argument( '--selfcheck', type=_bool, nargs='?', const=True, metavar='BOOL', default=core.globalproperties['selfcheck'], help='active self checks (slow!)' )
  subparsers = parser.add_subparsers( dest='command', help='command (add -h for command-specific help)' )
  subparsers.required = True
  for func in functions:
    subparser = subparsers.add_parser( func.__name__, formatter_class=argparse.ArgumentDefaultsHelpFormatter )
    for parameter in inspect.signature( func ).parameters.values():
      subparser.add_argument( '--'+parameter.name,
        dest='='+parameter.name, # prefix with '=' to distinguish nutils/func args
        default=parameter.default,
        metavar=type(parameter.default).__name__.upper(),
        help=parameter.annotation if parameter.annotation is not inspect._empty else None,
        **{'type':_bool,'nargs':'?','const':True} if isinstance( parameter.default, bool ) else {'type':type(parameter.default)} )
  ns = parser.parse_args()

  # set properties
  __scriptname__ = os.path.basename(sys.argv[0])
  __nprocs__ = ns.nprocs
  __outrootdir__ = os.path.abspath(os.path.expanduser(ns.outrootdir))
  __cachedir__ = os.path.join( __outrootdir__, __scriptname__, 'cache' )
  __outdir__ = os.path.abspath(os.path.expanduser(ns.outdir)) if ns.outdir is not None \
          else os.path.join( __outrootdir__, __scriptname__, datetime.datetime.now().strftime('%Y/%m/%d/%H-%M-%S/') )
  __verbose__ = ns.verbose
  __richoutput__ = ns.richoutput
  __htmloutput__ = ns.htmloutput
  __pdb__ = ns.pdb
  __imagetype__ = ns.imagetype
  __symlink__ = ns.symlink
  __recache__ = ns.recache
  __dot__ = ns.dot
  __selfcheck__ = ns.selfcheck

  # call function
  func = { f.__name__: f for f in functions }[ ns.command ]
  kwargs = { key[1:]: val for key, val in vars(ns).items() if key[0] == '=' }
  status = call( func, **kwargs )
  sys.exit( status )

def call( func, **kwargs ):
  '''set up compute environment and call function'''

  outdir = core.getprop( 'outdir' )
  os.makedirs( outdir ) # asserts nonexistence

  symlink = core.getprop( 'symlink', None )
  if symlink:
    for base, relpath in _relative_paths( core.getprop('outrootdir'), outdir ):
      target = os.path.join( base, symlink )
      if os.path.islink( target ):
        os.remove( target )
      os.symlink( relpath, target )

  htmloutput = core.getprop( 'htmloutput', True )
  if htmloutput:
    for base, relpath in _relative_paths( core.getprop('outrootdir'), outdir ):
      with open( os.path.join(base,'log.html'), 'w' ) as redirlog:
        print( '', file=redirlog )
        print( '<meta http-equiv="cache-control" content="max-age=0" />', file=redirlog )
        print( '<meta http-equiv="cache-control" content="no-cache" />', file=redirlog )
        print( '<meta http-equiv="expires" content="0" />', file=redirlog )
        print( '<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />', file=redirlog )
        print( '<meta http-equiv="pragma" content="no-cache" />', file=redirlog )
        print( '<meta http-equiv="refresh" content="0;URL={}" />'.format(os.path.join(relpath,'log.html')), file=redirlog )
        print( '', file=redirlog )

  origdir = os.getcwd()
  orighandler = signal.signal( signal.SIGINT, _sigint_handler )
  try:
    os.chdir( outdir )

    scriptname = core.getprop( 'scriptname' )

    __log__ = log._mklog() if not htmloutput \
         else log.TeeLog( log._mklog(), log.HtmlLog( 'log.html', title=scriptname, scriptname=scriptname ) )

    with __log__:

      log.info( 'nutils v{}'.format( _version() ) )
      log.info( '' )
      log.info( '{} {}'.format( scriptname, func.__name__ ) )
      for arg, value in kwargs.items():
        log.info( '  --{}={}'.format( arg, value ) )

      starttime = datetime.datetime.now()

      log.info( '' )
      log.info( 'start {}'.format( starttime.ctime() ) )
      log.info( '' )

      func( **kwargs )

      endtime = datetime.datetime.now()
      minutes, seconds = divmod( (endtime-starttime).seconds, 60 )
      hours, minutes = divmod( minutes, 60 )

      log.info( '' )
      log.info( 'finish {}'.format( endtime.ctime() ) )
      log.info( 'elapsed {:.0f}:{:02.0f}:{:02.0f}'.format( hours, minutes, seconds ) )

  except (KeyboardInterrupt,SystemExit,pdb.bdb.BdbQuit):
    return 1
  except:
    if core.getprop( 'pdb', False ):
      print( _mkbox(
        'YOUR PROGRAM HAS DIED. The Python debugger',
        'allows you to examine its post-mortem state',
        'to figure out why this happened. Type "h"',
        'for an overview of commands to get going.' ) )
      pdb.post_mortem()
    return 2
  else:
    return 0
  finally:
    signal.signal( signal.SIGINT, orighandler )
    os.chdir( origdir )

# vim:shiftwidth=2:softtabstop=2:expandtab:foldmethod=indent:foldnestmax=2
<commit_msg>add annotations to log output<commit_after># -*- coding: utf8 -*-
#
# Module CLI
#
# Part of Nutils: open source numerical utilities for Python. Jointly developed
# by HvZ Computational Engineering, TU/e Multiscale Engineering Fluid Dynamics,
# and others. More info at http://nutils.org <info@nutils.org>. (c) 2014

"""
The cli (command line interface) module provides the `cli.run` function that
can be used set up properties, initiate an output environment, and execute a
python function based arguments specified on the command line.
"""

from . import log, core, version
import sys, inspect, os, datetime, argparse, pdb, signal, subprocess

def _version():
  try:
    githash = subprocess.check_output( ['git','rev-parse','--short','HEAD'], universal_newlines=True, cwd=os.path.dirname(__file__) ).strip()
    if subprocess.check_output( ['git','status','--untracked-files=no','--porcelain'], cwd=os.path.dirname(__file__) ):
      githash += '+'
  except:
    return version
  else:
    return '{} (git:{})'.format( version, githash )

def _bool( s ):
  if s in ('true','True'):
    return True
  if s in ('false','False'):
    return False
  raise argparse.ArgumentTypeError( 'invalid boolean value: {!r}'.format(s) )

def _relative_paths( basepath, path ):
  baseparts = basepath.rstrip( os.path.sep ).split( os.path.sep )
  parts = path.rstrip( os.path.sep ).split( os.path.sep )
  if parts[:len(baseparts)] == baseparts:
    for i in range( len(baseparts), len(parts) ):
      yield os.path.sep.join(parts[:i]), os.path.sep.join(parts[i:])

def _mkbox( *lines ):
  width = max( len(line) for line in lines )
  ul, ur, ll, lr, hh, vv = '┌┐└┘─│' if core.getprop('richoutput') else '++++-|'
  return '\n'.join( [ ul + hh * (width+2) + ur ]
                  + [ vv + (' '+line).ljust(width+2) + vv for line in lines ]
                  + [ ll + hh * (width+2) + lr ] )

def _sigint_handler( mysignal, frame ):
  _handler = signal.signal( mysignal, signal.SIG_IGN ) # temporarily disable handler
  try:
    while True:
      answer = input( 'interrupted. quit, continue or start debugger? [q/c/d]' )
      if answer == 'q':
        raise KeyboardInterrupt
      if answer == 'c' or answer == 'd':
        break
    if answer == 'd': # after break, to minimize code after set_trace
      print( _mkbox(
        'TRACING ACTIVATED. Use the Python debugger',
        'to step through the code at source line',
        'level, list source code, set breakpoints,',
        'and evaluate arbitrary Python code in the',
        'context of any stack frame. Type "h" for',
        'an overview of commands to get going, or',
        '"c" to continue uninterrupted execution.' ) )
      pdb.set_trace()
  finally:
    signal.signal( mysignal, _handler )

def run( *functions ):
  '''parse command line arguments and call function'''

  assert functions

  # parse command line arguments
  parser = argparse.ArgumentParser()
  parser.add_argument( '--nprocs', type=int, metavar='INT', default=core.globalproperties['nprocs'], help='number of processors' )
  parser.add_argument( '--outrootdir', type=str, metavar='PATH', default=core.globalproperties['outrootdir'], help='root directory for output' )
  parser.add_argument( '--outdir', type=str, metavar='PATH', default=None, help='custom directory for output' )
  parser.add_argument( '--verbose', type=int, metavar='INT', default=core.globalproperties['verbose'], help='verbosity level' )
  parser.add_argument( '--richoutput', type=_bool, nargs='?', const=True, metavar='BOOL', default=core.globalproperties['richoutput'], help='use rich output (colors, unicode)' )
  parser.add_argument( '--htmloutput', type=_bool, nargs='?', const=True, metavar='BOOL', default=core.globalproperties['htmloutput'], help='generate a HTML log' )
  parser.add_argument( '--pdb', type=_bool, nargs='?', const=True, metavar='BOOL', default=core.globalproperties['pdb'], help='start python debugger on error' )
  parser.add_argument( '--imagetype', type=str, metavar='STR', default=core.globalproperties['imagetype'], help='default image type' )
  parser.add_argument( '--symlink', type=str, metavar='STR', default=core.globalproperties['symlink'], help='create symlink to latest results' )
  parser.add_argument( '--recache', type=_bool, nargs='?', const=True, metavar='BOOL', default=core.globalproperties['recache'], help='overwrite existing cache' )
  parser.add_argument( '--dot', type=str, metavar='STR', default=core.globalproperties['dot'], help='graphviz executable' )
  parser.add_argument( '--selfcheck', type=_bool, nargs='?', const=True, metavar='BOOL', default=core.globalproperties['selfcheck'], help='active self checks (slow!)' )
  subparsers = parser.add_subparsers( dest='command', help='command (add -h for command-specific help)' )
  subparsers.required = True
  for func in functions:
    subparser = subparsers.add_parser( func.__name__, formatter_class=argparse.ArgumentDefaultsHelpFormatter )
    for parameter in inspect.signature( func ).parameters.values():
      subparser.add_argument( '--'+parameter.name,
        dest='='+parameter.name, # prefix with '=' to distinguish nutils/func args
        default=parameter.default,
        metavar=type(parameter.default).__name__.upper(),
        help=parameter.annotation if parameter.annotation is not inspect._empty else None,
        **{'type':_bool,'nargs':'?','const':True} if isinstance( parameter.default, bool ) else {'type':type(parameter.default)} )
  ns = parser.parse_args()

  # set properties
  __scriptname__ = os.path.basename(sys.argv[0])
  __nprocs__ = ns.nprocs
  __outrootdir__ = os.path.abspath(os.path.expanduser(ns.outrootdir))
  __cachedir__ = os.path.join( __outrootdir__, __scriptname__, 'cache' )
  __outdir__ = os.path.abspath(os.path.expanduser(ns.outdir)) if ns.outdir is not None \
          else os.path.join( __outrootdir__, __scriptname__, datetime.datetime.now().strftime('%Y/%m/%d/%H-%M-%S/') )
  __verbose__ = ns.verbose
  __richoutput__ = ns.richoutput
  __htmloutput__ = ns.htmloutput
  __pdb__ = ns.pdb
  __imagetype__ = ns.imagetype
  __symlink__ = ns.symlink
  __recache__ = ns.recache
  __dot__ = ns.dot
  __selfcheck__ = ns.selfcheck

  # call function
  func = { f.__name__: f for f in functions }[ ns.command ]
  kwargs = { key[1:]: val for key, val in vars(ns).items() if key[0] == '=' }
  status = call( func, **kwargs )
  sys.exit( status )

def call( func, **kwargs ):
  '''set up compute environment and call function'''

  outdir = core.getprop( 'outdir' )
  os.makedirs( outdir ) # asserts nonexistence

  symlink = core.getprop( 'symlink', None )
  if symlink:
    for base, relpath in _relative_paths( core.getprop('outrootdir'), outdir ):
      target = os.path.join( base, symlink )
      if os.path.islink( target ):
        os.remove( target )
      os.symlink( relpath, target )

  htmloutput = core.getprop( 'htmloutput', True )
  if htmloutput:
    for base, relpath in _relative_paths( core.getprop('outrootdir'), outdir ):
      with open( os.path.join(base,'log.html'), 'w' ) as redirlog:
        print( '', file=redirlog )
        print( '<meta http-equiv="cache-control" content="max-age=0" />', file=redirlog )
        print( '<meta http-equiv="cache-control" content="no-cache" />', file=redirlog )
        print( '<meta http-equiv="expires" content="0" />', file=redirlog )
        print( '<meta http-equiv="expires" content="Tue, 01 Jan 1980 1:00:00 GMT" />', file=redirlog )
        print( '<meta http-equiv="pragma" content="no-cache" />', file=redirlog )
        print( '<meta http-equiv="refresh" content="0;URL={}" />'.format(os.path.join(relpath,'log.html')), file=redirlog )
        print( '', file=redirlog )

  origdir = os.getcwd()
  orighandler = signal.signal( signal.SIGINT, _sigint_handler )
  try:
    os.chdir( outdir )

    scriptname = core.getprop( 'scriptname' )

    __log__ = log._mklog() if not htmloutput \
         else log.TeeLog( log._mklog(), log.HtmlLog( 'log.html', title=scriptname, scriptname=scriptname ) )

    with __log__:

      log.info( 'nutils v{}'.format( _version() ) )
      log.info( '' )
      log.info( '{} {}'.format( scriptname, func.__name__ ) )
      for parameter in inspect.signature( func ).parameters.values():
        argstr = '  --{}={}'.format( parameter.name, kwargs.get(parameter.name,parameter.default) )
        if parameter.annotation is not inspect._empty:
          argstr += ' ({})'.format( parameter.annotation )
        log.info( argstr )

      starttime = datetime.datetime.now()

      log.info( '' )
      log.info( 'start {}'.format( starttime.ctime() ) )
      log.info( '' )

      func( **kwargs )

      endtime = datetime.datetime.now()
      minutes, seconds = divmod( (endtime-starttime).seconds, 60 )
      hours, minutes = divmod( minutes, 60 )

      log.info( '' )
      log.info( 'finish {}'.format( endtime.ctime() ) )
      log.info( 'elapsed {:.0f}:{:02.0f}:{:02.0f}'.format( hours, minutes, seconds ) )

  except (KeyboardInterrupt,SystemExit,pdb.bdb.BdbQuit):
    return 1
  except:
    if core.getprop( 'pdb', False ):
      print( _mkbox(
        'YOUR PROGRAM HAS DIED. The Python debugger',
        'allows you to examine its post-mortem state',
        'to figure out why this happened. Type "h"',
        'for an overview of commands to get going.' ) )
      pdb.post_mortem()
    return 2
  else:
    return 0
  finally:
    signal.signal( signal.SIGINT, orighandler )
    os.chdir( origdir )

# vim:shiftwidth=2:softtabstop=2:expandtab:foldmethod=indent:foldnestmax=2
<|endoftext|>