<commit_before>## Copyright (c) 2012 Aldebaran Robotics. All rights reserved.
## Use of this source code is governed by a BSD-style license that can be
## found in the COPYING file.

"""Display the toolchains names.

"""

import qibuild
import qitoolchain

def configure_parser(parser):
    """Configure parser for this action """
    qibuild.parsers.default_parser(parser)

def do(args):
    """ Main method """
    tc_names = qitoolchain.get_tc_names()
    if not tc_names:
        print "No toolchain yet"
        print "Use `qitoolchain create` to create a new toolchain"
        return
    print "Known toolchains:"
    for tc_name in tc_names:
        print "  ", tc_name
    print
    print "Use ``qitoolchain info <tc_name>`` for more info"
<commit_msg>Display default toolchain for current worktree (or --worktree).<commit_after>## Copyright (c) 2012 Aldebaran Robotics. All rights reserved.
## Use of this source code is governed by a BSD-style license that can be
## found in the COPYING file.

"""Display the toolchains names.

"""

import qisrc
import qibuild
import qitoolchain

from qibuild import ui

def configure_parser(parser):
    """Configure parser for this action """
    qibuild.parsers.worktree_parser(parser)

def do(args):
    """ Main method """
    tc_names = qitoolchain.get_tc_names()
    if not tc_names:
        print "No toolchain yet"
        print "Use `qitoolchain create` to create a new toolchain"
        return
    default_toc_name = None
    try:
        worktree = qisrc.worktree.open_worktree(args.worktree)
        toc = qibuild.toc.Toc(worktree)
        default_toc_name = toc.config.local.defaults.config
    except qisrc.worktree.NotInWorktree, e:
        pass
    print "Known toolchains:"
    for tc_name in tc_names:
        print "* " if tc_name == default_toc_name else "  ", tc_name
    print
    if default_toc_name is not None:
        ui.info("Worktree", ui.green, worktree.root, ui.reset, "is using",
                ui.blue, default_toc_name, ui.reset,
                "as its default toolchain.")
    print "Use ``qitoolchain info <tc_name>`` for more info"
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# shows live output from invoked subprocess
# usually you only get output once process completes
#
# https://github.com/fabianlee/blogcode/blob/master/python/runProcessWithLiveOutput.py
#
import sys
import argparse
import subprocess
import shlex

# runs subprocess with Popen, but output only returned when process complete
def invoke_process_popen_blocking(command,shellType=False,stdoutType=subprocess.PIPE):
  try:
    process = subprocess.Popen(shlex.split(command),shell=shellType,stdout=stdoutType)
    (stdout,stderr) = process.communicate()
    print(stdout.decode())
  except:
    print("ERROR {} while running {}".format(sys.exc_info()[1],command))

# runs subprocess with Popen/poll so that live stdout is shown
def invoke_process_popen_poll_live(command,shellType=False,stdoutType=subprocess.PIPE):
  try:
    process = subprocess.Popen(shlex.split(command),shell=shellType,stdout=stdoutType)
  except:
    print("ERROR {} while running {}".format(sys.exc_info()[1],command))
    return None
  while True:
    output = process.stdout.readline()
    # used to check for empty output in Python2, but seems
    # to work with just poll in 2.7.12 and 3.5.2
    #if output == '' and process.poll() is not None:
    if process.poll() is not None:
      break
    if output:
      print(output.strip().decode())
  rc = process.poll()
  return rc

def main(argv):
  while True:

    # python2 uses raw_input(), python3 uses input()
    prompt = "Execute which commmand [./loopWithSleep.sh]: "
    try:
      cmd = raw_input(prompt)
    except NameError:
      cmd = input(prompt)
    if "quit"==cmd: break
    if ""==cmd: cmd="./loopWithSleep.sh"

    print("== invoke_process_popen_blocking  ==============")
    invoke_process_popen_blocking(cmd)

    print("== invoke_process_popen_poll_live  ==============")
    invoke_process_popen_poll_live(cmd)

if __name__=='__main__':main(sys.argv)
<commit_msg>PEP8 style<commit_after>#!/usr/bin/env python
#
# shows live output from invoked subprocess
# usually you only get output once process completes
#
# https://github.com/fabianlee/blogcode/blob/master/python/runProcessWithLiveOutput.py
#
import sys
import argparse
import subprocess
import shlex

def invoke_process_popen_blocking(command, shellType=False, stdoutType=subprocess.PIPE):
    """runs subprocess with Popen, but output only returned when process complete"""
    try:
        process = subprocess.Popen(
            shlex.split(command), shell=shellType, stdout=stdoutType)
        (stdout, stderr) = process.communicate()
        print(stdout.decode())
    except:
        print("ERROR {} while running {}".format(sys.exc_info()[1], command))

def invoke_process_popen_poll_live(command, shellType=False, stdoutType=subprocess.PIPE):
    """runs subprocess with Popen/poll so that live stdout is shown"""
    try:
        process = subprocess.Popen(
            shlex.split(command), shell=shellType, stdout=stdoutType)
    except:
        print("ERROR {} while running {}".format(sys.exc_info()[1], command))
        return None
    while True:
        output = process.stdout.readline()
        # used to check for empty output in Python2, but seems
        # to work with just poll in 2.7.12 and 3.5.2
        # if output == '' and process.poll() is not None:
        if process.poll() is not None:
            break
        if output:
            print(output.strip().decode())
    rc = process.poll()
    return rc

def main(argv):
    while True:

        # python2 uses raw_input(), python3 uses input()
        prompt = "Execute which commmand [./loopWithSleep.sh]: "
        try:
            cmd = raw_input(prompt)
        except NameError:
            cmd = input(prompt)
        if "quit" == cmd:
            break
        if "" == cmd:
            cmd = "./loopWithSleep.sh"

        print("== invoke_process_popen_blocking  ==============")
        invoke_process_popen_blocking(cmd)

        print("== invoke_process_popen_poll_live  ==============")
        invoke_process_popen_poll_live(cmd)

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from .functions import UserDefinedJythonFunction

from pyspark.sql.catalog import Catalog
from pyspark.sql.types import DataType, StringType

def registerJythonFunction(self, name, fn, returnType=StringType()):
    """
    Register a function to be executed using Jython on the workers.
    The function passed in must either be a string containing your python lambda expression,
    or if you have dill installed on the driver a lambda dill can extract the source for.
    Note that not all Python code will execute in Jython and not all Python
    code will execute well in Jython. However, for some UDFs, executing in Jython
    may be faster as we can avoid copying the data from the JVM to the Python
    executor.

    This is a very experimental feature, and may be removed in future versions
    once we figure out if it is a good idea or not.
    ...Note: Experimental

    :param name: name of the UDF
    :param f: String containing python lambda or python function
    :param returnType: a :class:`DataType` object
    """
    udf = UserDefinedJythonFunction(fn, returnType, name)
    jython = self._jsparkSession.jvm.org.apache.spark.sql.Jython

    pimpedRegistrar = jython.JythonUDFRegistration(self._jsparkSession.udf())

    pimpedRegistrar.registerJython(name, udf._judf)
    return udf

# Monkey patch in this function
Catalog.registerJythonFunction = registerJythonFunction
<commit_msg>Fixed incorrect internal usage for sparkSession<commit_after>from __future__ import absolute_import
from .functions import UserDefinedJythonFunction

from pyspark.sql.catalog import Catalog
from pyspark.sql.types import DataType, StringType

def registerJythonFunction(self, name, fn, returnType=StringType()):
    """
    Register a function to be executed using Jython on the workers.
    The function passed in must either be a string containing your python lambda expression,
    or if you have dill installed on the driver a lambda dill can extract the source for.
    Note that not all Python code will execute in Jython and not all Python
    code will execute well in Jython. However, for some UDFs, executing in Jython
    may be faster as we can avoid copying the data from the JVM to the Python
    executor.

    This is a very experimental feature, and may be removed in future versions
    once we figure out if it is a good idea or not.
    ...Note: Experimental

    :param name: name of the UDF
    :param f: String containing python lambda or python function
    :param returnType: a :class:`DataType` object
    """
    udf = UserDefinedJythonFunction(fn, returnType, name)
    jython = self._sparkSession.jvm.org.apache.spark.sql.Jython

    pimpedRegistrar = jython.JythonUDFRegistration(self._jsparkSession.udf())

    pimpedRegistrar.registerJython(name, udf._judf)
    return udf

# Monkey patch in this function
Catalog.registerJythonFunction = registerJythonFunction
<|endoftext|>


<commit_before>from django.test.client import Client
import unittest

class MingusClientTests(unittest.TestCase):

    fixtures = ['test_data.json',]

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_HomePage(self):
        '''Test if the homepage renders.'''
        c = Client()
        response = c.get('/')
        self.failUnlessEqual(response.status_code, 200)

    def test_Homepage_Paging(self):
        '''
        Test paging the homepage list.

        '''

        #need to update blog Settings page size, it defaults to 20 but we
        #don't have 20 records to page. So we'll set it to 1 and we should be ok.
        from basic.blog.models import Settings
        settings = Settings.get_current()
        settings.page_size = 1
        settings.save()

        c = Client()
        response = c.get('/', {'page': 2},)
        self.failUnlessEqual(response.status_code, 200)

    def test_Search(self):
        '''Test search view.'''

        c = Client()
        response = c.get('/search/', {'q':'django'})

        #test success request
        self.failUnlessEqual(response.status_code, 200)

        #test result as expected
        self.assertEquals(response.context['object_list'][0].title, 'Django Community')

    def test_About(self):
        'Test if the about page renders.'
        c = Client()
        response = c.get('/about/')
        self.failUnlessEqual(response.status_code, 200)

    def test_Contact(self):
        'Test if the contact page renders.'
        c = Client()
        response = c.get('/contact/')

        self.failUnlessEqual(response.status_code, 200)

    def test_ContactSubmit(self):
        '''
        Test submitting the contact form. Expect to return to the form sent
        template.

        The field 'fonzie_kungfu' is the honeypot field to protect you from
        spam. This feature is provided by django-honeypot.

        '''
        c = Client()
        response = c.post('/contact/', {'name': 'charles',
                    'email': 'foo@foo.com', 'body': 'hello.',
                    'fonzie_kungfu': ''},
                    follow=True)
        self.failUnlessEqual(response.status_code, 200)

    def test_ContactSubmit_WithHoneyPot(self):
        '''Test the @honeypot decorator which exists to reduce spam.

        HoneyPot will return a 400 response if the honeypot field is
        submited with a value.

        '''
        c = Client()
        response = c.post('/contact/', {'name': 'charles', 'email':
                    'foo@foo.com', 'body': 'hello.', 'fonzie_kungfu': 'evil'},
                    follow=True)
        self.failUnlessEqual(response.status_code, 400)

    def test_QuoteList(self):
        '''Test quote list page renders.'''

        c = Client()
        response = c.get('/quotes/')
        self.failUnlessEqual(response.status_code, 200)

    def test_QuoteDetail(self):
        '''Test quote list page renders.'''

        from quoteme.models import Quote
        quote = Quote.objects.all()[0]

        c = Client()
        response = c.get(quote.get_absolute_url())
        self.failUnlessEqual(response.status_code, 200)

    def test_RSS(self):
        '''Test the latest posts feed displays.'''

        c = Client()
        response = c.get('/feeds/latest/')
        self.failUnlessEqual(response.status_code, 200)

    def test_SpringsteenFeed_Posts(self):
        '''
        Test the latest Post springsteen feed for findjango integration
        displays.
        '''

        c = Client()
        response = c.get('/api/springsteen/posts/')
        self.failUnlessEqual(response.status_code, 200)

    def test_SpringsteenFeed_Posts_ByCategory(self):
        '''
        Test the latest Post By Category springsteen feed for findjango
        integration displays.

        '''

        c = Client()
        response = c.get('/api/springsteen/category/django/')
        self.failUnlessEqual(response.status_code, 200)

<commit_msg>now using django.test.TestCase<commit_after>from django.test.client import Client
from django.test import TestCase

class MingusClientTests(TestCase):

    fixtures = ['test_data.json', ]

    def setUp(self):
        pass

    def tearDown(self):
        pass

    def test_HomePage(self):
        '''Test if the homepage renders.'''
        c = Client()
        response = c.get('/')
        self.failUnlessEqual(response.status_code, 200)

    def test_Homepage_Paging(self):
        '''
        Test paging the homepage list.

        '''

        #need to update blog Settings page size, it defaults to 20 but we
        #don't have 20 records to page. So we'll set it to 1 and we should be ok.
        from basic.blog.models import Settings
        settings = Settings.get_current()
        settings.page_size = 1
        settings.save()

        c = Client()
        response = c.get('/', {'page': 2},)
        self.failUnlessEqual(response.status_code, 200)

    def test_Search(self):
        '''Test search view.'''

        c = Client()
        response = c.get('/search/', {'q':'django'})

        #test success request
        self.failUnlessEqual(response.status_code, 200)

        #test result as expected
        self.assertEquals(response.context['object_list'][0].title, 'Django Community')

    def test_About(self):
        'Test if the about page renders.'
        c = Client()
        response = c.get('/about/')
        self.failUnlessEqual(response.status_code, 200)

    def test_Contact(self):
        'Test if the contact page renders.'
        c = Client()
        response = c.get('/contact/')

        self.failUnlessEqual(response.status_code, 200)

    def test_ContactSubmit(self):
        '''
        Test submitting the contact form. Expect to return to the form sent
        template.

        The field 'fonzie_kungfu' is the honeypot field to protect you from
        spam. This feature is provided by django-honeypot.

        '''
        c = Client()
        response = c.post('/contact/', {'name': 'charles',
                    'email': 'foo@foo.com', 'body': 'hello.',
                    'fonzie_kungfu': ''},
                    follow=True)
        self.failUnlessEqual(response.status_code, 200)

    def test_ContactSubmit_WithHoneyPot(self):
        '''Test the @honeypot decorator which exists to reduce spam.

        HoneyPot will return a 400 response if the honeypot field is
        submited with a value.

        '''
        c = Client()
        response = c.post('/contact/', {'name': 'charles', 'email':
                    'foo@foo.com', 'body': 'hello.', 'fonzie_kungfu': 'evil'},
                    follow=True)
        self.failUnlessEqual(response.status_code, 400)

    def test_QuoteList(self):
        '''Test quote list page renders.'''

        c = Client()
        response = c.get('/quotes/')
        self.failUnlessEqual(response.status_code, 200)

    def test_QuoteDetail(self):
        '''Test quote list page renders.'''

        from quoteme.models import Quote
        quote = Quote.objects.all()[0]

        c = Client()
        response = c.get(quote.get_absolute_url())
        self.failUnlessEqual(response.status_code, 200)

    def test_RSS(self):
        '''Test the latest posts feed displays.'''

        c = Client()
        response = c.get('/feeds/latest/')
        self.failUnlessEqual(response.status_code, 200)

    def test_SpringsteenFeed_Posts(self):
        '''
        Test the latest Post springsteen feed for findjango integration
        displays.
        '''

        c = Client()
        response = c.get('/api/springsteen/posts/')
        self.failUnlessEqual(response.status_code, 200)

    def test_SpringsteenFeed_Posts_ByCategory(self):
        '''
        Test the latest Post By Category springsteen feed for findjango
        integration displays.

        '''

        c = Client()
        response = c.get('/api/springsteen/category/django/')
        self.failUnlessEqual(response.status_code, 200)

<|endoftext|>


<commit_before>"""
link.py: interface and link abstractions for mininet

It seems useful to bundle functionality for interfaces into a single
class.

Also it seems useful to enable the possibility of multiple flavors of
links, including:

- simple veth pairs
- tunneled links
- patchable links (which can be disconnected and reconnected via a patchbay)
- link simulators (e.g. wireless)

Basic division of labor:

  Nodes: know how to execute commands
  Intfs: know how to configure themselves
  Links: know how to connect nodes together

Intf: basic interface object that can configure itself
TCIntf: interface with bandwidth limiting and delay via tc

Link: basic link class for creating veth pairs
"""

from mininet.log import info, error, debug
from mininet.util import makeIntfPair, quietRun
#from time import sleep
import re

class Intf( object ):

    "Basic interface object that can configure itself."

    def __init__( self, name, node=None, port=None, link=None, **params ):
        """name: interface name (e.g. h1-eth0)
           node: owning node (where this intf most likely lives)
           link: parent link if we're part of a link
           other arguments are passed to config()"""
        self.node = node
        self.name = name
        self.link = link
        self.mac, self.ip, self.prefixLen = None, None, None
        # Add to node (and move ourselves if necessary )
        node.addIntf( self, port=port )
        # Save params for future reference
        self.params = params
        self.config( **params )

    def cmd( self, *args, **kwargs ):
        "Run a command in our owning node"
        return self.node.cmd( *args, **kwargs )

    def ifconfig( self, *args ):
        "Configure ourselves using ifconfig"
        return self.cmd( 'ifconfig', self.name, *args )

    def setIP( self, ipstr, prefixLen=None ):
        """Set our IP address"""
        # This is a sign that we should perhaps rethink our prefix
        # mechanism and/or the way we specify IP addresses
        if '/' in ipstr:
            self.ip, self.prefixLen = ipstr.split( '/' )
            return self.ifconfig( ipstr, 'up' )
        else:
            self.ip, self.prefixLen = ipstr, prefixLen
            return self.ifconfig( '%s/%s' % ( ipstr, prefixLen ) )

    def setMAC( self, macstr ):
        """Set the MAC address for an interface.
           macstr: MAC address as string"""
        self.mac = macstr
        return ( self.ifconfig( 'down' ) +
                 self.ifconfig( 'hw', 'ether', macstr ) +
                 self.ifconfig( 'up' ) )

    _ipMatchRegex = re.compile( r'\d+\.\d+\.\d+\.\d+' )
    _macMatchRegex = re.compile( r'..:..:..:..:..:..' )

    def updateIP( self ):
        "Return updated IP address based on ifconfig"
        ifconfig = self.ifconfig()
        ips = self._ipMatchRegex.findall( ifconfig )
        self.ip = ips[ 0 ] if ips else None
        return self.ip

    def updateMAC( self ):
        "Return updated MAC address based on ifconfig"
        ifconfig = self.ifconfig()
        macs = self._macMatchRegex.findall( ifconfig )
        self.mac = macs[ 0 ] if macs else None
        return self.mac

    def IP( self ):
        "Return IP address"
        return self.ip

    def MAC( self ):
        "Return MAC address"
        return self.mac

    def isUp( self, setUp=False ):
        "Return whether interface is up"
        if setUp:
            self.ifconfig( 'up' )
        return "UP" in self.ifconfig()

    def rename( self, newname ):
        "Rename interface"
        self.ifconfig( 'down' )
        result = self.cmd( 'ip link set', self.name, 'name', newname )
        self.name = newname
        self.ifconfig( 'up' )
        return result

    # The reason why we configure things in this way is so
    # That the parameters can be listed and documented in
    # the config method.
    # Dealing with subclasses and superclasses is slightly
    # annoying, but at least the information is there!

    def setParam( self, results, method, **param ):
        """Internal method: configure a *single* parameter
           results: dict of results to update
           method: config method name
           param: arg=value (ignore if value=None)
           value may also be list or dict"""
        name, value = param.items()[ 0 ]
        f = getattr( self, method, None )
        if not f or value is None:
            return
        if type( value ) is list:
            result = f( *value )
        elif type( value ) is dict:
            result = f( **value )
        else:
            result = f( value )
        results[ name ] = result
        return result

    def config( self, mac=None, ip=None, ifconfig=None,
                up=True, **_params ):
        """Configure Node according to (optional) parameters:
           mac: MAC address
           ip: IP address
           ifconfig: arbitrary interface configuration
           Subclasses should override this method and call
           the parent class's config(**params)"""
        # If we were overriding this method, we would call
        # the superclass config method here as follows:
        # r = Parent.config( **params )
        r = {}
        self.setParam( r, 'setMAC', mac=mac )
        self.setParam( r, 'setIP', ip=ip )
        self.setParam( r, 'isUp', up=up )
        self.setParam( r, 'ifconfig', ifconfig=ifconfig )
        self.updateIP()
        self.updateMAC()
        return r

    def delete( self ):
        "Delete interface"
        self.cmd( 'ip link del ' + self.name )
        if self.node.inNamespace:
            # Link may have been dumped into root NS
            quietRun( 'ip link del ' + self.name )

    def __repr__( self ):
        return '<%s %s>' % ( self.__class__.__name__, self.name )

    def __str__( self ):
        return self.name

class TCIntf( Intf ):
    """Interface customized by tc (traffic control) utility
       Allows specification of bandwidth limits (various methods)
       as well as delay, loss and max queue length"""

    def bwCmds( self, bw=None, speedup=0, use_hfsc=False, use_tbf=False,
                latency_ms=None, enable_ecn=False, enable_red=False ):
        "Return tc commands to set bandwidth"

        cmds, parent = [], ' root '

        if bw and ( bw < 0 or bw > 1000 ):
            error( 'Bandwidth', bw, 'is outside range 0..1000 Mbps\n' )

        elif bw is not None:
            # BL: this seems a bit brittle...
            if ( speedup > 0 and
                 self.node.name[0:1] == 's' ):
                bw = speedup
            # This may not be correct - we should look more closely
            # at the semantics of burst (and cburst) to make sure we
            # are specifying the correct sizes. For now I have used
            # the same settings we had in the mininet-hifi code.
            if use_hfsc:
                cmds += [ '%s qdisc add dev %s root handle 5:0 hfsc default 1',
                          '%s class add dev %s parent 5:0 classid 5:1 hfsc sc '
                          + 'rate %fMbit ul rate %fMbit' % ( bw, bw ) ]
            elif use_tbf:
                if latency_ms is None:
                    latency_ms = 15 * 8 / bw
                cmds += [ '%s qdisc add dev %s root handle 5: tbf ' +
                          'rate %fMbit burst 15000 latency %fms' %
                          ( bw, latency_ms ) ]
            else:
                cmds += [ '%s qdisc add dev %s root handle 5:0 htb default 1',
                          '%s class add dev %s parent 5:0 classid 5:1 htb ' +
                          'rate %fMbit burst 15k' % bw ]
            parent = ' parent 5:1 '

            # ECN or RED
            if enable_ecn:
                cmds += [ '%s qdisc add dev %s' + parent +
                          'handle 6: red limit 1000000 ' +
                          'min 30000 max 35000 avpkt 1500 ' +
                          'burst 20 ' +
                          'bandwidth %fmbit probability 1 ecn' % bw ]
                parent = ' parent 6: '
            elif enable_red:
                cmds += [ '%s qdisc add dev %s' + parent +
                          'handle 6: red limit 1000000 ' +
                          'min 30000 max 35000 avpkt 1500 ' +
                          'burst 20 ' +
                          'bandwidth %fmbit probability 1' % bw ]
                parent = ' parent 6: '
        return cmds, parent

    @staticmethod
    def delayCmds( parent, delay=None, jitter=None,
                   loss=None, max_queue_size=None ):
        "Internal method: return tc commands for delay and loss"
        cmds = []
        if delay and delay < 0:
            error( 'Negative delay', delay, '\n' )
        elif jitter and jitter < 0:
            error( 'Negative jitter', jitter, '\n' )
        elif loss and ( loss < 0 or loss > 100 ):
            error( 'Bad loss percentage', loss, '%%\n' )
        else:
            # Delay/jitter/loss/max queue size
            netemargs = '%s%s%s%s' % (
                'delay %s ' % delay if delay is not None else '',
                '%s ' % jitter if jitter is not None else '',
                'loss %d ' % loss if loss is not None else '',
                'limit %d' % max_queue_size if max_queue_size is not None
                else '' )
            if netemargs:
                cmds = [ '%s qdisc add dev %s ' + parent +
                         ' handle 10: netem ' +
                         netemargs ]
                parent = ' parent 10:1 '
        return cmds, parent

    def tc( self, cmd, tc='tc' ):
        "Execute tc command for our interface"
        c = cmd % (tc, self)  # Add in tc command and our name
        debug(" *** executing command: %s\n" % c)
        return self.cmd( c )

    def config( self, bw=None, delay=None, jitter=None, loss=None,
                disable_gro=True, speedup=0, use_hfsc=False, use_tbf=False,
                latency_ms=None, enable_ecn=False, enable_red=False,
                max_queue_size=None, **params ):
        "Configure the port and set its properties."

        result = Intf.config( self, **params)

        # Disable GRO
        if disable_gro:
            self.cmd( 'ethtool -K %s gro off' % self )

        # Optimization: return if nothing else to configure
        # Question: what happens if we want to reset things?
        if ( bw is None and not delay and not loss
             and max_queue_size is None ):
            return

        # Clear existing configuration
        cmds = [ '%s qdisc del dev %s root' ]

        # Bandwidth limits via various methods
        bwcmds, parent = self.bwCmds( bw=bw, speedup=speedup,
                                      use_hfsc=use_hfsc, use_tbf=use_tbf,
                                      latency_ms=latency_ms,
                                      enable_ecn=enable_ecn,
                                      enable_red=enable_red )
        cmds += bwcmds

        # Delay/jitter/loss/max_queue_size using netem
        delaycmds, parent = self.delayCmds( delay=delay, jitter=jitter,
                                loss=loss, max_queue_size=max_queue_size,
                                parent=parent )
        cmds += delaycmds

        # Ugly but functional: display configuration info
        stuff = ( ( [ '%.2fMbit' % bw ] if bw is not None else [] ) +
                  ( [ '%s delay' % delay ] if delay is not None else [] ) +
                  ( [ '%s jitter' % jitter ] if jitter is not None else [] ) +
                  ( ['%d%% loss' % loss ] if loss is not None else [] ) +
                  ( [ 'ECN' ] if enable_ecn else [ 'RED' ]
                    if enable_red else [] ) )
        info( '(' + ' '.join( stuff ) + ') ' )

        # Execute all the commands in our node
        debug("at map stage w/cmds: %s\n" % cmds)
        tcoutputs = [ self.tc(cmd) for cmd in cmds ]
        debug( "cmds:", cmds, '\n' )
        debug( "outputs:", tcoutputs, '\n' )
        result[ 'tcoutputs'] = tcoutputs
        result[ 'parent' ] = parent

        return result

class Link( object ):

    """A basic link is just a veth pair.
       Other types of links could be tunnels, link emulators, etc.."""

    def __init__( self, node1, node2, port1=None, port2=None,
                  intfName1=None, intfName2=None,
                  intf=Intf, cls1=None, cls2=None, params1=None,
                  params2=None ):
        """Create veth link to another node, making two new interfaces.
           node1: first node
           node2: second node
           port1: node1 port number (optional)
           port2: node2 port number (optional)
           intf: default interface class/constructor
           cls1, cls2: optional interface-specific constructors
           intfName1: node1 interface name (optional)
           intfName2: node2  interface name (optional)
           params1: parameters for interface 1
           params2: parameters for interface 2"""
        # This is a bit awkward; it seems that having everything in
        # params would be more orthogonal, but being able to specify
        # in-line arguments is more convenient!
        if port1 is None:
            port1 = node1.newPort()
        if port2 is None:
            port2 = node2.newPort()
        if not intfName1:
            intfName1 = self.intfName( node1, port1 )
        if not intfName2:
            intfName2 = self.intfName( node2, port2 )

        self.makeIntfPair( intfName1, intfName2 )

        if not cls1:
            cls1 = intf
        if not cls2:
            cls2 = intf
        if not params1:
            params1 = {}
        if not params2:
            params2 = {}

        intf1 = cls1( name=intfName1, node=node1, port=port1,
                      link=self, **params1  )
        intf2 = cls2( name=intfName2, node=node2, port=port2,
                      link=self, **params2 )

        # All we are is dust in the wind, and our two interfaces
        self.intf1, self.intf2 = intf1, intf2

    @classmethod
    def intfName( cls, node, n ):
        "Construct a canonical interface name node-ethN for interface n."
        return node.name + '-eth' + repr( n )

    @classmethod
    def makeIntfPair( cls, intf1, intf2 ):
        """Create pair of interfaces
           intf1: name of interface 1
           intf2: name of interface 2
           (override this class method [and possibly delete()]
           to change link type)"""
        makeIntfPair( intf1, intf2  )

    def delete( self ):
        "Delete this link"
        self.intf1.delete()
        self.intf2.delete()

    def __str__( self ):
        return '%s<->%s' % ( self.intf1, self.intf2 )

class TCLink( Link ):
    "Link with symmetric TC interfaces configured via opts"
    def __init__( self, node1, node2, port1=None, port2=None,
                  intfName1=None, intfName2=None, **params ):
        Link.__init__( self, node1, node2, port1=port1, port2=port2,
                       intfName1=intfName1, intfName2=intfName2,
                       cls1=TCIntf,
                       cls2=TCIntf,
                       params1=params,
                       params2=params)
<commit_msg>Removing unused import in mininet/link.py<commit_after>"""
link.py: interface and link abstractions for mininet

It seems useful to bundle functionality for interfaces into a single
class.

Also it seems useful to enable the possibility of multiple flavors of
links, including:

- simple veth pairs
- tunneled links
- patchable links (which can be disconnected and reconnected via a patchbay)
- link simulators (e.g. wireless)

Basic division of labor:

  Nodes: know how to execute commands
  Intfs: know how to configure themselves
  Links: know how to connect nodes together

Intf: basic interface object that can configure itself
TCIntf: interface with bandwidth limiting and delay via tc

Link: basic link class for creating veth pairs
"""

from mininet.log import info, error, debug
from mininet.util import makeIntfPair, quietRun
import re

class Intf( object ):

    "Basic interface object that can configure itself."

    def __init__( self, name, node=None, port=None, link=None, **params ):
        """name: interface name (e.g. h1-eth0)
           node: owning node (where this intf most likely lives)
           link: parent link if we're part of a link
           other arguments are passed to config()"""
        self.node = node
        self.name = name
        self.link = link
        self.mac, self.ip, self.prefixLen = None, None, None
        # Add to node (and move ourselves if necessary )
        node.addIntf( self, port=port )
        # Save params for future reference
        self.params = params
        self.config( **params )

    def cmd( self, *args, **kwargs ):
        "Run a command in our owning node"
        return self.node.cmd( *args, **kwargs )

    def ifconfig( self, *args ):
        "Configure ourselves using ifconfig"
        return self.cmd( 'ifconfig', self.name, *args )

    def setIP( self, ipstr, prefixLen=None ):
        """Set our IP address"""
        # This is a sign that we should perhaps rethink our prefix
        # mechanism and/or the way we specify IP addresses
        if '/' in ipstr:
            self.ip, self.prefixLen = ipstr.split( '/' )
            return self.ifconfig( ipstr, 'up' )
        else:
            self.ip, self.prefixLen = ipstr, prefixLen
            return self.ifconfig( '%s/%s' % ( ipstr, prefixLen ) )

    def setMAC( self, macstr ):
        """Set the MAC address for an interface.
           macstr: MAC address as string"""
        self.mac = macstr
        return ( self.ifconfig( 'down' ) +
                 self.ifconfig( 'hw', 'ether', macstr ) +
                 self.ifconfig( 'up' ) )

    _ipMatchRegex = re.compile( r'\d+\.\d+\.\d+\.\d+' )
    _macMatchRegex = re.compile( r'..:..:..:..:..:..' )

    def updateIP( self ):
        "Return updated IP address based on ifconfig"
        ifconfig = self.ifconfig()
        ips = self._ipMatchRegex.findall( ifconfig )
        self.ip = ips[ 0 ] if ips else None
        return self.ip

    def updateMAC( self ):
        "Return updated MAC address based on ifconfig"
        ifconfig = self.ifconfig()
        macs = self._macMatchRegex.findall( ifconfig )
        self.mac = macs[ 0 ] if macs else None
        return self.mac

    def IP( self ):
        "Return IP address"
        return self.ip

    def MAC( self ):
        "Return MAC address"
        return self.mac

    def isUp( self, setUp=False ):
        "Return whether interface is up"
        if setUp:
            self.ifconfig( 'up' )
        return "UP" in self.ifconfig()

    def rename( self, newname ):
        "Rename interface"
        self.ifconfig( 'down' )
        result = self.cmd( 'ip link set', self.name, 'name', newname )
        self.name = newname
        self.ifconfig( 'up' )
        return result

    # The reason why we configure things in this way is so
    # That the parameters can be listed and documented in
    # the config method.
    # Dealing with subclasses and superclasses is slightly
    # annoying, but at least the information is there!

    def setParam( self, results, method, **param ):
        """Internal method: configure a *single* parameter
           results: dict of results to update
           method: config method name
           param: arg=value (ignore if value=None)
           value may also be list or dict"""
        name, value = param.items()[ 0 ]
        f = getattr( self, method, None )
        if not f or value is None:
            return
        if type( value ) is list:
            result = f( *value )
        elif type( value ) is dict:
            result = f( **value )
        else:
            result = f( value )
        results[ name ] = result
        return result

    def config( self, mac=None, ip=None, ifconfig=None,
                up=True, **_params ):
        """Configure Node according to (optional) parameters:
           mac: MAC address
           ip: IP address
           ifconfig: arbitrary interface configuration
           Subclasses should override this method and call
           the parent class's config(**params)"""
        # If we were overriding this method, we would call
        # the superclass config method here as follows:
        # r = Parent.config( **params )
        r = {}
        self.setParam( r, 'setMAC', mac=mac )
        self.setParam( r, 'setIP', ip=ip )
        self.setParam( r, 'isUp', up=up )
        self.setParam( r, 'ifconfig', ifconfig=ifconfig )
        self.updateIP()
        self.updateMAC()
        return r

    def delete( self ):
        "Delete interface"
        self.cmd( 'ip link del ' + self.name )
        if self.node.inNamespace:
            # Link may have been dumped into root NS
            quietRun( 'ip link del ' + self.name )

    def __repr__( self ):
        return '<%s %s>' % ( self.__class__.__name__, self.name )

    def __str__( self ):
        return self.name

class TCIntf( Intf ):
    """Interface customized by tc (traffic control) utility
       Allows specification of bandwidth limits (various methods)
       as well as delay, loss and max queue length"""

    def bwCmds( self, bw=None, speedup=0, use_hfsc=False, use_tbf=False,
                latency_ms=None, enable_ecn=False, enable_red=False ):
        "Return tc commands to set bandwidth"

        cmds, parent = [], ' root '

        if bw and ( bw < 0 or bw > 1000 ):
            error( 'Bandwidth', bw, 'is outside range 0..1000 Mbps\n' )

        elif bw is not None:
            # BL: this seems a bit brittle...
            if ( speedup > 0 and
                 self.node.name[0:1] == 's' ):
                bw = speedup
            # This may not be correct - we should look more closely
            # at the semantics of burst (and cburst) to make sure we
            # are specifying the correct sizes. For now I have used
            # the same settings we had in the mininet-hifi code.
            if use_hfsc:
                cmds += [ '%s qdisc add dev %s root handle 5:0 hfsc default 1',
                          '%s class add dev %s parent 5:0 classid 5:1 hfsc sc '
                          + 'rate %fMbit ul rate %fMbit' % ( bw, bw ) ]
            elif use_tbf:
                if latency_ms is None:
                    latency_ms = 15 * 8 / bw
                cmds += [ '%s qdisc add dev %s root handle 5: tbf ' +
                          'rate %fMbit burst 15000 latency %fms' %
                          ( bw, latency_ms ) ]
            else:
                cmds += [ '%s qdisc add dev %s root handle 5:0 htb default 1',
                          '%s class add dev %s parent 5:0 classid 5:1 htb ' +
                          'rate %fMbit burst 15k' % bw ]
            parent = ' parent 5:1 '

            # ECN or RED
            if enable_ecn:
                cmds += [ '%s qdisc add dev %s' + parent +
                          'handle 6: red limit 1000000 ' +
                          'min 30000 max 35000 avpkt 1500 ' +
                          'burst 20 ' +
                          'bandwidth %fmbit probability 1 ecn' % bw ]
                parent = ' parent 6: '
            elif enable_red:
                cmds += [ '%s qdisc add dev %s' + parent +
                          'handle 6: red limit 1000000 ' +
                          'min 30000 max 35000 avpkt 1500 ' +
                          'burst 20 ' +
                          'bandwidth %fmbit probability 1' % bw ]
                parent = ' parent 6: '
        return cmds, parent

    @staticmethod
    def delayCmds( parent, delay=None, jitter=None,
                   loss=None, max_queue_size=None ):
        "Internal method: return tc commands for delay and loss"
        cmds = []
        if delay and delay < 0:
            error( 'Negative delay', delay, '\n' )
        elif jitter and jitter < 0:
            error( 'Negative jitter', jitter, '\n' )
        elif loss and ( loss < 0 or loss > 100 ):
            error( 'Bad loss percentage', loss, '%%\n' )
        else:
            # Delay/jitter/loss/max queue size
            netemargs = '%s%s%s%s' % (
                'delay %s ' % delay if delay is not None else '',
                '%s ' % jitter if jitter is not None else '',
                'loss %d ' % loss if loss is not None else '',
                'limit %d' % max_queue_size if max_queue_size is not None
                else '' )
            if netemargs:
                cmds = [ '%s qdisc add dev %s ' + parent +
                         ' handle 10: netem ' +
                         netemargs ]
                parent = ' parent 10:1 '
        return cmds, parent

    def tc( self, cmd, tc='tc' ):
        "Execute tc command for our interface"
        c = cmd % (tc, self)  # Add in tc command and our name
        debug(" *** executing command: %s\n" % c)
        return self.cmd( c )

    def config( self, bw=None, delay=None, jitter=None, loss=None,
                disable_gro=True, speedup=0, use_hfsc=False, use_tbf=False,
                latency_ms=None, enable_ecn=False, enable_red=False,
                max_queue_size=None, **params ):
        "Configure the port and set its properties."

        result = Intf.config( self, **params)

        # Disable GRO
        if disable_gro:
            self.cmd( 'ethtool -K %s gro off' % self )

        # Optimization: return if nothing else to configure
        # Question: what happens if we want to reset things?
        if ( bw is None and not delay and not loss
             and max_queue_size is None ):
            return

        # Clear existing configuration
        cmds = [ '%s qdisc del dev %s root' ]

        # Bandwidth limits via various methods
        bwcmds, parent = self.bwCmds( bw=bw, speedup=speedup,
                                      use_hfsc=use_hfsc, use_tbf=use_tbf,
                                      latency_ms=latency_ms,
                                      enable_ecn=enable_ecn,
                                      enable_red=enable_red )
        cmds += bwcmds

        # Delay/jitter/loss/max_queue_size using netem
        delaycmds, parent = self.delayCmds( delay=delay, jitter=jitter,
                                loss=loss, max_queue_size=max_queue_size,
                                parent=parent )
        cmds += delaycmds

        # Ugly but functional: display configuration info
        stuff = ( ( [ '%.2fMbit' % bw ] if bw is not None else [] ) +
                  ( [ '%s delay' % delay ] if delay is not None else [] ) +
                  ( [ '%s jitter' % jitter ] if jitter is not None else [] ) +
                  ( ['%d%% loss' % loss ] if loss is not None else [] ) +
                  ( [ 'ECN' ] if enable_ecn else [ 'RED' ]
                    if enable_red else [] ) )
        info( '(' + ' '.join( stuff ) + ') ' )

        # Execute all the commands in our node
        debug("at map stage w/cmds: %s\n" % cmds)
        tcoutputs = [ self.tc(cmd) for cmd in cmds ]
        debug( "cmds:", cmds, '\n' )
        debug( "outputs:", tcoutputs, '\n' )
        result[ 'tcoutputs'] = tcoutputs
        result[ 'parent' ] = parent

        return result

class Link( object ):

    """A basic link is just a veth pair.
       Other types of links could be tunnels, link emulators, etc.."""

    def __init__( self, node1, node2, port1=None, port2=None,
                  intfName1=None, intfName2=None,
                  intf=Intf, cls1=None, cls2=None, params1=None,
                  params2=None ):
        """Create veth link to another node, making two new interfaces.
           node1: first node
           node2: second node
           port1: node1 port number (optional)
           port2: node2 port number (optional)
           intf: default interface class/constructor
           cls1, cls2: optional interface-specific constructors
           intfName1: node1 interface name (optional)
           intfName2: node2  interface name (optional)
           params1: parameters for interface 1
           params2: parameters for interface 2"""
        # This is a bit awkward; it seems that having everything in
        # params would be more orthogonal, but being able to specify
        # in-line arguments is more convenient!
        if port1 is None:
            port1 = node1.newPort()
        if port2 is None:
            port2 = node2.newPort()
        if not intfName1:
            intfName1 = self.intfName( node1, port1 )
        if not intfName2:
            intfName2 = self.intfName( node2, port2 )

        self.makeIntfPair( intfName1, intfName2 )

        if not cls1:
            cls1 = intf
        if not cls2:
            cls2 = intf
        if not params1:
            params1 = {}
        if not params2:
            params2 = {}

        intf1 = cls1( name=intfName1, node=node1, port=port1,
                      link=self, **params1  )
        intf2 = cls2( name=intfName2, node=node2, port=port2,
                      link=self, **params2 )

        # All we are is dust in the wind, and our two interfaces
        self.intf1, self.intf2 = intf1, intf2

    @classmethod
    def intfName( cls, node, n ):
        "Construct a canonical interface name node-ethN for interface n."
        return node.name + '-eth' + repr( n )

    @classmethod
    def makeIntfPair( cls, intf1, intf2 ):
        """Create pair of interfaces
           intf1: name of interface 1
           intf2: name of interface 2
           (override this class method [and possibly delete()]
           to change link type)"""
        makeIntfPair( intf1, intf2  )

    def delete( self ):
        "Delete this link"
        self.intf1.delete()
        self.intf2.delete()

    def __str__( self ):
        return '%s<->%s' % ( self.intf1, self.intf2 )

class TCLink( Link ):
    "Link with symmetric TC interfaces configured via opts"
    def __init__( self, node1, node2, port1=None, port2=None,
                  intfName1=None, intfName2=None, **params ):
        Link.__init__( self, node1, node2, port1=port1, port2=port2,
                       intfName1=intfName1, intfName2=intfName2,
                       cls1=TCIntf,
                       cls2=TCIntf,
                       params1=params,
                       params2=params)
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
"""
@file
@brief Some automation helpers to test notebooks and check they are still working fine.
"""
import os
import sys
from pyquickhelper.loghelper import noLOG
from pyquickhelper.ipythonhelper.notebook_helper import run_notebook, install_python_kernel_for_unittest
import pyensae

def ls_notebooks(subfolder):
    """
    list the notebooks in a particular subfolder

    @param      subfolder   subfolder (related to this module)
    @return                 list of files
    """
    this = os.path.abspath(os.path.dirname(__file__))
    docnote = os.path.join(
        this,
        "..",
        "..",
        "..",
        "_doc",
        "notebooks",
        subfolder)
    notes = [
        os.path.normpath(
            os.path.join(
                docnote,
                _)) for _ in os.listdir(docnote)]

    keepnote = []
    for i, note in enumerate(notes):
        ext = os.path.splitext(note)[-1]
        if ext != ".ipynb":
            continue
        keepnote.append(note)
    return keepnote

def get_additional_paths():
    """
    returns a list of paths to add before running the notebooks,
    paths to pyquickhelper, pyensae, pymmails

    @return             list of paths
    """
    import pyquickhelper
    import pymyinstall
    import pyrsslocal
    addpath = [os.path.dirname(pyquickhelper.__file__),
               os.path.dirname(pyensae.__file__),
               os.path.dirname(pyrsslocal.__file__),
               os.path.dirname(pymyinstall.__file__),
               os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."),
               ]
    addpath = [os.path.normpath(os.path.join(_, "..")) for _ in addpath]
    return addpath

def clean_function_notebook(code):
    """
    function which clean cells when unittesting notebooks 1A

    @param      code        cell content
    @return                 modified code
    """
    code = code.replace(
        'run_cmd("exemple.xlsx"',
        'skip_run_cmd("exemple.xlsx"')

    skip = ["faire une chose avec la probabilité 0.7",
            "# déclenche une exception",
            "# pour lancer Excel",
            "for k in list_exercice_1 :",
            "return ....",
            "return [ .... ]",
            "def __init__(self, ...) :",
            "if random.random() <= 0.7 :",
            "dictionnaire_depart.items() [0]",
            "iterateur(0,10) [0]",
            "# ...... à remplir",
            'String.Join(",", a.Select(c=>c.ToString()).ToArray())',
            "# elle n'existe pas encore",
            "from ggplot import *",
            # ggplot calls method show and it opens window blocking the offline
            # execution
            ]
    rep = [("# ...", "pass # "),
           ("%timeit", "#%timeit"),
           ]
    spl = ["# ......",
           "# elle n'existe pas encore",
           ]

    for s in skip:
        if s in code:
            return ""

    for s in spl:
        if s in code:
            code = code.split(s)[0]

    for s in rep:
        code = code.replace(s[0], s[1])

    return code

def execute_notebooks(folder, notebooks, filter,
                      clean_function=None,
                      fLOG=noLOG,
                      deepfLOG=noLOG):
    """
    execute a list of notebooks

    @param      folder          folder
    @param      notebooks       list of notebooks
    @param      filter          function which validate the notebooks
    @param      clean_function  cleaning function to apply to the code before running it
    @param      fLOG            logging function
    @param      deepfLOG        logging function used to run the notebook
    @return                     dictionary tuple (statistics, { notebook_file: (isSuccess, outout) })

    The signature of function ``filter`` is::

        def filter( i, filename) : return True or False

    """

    def valid_cell(cell):
        if "%system" in cell:
            return False
        if "df.plot(...)" in cell:
            return False
        if 'df["difference"] = ...' in cell:
            return False
        if 'print(next(it))' in cell:
            return False
        return True

    kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
        "actuariat_python")
    addpath = get_additional_paths()
    results = {}
    for i, note in enumerate(notebooks):
        if filter(i, note):
            fLOG("******", i, os.path.split(note)[-1])
            outfile = os.path.join(folder, "out_" + os.path.split(note)[-1])
            try:
                stat, out = run_notebook(note, working_dir=folder, outfilename=outfile,
                                         additional_path=addpath,
                                         valid=valid_cell,
                                         clean_function=clean_function,
                                         fLOG=deepfLOG,
                                         kernel_name=kernel_name
                                         )
                if not os.path.exists(outfile):
                    raise FileNotFoundError(outfile)
                results[note] = (True, stat, out)
            except Exception as e:
                results[note] = (False, None, e)
    return results

def unittest_raise_exception_notebook(res, fLOG):
    """
    same code for all unit tests

    @param      res     output of @see fn execute_notebooks
    """
    assert len(res) > 0
    fails = [(os.path.split(k)[-1], ) + v
             for k, v in sorted(res.items()) if not v[0]]
    for f in fails:
        fLOG(f)
    if len(fails) > 0:
        raise fails[0][-1]
    for k, v in sorted(res.items()):
        fLOG("final", os.path.split(k)[-1], v[0], v[1])
<commit_msg>update unit test<commit_after>#-*- coding: utf-8 -*-
"""
@file
@brief Some automation helpers to test notebooks and check they are still working fine.
"""
import os
import sys
from pyquickhelper.loghelper import noLOG
from pyquickhelper.ipythonhelper.notebook_helper import install_python_kernel_for_unittest
from pyquickhelper.ipythonhelper import run_notebook
import pyensae

def ls_notebooks(subfolder):
    """
    list the notebooks in a particular subfolder

    @param      subfolder   subfolder (related to this module)
    @return                 list of files
    """
    this = os.path.abspath(os.path.dirname(__file__))
    docnote = os.path.join(
        this,
        "..",
        "..",
        "..",
        "_doc",
        "notebooks",
        subfolder)
    notes = [
        os.path.normpath(
            os.path.join(
                docnote,
                _)) for _ in os.listdir(docnote)]

    keepnote = []
    for i, note in enumerate(notes):
        ext = os.path.splitext(note)[-1]
        if ext != ".ipynb":
            continue
        keepnote.append(note)
    return keepnote

def get_additional_paths():
    """
    returns a list of paths to add before running the notebooks,
    paths to pyquickhelper, pyensae, pymmails

    @return             list of paths
    """
    import pyquickhelper
    import pymyinstall
    import pyrsslocal
    addpath = [os.path.dirname(pyquickhelper.__file__),
               os.path.dirname(pyensae.__file__),
               os.path.dirname(pyrsslocal.__file__),
               os.path.dirname(pymyinstall.__file__),
               os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."),
               ]
    addpath = [os.path.normpath(os.path.join(_, "..")) for _ in addpath]
    return addpath

def clean_function_notebook(code):
    """
    function which clean cells when unittesting notebooks 1A

    @param      code        cell content
    @return                 modified code
    """
    code = code.replace(
        'run_cmd("exemple.xlsx"',
        'skip_run_cmd("exemple.xlsx"')

    skip = ["faire une chose avec la probabilité 0.7",
            "# déclenche une exception",
            "# pour lancer Excel",
            "for k in list_exercice_1 :",
            "return ....",
            "return [ .... ]",
            "def __init__(self, ...) :",
            "if random.random() <= 0.7 :",
            "dictionnaire_depart.items() [0]",
            "iterateur(0,10) [0]",
            "# ...... à remplir",
            'String.Join(",", a.Select(c=>c.ToString()).ToArray())',
            "# elle n'existe pas encore",
            "from ggplot import *",
            # ggplot calls method show and it opens window blocking the offline
            # execution
            ]
    rep = [("# ...", "pass # "),
           ("%timeit", "#%timeit"),
           ]
    spl = ["# ......",
           "# elle n'existe pas encore",
           ]

    for s in skip:
        if s in code:
            return ""

    for s in spl:
        if s in code:
            code = code.split(s)[0]

    for s in rep:
        code = code.replace(s[0], s[1])

    return code

def execute_notebooks(folder, notebooks, filter,
                      clean_function=None,
                      fLOG=noLOG,
                      deepfLOG=noLOG):
    """
    execute a list of notebooks

    @param      folder          folder
    @param      notebooks       list of notebooks
    @param      filter          function which validate the notebooks
    @param      clean_function  cleaning function to apply to the code before running it
    @param      fLOG            logging function
    @param      deepfLOG        logging function used to run the notebook
    @return                     dictionary tuple (statistics, { notebook_file: (isSuccess, outout) })

    The signature of function ``filter`` is::

        def filter( i, filename) : return True or False

    """

    def valid_cell(cell):
        if "%system" in cell:
            return False
        if "df.plot(...)" in cell:
            return False
        if 'df["difference"] = ...' in cell:
            return False
        if 'print(next(it))' in cell:
            return False
        return True

    kernel_name = None if "travis" in sys.executable else install_python_kernel_for_unittest(
        "actuariat_python")
    addpath = get_additional_paths()
    results = {}
    for i, note in enumerate(notebooks):
        if filter(i, note):
            fLOG("******", i, os.path.split(note)[-1])
            outfile = os.path.join(folder, "out_" + os.path.split(note)[-1])
            try:
                stat, out = run_notebook(note, working_dir=folder, outfilename=outfile,
                                         additional_path=addpath,
                                         valid=valid_cell,
                                         clean_function=clean_function,
                                         fLOG=deepfLOG,
                                         kernel_name=kernel_name
                                         )
                if not os.path.exists(outfile):
                    raise FileNotFoundError(outfile)
                results[note] = (True, stat, out)
            except Exception as e:
                results[note] = (False, None, e)
    return results

def unittest_raise_exception_notebook(res, fLOG):
    """
    same code for all unit tests

    @param      res     output of @see fn execute_notebooks
    """
    assert len(res) > 0
    fails = [(os.path.split(k)[-1], ) + v
             for k, v in sorted(res.items()) if not v[0]]
    for f in fails:
        fLOG(f)
    if len(fails) > 0:
        raise fails[0][-1]
    for k, v in sorted(res.items()):
        fLOG("final", os.path.split(k)[-1], v[0], v[1])
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2015-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the Mandatory Activation of Segregated Witness (BIP148) soft-fork logic."""

from test_framework.mininode import wait_until
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *

import logging
import re

## Scenario:
#
# Four miners, whose hashrate and strategies are:
#
#     A ( 9%): mine longest chain, signalling segwit
#     B (27%): mine longest chain, not signalling segwit
#     C (37%): bip 148 enforcing, signalling segwit
#     D (27%): switches from (A) to (C) during August
#
# (1, 3, 4 and 3 elevenths of total hashrate respectively; mining
#  11 blocks a day, gives 143 blocks in 13 days, just one block
#  short of a retarget cycle)
#
# B, with 27% of hashrate is sufficient to block activation (which
#    requires 25% on regtest)
# A+B+D has 65% of hashrate prior to D switching
# C+D has 55% of hashrate post D's switch, and since C's hashrate is larger
#   than D's the original BIP-148 chain will eventually have more work
#   than D's chain, forcing a reorg and eventual consensus

def bip9_blockver(*bits):
    return "-blockversion=%d" % (0x20000000 + sum(1<<b for b in bits))

class BIP148Test(BitcoinTestFramework):
    AUG_1 = 1501545600

    def __init__(self):
        self.log = logging.getLogger('TestFramework')
        super().__init__()
        self.num_nodes = 8
        self.setup_clean_chain = False
        self.extra_args = [
             [bip9_blockver(0)],   # for initial CSV activation only
             [bip9_blockver(1)],
             [bip9_blockver()],
             [bip9_blockver(1), "-bip148"],
             [bip9_blockver(1)],
             [],           # non-mining user, restarts with -bip148 on Aug 5th
             [],           # well connected, non BIP148 peer
             ["-bip148"]]  # well connected, BIP148 peer
        for ea in self.extra_args:
            ea.append('-whitelist=::/0')

        self.day = -30 # 30 days before BIP148 activation
        self.blockrate = [0,1,3,4,3,0,0]
        self.ancestor_cache = {}

    def connect_all(self):
        connect_nodes(self.nodes[6],7)
        for i in range(6):
            connect_nodes(self.nodes[i], 6)
            connect_nodes(self.nodes[i], 7)
            self.log.info("connecting %d and 6, 7" % (i))

    def setup_network(self):
        self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=self.extra_args)
        self.connect_all()
        self.sync_all()

    def is_block_ancestor(self, rpc, tip, ancestor, ancestor_height):
        cache_key = tip + ancestor
        if cache_key in self.ancestor_cache:
            return self.ancestor_cache[cache_key]
        blockinfo = rpc.getblock(tip)
        while blockinfo['height'] > ancestor_height:
            blockinfo = rpc.getblock(blockinfo['previousblockhash'])
            parent_cache_key = blockinfo['hash'] + ancestor
            if parent_cache_key in self.ancestor_cache:
                rv = self.ancestor_cache[parent_cache_key]
                self.ancestor_cache[cache_key] = rv
                return rv
        rv = (blockinfo['hash'] == ancestor)
        self.ancestor_cache[cache_key] = rv
        return rv

    # Taken from master test_framework.py
    def sync_all(self, node_groups=None):
        if not node_groups:
            node_groups = [self.nodes]

        for group in node_groups:
            sync_blocks(group)
            sync_mempools(group)

    def sync_chaintips(self, nodes=None, timeout=60, extra_check=None):
        if nodes is None:
            nodes = list(range(len(self.nodes)))
        statuses_where_node_has_the_block = ('valid-fork', 'active')
        def chaintip_check():
            if extra_check is not None:
                if not extra_check():
                    return False
            all_known_tips = {}
            chaintips_replies = [(r, self.nodes[r].getchaintips()) for r in nodes]
            for (r, tips) in chaintips_replies:
                for tip in tips:
                    if tip['hash'] in all_known_tips and tip['status'] not in statuses_where_node_has_the_block:
                        continue
                    all_known_tips[tip['hash']] = (r, tip)

            # Make sure we know a node we can fetch the block from
            for tip in all_known_tips.keys():
                if all_known_tips[tip][1]['status'] not in statuses_where_node_has_the_block:
                    for r in nodes:
                        try:
                            all_known_tips[tip] = (r, self.nodes[r].getblock(tip))
                            break
                        except:
                            pass

            self.log.debug('There are %d tips: %s' % (len(all_known_tips), tuple(sorted(all_known_tips.keys()))))
            for (r, tips) in chaintips_replies:
                invalid_blocks = []
                my_known_tips = set()
                active = None
                # Ideally, best should use chainwork, but that's not in getchaintips...
                best = {'height':0}
                for tip in tips:
                    my_known_tips.add(tip['hash'])
                    if tip['status'] == 'invalid':
                        invalid_blocks.append(tip)
                    else:
                        if tip['height'] > best['height']:
                            best = tip
                    if tip['status'] == 'active':
                        active = tip
                        if tip['height'] == best['height']:
                            best = tip
                if best != active:
                    self.log.debug("Best potentially-valid block is not active on node %s" % (r,))
                    return False
                missing_tips = all_known_tips.keys() - my_known_tips
                for tip in set(missing_tips):
                    for inv_tip in invalid_blocks:
                        if self.is_block_ancestor(self.nodes[all_known_tips[tip][0]], tip, inv_tip['hash'], inv_tip['height']):
                            # One of our invalid tips is a parent of the missing tip
                            missing_tips.remove(tip)
                            break
                    for known_tip in my_known_tips:
                        # NOTE: Can't assume this node has the block, in case it's invalid
                        if self.is_block_ancestor(self.nodes[all_known_tips[known_tip][0]], known_tip, tip, all_known_tips[tip][1]['height']):
                            # We have a valid tip that descends from the missing tip
                            missing_tips.remove(tip)
                            break
                    if tip in missing_tips:
                        self.log.debug('Node %s missing tip %s' % (r, tip))
                        return False
            self.log.debug('All nodes have all syncable tips')
            return True
        assert wait_until(chaintip_check, timeout=timeout)

    def get_peerset(self, rpc):
        def extract_nodenum(subver):
            m = re.search(r'\((.*?)[;)]', subver)
            return m.group(1)
        return {pi['id']: extract_nodenum(pi['subver']) for pi in rpc.getpeerinfo()}

    def mining(self):
        time = self.AUG_1 + (self.day*24*60*60)
        to_mine = self.blockrate[:]

        while sum(to_mine) > 0:
            for peer,blks in enumerate(to_mine):
                if blks == 0: continue
                set_node_times(self.nodes, time)

                peerset_before = tuple(self.get_peerset(r) for r in self.nodes)
                def check_peerset():
                    peerset_after = tuple(self.get_peerset(r) for r in self.nodes)
                    if peerset_before != peerset_after:
                        raise AssertionError("Network break!\nBefore=%s\nAfter =%s" % (peerset_before, peerset_after))
                    return True

                to_mine[peer] -= 1
                self.nodes[peer].generate(1)

                self.sync_chaintips(extra_check=check_peerset)
                time += 10*60

        self.day += 1

    def bip148_restart(self, peer):
        self.log.info("Restarting node %d with -bip148" % (peer,))
        self.stop_node(peer)
        time = self.AUG_1 + (self.day*24*60*60)
        self.nodes[peer] = start_node(peer, self.options.tmpdir, self.extra_args[4] + ["-bip148", "-mocktime=%d" % (time)])

        n = self.nodes[peer]
        blk = n.getblock(n.getbestblockhash())
        while blk["mediantime"] >= self.AUG_1:
            if blk["version"] | 0x20000002 != blk["version"]:
                raise AssertionError("Invalid BIP148 block in node running with -bip148")
            blk = n.getblock(blk["previousblockhash"])

        self.log.info("Reconnecting nodes")
        self.connect_all()

    def run_test(self):
        cnt = self.nodes[0].getblockcount()

        # Lock in CSV
        self.nodes[0].generate(500)
        if (self.nodes[0].getblockcount() != cnt + 500):
            raise AssertionError("Failed to mine 500 bip9 bit 0 blocks")
        cnt += 500

        if get_bip9_status(self.nodes[0], 'csv')["status"] != "active":
            raise AssertionError("Failed to activate OP_CSV")

        self.sync_all()

        for d in range(120):
            swstatus = get_bip9_status(self.nodes[0], 'segwit')["status"]
            if self.day == 0:
                if swstatus != "started":
                    raise AssertionError("segwit soft-fork in state %s rather than started at day 0" % (swstatus))

            self.mining()
            tips = set(n.getbestblockhash() for n in self.nodes)
            heights = [n.getblockcount() for n in self.nodes]
            segwit = [get_bip9_status(n, 'segwit')["status"] for n in self.nodes]
            connect = [len(n.getpeerinfo()) for n in self.nodes]
            maxh = max(heights)
            out = ["%d:" % (maxh)]
            for c,h,s in zip(connect,heights,segwit):
                a = "S" if s == "active" else "n"
                a = "%d/%s" % (c,a)
                if h < maxh:
                    out.append("%s%d" % (a, h-maxh))
                else:
                    out.append(a)

            self.log.info("Day %d:%s: status: %s:" % (self.day, swstatus, out))

            if self.day == 7:
                self.bip148_restart(5)

            if self.day == 20:
                self.bip148_restart(4)

            for i in range(6):
                if connect[i] == 0:
                    raise AssertionError("Peer %d has no connected peers after %d blocks" % (i,maxh))

            if self.day > 5 and len(tips) == 1:
                synccount += 1
            else:
                synccount = 0
            if synccount > 5:
                self.log.info("In sync for five days, consensus achieved")
                break

        if len(tips) > 1:
            raise AssertionError("Chain split still exists at day %d" % (self.day))

        return

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO, stream=sys.stdout)
    BIP148Test().main()
<commit_msg>Bugfix: tests/bip148-segwit-uasf: Initialise mocktime before mining any blocks<commit_after>#!/usr/bin/env python3
# Copyright (c) 2015-2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test the Mandatory Activation of Segregated Witness (BIP148) soft-fork logic."""

from test_framework.mininode import wait_until
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *

import logging
import re

## Scenario:
#
# Four miners, whose hashrate and strategies are:
#
#     A ( 9%): mine longest chain, signalling segwit
#     B (27%): mine longest chain, not signalling segwit
#     C (37%): bip 148 enforcing, signalling segwit
#     D (27%): switches from (A) to (C) during August
#
# (1, 3, 4 and 3 elevenths of total hashrate respectively; mining
#  11 blocks a day, gives 143 blocks in 13 days, just one block
#  short of a retarget cycle)
#
# B, with 27% of hashrate is sufficient to block activation (which
#    requires 25% on regtest)
# A+B+D has 65% of hashrate prior to D switching
# C+D has 55% of hashrate post D's switch, and since C's hashrate is larger
#   than D's the original BIP-148 chain will eventually have more work
#   than D's chain, forcing a reorg and eventual consensus

def bip9_blockver(*bits):
    return "-blockversion=%d" % (0x20000000 + sum(1<<b for b in bits))

class BIP148Test(BitcoinTestFramework):
    AUG_1 = 1501545600

    def __init__(self):
        self.log = logging.getLogger('TestFramework')
        super().__init__()
        self.num_nodes = 8
        self.setup_clean_chain = False
        self.extra_args = [
             [bip9_blockver(0)],   # for initial CSV activation only
             [bip9_blockver(1)],
             [bip9_blockver()],
             [bip9_blockver(1), "-bip148"],
             [bip9_blockver(1)],
             [],           # non-mining user, restarts with -bip148 on Aug 5th
             [],           # well connected, non BIP148 peer
             ["-bip148"]]  # well connected, BIP148 peer
        for ea in self.extra_args:
            ea.append('-whitelist=::/0')

        self.day = -30 # 30 days before BIP148 activation
        self.blockrate = [0,1,3,4,3,0,0]
        self.ancestor_cache = {}

    def connect_all(self):
        connect_nodes(self.nodes[6],7)
        for i in range(6):
            connect_nodes(self.nodes[i], 6)
            connect_nodes(self.nodes[i], 7)
            self.log.info("connecting %d and 6, 7" % (i))

    def setup_network(self):
        self.nodes = start_nodes(self.num_nodes, self.options.tmpdir, extra_args=self.extra_args)
        self.connect_all()
        self.sync_all()

    def is_block_ancestor(self, rpc, tip, ancestor, ancestor_height):
        cache_key = tip + ancestor
        if cache_key in self.ancestor_cache:
            return self.ancestor_cache[cache_key]
        blockinfo = rpc.getblock(tip)
        while blockinfo['height'] > ancestor_height:
            blockinfo = rpc.getblock(blockinfo['previousblockhash'])
            parent_cache_key = blockinfo['hash'] + ancestor
            if parent_cache_key in self.ancestor_cache:
                rv = self.ancestor_cache[parent_cache_key]
                self.ancestor_cache[cache_key] = rv
                return rv
        rv = (blockinfo['hash'] == ancestor)
        self.ancestor_cache[cache_key] = rv
        return rv

    # Taken from master test_framework.py
    def sync_all(self, node_groups=None):
        if not node_groups:
            node_groups = [self.nodes]

        for group in node_groups:
            sync_blocks(group)
            sync_mempools(group)

    def sync_chaintips(self, nodes=None, timeout=60, extra_check=None):
        if nodes is None:
            nodes = list(range(len(self.nodes)))
        statuses_where_node_has_the_block = ('valid-fork', 'active')
        def chaintip_check():
            if extra_check is not None:
                if not extra_check():
                    return False
            all_known_tips = {}
            chaintips_replies = [(r, self.nodes[r].getchaintips()) for r in nodes]
            for (r, tips) in chaintips_replies:
                for tip in tips:
                    if tip['hash'] in all_known_tips and tip['status'] not in statuses_where_node_has_the_block:
                        continue
                    all_known_tips[tip['hash']] = (r, tip)

            # Make sure we know a node we can fetch the block from
            for tip in all_known_tips.keys():
                if all_known_tips[tip][1]['status'] not in statuses_where_node_has_the_block:
                    for r in nodes:
                        try:
                            all_known_tips[tip] = (r, self.nodes[r].getblock(tip))
                            break
                        except:
                            pass

            self.log.debug('There are %d tips: %s' % (len(all_known_tips), tuple(sorted(all_known_tips.keys()))))
            for (r, tips) in chaintips_replies:
                invalid_blocks = []
                my_known_tips = set()
                active = None
                # Ideally, best should use chainwork, but that's not in getchaintips...
                best = {'height':0}
                for tip in tips:
                    my_known_tips.add(tip['hash'])
                    if tip['status'] == 'invalid':
                        invalid_blocks.append(tip)
                    else:
                        if tip['height'] > best['height']:
                            best = tip
                    if tip['status'] == 'active':
                        active = tip
                        if tip['height'] == best['height']:
                            best = tip
                if best != active:
                    self.log.debug("Best potentially-valid block is not active on node %s" % (r,))
                    return False
                missing_tips = all_known_tips.keys() - my_known_tips
                for tip in set(missing_tips):
                    for inv_tip in invalid_blocks:
                        if self.is_block_ancestor(self.nodes[all_known_tips[tip][0]], tip, inv_tip['hash'], inv_tip['height']):
                            # One of our invalid tips is a parent of the missing tip
                            missing_tips.remove(tip)
                            break
                    for known_tip in my_known_tips:
                        # NOTE: Can't assume this node has the block, in case it's invalid
                        if self.is_block_ancestor(self.nodes[all_known_tips[known_tip][0]], known_tip, tip, all_known_tips[tip][1]['height']):
                            # We have a valid tip that descends from the missing tip
                            missing_tips.remove(tip)
                            break
                    if tip in missing_tips:
                        self.log.debug('Node %s missing tip %s' % (r, tip))
                        return False
            self.log.debug('All nodes have all syncable tips')
            return True
        assert wait_until(chaintip_check, timeout=timeout)

    def get_peerset(self, rpc):
        def extract_nodenum(subver):
            m = re.search(r'\((.*?)[;)]', subver)
            return m.group(1)
        return {pi['id']: extract_nodenum(pi['subver']) for pi in rpc.getpeerinfo()}

    def mining(self):
        time = self.AUG_1 + (self.day*24*60*60)
        to_mine = self.blockrate[:]

        while sum(to_mine) > 0:
            for peer,blks in enumerate(to_mine):
                if blks == 0: continue
                set_node_times(self.nodes, time)

                peerset_before = tuple(self.get_peerset(r) for r in self.nodes)
                def check_peerset():
                    peerset_after = tuple(self.get_peerset(r) for r in self.nodes)
                    if peerset_before != peerset_after:
                        raise AssertionError("Network break!\nBefore=%s\nAfter =%s" % (peerset_before, peerset_after))
                    return True

                to_mine[peer] -= 1
                self.nodes[peer].generate(1)

                self.sync_chaintips(extra_check=check_peerset)
                time += 10*60

        self.day += 1

    def bip148_restart(self, peer):
        self.log.info("Restarting node %d with -bip148" % (peer,))
        self.stop_node(peer)
        time = self.AUG_1 + (self.day*24*60*60)
        self.nodes[peer] = start_node(peer, self.options.tmpdir, self.extra_args[4] + ["-bip148", "-mocktime=%d" % (time)])

        n = self.nodes[peer]
        blk = n.getblock(n.getbestblockhash())
        while blk["mediantime"] >= self.AUG_1:
            if blk["version"] | 0x20000002 != blk["version"]:
                raise AssertionError("Invalid BIP148 block in node running with -bip148")
            blk = n.getblock(blk["previousblockhash"])

        self.log.info("Reconnecting nodes")
        self.connect_all()

    def run_test(self):
        time = self.AUG_1 + (self.day*24*60*60)
        set_node_times(self.nodes, time)

        cnt = self.nodes[0].getblockcount()

        # Lock in CSV
        self.nodes[0].generate(500)
        if (self.nodes[0].getblockcount() != cnt + 500):
            raise AssertionError("Failed to mine 500 bip9 bit 0 blocks")
        cnt += 500

        if get_bip9_status(self.nodes[0], 'csv')["status"] != "active":
            raise AssertionError("Failed to activate OP_CSV")

        self.sync_all()

        for d in range(120):
            swstatus = get_bip9_status(self.nodes[0], 'segwit')["status"]
            if self.day == 0:
                if swstatus != "started":
                    raise AssertionError("segwit soft-fork in state %s rather than started at day 0" % (swstatus))

            self.mining()
            tips = set(n.getbestblockhash() for n in self.nodes)
            heights = [n.getblockcount() for n in self.nodes]
            segwit = [get_bip9_status(n, 'segwit')["status"] for n in self.nodes]
            connect = [len(n.getpeerinfo()) for n in self.nodes]
            maxh = max(heights)
            out = ["%d:" % (maxh)]
            for c,h,s in zip(connect,heights,segwit):
                a = "S" if s == "active" else "n"
                a = "%d/%s" % (c,a)
                if h < maxh:
                    out.append("%s%d" % (a, h-maxh))
                else:
                    out.append(a)

            self.log.info("Day %d:%s: status: %s:" % (self.day, swstatus, out))

            if self.day == 7:
                self.bip148_restart(5)

            if self.day == 20:
                self.bip148_restart(4)

            for i in range(6):
                if connect[i] == 0:
                    raise AssertionError("Peer %d has no connected peers after %d blocks" % (i,maxh))

            if self.day > 5 and len(tips) == 1:
                synccount += 1
            else:
                synccount = 0
            if synccount > 5:
                self.log.info("In sync for five days, consensus achieved")
                break

        if len(tips) > 1:
            raise AssertionError("Chain split still exists at day %d" % (self.day))

        return

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO, stream=sys.stdout)
    BIP148Test().main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2019 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This class implements the proposed relation APIs from MSC 1849.

Since the MSC has not been approved all APIs here are unstable and may change at
any time to reflect changes in the MSC.
"""

import logging

from twisted.internet import defer

from synapse.api.constants import EventTypes
from synapse.api.errors import SynapseError
from synapse.http.servlet import (
    RestServlet,
    parse_integer,
    parse_json_object_from_request,
    parse_string,
)

from ._base import client_v2_patterns

logger = logging.getLogger(__name__)

class RelationSendServlet(RestServlet):
    """Helper API for sending events that have relation data.

    Example API shape to send a 👍 reaction to a room:

        POST /rooms/!foo/send_relation/$bar/m.annotation/m.reaction?key=%F0%9F%91%8D
        {}

        {
            "event_id": "$foobar"
        }
    """

    PATTERN = (
        "/rooms/(?P<room_id>[^/]*)/send_relation"
        "/(?P<parent_id>[^/]*)/(?P<relation_type>[^/]*)/(?P<event_type>[^/]*)"
    )

    def __init__(self, hs):
        super(RelationSendServlet, self).__init__()
        self.auth = hs.get_auth()
        self.event_creation_handler = hs.get_event_creation_handler()

    def register(self, http_server):
        http_server.register_paths(
            "POST",
            client_v2_patterns(self.PATTERN + "$", releases=()),
            self.on_PUT_or_POST,
        )
        http_server.register_paths(
            "PUT",
            client_v2_patterns(self.PATTERN + "/(?P<txn_id>[^/]*)$", releases=()),
            self.on_PUT_or_POST,
        )

    @defer.inlineCallbacks
    def on_PUT_or_POST(
        self, request, room_id, parent_id, relation_type, event_type, txn_id=None
    ):
        requester = yield self.auth.get_user_by_req(request, allow_guest=True)

        if event_type == EventTypes.Member:
            # Add relations to a membership is meaningless, so we just deny it
            # at the CS API rather than trying to handle it correctly.
            raise SynapseError(400, "Cannot send member events with relations")

        content = parse_json_object_from_request(request)

        aggregation_key = parse_string(request, "key", encoding="utf-8")

        content["m.relates_to"] = {
            "event_id": parent_id,
            "key": aggregation_key,
            "rel_type": relation_type,
        }

        event_dict = {
            "type": event_type,
            "content": content,
            "room_id": room_id,
            "sender": requester.user.to_string(),
        }

        event = yield self.event_creation_handler.create_and_send_nonmember_event(
            requester, event_dict=event_dict, txn_id=txn_id
        )

        defer.returnValue((200, {"event_id": event.event_id}))

class RelationPaginationServlet(RestServlet):
    """API to paginate relations on an event by topological ordering, optionally
    filtered by relation type and event type.
    """

    PATTERNS = client_v2_patterns(
        "/rooms/(?P<room_id>[^/]*)/relations/(?P<parent_id>[^/]*)"
        "(/(?P<relation_type>[^/]*)(/(?P<event_type>[^/]*))?)?$",
        releases=(),
    )

    def __init__(self, hs):
        super(RelationPaginationServlet, self).__init__()
        self.auth = hs.get_auth()
        self.store = hs.get_datastore()
        self.clock = hs.get_clock()
        self._event_serializer = hs.get_event_client_serializer()

    @defer.inlineCallbacks
    def on_GET(self, request, room_id, parent_id, relation_type=None, event_type=None):
        requester = yield self.auth.get_user_by_req(request, allow_guest=True)

        yield self.auth.check_in_room_or_world_readable(
            room_id, requester.user.to_string()
        )

        limit = parse_integer(request, "limit", default=5)

        result = yield self.store.get_relations_for_event(
            event_id=parent_id,
            relation_type=relation_type,
            event_type=event_type,
            limit=limit,
        )

        events = yield self.store.get_events_as_list(
            [c["event_id"] for c in result.chunk]
        )

        now = self.clock.time_msec()
        events = yield self._event_serializer.serialize_events(events, now)

        return_value = result.to_dict()
        return_value["chunk"] = events

        defer.returnValue((200, return_value))

def register_servlets(hs, http_server):
    RelationSendServlet(hs).register(http_server)
    RelationPaginationServlet(hs).register(http_server)
<commit_msg>Actually implement idempotency<commit_after># -*- coding: utf-8 -*-
# Copyright 2019 New Vector Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This class implements the proposed relation APIs from MSC 1849.

Since the MSC has not been approved all APIs here are unstable and may change at
any time to reflect changes in the MSC.
"""

import logging

from twisted.internet import defer

from synapse.api.constants import EventTypes
from synapse.api.errors import SynapseError
from synapse.http.servlet import (
    RestServlet,
    parse_integer,
    parse_json_object_from_request,
    parse_string,
)
from synapse.rest.client.transactions import HttpTransactionCache

from ._base import client_v2_patterns

logger = logging.getLogger(__name__)

class RelationSendServlet(RestServlet):
    """Helper API for sending events that have relation data.

    Example API shape to send a 👍 reaction to a room:

        POST /rooms/!foo/send_relation/$bar/m.annotation/m.reaction?key=%F0%9F%91%8D
        {}

        {
            "event_id": "$foobar"
        }
    """

    PATTERN = (
        "/rooms/(?P<room_id>[^/]*)/send_relation"
        "/(?P<parent_id>[^/]*)/(?P<relation_type>[^/]*)/(?P<event_type>[^/]*)"
    )

    def __init__(self, hs):
        super(RelationSendServlet, self).__init__()
        self.auth = hs.get_auth()
        self.event_creation_handler = hs.get_event_creation_handler()
        self.txns = HttpTransactionCache(hs)

    def register(self, http_server):
        http_server.register_paths(
            "POST",
            client_v2_patterns(self.PATTERN + "$", releases=()),
            self.on_PUT_or_POST,
        )
        http_server.register_paths(
            "PUT",
            client_v2_patterns(self.PATTERN + "/(?P<txn_id>[^/]*)$", releases=()),
            self.on_PUT,
        )

    def on_PUT(self, request, *args, **kwargs):
        return self.txns.fetch_or_execute_request(
            request, self.on_PUT_or_POST, request, *args, **kwargs
        )

    @defer.inlineCallbacks
    def on_PUT_or_POST(
        self, request, room_id, parent_id, relation_type, event_type, txn_id=None
    ):
        requester = yield self.auth.get_user_by_req(request, allow_guest=True)

        if event_type == EventTypes.Member:
            # Add relations to a membership is meaningless, so we just deny it
            # at the CS API rather than trying to handle it correctly.
            raise SynapseError(400, "Cannot send member events with relations")

        content = parse_json_object_from_request(request)

        aggregation_key = parse_string(request, "key", encoding="utf-8")

        content["m.relates_to"] = {
            "event_id": parent_id,
            "key": aggregation_key,
            "rel_type": relation_type,
        }

        event_dict = {
            "type": event_type,
            "content": content,
            "room_id": room_id,
            "sender": requester.user.to_string(),
        }

        event = yield self.event_creation_handler.create_and_send_nonmember_event(
            requester, event_dict=event_dict, txn_id=txn_id
        )

        defer.returnValue((200, {"event_id": event.event_id}))

class RelationPaginationServlet(RestServlet):
    """API to paginate relations on an event by topological ordering, optionally
    filtered by relation type and event type.
    """

    PATTERNS = client_v2_patterns(
        "/rooms/(?P<room_id>[^/]*)/relations/(?P<parent_id>[^/]*)"
        "(/(?P<relation_type>[^/]*)(/(?P<event_type>[^/]*))?)?$",
        releases=(),
    )

    def __init__(self, hs):
        super(RelationPaginationServlet, self).__init__()
        self.auth = hs.get_auth()
        self.store = hs.get_datastore()
        self.clock = hs.get_clock()
        self._event_serializer = hs.get_event_client_serializer()

    @defer.inlineCallbacks
    def on_GET(self, request, room_id, parent_id, relation_type=None, event_type=None):
        requester = yield self.auth.get_user_by_req(request, allow_guest=True)

        yield self.auth.check_in_room_or_world_readable(
            room_id, requester.user.to_string()
        )

        limit = parse_integer(request, "limit", default=5)

        result = yield self.store.get_relations_for_event(
            event_id=parent_id,
            relation_type=relation_type,
            event_type=event_type,
            limit=limit,
        )

        events = yield self.store.get_events_as_list(
            [c["event_id"] for c in result.chunk]
        )

        now = self.clock.time_msec()
        events = yield self._event_serializer.serialize_events(events, now)

        return_value = result.to_dict()
        return_value["chunk"] = events

        defer.returnValue((200, return_value))

def register_servlets(hs, http_server):
    RelationSendServlet(hs).register(http_server)
    RelationPaginationServlet(hs).register(http_server)
<|endoftext|>


<commit_before>"""Transaction changelog for resources."""
from collections import defaultdict
from pyramid.registry import Registry
from adhocracy_core.interfaces import ChangelogMetadata
from adhocracy_core.interfaces import VisibilityChange

changelog_metadata = ChangelogMetadata(False, False, None, None, None,
                                       False, False, VisibilityChange.visible)

def clear_changelog_after_commit_hook(success: bool, registry: Registry):
    """Delete all entries in the transaction changelog."""
    changelog = getattr(registry, 'changelog', dict())
    changelog.clear()

def create_changelog() -> dict:
    """Return dict that maps resource path to :class:`ChangelogMetadata`."""
    metadata = lambda: changelog_metadata
    return defaultdict(metadata)

def clear_modification_date_after_commit_hook(success: bool,
                                              registry: Registry):
    """Delete the shared modification date for the transaction.

    The date is set by :func:`adhocracy_utils.get_modification_date`.
    """
    if getattr(registry, '__modification_date__',
               None) is not None:  # pragma: no branch
        del registry.__modification_date__

def includeme(config):
    """Add transaction changelog to the registry and register subscribers."""
    changelog = create_changelog()
    config.registry.changelog = changelog
    config.include('.subscriber')
<commit_msg>Fix formatting<commit_after>"""Transaction changelog for resources."""
from collections import defaultdict
from pyramid.registry import Registry
from adhocracy_core.interfaces import ChangelogMetadata
from adhocracy_core.interfaces import VisibilityChange

changelog_metadata = ChangelogMetadata(False, False, None, None, None,
                                       False, False, VisibilityChange.visible)

def clear_changelog_after_commit_hook(success: bool, registry: Registry):
    """Delete all entries in the transaction changelog."""
    changelog = getattr(registry, 'changelog', dict())
    changelog.clear()

def create_changelog() -> dict:
    """Return dict that maps resource path to :class:`ChangelogMetadata`."""
    metadata = lambda: changelog_metadata
    return defaultdict(metadata)

def clear_modification_date_after_commit_hook(success: bool,
                                              registry: Registry):
    """Delete the shared modification date for the transaction.

    The date is set by :func:`adhocracy_utils.get_modification_date`.
    """
    if getattr(registry, '__modification_date__',  # pragma: no branch
               None) is not None:
        del registry.__modification_date__

def includeme(config):
    """Add transaction changelog to the registry and register subscribers."""
    changelog = create_changelog()
    config.registry.changelog = changelog
    config.include('.subscriber')
<|endoftext|>


<commit_before><commit_msg>work in progress, just some notes<commit_after><|endoftext|>