<commit_before>#!/usr/bin/env python

import os
import random
import string
import subprocess
import sys

from boto import rds2
from time import sleep

AWS_REGION = os.environ.get('AWS_REGION', 'us-east-1')
AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
DB_INSTANCE_CLASS = os.environ.get('DB_INSTANCE_CLASS')

def dump_postgres(db_instance, db_name, out_file_name):
    if not 'PGPASSWORD' in os.environ:
        os.environ['PGPASSWORD'] = os.environ.get('DB_PASSWORD')

    with open(
        '/out/%s.dump' % out_file_name, 'w'
    ) as outfile:
        return subprocess.check_call([
            'pg_dump', '-w', '-Fc',
            '-U', os.environ.get('DB_USER', db_instance['MasterUsername']),
            '-h', db_instance['Endpoint']['Address'],
            '-p', str(db_instance['Endpoint']['Port']),
            db_name
        ], stdout=outfile)

def dump_mysql(db_instance, db_name, out_file_name):
    with open(
        '/out/%s.sql' % out_file_name, 'w'
    ) as outfile:
        return subprocess.check_call([
            'mysqldump',
            '-u', os.environ.get('DB_USER', db_instance['MasterUsername']),
            '-p%s' % os.environ.get('DB_PASSWORD', ''),
            '-h', db_instance['Endpoint']['Address'],
            '-P', str(db_instance['Endpoint']['Port']),
            db_name
        ], stdout=outfile)

dump_cmds = {
    'postgres': dump_postgres,
    'mysql': dump_mysql,
}

if len(sys.argv) < 2:
    print 'Usage: %s db-instance-name [db-name ...]' % sys.argv[0]
    sys.exit(1)

conn = rds2.connect_to_region(AWS_REGION, aws_access_key_id=AWS_ACCESS_KEY_ID,
                              aws_secret_access_key=AWS_SECRET_ACCESS_KEY)

_, db_instance_name, db_names = sys.argv[0], sys.argv[1], sys.argv[2:]

snapshots = conn.describe_db_snapshots(db_instance_name)\
        ['DescribeDBSnapshotsResponse']\
        ['DescribeDBSnapshotsResult']\
        ['DBSnapshots']

snapshots = [ s for s in snapshots if s['Status'] == 'available' ]
snapshots = sorted(snapshots, key=lambda s: s['SnapshotCreateTime'])

if len(snapshots) == 0:
    print 'No snapshots found for instance "%s"' % db_instance_name
    sys.exit(2)

latest_snapshot = snapshots[-1]
latest_snapshot_name = latest_snapshot['DBSnapshotIdentifier'].split(':')[-1]

print 'Found snapshot "%s".' % latest_snapshot['DBSnapshotIdentifier']

dump_instance_identifier = 'dump-%s-%s' % (
    latest_snapshot_name,
    "".join([ random.choice(string.letters) for x in range(8) ])
)

conn.restore_db_instance_from_db_snapshot(
    dump_instance_identifier,
    latest_snapshot['DBSnapshotIdentifier'],
    publicly_accessible=True,
    db_instance_class=DB_INSTANCE_CLASS
)

print 'Launched instance "%s".' % dump_instance_identifier

TIMEOUT = 1200
SLEEP_INTERVAL = 30

print "Waiting for instance to become available."

while TIMEOUT > 0:
    dump_instance = conn.describe_db_instances(dump_instance_identifier)\
            ['DescribeDBInstancesResponse']\
            ['DescribeDBInstancesResult']\
            ['DBInstances'][0]

    if dump_instance['DBInstanceStatus'] == 'available':
        break

    TIMEOUT -= SLEEP_INTERVAL
    sleep(SLEEP_INTERVAL)

if dump_instance['DBInstanceStatus'] != 'available':
    print ('Instance "%s" did not become available within time limit. '
           'Aborting.' % dump_instance_identifier)
    conn.delete_db_instance(dump_instance_identifier,
                            skip_final_snapshot=True)
    exit(3)

print "Instance is available."

print 'Instance engine is "%s".' % dump_instance['Engine']

if not dump_instance['Engine'] in dump_cmds:
    print "Error: Can't handle databases of this type. Aborting."
    sys.exit(4)

if len(db_names) == 0:
    print 'Dumping "%s".' % dump_instance['DBName']
    dump_cmds[dump_instance['Engine']](
        dump_instance, dump_instance['DBName'], latest_snapshot_name
    )
else:
    for db_name in db_names:
        print 'Dumping "%s".' % db_name
        dump_cmds[dump_instance['Engine']](
            dump_instance, db_name, '%s-%s' % (db_name, latest_snapshot_name)
        )

print "Dump completed."

conn.delete_db_instance(dump_instance_identifier,
                        skip_final_snapshot=True)

print 'Terminated "%s".' % dump_instance_identifier
<commit_msg>Increase DB creation timeout to 1 hour<commit_after>#!/usr/bin/env python

import os
import random
import string
import subprocess
import sys

from boto import rds2
from time import sleep

AWS_REGION = os.environ.get('AWS_REGION', 'us-east-1')
AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
DB_INSTANCE_CLASS = os.environ.get('DB_INSTANCE_CLASS')

def dump_postgres(db_instance, db_name, out_file_name):
    if not 'PGPASSWORD' in os.environ:
        os.environ['PGPASSWORD'] = os.environ.get('DB_PASSWORD')

    with open(
        '/out/%s.dump' % out_file_name, 'w'
    ) as outfile:
        return subprocess.check_call([
            'pg_dump', '-w', '-Fc',
            '-U', os.environ.get('DB_USER', db_instance['MasterUsername']),
            '-h', db_instance['Endpoint']['Address'],
            '-p', str(db_instance['Endpoint']['Port']),
            db_name
        ], stdout=outfile)

def dump_mysql(db_instance, db_name, out_file_name):
    with open(
        '/out/%s.sql' % out_file_name, 'w'
    ) as outfile:
        return subprocess.check_call([
            'mysqldump',
            '-u', os.environ.get('DB_USER', db_instance['MasterUsername']),
            '-p%s' % os.environ.get('DB_PASSWORD', ''),
            '-h', db_instance['Endpoint']['Address'],
            '-P', str(db_instance['Endpoint']['Port']),
            db_name
        ], stdout=outfile)

dump_cmds = {
    'postgres': dump_postgres,
    'mysql': dump_mysql,
}

if len(sys.argv) < 2:
    print 'Usage: %s db-instance-name [db-name ...]' % sys.argv[0]
    sys.exit(1)

conn = rds2.connect_to_region(AWS_REGION, aws_access_key_id=AWS_ACCESS_KEY_ID,
                              aws_secret_access_key=AWS_SECRET_ACCESS_KEY)

_, db_instance_name, db_names = sys.argv[0], sys.argv[1], sys.argv[2:]

snapshots = conn.describe_db_snapshots(db_instance_name)\
        ['DescribeDBSnapshotsResponse']\
        ['DescribeDBSnapshotsResult']\
        ['DBSnapshots']

snapshots = [ s for s in snapshots if s['Status'] == 'available' ]
snapshots = sorted(snapshots, key=lambda s: s['SnapshotCreateTime'])

if len(snapshots) == 0:
    print 'No snapshots found for instance "%s"' % db_instance_name
    sys.exit(2)

latest_snapshot = snapshots[-1]
latest_snapshot_name = latest_snapshot['DBSnapshotIdentifier'].split(':')[-1]

print 'Found snapshot "%s".' % latest_snapshot['DBSnapshotIdentifier']

dump_instance_identifier = 'dump-%s-%s' % (
    latest_snapshot_name,
    "".join([ random.choice(string.letters) for x in range(8) ])
)

conn.restore_db_instance_from_db_snapshot(
    dump_instance_identifier,
    latest_snapshot['DBSnapshotIdentifier'],
    publicly_accessible=True,
    db_instance_class=DB_INSTANCE_CLASS
)

print 'Launched instance "%s".' % dump_instance_identifier

TIMEOUT = 3600
SLEEP_INTERVAL = 30

print "Waiting for instance to become available."

while TIMEOUT > 0:
    dump_instance = conn.describe_db_instances(dump_instance_identifier)\
            ['DescribeDBInstancesResponse']\
            ['DescribeDBInstancesResult']\
            ['DBInstances'][0]

    if dump_instance['DBInstanceStatus'] == 'available':
        break

    TIMEOUT -= SLEEP_INTERVAL
    sleep(SLEEP_INTERVAL)

if dump_instance['DBInstanceStatus'] != 'available':
    print ('Instance "%s" did not become available within time limit. '
           'Aborting.' % dump_instance_identifier)
    conn.delete_db_instance(dump_instance_identifier,
                            skip_final_snapshot=True)
    exit(3)

print "Instance is available."

print 'Instance engine is "%s".' % dump_instance['Engine']

if not dump_instance['Engine'] in dump_cmds:
    print "Error: Can't handle databases of this type. Aborting."
    sys.exit(4)

if len(db_names) == 0:
    print 'Dumping "%s".' % dump_instance['DBName']
    dump_cmds[dump_instance['Engine']](
        dump_instance, dump_instance['DBName'], latest_snapshot_name
    )
else:
    for db_name in db_names:
        print 'Dumping "%s".' % db_name
        dump_cmds[dump_instance['Engine']](
            dump_instance, db_name, '%s-%s' % (db_name, latest_snapshot_name)
        )

print "Dump completed."

conn.delete_db_instance(dump_instance_identifier,
                        skip_final_snapshot=True)

print 'Terminated "%s".' % dump_instance_identifier
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import zlib

class PNGWrongHeaderError(Exception):
    """Exception, that says the input file is probably not PNG file."""
    pass

class PNGNotImplementedError(Exception):
    """Exception, that says the PNG image has a structure we cannot process."""
    pass

class PngReader():
    """Class for reading PNG images."""

    def __init__(self, filepath):

        # RGB-data from image as a list of rows,
        #   each row contains pixels - triplets of color values
        self.rgb = []

        # save the data from the file
        self.binary = open(filepath, mode='rb').read()

        self._parse_png()

    def _check_png(self):
        """Checks, if it is PNG, strip theheader."""
        if (self.binary[:8] != b'\x89PNG\r\n\x1a\n'):
            raise PNGWrongHeaderError()
            self.binary = self.binary[8:]
        return self

    def _bytes_to_num(self,bytes):
        n = 0
        for b in bytes:
            n = n*256 + b
        return n

    def _parse_png(self):
        self._check_png()
        p = 0
        self.data = []
        while p < len(self.binary):
            l = self._bytes_to_num(self.binary[p:p+4])
            p += 4

            self.data += [{'head':self.binary[p:p+4],
                           'length':l,
                           'data':self.binary[p+4:p+l+4],
                           'crc':self.binary[p+l+4:p+l+8]}]

            p += l+8
        return self

    def _get_size(self):
        for chunk in self.data:
            if (chunk['head'] == b'IHDR'):
                ihdr = chunk['data']
                break
        self.width = self._bytes_to_num(ihdr[0:4]);
        self.height = self._bytes_to_num(ihdr[4:8]);
        return self;

#
# just for testing
#
if __name__ == '__main__':
    PngReader('test_data/HelloWorld.png')
<commit_msg>Get IDAT, corrected bad header stripping inside bad if<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import zlib

class PNGWrongHeaderError(Exception):
    """Exception, that says the input file is probably not PNG file."""
    pass

class PNGNotImplementedError(Exception):
    """Exception, that says the PNG image has a structure we cannot process."""
    pass

class PngReader():
    """Class for reading PNG images."""

    def __init__(self, filepath):

        # RGB-data from image as a list of rows,
        #   each row contains pixels - triplets of color values
        self.rgb = []

        # save the data from the file
        self.binary = open(filepath, mode='rb').read()

        self._save()

    def _check_png(self):
        """Checks, if it is PNG, strip theheader."""
        if (self.binary[:8] != b'\x89PNG\r\n\x1a\n'):
            raise PNGWrongHeaderError()
        self.binary = self.binary[8:]
        return self

    def _bytes_to_num(self,bytes):
        n = 0
        for b in bytes:
            n = n*256 + b
        return n

    def _parse_png(self):
        self._check_png()
        p = 0
        self.data = []
        while p < len(self.binary):
            l = self._bytes_to_num(self.binary[p:p+4])
            p += 4

            self.data += [{'head':self.binary[p:p+4],
                           'length':l,
                           'data':self.binary[p+4:p+l+4],
                           'crc':self.binary[p+l+4:p+l+8]}]

            p += l+8
        return self

    def _get_size(self):
        for chunk in self.data:
            if (chunk['head'] == b'IHDR'):
                ihdr = chunk['data']
                break
        self.width = self._bytes_to_num(ihdr[0:4]);
        self.height = self._bytes_to_num(ihdr[4:8]);
        return self;

    def _get_idat(self):
        idat = b''
        for chunk in self.data:
            if (chunk['head'] == b'IDAT'):
                idat += chunk['data']

        return zlib.decompress(idat)

    def _save(self):
        self._parse_png()
        self._get_size()
        idat = self._get_idat()

        return self

#
# just for testing
#
if __name__ == '__main__':
    PngReader('test_data/HelloWorld.png')
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Siconos is a program dedicated to modeling, simulation and control
# of non smooth dynamical systems.
#
# Copyright 2018 INRIA.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from siconos.kernel import FirstOrderLinearDS, getMatrix, SimpleMatrix
from siconos.control.simulation import ControlZOHSimulation
from siconos.control.sensor import LinearSensor
from siconos.control.controller import LinearSMCOT2

import matplotlib
matplotlib.use('Agg')
from matplotlib.pyplot import subplot, title, plot, grid, savefig
from numpy import eye, empty, zeros, savetxt
from math import ceil, sin
from numpy.linalg import norm

# Derive our own version of FirstOrderLinearDS
class MyFOLDS(FirstOrderLinearDS):
    ''' derived FirstOrderLinearDS class to show how to override a method '''
    def computeb(self, time):
        t = sin(50*time)
        u = [t, -t]
        self.setbPtr(u)

# variable declaration
ndof = 2   # Number of degrees of freedom of your system
t0 = 0.0   # start time
T = 1    # end time
h = 1.0e-4  # time step for simulation
hControl = 1.0e-2 # time step for control
Xinit = 1.0 # initial position
theta = 0.5
N = 2*int(ceil((T-t0)/h)) # number of time steps
outputSize = 5 # number of variable to store at each time step

# Matrix declaration
A = zeros((ndof, ndof))
x0 = [Xinit, -Xinit]
sensorC = eye(ndof)
Csurface = [[0, 1.0]]
Brel = [[0], [2]]

# Simple check
if h > hControl:
    print("hControl must be bigger than h")
    exit(1)

# Declaration of the Dynamical System
processDS = MyFOLDS(x0, A)
# XXX b is not automatically created ...
processDS.setbPtr([0, 0])
# Control simulation
sim = ControlZOHSimulation(t0, T, h)
sim.setSaveOnlyMainSimulation(True)
sim.addDynamicalSystem(processDS)
# Actuator, Sensor & ControlManager
sens = LinearSensor(processDS, sensorC)
sim.addSensor(sens, hControl)
act = LinearSMCOT2(sens)
act.setCsurface(Csurface)
act.setB(Brel)
sim.addActuator(act, hControl)

# Initialization
sim.initialize()

# Run simulation
sim.run()

# Get data
dataPlot = sim.data()

# Save to disk
savetxt('SMCExampleImplicitOT2-noCplugin-py.dat', dataPlot)
# Plot interesting data
subplot(411)
title('x1')
plot(dataPlot[:, 0], dataPlot[:, 1])
grid()
subplot(412)
title('x2')
plot(dataPlot[:, 0], dataPlot[:, 2])
grid()
subplot(413)
title('u')
plot(dataPlot[:, 0], dataPlot[:, 3])
grid()
savefig('ismcOT2-noCplugin.png')

# compare with the reference
ref = getMatrix(SimpleMatrix("SMCExampleImplicitOT2-py.ref"))
print("%e" % norm(dataPlot - ref))
if (norm(dataPlot - ref) > 1e-12):
    print(dataPlot - ref)
    print("Warning. The result is rather different from the reference file.")
<commit_msg>[examples] return error code<commit_after>#!/usr/bin/env python

# Siconos is a program dedicated to modeling, simulation and control
# of non smooth dynamical systems.
#
# Copyright 2018 INRIA.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from siconos.kernel import FirstOrderLinearDS, getMatrix, SimpleMatrix
from siconos.control.simulation import ControlZOHSimulation
from siconos.control.sensor import LinearSensor
from siconos.control.controller import LinearSMCOT2

import matplotlib
matplotlib.use('Agg')
from matplotlib.pyplot import subplot, title, plot, grid, savefig
from numpy import eye, empty, zeros, savetxt
from math import ceil, sin
from numpy.linalg import norm

# Derive our own version of FirstOrderLinearDS
class MyFOLDS(FirstOrderLinearDS):
    ''' derived FirstOrderLinearDS class to show how to override a method '''
    def computeb(self, time):
        t = sin(50*time)
        u = [t, -t]
        self.setbPtr(u)

# variable declaration
ndof = 2   # Number of degrees of freedom of your system
t0 = 0.0   # start time
T = 1    # end time
h = 1.0e-4  # time step for simulation
hControl = 1.0e-2 # time step for control
Xinit = 1.0 # initial position
theta = 0.5
N = 2*int(ceil((T-t0)/h)) # number of time steps
outputSize = 5 # number of variable to store at each time step

# Matrix declaration
A = zeros((ndof, ndof))
x0 = [Xinit, -Xinit]
sensorC = eye(ndof)
Csurface = [[0, 1.0]]
Brel = [[0], [2]]

# Simple check
if h > hControl:
    print("hControl must be bigger than h")
    exit(1)

# Declaration of the Dynamical System
processDS = MyFOLDS(x0, A)
# XXX b is not automatically created ...
processDS.setbPtr([0, 0])
# Control simulation
sim = ControlZOHSimulation(t0, T, h)
sim.setSaveOnlyMainSimulation(True)
sim.addDynamicalSystem(processDS)
# Actuator, Sensor & ControlManager
sens = LinearSensor(processDS, sensorC)
sim.addSensor(sens, hControl)
act = LinearSMCOT2(sens)
act.setCsurface(Csurface)
act.setB(Brel)
sim.addActuator(act, hControl)

# Initialization
sim.initialize()

# Run simulation
sim.run()

# Get data
dataPlot = sim.data()

# Save to disk
savetxt('SMCExampleImplicitOT2-noCplugin-py.dat', dataPlot)
# Plot interesting data
subplot(411)
title('x1')
plot(dataPlot[:, 0], dataPlot[:, 1])
grid()
subplot(412)
title('x2')
plot(dataPlot[:, 0], dataPlot[:, 2])
grid()
subplot(413)
title('u')
plot(dataPlot[:, 0], dataPlot[:, 3])
grid()
savefig('ismcOT2-noCplugin.png')

# compare with the reference
ref = getMatrix(SimpleMatrix("SMCExampleImplicitOT2-py.ref"))
print("%e" % norm(dataPlot - ref))
if (norm(dataPlot - ref) > 1e-12):
    print(dataPlot - ref)
    print("Warning. The result is rather different from the reference file.")
    return 1
<|endoftext|>


<commit_before>import ast
import inspect
#import astpp
import copy

def coroutine(func):
    def start(*args, **kwargs):
        g = func(*args, **kwargs)
        g.next()
        return g
    return start

def invertibleGenerator(globalEnv):
    """ Add a co method to a generator function, that is the equivalent
    coroutine. """

    def decorator(func):
        func.co = coroutine(transformAstWith(
            globalEnv, [InvertGenerator])(func)
        )
        return func

    return decorator

class InvertGenerator(ast.NodeTransformer):
    # TODO: should probably be a two-pass procedure:
    # 1. Find what argument to turn from iterable to target
    # 2. Carry out transformations
    #
    # This needs to be done, if first yield is before first use of iterable,
    # in which case this code won't know how to conver the yield statement

    def __init__(self):
        self.functionArguments = None
        self.iterable = None

    def visit_FunctionDef(self, node):
        """ Gather function arguments for use later. """
        if self.functionArguments is None:
            self.functionArguments = node.args.args

        self.generic_visit(node)

        return node

    def isForStatementCandidate(self, node):

        return self.functionArguments is not None and \
            isinstance(node.iter, ast.Name) and \
            node.iter.id in (arg.id for arg in self.functionArguments)

    def visit_For(self, node):
        """ Change iteration into while-yield statements """
        # TODO: take care of GeneratorExit handling.
        # TODO: take care of StopIteration conversion?

        newnode = node
        if self.isForStatementCandidate(node):
            # For(expr target, expr iter, stmt* body, stmt* orelse)
            # While(expr test, stmt* body, stmt* orelse)

            # save the iterable, which will be now used as a target instead.
            self.target = node.iter

            newbody = [ast.Assign(targets=[node.target],
                                  value=ast.Yield(value=None))] + node.body
            newnode = ast.While(
                test=ast.Name(id='True', ctx=ast.Load()),
                body=newbody,
                orelse=[])  # TODO: what to do with orelse

        self.generic_visit(newnode)

        return ast.copy_location(newnode, node)

    def coroutineSendExpression(self, target, exprToSend):
        return ast.Expr(value=ast.Call(
            func=ast.Attribute(
                value=target,
                attr='send',
                ctx=ast.Load()),
            args=[exprToSend],
            keywords=[],
            starargs=None,
            kwargs=None
            ))

    def extractValueFromYieldExpr(self, expr):

        if isinstance(expr.value, ast.Yield):
            return expr.value.value

        return None

    def visit_Expr(self, node):

        newnode = node

        yieldValue = self.extractValueFromYieldExpr(node)
        if yieldValue is not None:
            newnode = self.coroutineSendExpression(
                self.target,
                yieldValue)

        self.generic_visit(newnode)
        return newnode

class RemoveDecorators(ast.NodeTransformer):

    def visit_FunctionDef(self, node):
        node.decorator_list = []
        return node

    def visit_ClassDef(self, node):
        node.decorator_list = []
        return node

def transformAstWith(globalEnv, transformers):

    transformers.insert(0, RemoveDecorators)

    def transformDecorator(func):
        funcName = func.__name__

        # TODO: need to unindent if method?
        node = ast.parse(inspect.getsource(func))

        #print "BEFORE: "
        #print astpp.dump(node)

        for transformer in transformers:
            transformer().visit(node)

        #print "AFTER: "
        #print astpp.dump(node)

        ast.fix_missing_locations(node)
        compiled = compile(node, '<string>', 'exec')

        tempNamespace = copy.copy(globalEnv)
        exec compiled in tempNamespace

        return tempNamespace[funcName]

    return transformDecorator
<commit_msg>Adding more comments<commit_after>import ast
import inspect
#import astpp
import copy

def coroutine(func):
    def start(*args, **kwargs):
        g = func(*args, **kwargs)
        g.next()
        return g
    return start

def invertibleGenerator(globalEnv):
    """ Add a co method to a generator function, that is the equivalent
    coroutine. """

    def decorator(func):
        func.co = coroutine(
            transformAstWith(globalEnv, [InvertGenerator])(func)
        )
        return func

    return decorator

class InvertGenerator(ast.NodeTransformer):
    """ Transform a function AST, from a generator into a coroutine (from pull
    to push). The iterable parameter to the generator that was pulled from, now
    becomes the "target" parameter of the coroutine that is pushed to."""
    # TODO: should probably be a two-pass procedure:
    # 1. Find what argument to turn from iterable to target
    # 2. Carry out transformations
    #
    # This needs to be done, if first yield is before first use of iterable,
    # in which case this code won't know how to conver the yield statement

    def __init__(self):
        self.functionArguments = None
        self.iterable = None

    def visit_FunctionDef(self, node):
        """ Gather function arguments for use later. """
        if self.functionArguments is None:
            self.functionArguments = node.args.args

        self.generic_visit(node)

        return node

    def isForStatementCandidate(self, node):
        """ Return True if For statement is a candidate for transformation """

        return self.functionArguments is not None and \
            isinstance(node.iter, ast.Name) and \
            node.iter.id in (arg.id for arg in self.functionArguments)

    def visit_For(self, node):
        """ Change iteration into while-yield statements """
        # TODO: take care of GeneratorExit handling.
        # TODO: take care of StopIteration conversion?

        newnode = node
        if self.isForStatementCandidate(node):
            # For(expr target, expr iter, stmt* body, stmt* orelse)
            # While(expr test, stmt* body, stmt* orelse)

            # save the iterable, which will be now used as a target instead.
            self.target = node.iter

            # prepend statement to await a value in the coroutine
            newbody = [ast.Assign(targets=[node.target],
                                  value=ast.Yield(value=None))] + node.body
            newnode = ast.While(
                test=ast.Name(id='True', ctx=ast.Load()),
                body=newbody,
                orelse=[])  # TODO: what to do with orelse

        self.generic_visit(newnode)

        return ast.copy_location(newnode, node)

    def coroutineSendExpression(self, target, exprToSend):
        """ Create an expression like
            target.send(exprToSend)
        """
        return ast.Expr(value=ast.Call(
            func=ast.Attribute(
                value=target,
                attr='send',
                ctx=ast.Load()),
            args=[exprToSend],
            keywords=[],
            starargs=None,
            kwargs=None
            ))

    def extractValueFromYieldExpr(self, expr):

        if isinstance(expr.value, ast.Yield):
            return expr.value.value

        return None

    def visit_Expr(self, node):

        newnode = node

        yieldValue = self.extractValueFromYieldExpr(node)
        if yieldValue is not None:
            newnode = self.coroutineSendExpression(
                self.target,
                yieldValue)

        self.generic_visit(newnode)
        return newnode

class RemoveDecorators(ast.NodeTransformer):

    def visit_FunctionDef(self, node):
        node.decorator_list = []
        return node

    def visit_ClassDef(self, node):
        node.decorator_list = []
        return node

def transformAstWith(globalEnv, transformers):
    """ Create a decorator that transforms functions in a given global
    environment, using the provided NodeTransformer classes """

    # Always remove decorators when transforming
    # TODO: necessary?
    transformers.insert(0, RemoveDecorators)

    def transformDecorator(func):
        funcName = func.__name__

        # TODO: need to unindent if method or local function
        node = ast.parse(inspect.getsource(func))

        #print "BEFORE: "
        #print astpp.dump(node)

        for transformer in transformers:
            transformer().visit(node)

        #print "AFTER: "
        #print astpp.dump(node)

        ast.fix_missing_locations(node)
        compiled = compile(node, '<string>', 'exec')

        tempNamespace = copy.copy(globalEnv)
        exec compiled in tempNamespace

        return tempNamespace[funcName]

    return transformDecorator
<|endoftext|>


<commit_before>import os, sys
import vtk, ctk, slicer
import logging

#
# MRMLUtility
#

'''
This class harbors all the utility functions to load, add, save and remove mrml nodes
'''

class MRMLUtility(object):
  @staticmethod
  def loadMRMLNode(node_name, file_dir, file_name, file_type):
    node = slicer.util.getNode(node_name)
    if node is None:
      properties = {}
      file_path = os.path.join(file_dir, file_name)
      if file_type == 'LabelMap':
        file_type = 'VolumeFile'
        properties['labelmap'] = True
      node = slicer.util.loadNodeFromFile(file_path, file_type, properties, returnNode=True)
      node = node[1]
      if node is None:
        logging.error('!!! Failed to load: %s', file_path)
        return -1
      if file_type == 'MarkupsFiducials':
        node.SetLocked(1)
      node.SetName(node_name)
    return node

  @staticmethod
  def createNewMRMLNode(node_name, mrml_type, copy_node=None, transform=None):
    mrml_node = slicer.mrmlScene.AddNode(mrml_type)
    if copy_node is not None:
      mrml_node.Copy(copy_node)
      if mrml_type.GetClassName() is 'vtkMRMLModelNode':
        display_node = slicer.mrmlScene.CreateNodeByClass('vtkMRMLModelDisplayNode')
        slicer.mrmlScene.AddNode(display_node)
        display_node.UnRegister(slicer.mrmlScene)
        mrml_node.SetAndObserveDisplayNodeID(display_node.GetID())
        mrml_node.SetAndObserveStorageNodeID(None)
      elif mrml_type.GetClassName() is 'vtkMRMLMarkupsFiducialNode':
        display_node = slicer.mrmlScene.CreateNodeByClass('vtkMRMLMarkupsDisplayNode')
        slicer.mrmlScene.AddNode(display_node)
        display_node.UnRegister(slicer.mrmlScene)
        mrml_node.SetAndObserveDisplayNodeID(display_node.GetID())
        mrml_node.SetAndObserveStorageNodeID(None)
    else:
      if mrml_type.GetClassName() is 'vtkMRMLModelNode':
        mrml_node.CreateDefaultDisplayNodes()

    if transform is not None:
      mrml_node.ApplyTransform(transform)
    mrml_node.SetName(node_name)
    return mrml_node

  @staticmethod
  def getMRMLNode(node_name, mrml_type, copy_node=None, transform=None):
    mrml_node = slicer.util.getNode(node_name)
    if mrml_node is None:
      mrml_node = MRMLUtility.createNewMRMLNode(node_name, mrml_type, copy_node, transform)
      already_exists = False
    else:
      already_exists = True
    return (mrml_node, already_exists)

  @staticmethod
  def isMRMLNodeEmpty(mrml_node, mrml_type):
    node_empty = True
    if mrml_node is not None:
      if mrml_type == 'vtkMRMLModelNode':
        if mrml_node.GetPolyData() is not None:
          node_empty = False
          print "***", mrml_node.GetName(), "already has a polyData attribute"
      if mrml_type == 'vtkMRMLModelHierarchyNode':
        if mrml_node.GetNumberOfChildrenNodes() != 0:
          node_empty = False
          print "***", mrml_node.GetName(), "already has children models"
      elif mrml_type == 'vtkMRMLTransformNode' or mrml_type == 'vtkMRMLLinearTransformNode':
        if mrml_node.GetTransformToParent() is not None:
          node_empty = False
          print "***", mrml_node.GetName(), "already has a transform attribute"
      elif mrml_type == 'vtkMRMLScalarVolumeNode' or mrml_type == 'vtkMRMLLabelMapVolumeNode':
        if mrml_node.GetImageData() is not None:
          node_empty = False
          print "***", mrml_node.GetName(), "already has an image attribute"
    return node_empty

  @staticmethod
  def saveMRMLNodes(nodes, case_dir):
    for i in range(len(nodes)):
      MRMLUtility.saveMRMLNode(nodes[i], case_dir)

  @staticmethod
  def saveMRMLNode(node, case_dir):
    if slicer.util.getNode(node.GetName()):
      file_name = node.GetName()
      class_name = node.GetClassName()
      if class_name == 'vtkMRMLScalarVolumeNode' or class_name == 'vtkMRMLLabelMapVolumeNode':
        file_name += '.nrrd'
      elif class_name == 'vtkMRMLModelNode':
        file_name += '.vtk'
      elif class_name == 'vtkMRMLLinearTransformNode' or class_name == 'vtkMRMLTransformNode':
        file_name += '.mat'
      elif class_name == 'vtkMRMLMarkupsFiducialNode':
        file_name += '.fcsv'
      elif class_name == 'vtkMRMLDoubleArrayNode':
        file_name += '.mcsv'
      file_path = os.path.join(case_dir, file_name)
      logging.info("Saving %s in %s", file_name, case_dir)
      slicer.util.saveNode(node, file_path)

  @staticmethod
  def removeMRMLNodes(nodes):
    for i in range(len(nodes)):
      MRMLUtility.removeMRMLNode(nodes[i])

  @staticmethod
  def removeMRMLNode(node):
    if slicer.util.getNode(node.GetName()):
      slicer.mrmlScene.RemoveNode(node)

<commit_msg>Fix slicer.util.MRMLNodeNotFoundException in shape analysis module<commit_after>import os, sys
import vtk, ctk, slicer
import logging

#
# MRMLUtility
#

'''
This class harbors all the utility functions to load, add, save and remove mrml nodes
'''

class MRMLUtility(object):
  @staticmethod
  def loadMRMLNode(node_name, file_dir, file_name, file_type):
    node = slicer.mrmlScene.GetFirstNodeByName(node_name)
    if node is None:
      properties = {}
      file_path = os.path.join(file_dir, file_name)
      if file_type == 'LabelMap':
        file_type = 'VolumeFile'
        properties['labelmap'] = True
      node = slicer.util.loadNodeFromFile(file_path, file_type, properties, returnNode=True)
      node = node[1]
      if node is None:
        logging.error('!!! Failed to load: %s', file_path)
        return -1
      if file_type == 'MarkupsFiducials':
        node.SetLocked(1)
      node.SetName(node_name)
    return node

  @staticmethod
  def createNewMRMLNode(node_name, mrml_type, copy_node=None, transform=None):
    mrml_node = slicer.mrmlScene.AddNode(mrml_type)
    if copy_node is not None:
      mrml_node.Copy(copy_node)
      if mrml_type.GetClassName() is 'vtkMRMLModelNode':
        display_node = slicer.mrmlScene.CreateNodeByClass('vtkMRMLModelDisplayNode')
        slicer.mrmlScene.AddNode(display_node)
        display_node.UnRegister(slicer.mrmlScene)
        mrml_node.SetAndObserveDisplayNodeID(display_node.GetID())
        mrml_node.SetAndObserveStorageNodeID(None)
      elif mrml_type.GetClassName() is 'vtkMRMLMarkupsFiducialNode':
        display_node = slicer.mrmlScene.CreateNodeByClass('vtkMRMLMarkupsDisplayNode')
        slicer.mrmlScene.AddNode(display_node)
        display_node.UnRegister(slicer.mrmlScene)
        mrml_node.SetAndObserveDisplayNodeID(display_node.GetID())
        mrml_node.SetAndObserveStorageNodeID(None)
    else:
      if mrml_type.GetClassName() is 'vtkMRMLModelNode':
        mrml_node.CreateDefaultDisplayNodes()

    if transform is not None:
      mrml_node.ApplyTransform(transform)
    mrml_node.SetName(node_name)
    return mrml_node

  @staticmethod
  def getMRMLNode(node_name, mrml_type, copy_node=None, transform=None):
    mrml_node = slicer.mrmlScene.GetFirstNodeByName(node_name)
    if mrml_node is None:
      mrml_node = MRMLUtility.createNewMRMLNode(node_name, mrml_type, copy_node, transform)
      already_exists = False
    else:
      already_exists = True
    return (mrml_node, already_exists)

  @staticmethod
  def isMRMLNodeEmpty(mrml_node, mrml_type):
    node_empty = True
    if mrml_node is not None:
      if mrml_type == 'vtkMRMLModelNode':
        if mrml_node.GetPolyData() is not None:
          node_empty = False
          print "***", mrml_node.GetName(), "already has a polyData attribute"
      if mrml_type == 'vtkMRMLModelHierarchyNode':
        if mrml_node.GetNumberOfChildrenNodes() != 0:
          node_empty = False
          print "***", mrml_node.GetName(), "already has children models"
      elif mrml_type == 'vtkMRMLTransformNode' or mrml_type == 'vtkMRMLLinearTransformNode':
        if mrml_node.GetTransformToParent() is not None:
          node_empty = False
          print "***", mrml_node.GetName(), "already has a transform attribute"
      elif mrml_type == 'vtkMRMLScalarVolumeNode' or mrml_type == 'vtkMRMLLabelMapVolumeNode':
        if mrml_node.GetImageData() is not None:
          node_empty = False
          print "***", mrml_node.GetName(), "already has an image attribute"
    return node_empty

  @staticmethod
  def saveMRMLNodes(nodes, case_dir):
    for i in range(len(nodes)):
      MRMLUtility.saveMRMLNode(nodes[i], case_dir)

  @staticmethod
  def saveMRMLNode(node, case_dir):
    if not node:
      return
    file_name = node.GetName()
    class_name = node.GetClassName()
    if class_name == 'vtkMRMLScalarVolumeNode' or class_name == 'vtkMRMLLabelMapVolumeNode':
      file_name += '.nrrd'
    elif class_name == 'vtkMRMLModelNode':
      file_name += '.vtk'
    elif class_name == 'vtkMRMLLinearTransformNode' or class_name == 'vtkMRMLTransformNode':
      file_name += '.mat'
    elif class_name == 'vtkMRMLMarkupsFiducialNode':
      file_name += '.fcsv'
    elif class_name == 'vtkMRMLDoubleArrayNode':
      file_name += '.mcsv'
    file_path = os.path.join(case_dir, file_name)
    logging.info("Saving %s in %s", file_name, case_dir)
    slicer.util.saveNode(node, file_path)

  @staticmethod
  def removeMRMLNodes(nodes):
    for i in range(len(nodes)):
      MRMLUtility.removeMRMLNode(nodes[i])

  @staticmethod
  def removeMRMLNode(node):
    slicer.mrmlScene.RemoveNode(node)

<|endoftext|>


<commit_before># Copyright (c) 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import threading

from networking_arista.common import db_lib
from networking_arista.ml2 import arista_ml2
from oslo_config import cfg
from oslo_log import log as logging

from neutron.common import constants as n_const
from neutron.i18n import _LI
from neutron.plugins.ml2.common import exceptions as ml2_exc
from neutron.plugins.ml2 import driver_api
from neutron.plugins.ml2.drivers.arista import config  # noqa
from neutron.plugins.ml2.drivers.arista import db
from neutron.plugins.ml2.drivers.arista import exceptions as arista_exc

LOG = logging.getLogger(__name__)

EOS_UNREACHABLE_MSG = _('Unable to reach EOS')

class AristaDriver(driver_api.MechanismDriver):
    """Ml2 Mechanism driver for Arista networking hardware.

    Remembers all networks and VMs that are provisioned on Arista Hardware.
    Does not send network provisioning request if the network has already been
    provisioned before for the given port.
    """
    def __init__(self, rpc=None):

        self.rpc = rpc or arista_ml2.AristaRPCWrapper()
        self.db_nets = db.AristaProvisionedNets()
        self.db_vms = db.AristaProvisionedVms()
        self.db_tenants = db.AristaProvisionedTenants()
        self.ndb = db_lib.NeutronNets()

        confg = cfg.CONF.ml2_arista
        self.segmentation_type = db_lib.VLAN_SEGMENTATION
        self.timer = None
        self.eos = arista_ml2.SyncService(self.rpc, self.ndb)
        self.sync_timeout = confg['sync_interval']
        self.eos_sync_lock = threading.Lock()

    def initialize(self):
        self.rpc.register_with_eos()
        self._cleanup_db()
        self.rpc.check_cli_commands()
        # Registering with EOS updates self.rpc.region_updated_time. Clear it
        # to force an initial sync
        self.rpc.clear_region_updated_time()
        self._synchronization_thread()

    def create_network_precommit(self, context):
        """Remember the tenant, and network information."""

        network = context.current
        segments = context.network_segments
        network_id = network['id']
        tenant_id = network['tenant_id']
        segmentation_id = segments[0]['segmentation_id']
        with self.eos_sync_lock:
            db_lib.remember_tenant(tenant_id)
            db_lib.remember_network(tenant_id,
                                network_id,
                                segmentation_id)

    def create_network_postcommit(self, context):
        """Provision the network on the Arista Hardware."""

        network = context.current
        network_id = network['id']
        network_name = network['name']
        tenant_id = network['tenant_id']
        segments = context.network_segments
        vlan_id = segments[0]['segmentation_id']
        shared_net = network['shared']
        with self.eos_sync_lock:
            if db_lib.is_network_provisioned(tenant_id, network_id):
                try:
                    network_dict = {
                        'network_id': network_id,
                        'segmentation_id': vlan_id,
                        'network_name': network_name,
                        'shared': shared_net}
                    self.rpc.create_network(tenant_id, network_dict)
                except arista_exc.AristaRpcError:
                    LOG.info(EOS_UNREACHABLE_MSG)
                    raise ml2_exc.MechanismDriverError()
            else:
                LOG.info(_LI('Network %s is not created as it is not found in '
                             'Arista DB'), network_id)

    def update_network_precommit(self, context):
        """At the moment we only support network name change

        Any other change in network is not supported at this time.
        We do not store the network names, therefore, no DB store
        action is performed here.
        """
        new_network = context.current
        orig_network = context.original
        if new_network['name'] != orig_network['name']:
            LOG.info(_LI('Network name changed to %s'), new_network['name'])

    def update_network_postcommit(self, context):
        """At the moment we only support network name change

        If network name is changed, a new network create request is
        sent to the Arista Hardware.
        """
        new_network = context.current
        orig_network = context.original
        if ((new_network['name'] != orig_network['name']) or
           (new_network['shared'] != orig_network['shared'])):
            network_id = new_network['id']
            network_name = new_network['name']
            tenant_id = new_network['tenant_id']
            vlan_id = new_network['provider:segmentation_id']
            shared_net = new_network['shared']
            with self.eos_sync_lock:
                if db_lib.is_network_provisioned(tenant_id, network_id):
                    try:
                        network_dict = {
                            'network_id': network_id,
                            'segmentation_id': vlan_id,
                            'network_name': network_name,
                            'shared': shared_net}
                        self.rpc.create_network(tenant_id, network_dict)
                    except arista_exc.AristaRpcError:
                        LOG.info(EOS_UNREACHABLE_MSG)
                        raise ml2_exc.MechanismDriverError()
                else:
                    LOG.info(_LI('Network %s is not updated as it is not found'
                                 ' in Arista DB'), network_id)

    def delete_network_precommit(self, context):
        """Delete the network infromation from the DB."""
        network = context.current
        network_id = network['id']
        tenant_id = network['tenant_id']
        with self.eos_sync_lock:
            if db_lib.is_network_provisioned(tenant_id, network_id):
                db_lib.forget_network(tenant_id, network_id)

    def delete_network_postcommit(self, context):
        """Send network delete request to Arista HW."""
        network = context.current
        network_id = network['id']
        tenant_id = network['tenant_id']
        with self.eos_sync_lock:

            # Succeed deleting network in case EOS is not accessible.
            # EOS state will be updated by sync thread once EOS gets
            # alive.
            try:
                self.rpc.delete_network(tenant_id, network_id)
                # if necessary, delete tenant as well.
                self.delete_tenant(tenant_id)
            except arista_exc.AristaRpcError:
                LOG.info(EOS_UNREACHABLE_MSG)
                raise ml2_exc.MechanismDriverError()

    def create_port_precommit(self, context):
        """Remember the infromation about a VM and its ports

        A VM information, along with the physical host information
        is saved.
        """
        port = context.current
        device_id = port['device_id']
        device_owner = port['device_owner']
        host = context.host

        # device_id and device_owner are set on VM boot
        is_vm_boot = device_id and device_owner
        if host and is_vm_boot:
            port_id = port['id']
            network_id = port['network_id']
            tenant_id = port['tenant_id']
            with self.eos_sync_lock:
                db_lib.remember_tenant(tenant_id)
                db_lib.remember_vm(device_id, host, port_id,
                               network_id, tenant_id)

    def create_port_postcommit(self, context):
        """Plug a physical host into a network.

        Send provisioning request to Arista Hardware to plug a host
        into appropriate network.
        """
        port = context.current
        device_id = port['device_id']
        device_owner = port['device_owner']
        host = context.host

        # device_id and device_owner are set on VM boot
        is_vm_boot = device_id and device_owner
        if host and is_vm_boot:
            port_id = port['id']
            port_name = port['name']
            network_id = port['network_id']
            tenant_id = port['tenant_id']
            with self.eos_sync_lock:
                hostname = self._host_name(host)
                vm_provisioned = db_lib.is_vm_provisioned(device_id,
                                                      host,
                                                      port_id,
                                                      network_id,
                                                      tenant_id)
                # If network does not exist under this tenant,
                # it may be a shared network. Get shared network owner Id
                net_provisioned = (
                    db_lib.is_network_provisioned(tenant_id, network_id) or
                    self.ndb.get_shared_network_owner_id(network_id)
                )
                if vm_provisioned and net_provisioned:
                    try:
                        self.rpc.plug_port_into_network(device_id,
                                                        hostname,
                                                        port_id,
                                                        network_id,
                                                        tenant_id,
                                                        port_name,
                                                        device_owner)
                    except arista_exc.AristaRpcError:
                        LOG.info(EOS_UNREACHABLE_MSG)
                        raise ml2_exc.MechanismDriverError()
                else:
                    LOG.info(_LI('VM %s is not created as it is not found in '
                                 'Arista DB'), device_id)

    def update_port_precommit(self, context):
        """Update the name of a given port.

        At the moment we only support port name change.
        Any other change to port is not supported at this time.
        We do not store the port names, therefore, no DB store
        action is performed here.
        """
        new_port = context.current
        orig_port = context.original
        if new_port['name'] != orig_port['name']:
            LOG.info(_LI('Port name changed to %s'), new_port['name'])

    def update_port_postcommit(self, context):
        """Update the name of a given port in EOS.

        At the moment we only support port name change
        Any other change to port is not supported at this time.
        """
        port = context.current
        orig_port = context.original
        if port['name'] == orig_port['name']:
            # nothing to do
            return

        device_id = port['device_id']
        device_owner = port['device_owner']
        host = context.host
        is_vm_boot = device_id and device_owner

        if host and is_vm_boot:
            port_id = port['id']
            port_name = port['name']
            network_id = port['network_id']
            tenant_id = port['tenant_id']
            with self.eos_sync_lock:
                hostname = self._host_name(host)
                segmentation_id = db_lib.get_segmentation_id(tenant_id,
                                                         network_id)
                vm_provisioned = db_lib.is_vm_provisioned(device_id,
                                                      host,
                                                      port_id,
                                                      network_id,
                                                      tenant_id)
                # If network does not exist under this tenant,
                # it may be a shared network. Get shared network owner Id
                net_provisioned = (
                    db_lib.is_network_provisioned(tenant_id, network_id,
                                                  segmentation_id) or
                    self.ndb.get_shared_network_owner_id(network_id)
                )
                if vm_provisioned and net_provisioned:
                    try:
                        self.rpc.plug_port_into_network(device_id,
                                                        hostname,
                                                        port_id,
                                                        network_id,
                                                        tenant_id,
                                                        port_name,
                                                        device_owner)
                    except arista_exc.AristaRpcError:
                        LOG.info(EOS_UNREACHABLE_MSG)
                        raise ml2_exc.MechanismDriverError()
                else:
                    LOG.info(_LI('VM %s is not updated as it is not found in '
                                 'Arista DB'), device_id)

    def delete_port_precommit(self, context):
        """Delete information about a VM and host from the DB."""
        port = context.current

        host_id = context.host
        device_id = port['device_id']
        tenant_id = port['tenant_id']
        network_id = port['network_id']
        port_id = port['id']
        with self.eos_sync_lock:
            if db_lib.is_vm_provisioned(device_id, host_id, port_id,
                                        network_id, tenant_id):
                db_lib.forget_vm(device_id, host_id, port_id,
                             network_id, tenant_id)

    def delete_port_postcommit(self, context):
        """unPlug a physical host from a network.

        Send provisioning request to Arista Hardware to unplug a host
        from appropriate network.
        """
        port = context.current
        device_id = port['device_id']
        host = context.host
        port_id = port['id']
        network_id = port['network_id']
        tenant_id = port['tenant_id']
        device_owner = port['device_owner']

        try:
            with self.eos_sync_lock:
                hostname = self._host_name(host)
                if device_owner == n_const.DEVICE_OWNER_DHCP:
                    self.rpc.unplug_dhcp_port_from_network(device_id,
                                                           hostname,
                                                           port_id,
                                                           network_id,
                                                           tenant_id)
                else:
                    self.rpc.unplug_host_from_network(device_id,
                                                      hostname,
                                                      port_id,
                                                      network_id,
                                                      tenant_id)
            # if necessary, delete tenant as well.
            self.delete_tenant(tenant_id)
        except arista_exc.AristaRpcError:
            LOG.info(EOS_UNREACHABLE_MSG)
            raise ml2_exc.MechanismDriverError()

    def delete_tenant(self, tenant_id):
        """delete a tenant from DB.

        A tenant is deleted only if there is no network or VM configured
        configured for this tenant.
        """
        objects_for_tenant = (db_lib.num_nets_provisioned(tenant_id) +
                              db_lib.num_vms_provisioned(tenant_id))
        if not objects_for_tenant:
            db_lib.forget_tenant(tenant_id)
            try:
                self.rpc.delete_tenant(tenant_id)
            except arista_exc.AristaRpcError:
                LOG.info(EOS_UNREACHABLE_MSG)
                raise ml2_exc.MechanismDriverError()

    def _host_name(self, hostname):
        fqdns_used = cfg.CONF.ml2_arista['use_fqdn']
        return hostname if fqdns_used else hostname.split('.')[0]

    def _synchronization_thread(self):
        with self.eos_sync_lock:
            self.eos.do_synchronize()

        self.timer = threading.Timer(self.sync_timeout,
                                     self._synchronization_thread)
        self.timer.start()

    def stop_synchronization_thread(self):
        if self.timer:
            self.timer.cancel()
            self.timer = None

    def _cleanup_db(self):
        """Clean up any uncessary entries in our DB."""
        db_tenants = db_lib.get_tenants()
        for tenant in db_tenants:
            neutron_nets = self.ndb.get_all_networks_for_tenant(tenant)
            neutron_nets_id = []
            for net in neutron_nets:
                neutron_nets_id.append(net['id'])
            db_nets = db_lib.get_networks(tenant)
            for net_id in db_nets.keys():
                if net_id not in neutron_nets_id:
                    db_lib.forget_network(tenant, net_id)
<commit_msg>Fix tenant-id in Arista ML2 driver to support HA router<commit_after># Copyright (c) 2013 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import threading

from networking_arista.common import db_lib
from networking_arista.ml2 import arista_ml2
from oslo_config import cfg
from oslo_log import log as logging

from neutron.common import constants as n_const
from neutron.i18n import _LI
from neutron.plugins.ml2.common import exceptions as ml2_exc
from neutron.plugins.ml2 import driver_api
from neutron.plugins.ml2.drivers.arista import config  # noqa
from neutron.plugins.ml2.drivers.arista import db
from neutron.plugins.ml2.drivers.arista import exceptions as arista_exc

LOG = logging.getLogger(__name__)

EOS_UNREACHABLE_MSG = _('Unable to reach EOS')

class AristaDriver(driver_api.MechanismDriver):
    """Ml2 Mechanism driver for Arista networking hardware.

    Remembers all networks and VMs that are provisioned on Arista Hardware.
    Does not send network provisioning request if the network has already been
    provisioned before for the given port.
    """
    def __init__(self, rpc=None):

        self.rpc = rpc or arista_ml2.AristaRPCWrapper()
        self.db_nets = db.AristaProvisionedNets()
        self.db_vms = db.AristaProvisionedVms()
        self.db_tenants = db.AristaProvisionedTenants()
        self.ndb = db_lib.NeutronNets()

        confg = cfg.CONF.ml2_arista
        self.segmentation_type = db_lib.VLAN_SEGMENTATION
        self.timer = None
        self.eos = arista_ml2.SyncService(self.rpc, self.ndb)
        self.sync_timeout = confg['sync_interval']
        self.eos_sync_lock = threading.Lock()

    def initialize(self):
        self.rpc.register_with_eos()
        self._cleanup_db()
        self.rpc.check_cli_commands()
        # Registering with EOS updates self.rpc.region_updated_time. Clear it
        # to force an initial sync
        self.rpc.clear_region_updated_time()
        self._synchronization_thread()

    def create_network_precommit(self, context):
        """Remember the tenant, and network information."""

        network = context.current
        segments = context.network_segments
        network_id = network['id']
        tenant_id = network['tenant_id']
        if not tenant_id:
            tenant_id = context._plugin_context.tenant_id
        segmentation_id = segments[0]['segmentation_id']
        with self.eos_sync_lock:
            db_lib.remember_tenant(tenant_id)
            db_lib.remember_network(tenant_id,
                                network_id,
                                segmentation_id)

    def create_network_postcommit(self, context):
        """Provision the network on the Arista Hardware."""

        network = context.current
        network_id = network['id']
        network_name = network['name']
        tenant_id = network['tenant_id']
        if not tenant_id:
            tenant_id = context._plugin_context.tenant_id
        segments = context.network_segments
        vlan_id = segments[0]['segmentation_id']
        shared_net = network['shared']
        with self.eos_sync_lock:
            if db_lib.is_network_provisioned(tenant_id, network_id):
                try:
                    network_dict = {
                        'network_id': network_id,
                        'segmentation_id': vlan_id,
                        'network_name': network_name,
                        'shared': shared_net}
                    self.rpc.create_network(tenant_id, network_dict)
                except arista_exc.AristaRpcError:
                    LOG.info(EOS_UNREACHABLE_MSG)
                    raise ml2_exc.MechanismDriverError()
            else:
                LOG.info(_LI('Network %s is not created as it is not found in '
                             'Arista DB'), network_id)

    def update_network_precommit(self, context):
        """At the moment we only support network name change

        Any other change in network is not supported at this time.
        We do not store the network names, therefore, no DB store
        action is performed here.
        """
        new_network = context.current
        orig_network = context.original
        if new_network['name'] != orig_network['name']:
            LOG.info(_LI('Network name changed to %s'), new_network['name'])

    def update_network_postcommit(self, context):
        """At the moment we only support network name change

        If network name is changed, a new network create request is
        sent to the Arista Hardware.
        """
        new_network = context.current
        orig_network = context.original
        if ((new_network['name'] != orig_network['name']) or
           (new_network['shared'] != orig_network['shared'])):
            network_id = new_network['id']
            network_name = new_network['name']
            tenant_id = new_network['tenant_id']
            if not tenant_id:
                tenant_id = context._plugin_context.tenant_id
            vlan_id = new_network['provider:segmentation_id']
            shared_net = new_network['shared']
            with self.eos_sync_lock:
                if db_lib.is_network_provisioned(tenant_id, network_id):
                    try:
                        network_dict = {
                            'network_id': network_id,
                            'segmentation_id': vlan_id,
                            'network_name': network_name,
                            'shared': shared_net}
                        self.rpc.create_network(tenant_id, network_dict)
                    except arista_exc.AristaRpcError:
                        LOG.info(EOS_UNREACHABLE_MSG)
                        raise ml2_exc.MechanismDriverError()
                else:
                    LOG.info(_LI('Network %s is not updated as it is not found'
                                 ' in Arista DB'), network_id)

    def delete_network_precommit(self, context):
        """Delete the network infromation from the DB."""
        network = context.current
        network_id = network['id']
        tenant_id = network['tenant_id']
        with self.eos_sync_lock:
            if db_lib.is_network_provisioned(tenant_id, network_id):
                db_lib.forget_network(tenant_id, network_id)

    def delete_network_postcommit(self, context):
        """Send network delete request to Arista HW."""
        network = context.current
        network_id = network['id']
        tenant_id = network['tenant_id']
        if not tenant_id:
            tenant_id = context._plugin_context.tenant_id
        with self.eos_sync_lock:

            # Succeed deleting network in case EOS is not accessible.
            # EOS state will be updated by sync thread once EOS gets
            # alive.
            try:
                self.rpc.delete_network(tenant_id, network_id)
                # if necessary, delete tenant as well.
                self.delete_tenant(tenant_id)
            except arista_exc.AristaRpcError:
                LOG.info(EOS_UNREACHABLE_MSG)
                raise ml2_exc.MechanismDriverError()

    def create_port_precommit(self, context):
        """Remember the infromation about a VM and its ports

        A VM information, along with the physical host information
        is saved.
        """
        port = context.current
        device_id = port['device_id']
        device_owner = port['device_owner']
        host = context.host

        # device_id and device_owner are set on VM boot
        is_vm_boot = device_id and device_owner
        if host and is_vm_boot:
            port_id = port['id']
            network_id = port['network_id']
            tenant_id = port['tenant_id']
            if not tenant_id:
                tenant_id = context._plugin_context.tenant_id
            with self.eos_sync_lock:
                db_lib.remember_tenant(tenant_id)
                db_lib.remember_vm(device_id, host, port_id,
                               network_id, tenant_id)

    def create_port_postcommit(self, context):
        """Plug a physical host into a network.

        Send provisioning request to Arista Hardware to plug a host
        into appropriate network.
        """
        port = context.current
        device_id = port['device_id']
        device_owner = port['device_owner']
        host = context.host

        # device_id and device_owner are set on VM boot
        is_vm_boot = device_id and device_owner
        if host and is_vm_boot:
            port_id = port['id']
            port_name = port['name']
            network_id = port['network_id']
            tenant_id = port['tenant_id']
            if not tenant_id:
                tenant_id = context._plugin_context.tenant_id
            with self.eos_sync_lock:
                hostname = self._host_name(host)
                vm_provisioned = db_lib.is_vm_provisioned(device_id,
                                                      host,
                                                      port_id,
                                                      network_id,
                                                      tenant_id)
                # If network does not exist under this tenant,
                # it may be a shared network. Get shared network owner Id
                net_provisioned = (
                    db_lib.is_network_provisioned(tenant_id, network_id) or
                    self.ndb.get_shared_network_owner_id(network_id)
                )
                if vm_provisioned and net_provisioned:
                    try:
                        self.rpc.plug_port_into_network(device_id,
                                                        hostname,
                                                        port_id,
                                                        network_id,
                                                        tenant_id,
                                                        port_name,
                                                        device_owner)
                    except arista_exc.AristaRpcError:
                        LOG.info(EOS_UNREACHABLE_MSG)
                        raise ml2_exc.MechanismDriverError()
                else:
                    LOG.info(_LI('VM %s is not created as it is not found in '
                                 'Arista DB'), device_id)

    def update_port_precommit(self, context):
        """Update the name of a given port.

        At the moment we only support port name change.
        Any other change to port is not supported at this time.
        We do not store the port names, therefore, no DB store
        action is performed here.
        """
        new_port = context.current
        orig_port = context.original
        if new_port['name'] != orig_port['name']:
            LOG.info(_LI('Port name changed to %s'), new_port['name'])

    def update_port_postcommit(self, context):
        """Update the name of a given port in EOS.

        At the moment we only support port name change
        Any other change to port is not supported at this time.
        """
        port = context.current
        orig_port = context.original
        if port['name'] == orig_port['name']:
            # nothing to do
            return

        device_id = port['device_id']
        device_owner = port['device_owner']
        host = context.host
        is_vm_boot = device_id and device_owner

        if host and is_vm_boot:
            port_id = port['id']
            port_name = port['name']
            network_id = port['network_id']
            tenant_id = port['tenant_id']
            if not tenant_id:
                tenant_id = context._plugin_context.tenant_id
            with self.eos_sync_lock:
                hostname = self._host_name(host)
                segmentation_id = db_lib.get_segmentation_id(tenant_id,
                                                         network_id)
                vm_provisioned = db_lib.is_vm_provisioned(device_id,
                                                      host,
                                                      port_id,
                                                      network_id,
                                                      tenant_id)
                # If network does not exist under this tenant,
                # it may be a shared network. Get shared network owner Id
                net_provisioned = (
                    db_lib.is_network_provisioned(tenant_id, network_id,
                                                  segmentation_id) or
                    self.ndb.get_shared_network_owner_id(network_id)
                )
                if vm_provisioned and net_provisioned:
                    try:
                        self.rpc.plug_port_into_network(device_id,
                                                        hostname,
                                                        port_id,
                                                        network_id,
                                                        tenant_id,
                                                        port_name,
                                                        device_owner)
                    except arista_exc.AristaRpcError:
                        LOG.info(EOS_UNREACHABLE_MSG)
                        raise ml2_exc.MechanismDriverError()
                else:
                    LOG.info(_LI('VM %s is not updated as it is not found in '
                                 'Arista DB'), device_id)

    def delete_port_precommit(self, context):
        """Delete information about a VM and host from the DB."""
        port = context.current

        host_id = context.host
        device_id = port['device_id']
        tenant_id = port['tenant_id']
        if not tenant_id:
            tenant_id = context._plugin_context.tenant_id
        network_id = port['network_id']
        port_id = port['id']
        with self.eos_sync_lock:
            if db_lib.is_vm_provisioned(device_id, host_id, port_id,
                                        network_id, tenant_id):
                db_lib.forget_vm(device_id, host_id, port_id,
                             network_id, tenant_id)

    def delete_port_postcommit(self, context):
        """unPlug a physical host from a network.

        Send provisioning request to Arista Hardware to unplug a host
        from appropriate network.
        """
        port = context.current
        device_id = port['device_id']
        host = context.host
        port_id = port['id']
        network_id = port['network_id']
        tenant_id = port['tenant_id']
        if not tenant_id:
            tenant_id = context._plugin_context.tenant_id
        device_owner = port['device_owner']

        try:
            with self.eos_sync_lock:
                hostname = self._host_name(host)
                if device_owner == n_const.DEVICE_OWNER_DHCP:
                    self.rpc.unplug_dhcp_port_from_network(device_id,
                                                           hostname,
                                                           port_id,
                                                           network_id,
                                                           tenant_id)
                else:
                    self.rpc.unplug_host_from_network(device_id,
                                                      hostname,
                                                      port_id,
                                                      network_id,
                                                      tenant_id)
            # if necessary, delete tenant as well.
            self.delete_tenant(tenant_id)
        except arista_exc.AristaRpcError:
            LOG.info(EOS_UNREACHABLE_MSG)
            raise ml2_exc.MechanismDriverError()

    def delete_tenant(self, tenant_id):
        """delete a tenant from DB.

        A tenant is deleted only if there is no network or VM configured
        configured for this tenant.
        """
        objects_for_tenant = (db_lib.num_nets_provisioned(tenant_id) +
                              db_lib.num_vms_provisioned(tenant_id))
        if not objects_for_tenant:
            db_lib.forget_tenant(tenant_id)
            try:
                self.rpc.delete_tenant(tenant_id)
            except arista_exc.AristaRpcError:
                LOG.info(EOS_UNREACHABLE_MSG)
                raise ml2_exc.MechanismDriverError()

    def _host_name(self, hostname):
        fqdns_used = cfg.CONF.ml2_arista['use_fqdn']
        return hostname if fqdns_used else hostname.split('.')[0]

    def _synchronization_thread(self):
        with self.eos_sync_lock:
            self.eos.do_synchronize()

        self.timer = threading.Timer(self.sync_timeout,
                                     self._synchronization_thread)
        self.timer.start()

    def stop_synchronization_thread(self):
        if self.timer:
            self.timer.cancel()
            self.timer = None

    def _cleanup_db(self):
        """Clean up any uncessary entries in our DB."""
        db_tenants = db_lib.get_tenants()
        for tenant in db_tenants:
            neutron_nets = self.ndb.get_all_networks_for_tenant(tenant)
            neutron_nets_id = []
            for net in neutron_nets:
                neutron_nets_id.append(net['id'])
            db_nets = db_lib.get_networks(tenant)
            for net_id in db_nets.keys():
                if net_id not in neutron_nets_id:
                    db_lib.forget_network(tenant, net_id)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from anormbookmarker.test.test_enviroment import *
from sqlalchemy.exc import IntegrityError

ed = Word.construct(session=SESSION, word='Eucalyptus deglupta')
try:
    SESSION.commit()
except IntegrityError:
    print("Correctly raises IntegrityError")

db_result = [('select COUNT(*) from alias;', 0),
             ('select COUNT(*) from aliasword;', 0),
             ('select COUNT(*) from bookmark;', 0),
             ('select COUNT(*) from filename;', 0),
             ('select COUNT(*) from tag;', 0),
             ('select COUNT(*) from tag_relationship;', 0),
             ('select COUNT(*) from tagbookmarks;', 0),
             ('select COUNT(*) from tagword;', 0),
             ('select COUNT(*) from word;', 0),
             ('select COUNT(*) from wordmisspelling;', 0)]

check_db_result(config=CONFIG, db_result=db_result, session=SESSION)
<commit_msg>auto-commit<commit_after>#!/usr/bin/env python3

from anormbookmarker.test.test_enviroment import *
from sqlalchemy.exc import IntegrityError

try:
    ed = Word.construct(session=SESSION, word='Eucalyptus deglupta')
except IntegrityError:
    print("Correctly raises IntegrityError")
SESSION.commit()

db_result = [('select COUNT(*) from alias;', 0),
             ('select COUNT(*) from aliasword;', 0),
             ('select COUNT(*) from bookmark;', 0),
             ('select COUNT(*) from filename;', 0),
             ('select COUNT(*) from tag;', 0),
             ('select COUNT(*) from tag_relationship;', 0),
             ('select COUNT(*) from tagbookmarks;', 0),
             ('select COUNT(*) from tagword;', 0),
             ('select COUNT(*) from word;', 0),
             ('select COUNT(*) from wordmisspelling;', 0)]

check_db_result(config=CONFIG, db_result=db_result, session=SESSION)
<|endoftext|>


<commit_before>from Cython.TestUtils import TransformTest
from Cython.Compiler.ParseTreeTransforms import *
from Cython.Compiler.Nodes import *

class TestNormalizeTree(TransformTest):
    def test_parserbehaviour_is_what_we_coded_for(self):
        t = self.fragment(u"if x: y").root
        self.assertLines(u"""
(root): StatListNode
  stats[0]: IfStatNode
    if_clauses[0]: IfClauseNode
      condition: NameNode
      body: ExprStatNode
        expr: NameNode
""", self.treetypes(t))

    def test_wrap_singlestat(self):
    	t = self.run_pipeline([NormalizeTree(None)], u"if x: y")
        self.assertLines(u"""
(root): StatListNode
  stats[0]: IfStatNode
    if_clauses[0]: IfClauseNode
      condition: NameNode
      body: StatListNode
        stats[0]: ExprStatNode
          expr: NameNode
""", self.treetypes(t))

    def test_wrap_multistat(self):
        t = self.run_pipeline([NormalizeTree(None)], u"""
            if z:
                x
                y
        """)
        self.assertLines(u"""
(root): StatListNode
  stats[0]: IfStatNode
    if_clauses[0]: IfClauseNode
      condition: NameNode
      body: StatListNode
        stats[0]: ExprStatNode
          expr: NameNode
        stats[1]: ExprStatNode
          expr: NameNode
""", self.treetypes(t))

    def test_statinexpr(self):
        t = self.run_pipeline([NormalizeTree(None)], u"""
            a, b = x, y
        """)
        self.assertLines(u"""
(root): StatListNode
  stats[0]: ParallelAssignmentNode
    stats[0]: SingleAssignmentNode
      lhs: NameNode
      rhs: NameNode
    stats[1]: SingleAssignmentNode
      lhs: NameNode
      rhs: NameNode
""", self.treetypes(t))

    def test_wrap_offagain(self):
        t = self.run_pipeline([NormalizeTree(None)], u"""
            x
            y
            if z:
                x
        """)
        self.assertLines(u"""
(root): StatListNode
  stats[0]: ExprStatNode
    expr: NameNode
  stats[1]: ExprStatNode
    expr: NameNode
  stats[2]: IfStatNode
    if_clauses[0]: IfClauseNode
      condition: NameNode
      body: StatListNode
        stats[0]: ExprStatNode
          expr: NameNode
""", self.treetypes(t))

    def test_pass_eliminated(self):
        t = self.run_pipeline([NormalizeTree(None)], u"pass")
        self.assert_(len(t.stats) == 0)

class TestWithTransform(TransformTest):

    def test_simplified(self):
        t = self.run_pipeline([WithTransform(None)], u"""
        with x:
            y = z ** 3
        """)

        self.assertCode(u"""

        $1_0 = x
        $1_2 = $1_0.__exit__
        $1_0.__enter__()
        $1_1 = True
        $0_0 = None
        try:
            try:
                y = z ** 3
            except:
                $1_1 = False
                if (not $1_2($0_0)):
                    raise
        finally:
            if $1_1:
                $1_2(None, None, None)

        """, t)

    def test_basic(self):
        t = self.run_pipeline([WithTransform(None)], u"""
        with x as y:
            y = z ** 3
        """)
        self.assertCode(u"""

        $1_0 = x
        $1_2 = $1_0.__exit__
        $1_3 = $1_0.__enter__()
        $1_1 = True
        $0_0 = None
        try:
            try:
                y = $1_3
                y = z ** 3
            except:
                $1_1 = False
                if (not $1_2($0_0)):
                    raise
        finally:
            if $1_1:
                $1_2(None, None, None)

        """, t)

if __name__ == "__main__":
    import unittest
    unittest.main()
<commit_msg>test fixes after latest with transform changes<commit_after>from Cython.TestUtils import TransformTest
from Cython.Compiler.ParseTreeTransforms import *
from Cython.Compiler.Nodes import *

class TestNormalizeTree(TransformTest):
    def test_parserbehaviour_is_what_we_coded_for(self):
        t = self.fragment(u"if x: y").root
        self.assertLines(u"""
(root): StatListNode
  stats[0]: IfStatNode
    if_clauses[0]: IfClauseNode
      condition: NameNode
      body: ExprStatNode
        expr: NameNode
""", self.treetypes(t))

    def test_wrap_singlestat(self):
    	t = self.run_pipeline([NormalizeTree(None)], u"if x: y")
        self.assertLines(u"""
(root): StatListNode
  stats[0]: IfStatNode
    if_clauses[0]: IfClauseNode
      condition: NameNode
      body: StatListNode
        stats[0]: ExprStatNode
          expr: NameNode
""", self.treetypes(t))

    def test_wrap_multistat(self):
        t = self.run_pipeline([NormalizeTree(None)], u"""
            if z:
                x
                y
        """)
        self.assertLines(u"""
(root): StatListNode
  stats[0]: IfStatNode
    if_clauses[0]: IfClauseNode
      condition: NameNode
      body: StatListNode
        stats[0]: ExprStatNode
          expr: NameNode
        stats[1]: ExprStatNode
          expr: NameNode
""", self.treetypes(t))

    def test_statinexpr(self):
        t = self.run_pipeline([NormalizeTree(None)], u"""
            a, b = x, y
        """)
        self.assertLines(u"""
(root): StatListNode
  stats[0]: ParallelAssignmentNode
    stats[0]: SingleAssignmentNode
      lhs: NameNode
      rhs: NameNode
    stats[1]: SingleAssignmentNode
      lhs: NameNode
      rhs: NameNode
""", self.treetypes(t))

    def test_wrap_offagain(self):
        t = self.run_pipeline([NormalizeTree(None)], u"""
            x
            y
            if z:
                x
        """)
        self.assertLines(u"""
(root): StatListNode
  stats[0]: ExprStatNode
    expr: NameNode
  stats[1]: ExprStatNode
    expr: NameNode
  stats[2]: IfStatNode
    if_clauses[0]: IfClauseNode
      condition: NameNode
      body: StatListNode
        stats[0]: ExprStatNode
          expr: NameNode
""", self.treetypes(t))

    def test_pass_eliminated(self):
        t = self.run_pipeline([NormalizeTree(None)], u"pass")
        self.assert_(len(t.stats) == 0)

class TestWithTransform(TransformTest):

    def test_simplified(self):
        t = self.run_pipeline([WithTransform(None)], u"""
        with x:
            y = z ** 3
        """)

        self.assertCode(u"""

        $0_0 = x
        $0_2 = $0_0.__exit__
        $0_0.__enter__()
        $0_1 = True
        try:
            try:
                $1_0 = None
                y = z ** 3
            except:
                $0_1 = False
                if (not $0_2($1_0)):
                    raise
        finally:
            if $0_1:
                $0_2(None, None, None)

        """, t)

    def test_basic(self):
        t = self.run_pipeline([WithTransform(None)], u"""
        with x as y:
            y = z ** 3
        """)
        self.assertCode(u"""

        $0_0 = x
        $0_2 = $0_0.__exit__
        $0_3 = $0_0.__enter__()
        $0_1 = True
        try:
            try:
                $1_0 = None
                y = $0_3
                y = z ** 3
            except:
                $0_1 = False
                if (not $0_2($1_0)):
                    raise
        finally:
            if $0_1:
                $0_2(None, None, None)

        """, t)

if __name__ == "__main__":
    import unittest
    unittest.main()
<|endoftext|>


<commit_before>import numpy as np
import theano
import theano.tensor as T
import nntools

# Sequence length
LENGTH = 10
# Number of units in the hidden (recurrent) layer
N_HIDDEN = 4
# Number of training sequences in each batch
N_BATCH = 30
# Delay used to generate artificial training data
DELAY = 2

def gen_data(length=LENGTH, n_batch=N_BATCH, delay=DELAY):
    '''
    Generate a simple lag sequence

    :parameters:
        - length : int
            Length of sequences to generate
        - n_batch : int
            Number of training sequences per batch
        - delay : int
            How much to delay one feature dimension in the target

    :returns:
        - X : np.ndarray, shape=(n_batch, length, 2)
            Input sequence
        - y : np.ndarray, shape=(n_batch, length, 1)
            Target sequence, where
            y[n] = X[:, n, 0] - X[:, n - delay, 1] + noise
    '''
    X = np.random.rand(n_batch, length, 2)
    y = X[:, :, 0].reshape((n_batch, length, 1))
    # Compute y[n] = X[:, n, 0] - X[:, n - delay, 1] + noise
    y[:, delay:, 0] -= (X[:, :-delay, 1]
                        + .01*np.random.randn(n_batch, length - delay))
    return X, y

# Generate a "validation" sequence whose cost we will periodically compute
X_val, y_val = gen_data()

# Construct vanilla RNN: One recurrent layer (with input weights) and one
# dense output layer
l_in = nntools.layers.InputLayer(shape=(N_BATCH, LENGTH, X_val.shape[-1]))

l_recurrent_in = nntools.layers.InputLayer(shape=(N_BATCH, X_val.shape[-1]))
l_input_to_hidden = nntools.layers.DenseLayer(l_recurrent_in, N_HIDDEN,
                                              nonlinearity=None)

l_recurrent_hid = nntools.layers.InputLayer(shape=(N_BATCH, N_HIDDEN))
l_hidden_to_hidden = nntools.layers.DenseLayer(l_recurrent_hid, N_HIDDEN,
                                               nonlinearity=None,
                                               b=nntools.init.Constant(1.))

l_recurrent = nntools.layers.RecurrentLayer(l_in,
                                            l_input_to_hidden,
                                            l_hidden_to_hidden,
                                            nonlinearity=None)
l_reshape = nntools.layers.ReshapeLayer(l_recurrent,
                                        (N_BATCH*LENGTH, N_HIDDEN))

l_recurrent_out = nntools.layers.DenseLayer(l_reshape,
                                            num_units=y_val.shape[-1],
                                            nonlinearity=None)
l_out = nntools.layers.ReshapeLayer(l_recurrent_out,
                                    (N_BATCH, LENGTH, y_val.shape[-1]))

# Cost function is mean squared error
input = T.tensor3('input')
target_output = T.tensor3('target_output')
# Cost = mean squared error, starting from delay point
cost = T.mean((l_out.get_output(input)[:, DELAY:, :]
               - target_output[:, DELAY:, :])**2)
# Use SGD for training
learning_rate = 1e-5
all_params = nntools.layers.get_all_params(l_out)
updates = nntools.updates.nesterov_momentum(cost, all_params, learning_rate)
# Theano functions for training, getting output, and computing cost
train = theano.function([input, target_output], cost, updates=updates)
y_pred = theano.function([input], l_out.get_output(input))
compute_cost = theano.function([input, target_output], cost)

# Train the net
n_iterations = 1000
costs = np.zeros(n_iterations)
for n in range(n_iterations):
    X, y = gen_data()
    costs[n] = train(X, y)
    if not n % 100:
        cost_val = compute_cost(X_val, y_val)
        print "Iteration {} validation cost = {}".format(n, cost_val)
<commit_msg>More cleanup/comments<commit_after>import numpy as np
import theano
import theano.tensor as T
import nntools

# Sequence length
LENGTH = 10
# Number of units in the hidden (recurrent) layer
N_HIDDEN = 4
# Number of training sequences in each batch
N_BATCH = 30
# Delay used to generate artificial training data
DELAY = 2
# SGD learning rate
LEARNING_RATE = 1e-5
# Number of iterations to train the net
N_ITERATIONS = 1000

def gen_data(length=LENGTH, n_batch=N_BATCH, delay=DELAY):
    '''
    Generate a simple lag sequence

    :parameters:
        - length : int
            Length of sequences to generate
        - n_batch : int
            Number of training sequences per batch
        - delay : int
            How much to delay one feature dimension in the target

    :returns:
        - X : np.ndarray, shape=(n_batch, length, 2)
            Input sequence
        - y : np.ndarray, shape=(n_batch, length, 1)
            Target sequence, where
            y[n] = X[:, n, 0] - X[:, n - delay, 1] + noise
    '''
    X = np.random.rand(n_batch, length, 2)
    y = X[:, :, 0].reshape((n_batch, length, 1))
    # Compute y[n] = X[:, n, 0] - X[:, n - delay, 1] + noise
    y[:, delay:, 0] -= (X[:, :-delay, 1]
                        + .01*np.random.randn(n_batch, length - delay))
    return X, y

# Generate a "validation" sequence whose cost we will periodically compute
X_val, y_val = gen_data()

# Construct vanilla RNN: One recurrent layer (with input weights) and one
# dense output layer
l_in = nntools.layers.InputLayer(shape=(N_BATCH, LENGTH, X_val.shape[-1]))

# This input layer is used to tell the input-to-hidden what shape to expect
# As we iterate over time steps, the input will be batch size x feature dim
l_recurrent_in = nntools.layers.InputLayer(shape=(N_BATCH, X_val.shape[-1]))
l_input_to_hidden = nntools.layers.DenseLayer(l_recurrent_in, N_HIDDEN,
                                              nonlinearity=None)

# As above, we need to tell the hidden-to-hidden layer what shape to expect
l_recurrent_hid = nntools.layers.InputLayer(shape=(N_BATCH, N_HIDDEN))
l_hidden_to_hidden = nntools.layers.DenseLayer(l_recurrent_hid, N_HIDDEN,
                                               nonlinearity=None,
                                               b=nntools.init.Constant(1.))

l_recurrent = nntools.layers.RecurrentLayer(l_in,
                                            l_input_to_hidden,
                                            l_hidden_to_hidden,
                                            nonlinearity=None)
l_reshape = nntools.layers.ReshapeLayer(l_recurrent,
                                        (N_BATCH*LENGTH, N_HIDDEN))

l_recurrent_out = nntools.layers.DenseLayer(l_reshape,
                                            num_units=y_val.shape[-1],
                                            nonlinearity=None)
l_out = nntools.layers.ReshapeLayer(l_recurrent_out,
                                    (N_BATCH, LENGTH, y_val.shape[-1]))

# Cost function is mean squared error
input = T.tensor3('input')
target_output = T.tensor3('target_output')
# Cost = mean squared error, starting from delay point
cost = T.mean((l_out.get_output(input)[:, DELAY:, :]
               - target_output[:, DELAY:, :])**2)
# Use NAG for training
all_params = nntools.layers.get_all_params(l_out)
updates = nntools.updates.nesterov_momentum(cost, all_params, LEARNING_RATE)
# Theano functions for training, getting output, and computing cost
train = theano.function([input, target_output], cost, updates=updates)
y_pred = theano.function([input], l_out.get_output(input))
compute_cost = theano.function([input, target_output], cost)

# Train the net
costs = np.zeros(N_ITERATIONS)
for n in range(N_ITERATIONS):
    X, y = gen_data()
    costs[n] = train(X, y)
    if not n % 100:
        cost_val = compute_cost(X_val, y_val)
        print "Iteration {} validation cost = {}".format(n, cost_val)
<|endoftext|>