<commit_before># Copyright (c) 2016 Cisco Systems, Inc.
# All Rights Reserved
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

# This is a simple Flask application that provides REST APIs by which
# compute and network services can communicate, plus a REST API for
# debugging using a CLI client.

# Note that it does *NOT* at this point have a persistent database, so
# restarting this process will make Gluon forget about every port it's
# learned, which will not do your system much good (the data is in the
# global 'backends' and 'ports' objects).  This is for simplicity of
# demonstration; we have a second codebase already defined that is
# written to OpenStack endpoint principles and includes its ORM, so
# that work was not repeated here where the aim was to get the APIs
# worked out.  The two codebases will merge in the future.

from flask import Flask
from flask_restful import Api
from flask_restful import reqparse
from flask_restful import Resource
import os
import sys
import vpp
from neutron.agent.linux import bridge_lib
from neutron.agent.linux import ip_lib
from neutron.common import constants as n_const
from networking_vpp import config_opts
from oslo_config import cfg
#from oslo_log import log as logging
import logging
import time

######################################################################

# This mirrors functionality in Neutron so that we're creating a name
# that Neutron can find for its agents.

DEV_NAME_PREFIX = n_const.TAP_DEVICE_PREFIX

def get_tap_name(uuid):
    return n_const.TAP_DEVICE_PREFIX + uuid[0:11]

# This mirrors functionality in Nova so that we're creating a vhostuser
# name that it will be able to locate

VHOSTUSER_DIR = '/tmp'

def get_vhostuser_name(uuid):
    return os.path.join(VHOSTUSER_DIR, uuid)

######################################################################

class VPPForwarder(object):

    def __init__(self,
                 flat_network_if=None,
                 vlan_trunk_if=None,
                 vxlan_src_addr=None,
                 vxlan_bcast_addr=None,
                 vxlan_vrf=None):
        self.vpp = vpp.VPPInterface()
        # This is the flat network interface for providing FLAT networking
        self.flat_if = flat_network_if
        # This is the trunk interface for VLAN networking
        self.trunk_if = vlan_trunk_if

        # This is the address we'll use if we plan on broadcasting
        # vxlan packets
        self.vxlan_bcast_addr = vxlan_bcast_addr
        self.vxlan_src_addr = vxlan_src_addr
        self.vxlan_vrf = vxlan_vrf

        # Used as a unique number for bridge IDs
        self.next_bridge_id = 5678

        self.networks = {}      # vlan: bridge index
        self.interfaces = {}    # uuid: if idx

        for (ifname, f) in self.vpp.get_interfaces():
            # Clean up interfaces from previous runs
            # TODO(ijw) can't easily SPOT VLAN subifs to delete
            if ifname.startswith('tap-'):
                # all VPP tap interfaces are of this form
                self.vpp.delete_tap(f.sw_if_index)
            elif ifname.startswith('VirtualEthernet'):
                # all VPP vhostuser interfaces are of this form
                self.vpp.delete_vhostuser(f.sw_if_index)
        trunk_ifstruct = self.vpp.get_interface(self.trunk_if) if self.trunk_if else None
        flat_ifstruct = self.vpp.get_interface(self.flat_if) if self.flat_if else None
        if trunk_ifstruct is None and flat_ifstruct is None:
		    raise Exception("Could not find a VPP uplink interface:%s" % self.trunk_if or self.flat_if)
        if trunk_ifstruct is not None:
            self.trunk_ifidx = trunk_ifstruct.sw_if_index
            # This interface is not automatically up just because
            # we've started and we need to ensure it is before
            # proceeding.

            # TODO(ijw): when we start up in a recovery mode we may
            # want to check the local VPP config and bring it up when
            # confirmed.
            self.vpp.ifup(self.trunk_ifidx)
        if flat_ifstruct is not None:
            self.flat_ifidx = flat_ifstruct.sw_if_index
            self.vpp.ifup(self.flat_ifidx)

    # This, here, is us creating a VLAN backed network
    def network_on_host(self, type, seg_id):
        if (type, seg_id) not in self.networks:
            # TODO(ijw): bridge domains have no distinguishing marks.
            # VPP needs to allow us to name or label them so that we
            # can find them when we restart
            if type == 'flat':
                if_upstream = self.flat_ifidx
                app.logger.debug('Adding upstream interface:%s to bridge for flat networking' % self.flat_if)
            elif type == 'vlan':
                # TODO(ijw): this VLAN subinterface may already exist, and
                # may even be in another bridge domain already (see
                # above).
                if_upstream = self.vpp.create_vlan_subif(self.trunk_ifidx,
                                                         seg_id)
                app.logger.debug('Adding upstream trunk interface:%s.%s \
                to bridge for vlan networking' % (self.trunk_if,seg_id))
            elif type == 'vxlan':
                if_upstream = \
                    self.vpp.create_srcrep_vxlan_subif(self, self.vxlan_vrf,
                                                       self.vxlan_src_addr,
                                                       self.vxlan_bcast_addr,
                                                       seg_id)
            else:
                raise Exception('type %s not supported', type)

            self.vpp.ifup(if_upstream)

            # May not remain this way but we use the VLAN ID as the
            # bridge ID; TODO(ijw): bridge ID can already exist, we
            # should check till we find a free one
            id = self.next_bridge_id
            self.next_bridge_id += 1

            self.vpp.create_bridge_domain(id)

            self.vpp.add_to_bridge(id, if_upstream)

            self.networks[(type, seg_id)] = id

        return self.networks[(type, seg_id)]

    ########################################
    # stolen from LB driver
    def _bridge_exists_and_ensure_up(self, bridge_name):
        """Check if the bridge exists and make sure it is up."""
        br = ip_lib.IPDevice(bridge_name)
        br.set_log_fail_as_error(False)
        try:
            # If the device doesn't exist this will throw a RuntimeError
            br.link.set_up()
        except RuntimeError:
            return False
        return True

    def ensure_bridge(self, bridge_name):
        """Create a bridge unless it already exists."""
        # _bridge_exists_and_ensure_up instead of device_exists is used here
        # because there are cases where the bridge exists but it's not UP,
        # for example:
        # 1) A greenthread was executing this function and had not yet executed
        # "ip link set bridge_name up" before eventlet switched to this
        # thread running the same function
        # 2) The Nova VIF driver was running concurrently and had just created
        #    the bridge, but had not yet put it UP
        if not self._bridge_exists_and_ensure_up(bridge_name):
            bridge_device = bridge_lib.BridgeDevice.addbr(bridge_name)
            if bridge_device.setfd(0):
                return
            if bridge_device.disable_stp():
                return
            if bridge_device.disable_ipv6():
                return
            if bridge_device.link.set_up():
                return
        else:
            bridge_device = bridge_lib.BridgeDevice(bridge_name)
        return bridge_device

        # TODO(ijw): should be checking this all succeeded

    # end theft
    ########################################
    def wait_for_tap(self, device_name):
        """Wait for the external tap device to be created by the DHCP agent"""
        wait_time = 180
        found = False
        while wait_time > 0:
            if ip_lib.device_exists(device_name):
                app.logger.debug('External tap device %s found!' % device_name)
                found = True
                break
            else:
                app.logger.debug('Waiting for external tap device %s to be created' % device_name)
                time.sleep(5)
                wait_time = wait_time - 5
        return found

    def create_interface_on_host(self, if_type, uuid, mac):
        if uuid in self.interfaces:
            app.logger.debug('port %s repeat binding request - ignored' % uuid)
        else:
            app.logger.debug('binding port %s as type %s' % (uuid, if_type))

            # TODO(ijw): naming not obviously consistent with
            # Neutron's naming
            name = uuid[0:11]
            bridge_name = 'br-' + name
            tap_name = 'tap' + name

            if if_type == 'maketap' or if_type == 'plugtap':
                if if_type == 'maketap':
                    iface = self.vpp.create_tap(tap_name, mac)
                    props = {'bind_type': 'maketap', 'name': tap_name}
                else:
                    int_tap_name = 'vpp' + name

                    props = {'bind_type': 'plugtap',
                             'bridge_name': bridge_name,
                             'ext_tap_name': tap_name,
                             'int_tap_name': int_tap_name
                    }
                    app.logger.debug('Creating tap interface %s with mac %s' % (int_tap_name, mac))

                    iface = self.vpp.create_tap(int_tap_name, mac)
                    # TODO(ijw): someone somewhere ought to be sorting
                    # the MTUs out
                    br = self.ensure_bridge(bridge_name)
                    # This is the external TAP device that will be created by an agent, say the DHCP agent
                    if self.wait_for_tap(tap_name):
                        app.logger.debug('Adding tap interface %s to bridge %s' % (tap_name, bridge_name))
                        br.addif(tap_name)
                    else:
                        app.logger.error('Could not find the external tap interface %s to bridge' % tap_name)
                    # This is the device that we just created with VPP
                    br.addif(int_tap_name)

            elif if_type == 'vhostuser':
                path = get_vhostuser_name(uuid)
                iface = self.vpp.create_vhostuser(path, mac)
                props = {'bind_type': 'vhostuser', 'path': uuid}
            else:
                raise Exception('unsupported interface type')

            self.interfaces[uuid] = (iface, props)
        return self.interfaces[uuid]

    def bind_interface_on_host(self, if_type, uuid, mac, net_type, seg_id):
        net_br_idx = self.network_on_host(net_type, seg_id)
        (iface, props) = self.create_interface_on_host(if_type, uuid, mac)

        self.vpp.ifup(iface)
        self.vpp.add_to_bridge(net_br_idx, iface)

        return props

    def unbind_interface_on_host(self, uuid):
        if uuid not in self.interfaces:
            app.logger.debug("unknown port %s unbinding request - ignored" % uuid)
        else:

            iface_idx, props = self.interfaces[uuid]

            app.logger.debug("unbinding port %s, recorded as type %s" % (uuid, props['bind_type']))

            # We no longer need this interface.  Specifically if it's
            # a vhostuser interface it's annoying to have it around
            # because the VM's memory (hugepages) will not be
            # released.  So, here, we destroy it.

            # TODO(ijw): I'm assuming deleting an interface in a
            # bridge domain works...
            if props['bind_type'] == 'vhostuser':
                self.vpp.delete_vhostuser(iface_idx)
            elif props['bind_type'] in ['maketap', 'plugtap']:
                self.vpp.delete_tap(iface_idx)
                if props['bind_type'] == 'plugtap':
                    bridge = bridge_lib.BridgeDevice(bridge_name)
                    if bridge_device.exists():
                        # These may fail, don't care much
                        try:
                            bridge.delif(props['int_tap_name'])
                            bridge.delif(props['ext_tap_name'])
                            bridge.delbr(props['bridge_name'])
                        except:
                            pass
            else:
                app.logger.error('Unknown port type %s during unbind' % props['bind_type'])

######################################################################

class PortBind(Resource):
    bind_args = reqparse.RequestParser()
    bind_args.add_argument('mac_address', type=str, required=True)
    bind_args.add_argument('mtu', type=str, required=True)
    bind_args.add_argument('segmentation_id', type=int, required=True)
    bind_args.add_argument('network_type', type=str, required=True)
    bind_args.add_argument('host', type=str, required=True)
    bind_args.add_argument('binding_type', type=str, required=True)

    def put(self, id):
        global vppf

        args = self.bind_args.parse_args()
        app.logger.debug('on host %s, binding %s %d to mac %s id %s as binding_type %s'
                         % (args['host'],
                            args['network_type'],
                            args['segmentation_id'],
                            args['mac_address'],
                            id,
                            args['binding_type'])
                         )
        if args['binding_type'] in 'vhostuser':
            app.logger.debug('Creating a vhostuser port:%s binding on host %s' % (id, args['host']))
            vppf.bind_interface_on_host('vhostuser',
                                    id,
                                    args['mac_address'],
                                    args['network_type'],
                                    args['segmentation_id'])
        elif args['binding_type'] in 'plugtap':
            app.logger.debug('Creating a plugtap port:%s binding on host %s' % (id, args['host']))
            vppf.bind_interface_on_host('plugtap',
                                    id,
                                    args['mac_address'],
                                    args['network_type'],
                                    args['segmentation_id'])
        else:
            app.logger.error('Unsupported binding type :%s requested' % args['binding_type'])

class PortUnbind(Resource):

    def __init(self, *args, **kwargs):
        super('PortBind', self).__init__(*args, **kwargs)

    def put(self, id):
        global vppf
        vppf.unbind_interface_on_host(id)

# Basic Flask RESTful app setup with logging
app = Flask('vpp-agent')
app.debug = True
ch = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(message)s')
ch.setFormatter(formatter)
app.logger.addHandler(ch)
app.logger.debug('Debug logging enabled')

def main():
    cfg.CONF(sys.argv[1:])
    global vppf
    vppf = VPPForwarder(flat_network_if=cfg.CONF.ml2_vpp.flat_network_if,
                        vlan_trunk_if=cfg.CONF.ml2_vpp.vlan_trunk_if,
                        vxlan_src_addr=cfg.CONF.ml2_vpp.vxlan_src_addr,
                        vxlan_bcast_addr=cfg.CONF.ml2_vpp.vxlan_bcast_addr,
                        vxlan_vrf=cfg.CONF.ml2_vpp.vxlan_vrf)
    api = Api(app)
    api.add_resource(PortBind, '/ports/<id>/bind')
    api.add_resource(PortUnbind, '/ports/<id>/unbind')
    app.logger.debug("Starting VPP agent on host address: 0.0.0.0 and port 2704")
    app.run(host='0.0.0.0',port=2704)

if __name__ == '__main__':
    main()
<commit_msg>dhcp fixes<commit_after># Copyright (c) 2016 Cisco Systems, Inc.
# All Rights Reserved
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

# This is a simple Flask application that provides REST APIs by which
# compute and network services can communicate, plus a REST API for
# debugging using a CLI client.

# Note that it does *NOT* at this point have a persistent database, so
# restarting this process will make Gluon forget about every port it's
# learned, which will not do your system much good (the data is in the
# global 'backends' and 'ports' objects).  This is for simplicity of
# demonstration; we have a second codebase already defined that is
# written to OpenStack endpoint principles and includes its ORM, so
# that work was not repeated here where the aim was to get the APIs
# worked out.  The two codebases will merge in the future.

from flask import Flask
from flask_restful import Api
from flask_restful import reqparse
from flask_restful import Resource
import os
import sys
import vpp
from neutron.agent.linux import bridge_lib
from neutron.agent.linux import ip_lib
from neutron.common import constants as n_const
from networking_vpp import config_opts
from oslo_config import cfg
#from oslo_log import log as logging
import logging
import time

######################################################################

# This mirrors functionality in Neutron so that we're creating a name
# that Neutron can find for its agents.

DEV_NAME_PREFIX = n_const.TAP_DEVICE_PREFIX

def get_tap_name(uuid):
    return n_const.TAP_DEVICE_PREFIX + uuid[0:11]

# This mirrors functionality in Nova so that we're creating a vhostuser
# name that it will be able to locate

VHOSTUSER_DIR = '/tmp'

def get_vhostuser_name(uuid):
    return os.path.join(VHOSTUSER_DIR, uuid)

######################################################################

class VPPForwarder(object):

    def __init__(self,
                 flat_network_if=None,
                 vlan_trunk_if=None,
                 vxlan_src_addr=None,
                 vxlan_bcast_addr=None,
                 vxlan_vrf=None):
        self.vpp = vpp.VPPInterface()
        # This is the flat network interface for providing FLAT networking
        self.flat_if = flat_network_if
        # This is the trunk interface for VLAN networking
        self.trunk_if = vlan_trunk_if

        # This is the address we'll use if we plan on broadcasting
        # vxlan packets
        self.vxlan_bcast_addr = vxlan_bcast_addr
        self.vxlan_src_addr = vxlan_src_addr
        self.vxlan_vrf = vxlan_vrf

        # Used as a unique number for bridge IDs
        self.next_bridge_id = 5678

        self.networks = {}      # vlan: bridge index
        self.interfaces = {}    # uuid: if idx

        for (ifname, f) in self.vpp.get_interfaces():
            # Clean up interfaces from previous runs
            # TODO(ijw) can't easily SPOT VLAN subifs to delete
            if ifname.startswith('tap-'):
                # all VPP tap interfaces are of this form
                self.vpp.delete_tap(f.sw_if_index)
            elif ifname.startswith('VirtualEthernet'):
                # all VPP vhostuser interfaces are of this form
                self.vpp.delete_vhostuser(f.sw_if_index)
        trunk_ifstruct = self.vpp.get_interface(self.trunk_if) if self.trunk_if else None
        flat_ifstruct = self.vpp.get_interface(self.flat_if) if self.flat_if else None
        if trunk_ifstruct is None and flat_ifstruct is None:
		    raise Exception("Could not find a VPP uplink interface:%s" % self.trunk_if or self.flat_if)
        if trunk_ifstruct is not None:
            self.trunk_ifidx = trunk_ifstruct.sw_if_index
            # This interface is not automatically up just because
            # we've started and we need to ensure it is before
            # proceeding.

            # TODO(ijw): when we start up in a recovery mode we may
            # want to check the local VPP config and bring it up when
            # confirmed.
            self.vpp.ifup(self.trunk_ifidx)
        if flat_ifstruct is not None:
            self.flat_ifidx = flat_ifstruct.sw_if_index
            self.vpp.ifup(self.flat_ifidx)

    # This, here, is us creating a VLAN backed network
    def network_on_host(self, type, seg_id):
        if (type, seg_id) not in self.networks:
            # TODO(ijw): bridge domains have no distinguishing marks.
            # VPP needs to allow us to name or label them so that we
            # can find them when we restart
            if type == 'flat':
                if_upstream = self.flat_ifidx
                app.logger.debug('Adding upstream interface:%s to bridge for flat networking' % self.flat_if)
            elif type == 'vlan':
                # TODO(ijw): this VLAN subinterface may already exist, and
                # may even be in another bridge domain already (see
                # above).
                if_upstream = self.vpp.create_vlan_subif(self.trunk_ifidx,
                                                         seg_id)
                app.logger.debug('Adding upstream trunk interface:%s.%s \
                to bridge for vlan networking' % (self.trunk_if,seg_id))
            elif type == 'vxlan':
                if_upstream = \
                    self.vpp.create_srcrep_vxlan_subif(self, self.vxlan_vrf,
                                                       self.vxlan_src_addr,
                                                       self.vxlan_bcast_addr,
                                                       seg_id)
            else:
                raise Exception('type %s not supported', type)

            self.vpp.ifup(if_upstream)

            # May not remain this way but we use the VLAN ID as the
            # bridge ID; TODO(ijw): bridge ID can already exist, we
            # should check till we find a free one
            id = self.next_bridge_id
            self.next_bridge_id += 1

            self.vpp.create_bridge_domain(id)

            self.vpp.add_to_bridge(id, if_upstream)

            self.networks[(type, seg_id)] = id

        return self.networks[(type, seg_id)]

    ########################################
    # stolen from LB driver
    def _bridge_exists_and_ensure_up(self, bridge_name):
        """Check if the bridge exists and make sure it is up."""
        br = ip_lib.IPDevice(bridge_name)
        br.set_log_fail_as_error(False)
        try:
            # If the device doesn't exist this will throw a RuntimeError
            br.link.set_up()
        except RuntimeError:
            return False
        return True

    def ensure_bridge(self, bridge_name):
        """Create a bridge unless it already exists."""
        # _bridge_exists_and_ensure_up instead of device_exists is used here
        # because there are cases where the bridge exists but it's not UP,
        # for example:
        # 1) A greenthread was executing this function and had not yet executed
        # "ip link set bridge_name up" before eventlet switched to this
        # thread running the same function
        # 2) The Nova VIF driver was running concurrently and had just created
        #    the bridge, but had not yet put it UP
        if not self._bridge_exists_and_ensure_up(bridge_name):
            bridge_device = bridge_lib.BridgeDevice.addbr(bridge_name)
            if bridge_device.setfd(0):
                return
            if bridge_device.disable_stp():
                return
            if bridge_device.disable_ipv6():
                return
            if bridge_device.link.set_up():
                return
        else:
            bridge_device = bridge_lib.BridgeDevice(bridge_name)
        return bridge_device

        # TODO(ijw): should be checking this all succeeded

    # end theft
    ########################################

    ##TODO(njoy): make wait_time configurable
    def wait_for_tap(self, device_name):
        """Wait for the external tap device to be created by the DHCP agent"""
        wait_time = 180
        found = False
        while wait_time > 0:
            if ip_lib.device_exists(device_name):
                app.logger.debug('External tap device %s found!' % device_name)
                found = True
                break
            else:
                app.logger.debug('Waiting for external tap device %s to be created' % device_name)
                time.sleep(5)
                wait_time = wait_time - 5
        return found

    def create_interface_on_host(self, if_type, uuid, mac):
        if uuid in self.interfaces:
            app.logger.debug('port %s repeat binding request - ignored' % uuid)
        else:
            app.logger.debug('binding port %s as type %s' % (uuid, if_type))

            # TODO(ijw): naming not obviously consistent with
            # Neutron's naming
            name = uuid[0:11]
            bridge_name = 'br-' + name
            tap_name = 'tap' + name

            if if_type == 'maketap' or if_type == 'plugtap':
                if if_type == 'maketap':
                    iface = self.vpp.create_tap(tap_name, mac)
                    props = {'bind_type': 'maketap', 'name': tap_name}
                else:
                    int_tap_name = 'vpp' + name

                    props = {'bind_type': 'plugtap',
                             'bridge_name': bridge_name,
                             'ext_tap_name': tap_name,
                             'int_tap_name': int_tap_name
                    }
                    app.logger.debug('Creating tap interface %s with mac %s' % (int_tap_name, mac))

                    iface = self.vpp.create_tap(int_tap_name, mac)
                    # TODO(ijw): someone somewhere ought to be sorting
                    # the MTUs out
                    br = self.ensure_bridge(bridge_name)
                    # This is the external TAP device that will be created by an agent, say the DHCP agent
                    if self.wait_for_tap(tap_name):
                        app.logger.debug('Adding tap interface %s to bridge %s' % (tap_name, bridge_name))
                        br.addif(tap_name)
                    else:
                        app.logger.error('Could not find the external tap interface %s to bridge' % tap_name)
                    # This is the device that we just created with VPP
                    br.addif(int_tap_name)

            elif if_type == 'vhostuser':
                path = get_vhostuser_name(uuid)
                iface = self.vpp.create_vhostuser(path, mac)
                props = {'bind_type': 'vhostuser', 'path': uuid}
            else:
                raise Exception('unsupported interface type')

            self.interfaces[uuid] = (iface, props)
        return self.interfaces[uuid]

    def bind_interface_on_host(self, if_type, uuid, mac, net_type, seg_id):
        net_br_idx = self.network_on_host(net_type, seg_id)
        (iface, props) = self.create_interface_on_host(if_type, uuid, mac)

        self.vpp.ifup(iface)
        self.vpp.add_to_bridge(net_br_idx, iface)

        return props

    def unbind_interface_on_host(self, uuid):
        if uuid not in self.interfaces:
            app.logger.debug("unknown port %s unbinding request - ignored" % uuid)
        else:

            iface_idx, props = self.interfaces[uuid]

            app.logger.debug("unbinding port %s, recorded as type %s" % (uuid, props['bind_type']))

            # We no longer need this interface.  Specifically if it's
            # a vhostuser interface it's annoying to have it around
            # because the VM's memory (hugepages) will not be
            # released.  So, here, we destroy it.

            # TODO(ijw): I'm assuming deleting an interface in a
            # bridge domain works...
            if props['bind_type'] == 'vhostuser':
                self.vpp.delete_vhostuser(iface_idx)
            elif props['bind_type'] in ['maketap', 'plugtap']:
                self.vpp.delete_tap(iface_idx)
                if props['bind_type'] == 'plugtap':
                    bridge = bridge_lib.BridgeDevice(bridge_name)
                    if bridge_device.exists():
                        # These may fail, don't care much
                        try:
                            bridge.delif(props['int_tap_name'])
                            bridge.delif(props['ext_tap_name'])
                            bridge.delbr(props['bridge_name'])
                        except:
                            pass
            else:
                app.logger.error('Unknown port type %s during unbind' % props['bind_type'])

######################################################################

class PortBind(Resource):
    bind_args = reqparse.RequestParser()
    bind_args.add_argument('mac_address', type=str, required=True)
    bind_args.add_argument('mtu', type=str, required=True)
    bind_args.add_argument('segmentation_id', type=int, required=True)
    bind_args.add_argument('network_type', type=str, required=True)
    bind_args.add_argument('host', type=str, required=True)
    bind_args.add_argument('binding_type', type=str, required=True)

    def put(self, id):
        global vppf

        args = self.bind_args.parse_args()
        app.logger.debug('on host %s, binding %s %d to mac %s id %s as binding_type %s'
                         % (args['host'],
                            args['network_type'],
                            args['segmentation_id'],
                            args['mac_address'],
                            id,
                            args['binding_type'])
                         )
        if args['binding_type'] in 'vhostuser':
            app.logger.debug('Creating a vhostuser port:%s binding on host %s' % (id, args['host']))
            vppf.bind_interface_on_host('vhostuser',
                                    id,
                                    args['mac_address'],
                                    args['network_type'],
                                    args['segmentation_id'])
        elif args['binding_type'] in 'plugtap':
            app.logger.debug('Creating a plugtap port:%s binding on host %s' % (id, args['host']))
            vppf.bind_interface_on_host('plugtap',
                                    id,
                                    args['mac_address'],
                                    args['network_type'],
                                    args['segmentation_id'])
        else:
            app.logger.error('Unsupported binding type :%s requested' % args['binding_type'])

class PortUnbind(Resource):

    def __init(self, *args, **kwargs):
        super('PortBind', self).__init__(*args, **kwargs)

    def put(self, id):
        global vppf
        vppf.unbind_interface_on_host(id)

# Basic Flask RESTful app setup with logging
app = Flask('vpp-agent')
app.debug = True
ch = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(message)s')
ch.setFormatter(formatter)
app.logger.addHandler(ch)
app.logger.debug('Debug logging enabled')

def main():
    cfg.CONF(sys.argv[1:])
    global vppf
    vppf = VPPForwarder(flat_network_if=cfg.CONF.ml2_vpp.flat_network_if,
                        vlan_trunk_if=cfg.CONF.ml2_vpp.vlan_trunk_if,
                        vxlan_src_addr=cfg.CONF.ml2_vpp.vxlan_src_addr,
                        vxlan_bcast_addr=cfg.CONF.ml2_vpp.vxlan_bcast_addr,
                        vxlan_vrf=cfg.CONF.ml2_vpp.vxlan_vrf)
    api = Api(app)
    api.add_resource(PortBind, '/ports/<id>/bind')
    api.add_resource(PortUnbind, '/ports/<id>/unbind')
    app.logger.debug("Starting VPP agent on host address: 0.0.0.0 and port 2704")
    app.run(host='0.0.0.0',port=2704)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
**********
Edge Lists
**********
Read and write NetworkX graphs as edge lists.

The multi-line adjacency list format is useful for graphs with nodes
that can be meaningfully represented as strings.  With the edgelist
format simple edge data can be stored but node or graph data is not.
There is no way of representing isolated nodes unless the node has a
self-loop edge.

Format
------
You can read or write three formats of edge lists with these functions.

Node pairs with no data::

 1 2

Python dictionary as data::

 1 2 {'weight':7, 'color':'green'}

Arbitrary data::

 1 2 7 green
"""
__author__ = """Aric Hagberg (hagberg@lanl.gov)\nDan Schult (dschult@colgate.edu)"""
#    Copyright (C) 2004-2011 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.

__all__ = ['generate_edgelist',
           'write_edgelist',
           'parse_edgelist',
           'read_edgelist',
           'read_weighted_edgelist',
           'write_weighted_edgelist']

from networkx.utils import open_file, make_str
import networkx as nx

def generate_edgelist(G, delimiter=' ', data=True):
    """Generate a single line of the graph G in edge list format.

    Parameters
    ----------
    G : NetworkX graph

    delimiter : string, optional
       Separator for node labels

    data : bool or list of keys
       If False generate no edge data.  If True use a dictionary
       representation of edge data.  If a list of keys use a list of data
       values corresponding to the keys.

    Returns
    -------
    lines : string
        Lines of data in adjlist format.

    Examples
    --------
    >>> G = nx.lollipop_graph(4, 3)
    >>> G[1][2]['weight'] = 3
    >>> G[3][4]['capacity'] = 12
    >>> for line in nx.generate_edgelist(G, data=False):
    ...     print(line)
    0 1
    0 2
    0 3
    1 2
    1 3
    2 3
    3 4
    4 5
    5 6

    >>> for line in nx.generate_edgelist(G):
    ...     print(line)
    0 1 {}
    0 2 {}
    0 3 {}
    1 2 {'weight': 3}
    1 3 {}
    2 3 {}
    3 4 {'capacity': 12}
    4 5 {}
    5 6 {}

    >>> for line in nx.generate_edgelist(G,data=['weight']):
    ...     print(line)
    0 1
    0 2
    0 3
    1 2 3
    1 3
    2 3
    3 4
    4 5
    5 6

    See Also
    --------
    write_adjlist, read_adjlist
    """
    if data is True or data is False:
        for e in G.edges(data=data):
            yield delimiter.join(map(make_str,e))
    else:
        for u,v,d in G.edges(data=True):
            e=[u,v]
            try:
                e.extend(d[k] for k in data)
            except KeyError:
                pass # missing data for this edge, should warn?
            yield delimiter.join(map(make_str,e))

@open_file(1,mode='wb')
def write_edgelist(G, path, comments="#", delimiter=' ', data=True,
                   encoding = 'utf-8'):
    """Write graph as a list of edges.

    Parameters
    ----------
    G : graph
       A NetworkX graph
    path : file or string
       File or filename to write. If a file is provided, it must be
       opened in 'wb' mode. Filenames ending in .gz or .bz2 will be compressed.
    comments : string, optional
       The character used to indicate the start of a comment
    delimiter : string, optional
       The string used to separate values.  The default is whitespace.
    data : bool or list, optional
       If False write no edge data.
       If True write a string representation of the edge data dictionary..
       If a list (or other iterable) is provided, write the  keys specified
       in the list.
    encoding: string, optional
       Specify which encoding to use when writing file.

    Examples
    --------
    >>> G=nx.path_graph(4)
    >>> nx.write_edgelist(G, "test.edgelist")
    >>> G=nx.path_graph(4)
    >>> fh=open("test.edgelist",'wb')
    >>> nx.write_edgelist(G, fh)
    >>> nx.write_edgelist(G, "test.edgelist.gz")
    >>> nx.write_edgelist(G, "test.edgelist.gz", data=False)

    >>> G=nx.Graph()
    >>> G.add_edge(1,2,weight=7,color='red')
    >>> nx.write_edgelist(G,'test.edgelist',data=False)
    >>> nx.write_edgelist(G,'test.edgelist',data=['color'])
    >>> nx.write_edgelist(G,'test.edgelist',data=['color','weight'])

    See Also
    --------
    write_edgelist()
    write_weighted_edgelist()
    """

    for line in generate_edgelist(G, delimiter, data):
        line+='\n'
        path.write(line.encode(encoding))

def parse_edgelist(lines, comments='#', delimiter=None,
                   create_using=None, nodetype=None, data=True):
    """Parse lines of an edge list representation of a graph.

    Returns
    -------
    G: NetworkX Graph
        The graph corresponding to lines
    data : bool or list of (label,type) tuples
       If False generate no edge data or if True use a dictionary
       representation of edge data or a list tuples specifying dictionary
       key names and types for edge data.
    create_using: NetworkX graph container, optional
       Use given NetworkX graph for holding nodes or edges.
    nodetype : Python type, optional
       Convert nodes to this type.
    comments : string, optional
       Marker for comment lines
    delimiter : string, optional
       Separator for node labels
    create_using: NetworkX graph container
       Use given NetworkX graph for holding nodes or edges.

    Examples
    --------
    Edgelist with no data:

    >>> lines = ["1 2",
    ...          "2 3",
    ...          "3 4"]
    >>> G = nx.parse_edgelist(lines, nodetype = int)
    >>> G.nodes()
    [1, 2, 3, 4]
    >>> G.edges()
    [(1, 2), (2, 3), (3, 4)]

    Edgelist with data in Python dictionary representation:

    >>> lines = ["1 2 {'weight':3}",
    ...          "2 3 {'weight':27}",
    ...          "3 4 {'weight':3.0}"]
    >>> G = nx.parse_edgelist(lines, nodetype = int)
    >>> G.nodes()
    [1, 2, 3, 4]
    >>> G.edges(data = True)
    [(1, 2, {'weight': 3}), (2, 3, {'weight': 27}), (3, 4, {'weight': 3.0})]

    Edgelist with data in a list:

    >>> lines = ["1 2 3",
    ...          "2 3 27",
    ...          "3 4 3.0"]
    >>> G = nx.parse_edgelist(lines, nodetype = int, data=(('weight',float),))
    >>> G.nodes()
    [1, 2, 3, 4]
    >>> G.edges(data = True)
    [(1, 2, {'weight': 3.0}), (2, 3, {'weight': 27.0}), (3, 4, {'weight': 3.0})]

    See Also
    --------
    read_weighted_edgelist

    """
    from ast import literal_eval
    if create_using is None:
        G=nx.Graph()
    else:
        try:
            G=create_using
            G.clear()
        except:
            raise TypeError("create_using input is not a NetworkX graph type")

    for line in lines:
        p=line.find(comments)
        if p>=0:
            line = line[:p]
        if not len(line):
            continue
        # split line, should have 2 or more
        s=line.strip().split(delimiter)
        if len(s)<2:
            continue
        u=s.pop(0)
        v=s.pop(0)
        d=s
        if nodetype is not None:
            try:
                u=nodetype(u)
                v=nodetype(v)
            except:
                raise TypeError("Failed to convert nodes %s,%s to type %s."
                                %(u,v,nodetype))

        if len(d)==0 or data is False:
            # no data or data type specified
            edgedata={}
        elif data is True:
            # no edge types specified
            try: # try to evaluate as dictionary
                edgedata=dict(literal_eval(' '.join(d)))
            except:
                raise TypeError(
                    "Failed to convert edge data (%s) to dictionary."%(d))
        else:
            # convert edge data to dictionary with specified keys and type
            if len(d)!=len(data):
                raise IndexError(
                    "Edge data %s and data_keys %s are not the same length"%
                    (d, data))
            edgedata={}
            for (edge_key,edge_type),edge_value in zip(data,d):
                try:
                    edge_value=edge_type(edge_value)
                except:
                    raise TypeError(
                        "Failed to convert %s data %s to type %s."
                        %(edge_key, edge_value, edge_type))
                edgedata.update({edge_key:edge_value})
        G.add_edge(u, v, attr_dict=edgedata)
    return G

@open_file(0,mode='rb')
def read_edgelist(path, comments="#", delimiter=None, create_using=None,
                  nodetype=None, data=True, edgetype=None, encoding='utf-8'):
    """Read a graph from a list of edges.

    Parameters
    ----------
    path : file or string
       File or filename to write. If a file is provided, it must be
       opened in 'rb' mode.
       Filenames ending in .gz or .bz2 will be uncompressed.
    comments : string, optional
       The character used to indicate the start of a comment.
    delimiter : string, optional
       The string used to separate values.  The default is whitespace.
    create_using : Graph container, optional,
       Use specified container to build graph.  The default is networkx.Graph,
       an undirected graph.
    nodetype : int, float, str, Python type, optional
       Convert node data from strings to specified type
    data : bool or list of (label,type) tuples
       Tuples specifying dictionary key names and types for edge data
    edgetype : int, float, str, Python type, optional OBSOLETE
       Convert edge data from strings to specified type and use as 'weight'
    encoding: string, optional
       Specify which encoding to use when reading file.

    Returns
    -------
    G : graph
       A networkx Graph or other type specified with create_using

    Examples
    --------
    >>> nx.write_edgelist(nx.path_graph(4), "test.edgelist")
    >>> G=nx.read_edgelist("test.edgelist")

    >>> fh=open("test.edgelist", 'rb')
    >>> G=nx.read_edgelist(fh)
    >>> fh.close()

    >>> G=nx.read_edgelist("test.edgelist", nodetype=int)
    >>> G=nx.read_edgelist("test.edgelist",create_using=nx.DiGraph())

    Edgelist with data in a list:

    >>> textline = '1 2 3'
    >>> fh = open('test.edgelist','w')
    >>> d = fh.write(textline)
    >>> fh.close()
    >>> G = nx.read_edgelist('test.edgelist', nodetype=int, data=(('weight',float),))
    >>> G.nodes()
    [1, 2]
    >>> G.edges(data = True)
    [(1, 2, {'weight': 3.0})]

    See parse_edgelist() for more examples of formatting.

    See Also
    --------
    parse_edgelist

    Notes
    -----
    Since nodes must be hashable, the function nodetype must return hashable
    types (e.g. int, float, str, frozenset - or tuples of those, etc.)
    """
    lines = (line.decode(encoding) for line in path)
    return parse_edgelist(lines,comments=comments, delimiter=delimiter,
                          create_using=create_using, nodetype=nodetype,
                          data=data)

def write_weighted_edgelist(G, path, comments="#",
                            delimiter=' ', encoding='utf-8'):
    """Write graph G as a list of edges with numeric weights.

    Parameters
    ----------
    G : graph
       A NetworkX graph
    path : file or string
       File or filename to write. If a file is provided, it must be
       opened in 'wb' mode.
       Filenames ending in .gz or .bz2 will be compressed.
    comments : string, optional
       The character used to indicate the start of a comment
    delimiter : string, optional
       The string used to separate values.  The default is whitespace.
    encoding: string, optional
       Specify which encoding to use when writing file.

    Examples
    --------
    >>> G=nx.Graph()
    >>> G.add_edge(1,2,weight=7)
    >>> nx.write_weighted_edgelist(G, 'test.weighted.edgelist')

    See Also
    --------
    read_edgelist()
    write_edgelist()
    write_weighted_edgelist()

    """
    write_edgelist(G,path, comments=comments, delimiter=delimiter,
                   data=('weight',), encoding = encoding)

def read_weighted_edgelist(path, comments="#", delimiter=None,
                           create_using=None, nodetype=None, encoding='utf-8'):

    """Read a graph as list of edges with numeric weights.

    Parameters
    ----------
    path : file or string
       File or filename to write. If a file is provided, it must be
       opened in 'rb' mode.
       Filenames ending in .gz or .bz2 will be uncompressed.
    comments : string, optional
       The character used to indicate the start of a comment.
    delimiter : string, optional
       The string used to separate values.  The default is whitespace.
    create_using : Graph container, optional,
       Use specified container to build graph.  The default is networkx.Graph,
       an undirected graph.
    nodetype : int, float, str, Python type, optional
       Convert node data from strings to specified type
    encoding: string, optional
       Specify which encoding to use when reading file.

    Returns
    -------
    G : graph
       A networkx Graph or other type specified with create_using

    Notes
    -----
    Since nodes must be hashable, the function nodetype must return hashable
    types (e.g. int, float, str, frozenset - or tuples of those, etc.)

    Example edgelist file format.

    With numeric edge data::

     # read with
     # >>> G=nx.read_weighted_edgelist(fh)
     # source target data
     a b 1
     a c 3.14159
     d e 42
    """
    return read_edgelist(path,
                         comments=comments,
                         delimiter=delimiter,
                         create_using=create_using,
                         nodetype=nodetype,
                         data=(('weight',float),),
                         encoding = encoding
                         )

# fixture for nose tests
def teardown_module(module):
    import os
    os.unlink('test.edgelist')
    os.unlink('test.edgelist.gz')
    os.unlink('test.weighted.edgelist')
<commit_msg>Fix. parse_edgelist documentation<commit_after>"""
**********
Edge Lists
**********
Read and write NetworkX graphs as edge lists.

The multi-line adjacency list format is useful for graphs with nodes
that can be meaningfully represented as strings.  With the edgelist
format simple edge data can be stored but node or graph data is not.
There is no way of representing isolated nodes unless the node has a
self-loop edge.

Format
------
You can read or write three formats of edge lists with these functions.

Node pairs with no data::

 1 2

Python dictionary as data::

 1 2 {'weight':7, 'color':'green'}

Arbitrary data::

 1 2 7 green
"""
__author__ = """Aric Hagberg (hagberg@lanl.gov)\nDan Schult (dschult@colgate.edu)"""
#    Copyright (C) 2004-2011 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.

__all__ = ['generate_edgelist',
           'write_edgelist',
           'parse_edgelist',
           'read_edgelist',
           'read_weighted_edgelist',
           'write_weighted_edgelist']

from networkx.utils import open_file, make_str
import networkx as nx

def generate_edgelist(G, delimiter=' ', data=True):
    """Generate a single line of the graph G in edge list format.

    Parameters
    ----------
    G : NetworkX graph

    delimiter : string, optional
       Separator for node labels

    data : bool or list of keys
       If False generate no edge data.  If True use a dictionary
       representation of edge data.  If a list of keys use a list of data
       values corresponding to the keys.

    Returns
    -------
    lines : string
        Lines of data in adjlist format.

    Examples
    --------
    >>> G = nx.lollipop_graph(4, 3)
    >>> G[1][2]['weight'] = 3
    >>> G[3][4]['capacity'] = 12
    >>> for line in nx.generate_edgelist(G, data=False):
    ...     print(line)
    0 1
    0 2
    0 3
    1 2
    1 3
    2 3
    3 4
    4 5
    5 6

    >>> for line in nx.generate_edgelist(G):
    ...     print(line)
    0 1 {}
    0 2 {}
    0 3 {}
    1 2 {'weight': 3}
    1 3 {}
    2 3 {}
    3 4 {'capacity': 12}
    4 5 {}
    5 6 {}

    >>> for line in nx.generate_edgelist(G,data=['weight']):
    ...     print(line)
    0 1
    0 2
    0 3
    1 2 3
    1 3
    2 3
    3 4
    4 5
    5 6

    See Also
    --------
    write_adjlist, read_adjlist
    """
    if data is True or data is False:
        for e in G.edges(data=data):
            yield delimiter.join(map(make_str,e))
    else:
        for u,v,d in G.edges(data=True):
            e=[u,v]
            try:
                e.extend(d[k] for k in data)
            except KeyError:
                pass # missing data for this edge, should warn?
            yield delimiter.join(map(make_str,e))

@open_file(1,mode='wb')
def write_edgelist(G, path, comments="#", delimiter=' ', data=True,
                   encoding = 'utf-8'):
    """Write graph as a list of edges.

    Parameters
    ----------
    G : graph
       A NetworkX graph
    path : file or string
       File or filename to write. If a file is provided, it must be
       opened in 'wb' mode. Filenames ending in .gz or .bz2 will be compressed.
    comments : string, optional
       The character used to indicate the start of a comment
    delimiter : string, optional
       The string used to separate values.  The default is whitespace.
    data : bool or list, optional
       If False write no edge data.
       If True write a string representation of the edge data dictionary..
       If a list (or other iterable) is provided, write the  keys specified
       in the list.
    encoding: string, optional
       Specify which encoding to use when writing file.

    Examples
    --------
    >>> G=nx.path_graph(4)
    >>> nx.write_edgelist(G, "test.edgelist")
    >>> G=nx.path_graph(4)
    >>> fh=open("test.edgelist",'wb')
    >>> nx.write_edgelist(G, fh)
    >>> nx.write_edgelist(G, "test.edgelist.gz")
    >>> nx.write_edgelist(G, "test.edgelist.gz", data=False)

    >>> G=nx.Graph()
    >>> G.add_edge(1,2,weight=7,color='red')
    >>> nx.write_edgelist(G,'test.edgelist',data=False)
    >>> nx.write_edgelist(G,'test.edgelist',data=['color'])
    >>> nx.write_edgelist(G,'test.edgelist',data=['color','weight'])

    See Also
    --------
    write_edgelist()
    write_weighted_edgelist()
    """

    for line in generate_edgelist(G, delimiter, data):
        line+='\n'
        path.write(line.encode(encoding))

def parse_edgelist(lines, comments='#', delimiter=None,
                   create_using=None, nodetype=None, data=True):
    """Parse lines of an edge list representation of a graph.

    Parameters
    ----------
    lines : list or iterator of strings
        Input data in edgelist format
    comments : string, optional
       Marker for comment lines
    delimiter : string, optional
       Separator for node labels
    create_using: NetworkX graph container, optional
       Use given NetworkX graph for holding nodes or edges.
    nodetype : Python type, optional
       Convert nodes to this type.
    data : bool or list of (label,type) tuples
       If False generate no edge data or if True use a dictionary
       representation of edge data or a list tuples specifying dictionary
       key names and types for edge data.

    Returns
    -------
    G: NetworkX Graph
        The graph corresponding to lines

    Examples
    --------
    Edgelist with no data:

    >>> lines = ["1 2",
    ...          "2 3",
    ...          "3 4"]
    >>> G = nx.parse_edgelist(lines, nodetype = int)
    >>> G.nodes()
    [1, 2, 3, 4]
    >>> G.edges()
    [(1, 2), (2, 3), (3, 4)]

    Edgelist with data in Python dictionary representation:

    >>> lines = ["1 2 {'weight':3}",
    ...          "2 3 {'weight':27}",
    ...          "3 4 {'weight':3.0}"]
    >>> G = nx.parse_edgelist(lines, nodetype = int)
    >>> G.nodes()
    [1, 2, 3, 4]
    >>> G.edges(data = True)
    [(1, 2, {'weight': 3}), (2, 3, {'weight': 27}), (3, 4, {'weight': 3.0})]

    Edgelist with data in a list:

    >>> lines = ["1 2 3",
    ...          "2 3 27",
    ...          "3 4 3.0"]
    >>> G = nx.parse_edgelist(lines, nodetype = int, data=(('weight',float),))
    >>> G.nodes()
    [1, 2, 3, 4]
    >>> G.edges(data = True)
    [(1, 2, {'weight': 3.0}), (2, 3, {'weight': 27.0}), (3, 4, {'weight': 3.0})]

    See Also
    --------
    read_weighted_edgelist

    """
    from ast import literal_eval
    if create_using is None:
        G=nx.Graph()
    else:
        try:
            G=create_using
            G.clear()
        except:
            raise TypeError("create_using input is not a NetworkX graph type")

    for line in lines:
        p=line.find(comments)
        if p>=0:
            line = line[:p]
        if not len(line):
            continue
        # split line, should have 2 or more
        s=line.strip().split(delimiter)
        if len(s)<2:
            continue
        u=s.pop(0)
        v=s.pop(0)
        d=s
        if nodetype is not None:
            try:
                u=nodetype(u)
                v=nodetype(v)
            except:
                raise TypeError("Failed to convert nodes %s,%s to type %s."
                                %(u,v,nodetype))

        if len(d)==0 or data is False:
            # no data or data type specified
            edgedata={}
        elif data is True:
            # no edge types specified
            try: # try to evaluate as dictionary
                edgedata=dict(literal_eval(' '.join(d)))
            except:
                raise TypeError(
                    "Failed to convert edge data (%s) to dictionary."%(d))
        else:
            # convert edge data to dictionary with specified keys and type
            if len(d)!=len(data):
                raise IndexError(
                    "Edge data %s and data_keys %s are not the same length"%
                    (d, data))
            edgedata={}
            for (edge_key,edge_type),edge_value in zip(data,d):
                try:
                    edge_value=edge_type(edge_value)
                except:
                    raise TypeError(
                        "Failed to convert %s data %s to type %s."
                        %(edge_key, edge_value, edge_type))
                edgedata.update({edge_key:edge_value})
        G.add_edge(u, v, attr_dict=edgedata)
    return G

@open_file(0,mode='rb')
def read_edgelist(path, comments="#", delimiter=None, create_using=None,
                  nodetype=None, data=True, edgetype=None, encoding='utf-8'):
    """Read a graph from a list of edges.

    Parameters
    ----------
    path : file or string
       File or filename to write. If a file is provided, it must be
       opened in 'rb' mode.
       Filenames ending in .gz or .bz2 will be uncompressed.
    comments : string, optional
       The character used to indicate the start of a comment.
    delimiter : string, optional
       The string used to separate values.  The default is whitespace.
    create_using : Graph container, optional,
       Use specified container to build graph.  The default is networkx.Graph,
       an undirected graph.
    nodetype : int, float, str, Python type, optional
       Convert node data from strings to specified type
    data : bool or list of (label,type) tuples
       Tuples specifying dictionary key names and types for edge data
    edgetype : int, float, str, Python type, optional OBSOLETE
       Convert edge data from strings to specified type and use as 'weight'
    encoding: string, optional
       Specify which encoding to use when reading file.

    Returns
    -------
    G : graph
       A networkx Graph or other type specified with create_using

    Examples
    --------
    >>> nx.write_edgelist(nx.path_graph(4), "test.edgelist")
    >>> G=nx.read_edgelist("test.edgelist")

    >>> fh=open("test.edgelist", 'rb')
    >>> G=nx.read_edgelist(fh)
    >>> fh.close()

    >>> G=nx.read_edgelist("test.edgelist", nodetype=int)
    >>> G=nx.read_edgelist("test.edgelist",create_using=nx.DiGraph())

    Edgelist with data in a list:

    >>> textline = '1 2 3'
    >>> fh = open('test.edgelist','w')
    >>> d = fh.write(textline)
    >>> fh.close()
    >>> G = nx.read_edgelist('test.edgelist', nodetype=int, data=(('weight',float),))
    >>> G.nodes()
    [1, 2]
    >>> G.edges(data = True)
    [(1, 2, {'weight': 3.0})]

    See parse_edgelist() for more examples of formatting.

    See Also
    --------
    parse_edgelist

    Notes
    -----
    Since nodes must be hashable, the function nodetype must return hashable
    types (e.g. int, float, str, frozenset - or tuples of those, etc.)
    """
    lines = (line.decode(encoding) for line in path)
    return parse_edgelist(lines,comments=comments, delimiter=delimiter,
                          create_using=create_using, nodetype=nodetype,
                          data=data)

def write_weighted_edgelist(G, path, comments="#",
                            delimiter=' ', encoding='utf-8'):
    """Write graph G as a list of edges with numeric weights.

    Parameters
    ----------
    G : graph
       A NetworkX graph
    path : file or string
       File or filename to write. If a file is provided, it must be
       opened in 'wb' mode.
       Filenames ending in .gz or .bz2 will be compressed.
    comments : string, optional
       The character used to indicate the start of a comment
    delimiter : string, optional
       The string used to separate values.  The default is whitespace.
    encoding: string, optional
       Specify which encoding to use when writing file.

    Examples
    --------
    >>> G=nx.Graph()
    >>> G.add_edge(1,2,weight=7)
    >>> nx.write_weighted_edgelist(G, 'test.weighted.edgelist')

    See Also
    --------
    read_edgelist()
    write_edgelist()
    write_weighted_edgelist()

    """
    write_edgelist(G,path, comments=comments, delimiter=delimiter,
                   data=('weight',), encoding = encoding)

def read_weighted_edgelist(path, comments="#", delimiter=None,
                           create_using=None, nodetype=None, encoding='utf-8'):

    """Read a graph as list of edges with numeric weights.

    Parameters
    ----------
    path : file or string
       File or filename to write. If a file is provided, it must be
       opened in 'rb' mode.
       Filenames ending in .gz or .bz2 will be uncompressed.
    comments : string, optional
       The character used to indicate the start of a comment.
    delimiter : string, optional
       The string used to separate values.  The default is whitespace.
    create_using : Graph container, optional,
       Use specified container to build graph.  The default is networkx.Graph,
       an undirected graph.
    nodetype : int, float, str, Python type, optional
       Convert node data from strings to specified type
    encoding: string, optional
       Specify which encoding to use when reading file.

    Returns
    -------
    G : graph
       A networkx Graph or other type specified with create_using

    Notes
    -----
    Since nodes must be hashable, the function nodetype must return hashable
    types (e.g. int, float, str, frozenset - or tuples of those, etc.)

    Example edgelist file format.

    With numeric edge data::

     # read with
     # >>> G=nx.read_weighted_edgelist(fh)
     # source target data
     a b 1
     a c 3.14159
     d e 42
    """
    return read_edgelist(path,
                         comments=comments,
                         delimiter=delimiter,
                         create_using=create_using,
                         nodetype=nodetype,
                         data=(('weight',float),),
                         encoding = encoding
                         )

# fixture for nose tests
def teardown_module(module):
    import os
    os.unlink('test.edgelist')
    os.unlink('test.edgelist.gz')
    os.unlink('test.weighted.edgelist')
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Module for feature detection and description.

Summary of implemented functions:

- FAST keypoints
- ORB keypoints
- MSER keypoints
- BRISK keypoints

We can now specify the desired no of keypoints. Fitness is based on 'response' value of keypoints.

TODO: -look into adaptive thresholds to generate keypoints if no points are detected with default parameters.
      -make sure if varying patchSize for ORB, descriptor uses same parameters! (DONE - we use **kwargs)
"""

import numpy as np
import cv2
from matplotlib import pyplot as plt

def sort_keypoints_by_response_and_get_n_best(keypoint_list, n=500):
    """
    Sorts keypoint list by "response" field and returns the first n best keypoints.
    If the length of the list is smaller than n, than return the whole list.
    input: keypoint_list - list of keypoints
           n - no of keypoints to be returned
    """
    sortedList = sorted(keypoint_list, key=lambda x: x.response, reverse=True)

    bestNKeypoints = []

    if (len(sortedList) > n):
        bestNKeypoints = sortedList[:n]
    else:
        bestNKeypoints = sortedList

    return bestNKeypoints

def get_FAST_keypoints(image, n=500):
    """
    Detects keypoints using FAST feature detector.
    (nonmaxSuppression prevents too many keypoints being detected in the same region. This is enabled by default)
    input:  image
            n - max number of returned keypoints (default 500)
    output: list of FAST keypoints
    """
    # Initiate FAST object with default values
    fast = cv2.FastFeatureDetector()

    # find keypoints
    keyPoints = fast.detect(image, None)

    # sort by response and return best n keypoints
    keyPoints = sort_keypoints_by_response_and_get_n_best(keyPoints, n)

    return image, keyPoints, {}

def get_ORB_keypoints(image, n=500, **kwargs):
    """
    Detects keypoints using ORB feature detector. Maximum no of keypoints returned is 500.
    input: image
           n - max number of returned keypoints (default 500)
    output: list of ORB keypoints
    """
    # blur using a Gaussian kernel
    #image = cv2.GaussianBlur(image,(3,3),0)
    #image = cv2.bilateralFilter(image,5,75,75)

    keyPoints = []
    thePatchSize = 31
    # find keypoints; if none found, decrease patchSize
    while ( len(keyPoints) == 0 ):
        # Initiate ORB detector
        orb = cv2.ORB(nfeatures = n, edgeThreshold = 0, patchSize = thePatchSize)
        keyPoints = orb.detect(image, None)
        if thePatchSize <= 3:
            print "Reached limit of patch size."
            break
        thePatchSize -= 2

    # sort by response and return best n keypoints
    # already scored by "scoreType" HARRIS_SCORE?
    keyPoints = sort_keypoints_by_response_and_get_n_best(keyPoints, n)

    return image, keyPoints, {"patchSize" : thePatchSize}

def get_ORB_descriptors(image, keyPoints, **kwargs):
    """
    Computes ORB descriptors for given keypoints.
    input:  image (that was returned with the keypoints!)
            keyPoints - detected keypoints
            **kwargs = detection arguments
    output: list of descriptors for given keypoints
    """
    orb = cv2.ORB(edgeThreshold = 0, patchSize = kwargs["patchSize"])
    # gets keypoints and descriptors
    kp, descriptors = orb.compute(image, keyPoints)

    if descriptors is None:
        descriptors = []

    return descriptors

def get_BRISK_keypoints(image, n=500):
    """
    Detects keypoints using BRISK feature detector.
    input:  image
            n - max no of returned keypoints (default 500)
    output: list of BRISK keypoints
    """
    # Initiate BRISK detector
    brisk = cv2.BRISK()

    # find keypoints
    keyPoints = brisk.detect(image, None)

    # sort by response and return best n keypoints
    keyPoints = sort_keypoints_by_response_and_get_n_best(keyPoints, n)

    return image, keyPoints, {}

def get_BRISK_descriptors(image, keyPoints, **kwargs):
    """
    Computes BRISK descriptors for given keypoints.
    input:  image (that was returned with the keypoints!)
            keyPoints - detected keypoints
            **kwargs = detection arguments
    output: list of descriptors for given keypoints
    """
    brisk = cv2.BRISK()
    # gets keypoints and descriptors
    kp, descriptors = brisk.compute(image, keyPoints)

    if descriptors is None:
        descriptors = []

    return descriptors

def get_MSER_keypoints(image, n=500):
    """
    Detects keypoints using MSER detector.
    input:  image
            n - max number of returned keypoints (default 500)
    output: list of MSER keypoints
    """
    # initiate MSER detector
    mser = cv2.FeatureDetector_create('MSER')

    # find keypoints
    keyPoints = mser.detect(image)

    # sort by response and return best n keypoints
    keyPoints = sort_keypoints_by_response_and_get_n_best(keyPoints, n)

    return image, keyPoints, {}<commit_msg>Remove unnecessary dependencies<commit_after>#!/usr/bin/env python
"""
Module for feature detection and description.

Summary of implemented functions:

- FAST keypoints
- ORB keypoints
- MSER keypoints
- BRISK keypoints

We can now specify the desired no of keypoints. Fitness is based on 'response' value of keypoints.

TODO: -look into adaptive thresholds to generate keypoints if no points are detected with default parameters.
      -make sure if varying patchSize for ORB, descriptor uses same parameters! (DONE - we use **kwargs)
"""

import cv2

def sort_keypoints_by_response_and_get_n_best(keypoint_list, n=500):
    """
    Sorts keypoint list by "response" field and returns the first n best keypoints.
    If the length of the list is smaller than n, than return the whole list.
    input: keypoint_list - list of keypoints
           n - no of keypoints to be returned
    """
    sortedList = sorted(keypoint_list, key=lambda x: x.response, reverse=True)

    bestNKeypoints = []

    if (len(sortedList) > n):
        bestNKeypoints = sortedList[:n]
    else:
        bestNKeypoints = sortedList

    return bestNKeypoints

def get_FAST_keypoints(image, n=500):
    """
    Detects keypoints using FAST feature detector.
    (nonmaxSuppression prevents too many keypoints being detected in the same region. This is enabled by default)
    input:  image
            n - max number of returned keypoints (default 500)
    output: list of FAST keypoints
    """
    # Initiate FAST object with default values
    fast = cv2.FastFeatureDetector()

    # find keypoints
    keyPoints = fast.detect(image, None)

    # sort by response and return best n keypoints
    keyPoints = sort_keypoints_by_response_and_get_n_best(keyPoints, n)

    return image, keyPoints, {}

def get_ORB_keypoints(image, n=500, **kwargs):
    """
    Detects keypoints using ORB feature detector. Maximum no of keypoints returned is 500.
    input: image
           n - max number of returned keypoints (default 500)
    output: list of ORB keypoints
    """
    # blur using a Gaussian kernel
    #image = cv2.GaussianBlur(image,(3,3),0)
    #image = cv2.bilateralFilter(image,5,75,75)

    keyPoints = []
    thePatchSize = 31
    # find keypoints; if none found, decrease patchSize
    while ( len(keyPoints) == 0 ):
        # Initiate ORB detector
        orb = cv2.ORB(nfeatures = n, edgeThreshold = 0, patchSize = thePatchSize)
        keyPoints = orb.detect(image, None)
        if thePatchSize <= 3:
            print "Reached limit of patch size."
            break
        thePatchSize -= 2

    # sort by response and return best n keypoints
    # already scored by "scoreType" HARRIS_SCORE?
    keyPoints = sort_keypoints_by_response_and_get_n_best(keyPoints, n)

    return image, keyPoints, {"patchSize" : thePatchSize}

def get_ORB_descriptors(image, keyPoints, **kwargs):
    """
    Computes ORB descriptors for given keypoints.
    input:  image (that was returned with the keypoints!)
            keyPoints - detected keypoints
            **kwargs = detection arguments
    output: list of descriptors for given keypoints
    """
    orb = cv2.ORB(edgeThreshold = 0, patchSize = kwargs["patchSize"])
    # gets keypoints and descriptors
    kp, descriptors = orb.compute(image, keyPoints)

    if descriptors is None:
        descriptors = []

    return descriptors

def get_BRISK_keypoints(image, n=500):
    """
    Detects keypoints using BRISK feature detector.
    input:  image
            n - max no of returned keypoints (default 500)
    output: list of BRISK keypoints
    """
    # Initiate BRISK detector
    brisk = cv2.BRISK()

    # find keypoints
    keyPoints = brisk.detect(image, None)

    # sort by response and return best n keypoints
    keyPoints = sort_keypoints_by_response_and_get_n_best(keyPoints, n)

    return image, keyPoints, {}

def get_BRISK_descriptors(image, keyPoints, **kwargs):
    """
    Computes BRISK descriptors for given keypoints.
    input:  image (that was returned with the keypoints!)
            keyPoints - detected keypoints
            **kwargs = detection arguments
    output: list of descriptors for given keypoints
    """
    brisk = cv2.BRISK()
    # gets keypoints and descriptors
    kp, descriptors = brisk.compute(image, keyPoints)

    if descriptors is None:
        descriptors = []

    return descriptors

def get_MSER_keypoints(image, n=500):
    """
    Detects keypoints using MSER detector.
    input:  image
            n - max number of returned keypoints (default 500)
    output: list of MSER keypoints
    """
    # initiate MSER detector
    mser = cv2.FeatureDetector_create('MSER')

    # find keypoints
    keyPoints = mser.detect(image)

    # sort by response and return best n keypoints
    keyPoints = sort_keypoints_by_response_and_get_n_best(keyPoints, n)

    return image, keyPoints, {}
<|endoftext|>


<commit_before>from .rgscat import RGscat
from .miescat import Mie
from .. import constants as c

"""
--------------------------------------------------------------
    API
--------------------------------------------------------------
 A dust scattering model should contain functions that take
 energy value, complex index of refraction object (see cmlib),
 and grain sizes

 Qsca ( E  : scalar or np.array [keV]
        cm : cmtype object from cmi.py
        a  : scalar [grain size, micron] ) :
 returns scalar or np.array [scattering efficiency, unitless]

 Diff ( cm : cmtype object from cmi.py
        theta : scalar or np.array [angle, arcsec]
        a  : scalar [grain size, micron]
        E  : scalar or np.array [energy, keV]
        ** if len(E) > 1 and len(theta) > 1, then len(E) must equal len(theta)
           returns dsigma/dOmega of values (E0,theta0), (E1,theta1) etc...

 Some (but not all) scattering models may also contain related extinction terms

 Qext ( E  : scalar or np.array [keV]
        cm : cmtype object cmi.py
        a  : scalar [grain size, micron] ) :
 returns scalar or np.array [extinction efficiency, unitless]

 Qabs ( E  : scalar or np.array [keV]
        cm : cmtype object cmi.py
        a  : scalar [grain size, micron] ) :
 returns scalar or np.array [absorption efficiency, unitless]
"""
<commit_msg>Updated scatmodel API docstring<commit_after>from .rgscat import RGscat
from .miescat import Mie
from .. import constants as c

"""
--------------------------------------------------------------
    API for Abstract Class 'ScatModel'
--------------------------------------------------------------
 A dust scattering model should contain the following attributes

calculate( lam : scalar or np.array [wavelength or energy grid, keV default]
           a   : scalar [grain size, micron]
           cm  : newdust.graindist.composition cm object (abstract class)
           unit = : string ['kev', 'angs']
           theta = : scalar or np.array [angles to calculate differential scattering, arcsec, default 0.0]
           **kwargs
           )

qsca : np.array, scattering efficiency [unitless]
qabs : np.array, absorption efficiency [unitless]
qext : np.array, extinction efficiency [unitless]
self.diff : np.array, differentifal scattering cross section [cm**2/ster]
"""
<|endoftext|>