<commit_before>#Copyright 2010-2011 Miquel Torres <tobami@googlemail.com>
#
#Licensed under the Apache License, Version 2.0 (the "License");
#you may not use this file except in compliance with the License.
#You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.
#
"""Node configuration and syncing
See http://wiki.opscode.com/display/chef/Anatomy+of+a+Chef+Run
"""
import os
import simplejson as json

from fabric.api import *
from fabric.contrib.files import append, exists
from fabric import colors
from fabric.utils import abort

import lib

# Upload sensitive files with secure permissions
_file_mode = 400

def save_config(save, data, hostname):
    """Saves node configuration
    if save=True, or if no hostname.json exists, it creates one
    if save=False, it saves it to tmp_node.json
    """
    filepath = os.path.join("nodes/", hostname + ".json")
    if os.path.exists(filepath) and not save:
        filepath = 'tmp_node.json'
    with open(filepath, 'w') as f:
        f.write(json.dumps(data, indent=4))
        f.write('\n')
    return filepath

def sync_node(filepath, cookbook_paths, node_work_path):
    """Builds, synchronizes and configures a node"""
    if env.ssh_config:
        credentials = env.ssh_config.lookup(env.host)
        # translate from paramiko params to fabric params
        credentials['key_filename'] = credentials['identityfile']
    else:
        credentials = env

    with settings(**credentials):
        _synchronize_node(filepath, cookbook_paths, node_work_path)
        _configure_node(filepath)

def _synchronize_node(configfile, cookbook_paths, node_work_path):
    """Performs the Synchronize step of a Chef run:
    Uploads needed cookbooks and all roles to a node
    """
    # Clean up node
    for path in ['roles'] + cookbook_paths:
        with hide('stdout'):
            sudo('rm -rf {0}/{1}'.format(node_work_path, path))

    cookbooks = []
    with open(configfile, 'r') as f:
        try:
            node = json.loads(f.read())
        except json.decoder.JSONDecodeError as e:
            msg = 'Little Chef found the following error in'
            msg += ' "{0}":\n                {1}'.format(configfile, str(e))
            abort(msg)
    # Fetch cookbooks needed for recipes
    for recipe in lib.get_recipes_in_node(node):
        recipe = recipe.split('::')[0]
        if recipe not in cookbooks:
            cookbooks.append(recipe)

    # Fetch cookbooks needed for role recipes
    for role in lib.get_roles_in_node(node):
        try:
            with open('roles/' + role + '.json', 'r') as f:
                try:
                    roles = json.loads(f.read())
                except json.decoder.JSONDecodeError as e:
                    msg = 'Little Chef found the following error in your'
                    msg += ' "{0}" role file:\n                {1}'.format(
                        role, str(e))
                    abort(msg)
                # Reuse _get_recipes_in_node to extract recipes in a role
                for recipe in lib.get_recipes_in_node(roles):
                    recipe = recipe.split('::')[0]
                    if recipe not in cookbooks:
                        cookbooks.append(recipe)
        except IOError:
            abort(colors.red("Role '{0}' not found".format(role)))

    # Fetch dependencies
    warnings = []
    for cookbook in cookbooks:
        for recipe in lib.get_recipes_in_cookbook(cookbook, cookbook_paths):
            for dep in recipe['dependencies']:
                if dep not in cookbooks:
                    try:
                        lib.get_cookbook_path(dep, cookbook_paths)
                        cookbooks.append(dep)
                    except IOError:
                        if dep not in warnings:
                            warnings.append(dep)
                            print "Warning: Possible error because of missing",
                            print "dependency for cookbook {0}".format(recipe['name'])
                            print "         Cookbook '{0}' not found".format(dep)
                            import time
                            time.sleep(1)

    cookbooks_by_path = {}
    for cookbook in cookbooks:
        for cookbook_path in cookbook_paths:
            path = os.path.join(cookbook_path, cookbook)
            if os.path.exists(path):
                cookbooks_by_path[path] = cookbook

    print "Uploading cookbooks... ({0})".format(
            ", ".join(c for c in cookbooks))
    _upload_and_unpack([p for p in cookbooks_by_path.keys()], node_work_path)

    print "Uploading roles..."
    _upload_and_unpack(['roles'], node_work_path)

def _configure_node(configfile):
    """Exectutes chef-solo to apply roles and recipes to a node"""
    with hide('running'):
        print "Uploading node.json..."
        remote_file = '/root/{0}'.format(configfile.split("/")[-1])
        put(configfile, remote_file, use_sudo=True, mode=_file_mode)
        sudo('chown root:root {0}'.format(remote_file)),
        sudo('mv {0} /etc/chef/node.json'.format(remote_file)),

        print "\n== Cooking ==\n"
        with settings(hide('warnings'), warn_only=True):
            output = sudo(
                'chef-solo -l {0} -j /etc/chef/node.json'.format(env.loglevel))
            if output.failed:
                if 'chef-solo: command not found' in output:
                    print(
                        colors.red(
                            "\nFAILED: Chef Solo is not installed on this node"))
                    print(
                        "Type 'cook nodes:{0} deploy_chef' to install it".format(
                            env.host))
                    abort("")
                else:
                    print(colors.red(
                        "\nFAILED: A problem occurred while executing chef-solo"))
                    abort("")
            else:
                print(colors.green("\nSUCCESS: Node correctly configured"))

def _upload_and_unpack(source, node_work_path):
    """Packs the given directories, uploads the tar.gz to the node
    and unpacks it in the node_work_path (typically '/var/chef-solo') directory
    """
    with hide('running', 'stdout'):
        # Local archive relative path
        local_archive = 'temp.tar.gz'
        # Remote archive absolute path
        remote_archive = '/root/{0}'.format(local_archive)
        # Remove existing temporary directory
        local('(chmod -R u+rwX tmp; rm -rf tmp) > /dev/null 2>&1')
        # Create temporary directory
        local('mkdir tmp')
        # Copy selected sources into temporary directory
        for item in source:
            local('mkdir -p tmp/{0}'.format(os.path.dirname(item)))
            local('cp -R {0} tmp/{1}'.format(item, item))
        # Set secure permissions on copied sources
        local('chmod -R u=rX,go= tmp')
        # Create archive locally
        local(
            'cd tmp && COPYFILE_DISABLE=true tar czf ../{0} --exclude=".svn" .'.format(
                local_archive))
        # Upload archive to remote
        put(local_archive, remote_archive, use_sudo=True, mode=_file_mode)
        # Remove local copy of archive and directory
        local('rm {0}'.format(local_archive))
        local('chmod -R u+w tmp')
        local('rm -rf tmp')
        if not exists(node_work_path):
            # Report error with remote paths
            msg = "the {0} directory was not found at ".format(node_work_path)
            msg += "the node. Is Chef correctly installed?"
            msg += "\nYou can deploy chef-solo by typing:\n"
            msg += "  cook node:{0} deploy_chef".format(env.host)
            abort(msg)
        with cd(node_work_path):
            # Install the remote copy of archive
            sudo('tar xzf {0}'.format(remote_archive))
            # Fix ownership
            sudo('chown -R root:root {0}'.format(node_work_path))
            # Remove the remote copy of archive
            sudo('rm {0}'.format(remote_archive))

<commit_msg>Handle case where identityfile is not set in .ssh/config<commit_after>#Copyright 2010-2011 Miquel Torres <tobami@googlemail.com>
#
#Licensed under the Apache License, Version 2.0 (the "License");
#you may not use this file except in compliance with the License.
#You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.
#
"""Node configuration and syncing
See http://wiki.opscode.com/display/chef/Anatomy+of+a+Chef+Run
"""
import os
import simplejson as json

from fabric.api import *
from fabric.contrib.files import append, exists
from fabric import colors
from fabric.utils import abort

import lib

# Upload sensitive files with secure permissions
_file_mode = 400

def save_config(save, data, hostname):
    """Saves node configuration
    if save=True, or if no hostname.json exists, it creates one
    if save=False, it saves it to tmp_node.json
    """
    filepath = os.path.join("nodes/", hostname + ".json")
    if os.path.exists(filepath) and not save:
        filepath = 'tmp_node.json'
    with open(filepath, 'w') as f:
        f.write(json.dumps(data, indent=4))
        f.write('\n')
    return filepath

def sync_node(filepath, cookbook_paths, node_work_path):
    """Builds, synchronizes and configures a node"""
    if env.ssh_config:
        credentials = env.ssh_config.lookup(env.host)
        # translate from paramiko params to fabric params
        if 'identityfile' in credentials:
            credentials['key_filename'] = credentials['identityfile']
    else:
        credentials = env

    with settings(**credentials):
        _synchronize_node(filepath, cookbook_paths, node_work_path)
        _configure_node(filepath)

def _synchronize_node(configfile, cookbook_paths, node_work_path):
    """Performs the Synchronize step of a Chef run:
    Uploads needed cookbooks and all roles to a node
    """
    # Clean up node
    for path in ['roles'] + cookbook_paths:
        with hide('stdout'):
            sudo('rm -rf {0}/{1}'.format(node_work_path, path))

    cookbooks = []
    with open(configfile, 'r') as f:
        try:
            node = json.loads(f.read())
        except json.decoder.JSONDecodeError as e:
            msg = 'Little Chef found the following error in'
            msg += ' "{0}":\n                {1}'.format(configfile, str(e))
            abort(msg)
    # Fetch cookbooks needed for recipes
    for recipe in lib.get_recipes_in_node(node):
        recipe = recipe.split('::')[0]
        if recipe not in cookbooks:
            cookbooks.append(recipe)

    # Fetch cookbooks needed for role recipes
    for role in lib.get_roles_in_node(node):
        try:
            with open('roles/' + role + '.json', 'r') as f:
                try:
                    roles = json.loads(f.read())
                except json.decoder.JSONDecodeError as e:
                    msg = 'Little Chef found the following error in your'
                    msg += ' "{0}" role file:\n                {1}'.format(
                        role, str(e))
                    abort(msg)
                # Reuse _get_recipes_in_node to extract recipes in a role
                for recipe in lib.get_recipes_in_node(roles):
                    recipe = recipe.split('::')[0]
                    if recipe not in cookbooks:
                        cookbooks.append(recipe)
        except IOError:
            abort(colors.red("Role '{0}' not found".format(role)))

    # Fetch dependencies
    warnings = []
    for cookbook in cookbooks:
        for recipe in lib.get_recipes_in_cookbook(cookbook, cookbook_paths):
            for dep in recipe['dependencies']:
                if dep not in cookbooks:
                    try:
                        lib.get_cookbook_path(dep, cookbook_paths)
                        cookbooks.append(dep)
                    except IOError:
                        if dep not in warnings:
                            warnings.append(dep)
                            print "Warning: Possible error because of missing",
                            print "dependency for cookbook {0}".format(recipe['name'])
                            print "         Cookbook '{0}' not found".format(dep)
                            import time
                            time.sleep(1)

    cookbooks_by_path = {}
    for cookbook in cookbooks:
        for cookbook_path in cookbook_paths:
            path = os.path.join(cookbook_path, cookbook)
            if os.path.exists(path):
                cookbooks_by_path[path] = cookbook

    print "Uploading cookbooks... ({0})".format(
            ", ".join(c for c in cookbooks))
    _upload_and_unpack([p for p in cookbooks_by_path.keys()], node_work_path)

    print "Uploading roles..."
    _upload_and_unpack(['roles'], node_work_path)

def _configure_node(configfile):
    """Exectutes chef-solo to apply roles and recipes to a node"""
    with hide('running'):
        print "Uploading node.json..."
        remote_file = '/root/{0}'.format(configfile.split("/")[-1])
        put(configfile, remote_file, use_sudo=True, mode=_file_mode)
        sudo('chown root:root {0}'.format(remote_file)),
        sudo('mv {0} /etc/chef/node.json'.format(remote_file)),

        print "\n== Cooking ==\n"
        with settings(hide('warnings'), warn_only=True):
            output = sudo(
                'chef-solo -l {0} -j /etc/chef/node.json'.format(env.loglevel))
            if output.failed:
                if 'chef-solo: command not found' in output:
                    print(
                        colors.red(
                            "\nFAILED: Chef Solo is not installed on this node"))
                    print(
                        "Type 'cook nodes:{0} deploy_chef' to install it".format(
                            env.host))
                    abort("")
                else:
                    print(colors.red(
                        "\nFAILED: A problem occurred while executing chef-solo"))
                    abort("")
            else:
                print(colors.green("\nSUCCESS: Node correctly configured"))

def _upload_and_unpack(source, node_work_path):
    """Packs the given directories, uploads the tar.gz to the node
    and unpacks it in the node_work_path (typically '/var/chef-solo') directory
    """
    with hide('running', 'stdout'):
        # Local archive relative path
        local_archive = 'temp.tar.gz'
        # Remote archive absolute path
        remote_archive = '/root/{0}'.format(local_archive)
        # Remove existing temporary directory
        local('(chmod -R u+rwX tmp; rm -rf tmp) > /dev/null 2>&1')
        # Create temporary directory
        local('mkdir tmp')
        # Copy selected sources into temporary directory
        for item in source:
            local('mkdir -p tmp/{0}'.format(os.path.dirname(item)))
            local('cp -R {0} tmp/{1}'.format(item, item))
        # Set secure permissions on copied sources
        local('chmod -R u=rX,go= tmp')
        # Create archive locally
        local(
            'cd tmp && COPYFILE_DISABLE=true tar czf ../{0} --exclude=".svn" .'.format(
                local_archive))
        # Upload archive to remote
        put(local_archive, remote_archive, use_sudo=True, mode=_file_mode)
        # Remove local copy of archive and directory
        local('rm {0}'.format(local_archive))
        local('chmod -R u+w tmp')
        local('rm -rf tmp')
        if not exists(node_work_path):
            # Report error with remote paths
            msg = "the {0} directory was not found at ".format(node_work_path)
            msg += "the node. Is Chef correctly installed?"
            msg += "\nYou can deploy chef-solo by typing:\n"
            msg += "  cook node:{0} deploy_chef".format(env.host)
            abort(msg)
        with cd(node_work_path):
            # Install the remote copy of archive
            sudo('tar xzf {0}'.format(remote_archive))
            # Fix ownership
            sudo('chown -R root:root {0}'.format(node_work_path))
            # Remove the remote copy of archive
            sudo('rm {0}'.format(remote_archive))

<|endoftext|>


<commit_before># Copyright (c) 2015 Intracom S.A. Telecom Solutions. All rights reserved.
#
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution,
# and is available at http://www.eclipse.org/legal/epl-v10.html

"""
Builds on Mininet to emulate large-scale SDN networks
by creating distributed Mininet topologies
"""

import logging
import time
import mininet
import mininet.util
import mininet.net
import mininet.node
import mininet.link
import mininet.clean
import itertools
import net.topologies
import socket
import struct

logging.basicConfig(level=logging.DEBUG)

class Multinet(mininet.net.Mininet):

    """
    Superclass representing a Mininet topology that is being booted in custom
    way. Switches are being added in groups with certain delay between each
    group.
    """

    """
    name - class correspondence for the topologies
    """
    TOPOS = {
        'disconnected': net.topologies.DisconnectedTopo,
        'linear': net.topologies.LinearTopo,
        'ring': net.topologies.RingTopo,
        'mesh': net.topologies.MeshTopo
    }

    """
    name - class correspondence for the soft switches
    """
    SWITCH_CLASSES = {
        'ovsk': mininet.node.OVSKernelSwitch,
        'user': mininet.node.UserSwitch
    }

    def __init__(self, controller_ip, controller_port, switch_type, topo_type,
                 num_switches, group_size, group_delay_ms, hosts_per_switch,
                 dpid_offset, traffic_generation_duration_ms,
                 interpacket_delay_ms, auto_detect_hosts=False):
        """
        Call the super constructor and initialize any extra properties we want to user

        Args:
            controller_ip (str): The IP address of the RemoteController
            controller_port (int): The OpenFlow port of the RemoteController
            switch_type (str): The type of the soft switch to use for the emulation
            topo_type (str): The type of the topology to build
            num_switches (int): The number of the switches in the topology
            group_size (int): The number of switches in a gorup for gradual bootup
            group_delay_ms (int): The delay between the bootup of each group
            hosts_per_switch (int): The number of hosts connected to each switch
            dpid_offset (int): The dpid offset of this worker
            traffic_generation_duration_ms (int): The interval of time, during
                                                  which transmission of
                                                  Packet_IN's occures
            interpacket_delay_ms (int): The interval of time between 2
                                        Packet_IN transmissions
            auto_detect_hosts (bool): Enable or disable automatic host detection
        """
        self.__network_mask_bits = 8
        self.__base_network = '10.0.0.0'
        self.__network_ip_range = long(2 ** (32 - self.__network_mask_bits))
        self.__available_networks = long(2 ** self.__network_mask_bits -
            (self.ip2long(self.__base_network) / self.__network_ip_range))
        # Initialize the Mininet network of the worker, based on the dpid.
        # Each worker has its own network.
        if dpid_offset <= self.__available_networks:
            self.__mininet_network = self.long2ip(
                self.ip2long(self.__base_network) +
                (dpid_offset * self.__network_ip_range))
        else:
            error('Worker Mininet network is out of range.')
            raise ValueError('Worker Mininet network is out of range.')

        self._topo_type = topo_type
        self._num_switches = num_switches
        self._dpid_offset = dpid_offset
        self._group_size = group_size
        self._group_delay = float(group_delay_ms) / 1000
        self._hosts_per_switch = hosts_per_switch
        self.auto_detect_hosts = auto_detect_hosts
        self._controller_ip = controller_ip
        self._controller_port = controller_port
        self.booted_switches = 0
        self._traffic_generation_duration_ms = traffic_generation_duration_ms
        self._interpacket_delay_ms = interpacket_delay_ms

        super(
            Multinet,
            self).__init__(
            topo=self.TOPOS[
                self._topo_type](
                k=self._num_switches,
                n=self._hosts_per_switch,
                dpid=self._dpid_offset),
            switch=self.SWITCH_CLASSES[switch_type],
            host=mininet.node.Host,
            controller=mininet.node.RemoteController,
            link=mininet.link.Link,
            intf=mininet.link.Intf,
            build=False,
            xterms=False,
            cleanup=False,
            ipBase=self.__mininet_network + '/' + str(self.__network_mask_bits),
            inNamespace=False,
            autoSetMacs=False,
            autoStaticArp=False,
            autoPinCpus=False,
            listenPort=6634,
            waitConnected=False)
        self.ipBaseNum += self._dpid_offset

    def buildFromTopo(self, topo=None):
        """
        Build mininet from a topology object
        At the end of this function, everything should be connected and up.
        Use the dpid offset to distinguise the nodes between Multinet Instances
        """

        info = logging.info
        # Possibly we should clean up here and/or validate
        # the topo
        if self.cleanup:
            pass

        info('*** Creating network\n')

        if not self.controllers and self.controller:
            # Add a default controller
            info('*** Adding controller\n')
            try:
                classes = self.controller
                if not isinstance(classes, list):
                    classes = [classes]
                for i, cls in enumerate(classes):
                    # Allow Controller objects because nobody understands partial()
                    if isinstance(cls, mininet.node.Controller):
                        self.addController(controller=cls,
                                           ip=self._controller_ip,
                                           port=self._controller_port)
                    else:
                        self.addController(name='c{0}'.format(i),
                                           controller=cls,
                                           ip=self._controller_ip,
                                           port=self._controller_port)
            except:
                self.addController(name='c{0}'.format(i), controller=mininet.node.DefaultController)

        info('*** Adding hosts:\n')
        for hostName in topo.hosts():
            kwargs_host = topo.nodeInfo(hostName)
            self.addHost(hostName, **kwargs_host)
            info(hostName + ' ')

        info('\n*** Adding switches:\n')
        for switchName in topo.switches():
            # A bit ugly: add batch parameter if appropriate
            params = topo.nodeInfo(switchName)
            cls = params.get('cls', self.switch)
            params['dpid'] = None
            params['protocols'] = 'OpenFlow13'
            #if hasattr(cls, 'batchStartup'):
            #    params.setdefault('batch', True)
            self.addSwitch(switchName, **params)
            info(switchName + ' ')

        info('\n*** Adding links:\n')
        for srcName, dstName, params in topo.links(
                sort=True, withInfo=True):
            self.addLink(**params)
            info('(%s, %s) ' % (srcName, dstName))

        info('\n')

    def init_topology(self):
        """
        Init the topology
        """

        logging.info("[mininet] Initializing topology.")
        self.build()
        logging.info('[mininet] Topology initialized successfully. '
                     'Booted up {0} switches'.format(self._num_switches))

    def start_topology(self):
        """
        Start controller and switches.
        Do a gradual bootup.
        """
        info = logging.info
        if not self.built:
            self.build()
        info('*** Starting controller\n')
        for controller in self.controllers:
            info(controller.name + ' ')
            controller.start()
        info('\n')
        info('*** Starting %s switches\n' % len(self.switches))

        for ind, switch in enumerate(self.switches):
            if ind % self._group_size == 0:
                time.sleep(self._group_delay)
            logging.debug('[mininet] Starting switch with index {0}'.
                          format(ind + 1))
            info(switch.name + ' ')
            switch.start(self.controllers)
            self.booted_switches += 1

        started = {}
        for swclass, switches in itertools.groupby(
                sorted(self.switches, key=type), type):
            switches = tuple(switches)
            if hasattr(swclass, 'batchStartup'):
                success = swclass.batchStartup(switches)
                started.update({s: s for s in success})
        info('\n')
        if self.waitConn:
            self.waitConnected()

        logging.info('[mininet] Topology started successfully. '
                     'Booted up {0} switches'.format(self._num_switches))
        time.sleep(self._group_delay * 2)

        if self.auto_detect_hosts:
            self.detect_hosts(ping_cnt=50)

    def detect_hosts(self, ping_cnt=50):
        """
        Do a ping from each host to the void to send a PACKET_IN to the controller
        and enable the controller host detector

        Args:
            ping_cnt: Number of pings to send from each host
        """
        for host in self.hosts:
            # ping the void
            host.sendCmd('ping -c{0} {1}'.format(str(ping_cnt),
                                                 str(self.controllers[0].IP())))

        logging.debug('[mininet] Hosts should be visible now')

    def get_switches(self):
        """Returns the total number of switches of the topology

        Returns:
            (int): number of switches in the topology
        """
        return self.booted_switches

    def stop_topology(self):
        """
        Stops the topology
        """

        logging.info('[mininet] Halting topology. Terminating switches.')
        for h in self.hosts:
            h.sendInt()
        mininet.clean.cleanup()
        self.switches = []
        self.hosts = []
        self.links = []
        self.controllers = []
        self.built = False
        self.booted_switches = 0
        logging.info('[mininet] Topology halted successfully')

    def ping_all(self):
        """
        All-to-all host pinging used for testing.
        """
        self.pingAll(timeout=None)

    def get_flows(self):
        """
        Getting flows from switches
        """
        logging.info('[get_flows] Getting flows from switches.')
        flow_number_total = 0
        t_start = time.time()
        for switch in self.switches:
            """
            logging.debug('[get_flows] Raw command output' + switch.dpctl('-O OpenFlow13 dump-aggregate'))
            for stat_item in switch.dpctl('-O OpenFlow13 dump-aggregate').split(' '):
                if stat_item.split('=')[0] == 'flow_count' and len(stat_item.split('=')) == 2:
                    flow_number_total += int(stat_item.split('=')[-1])
            """
            #-------------------------------------------------------------------
            flows_list = switch.dpctl('-O OpenFlow13 dump-flows').split('\n')
            flow_number = len(flows_list) - 2
            flow_number_total += flow_number
            #-------------------------------------------------------------------
        logging.debug('[get_flows] number of flows: {0}'.format(flow_number_total))
        get_flow_latency = time.time() - t_start
        logging.info('[get_flows] Flow latency interval on worker: {0} [sec]]'.
                     format(get_flow_latency))
        return flow_number_total

    def generate_mac_address_pairs(self, current_mac):
        """
        Generated tuple of source/destination mac addresses

        Args:
          current_mac (str): The last generated mac used for traffic
                             generation. It is used as reference to generate
                             the next pair of source and destination mac
                             addresses.
        """

        # We place an extra 11 in front of base_mac and these digits are
        # filtered out, in order to get a full range of mac addresses. We get
        # generated mac addresses as string separated with : for every 2
        # characters.
        base_mac = 0x11000000000000
        generated_mac = hex(base_mac + int(current_mac, 16))

        source_mac = ':'.join(''.join(pair) for pair in zip(*[iter(hex(int(generated_mac, 16) + 1))]*2))[6:]
        dest_mac = ':'.join(''.join(pair) for pair in zip(*[iter(hex(int(generated_mac, 16) + 2))]*2))[6:]
        return source_mac, dest_mac

    def generate_traffic(self):
        """
        Traffic generation from switches to controller
        """

        logging.info('[mininet] Generating traffic from switches.')

        if not self._hosts_per_switch>1:
            raise AssertionError(
                '_hosts_per_switch must be at least 2 or greater.')
        traffic_transmission_delay = self._interpacket_delay_ms / 1000
        traffic_transmission_interval = \
            self._traffic_generation_duration_ms / 1000
        host_index = 0

        transmission_start = time.time()
        last_mac = hex(int(hex(self._dpid_offset) + '00000000', 16) + 0xffffffff)
        current_mac = hex(int(last_mac, 16) - 0x0000ffffffff + 0x000000000001)

        while (time.time() - transmission_start) <= traffic_transmission_interval:
            src_mac, dst_mac = self.generate_mac_address_pairs(current_mac)
            current_mac = hex(int(current_mac, 16) + 2)
            # This is the place where generation of flows happens. In order to
            # work properly we must configure the ODL controller with L2Switch
            # plugin and each switch of the topology must have at least 2
            # hosts.
            # Step1:
            # From host1 of switch1 we initially send a Gratuitous ARP Reply.
            # We encapsulate this Reply in an ethernet frame with a specific
            # src and dst MAC addresses, generated from the MAC address
            # generator in this class.
            # Step2:
            # We repeat the above steps from host2 of switch1 reversing the src
            # and dst MAC addresses of the ethernet frame.
            # The above sequence has as a result to trigger ODL controller to
            # respond with 2 FlowMod messages in order to establish a datapath
            # between the 2 hosts
            self.hosts[host_index].sendCmd(
                'sudo mz -a {0} -b {1} -t arp'.format(src_mac, dst_mac))
            # We brake transmission delay and we place a delay between the
            # transmission of the 2 Gratuitous ARP messages in order to avoid
            # bursts of messages
            time.sleep(traffic_transmission_delay/2)
            self.hosts[host_index + 1].sendCmd(
                'sudo mz -a {0} -b {1} -t arp'.format(dst_mac, src_mac))
            time.sleep(traffic_transmission_delay/2)
            host_index += self._hosts_per_switch

            if host_index >= len(self.hosts):
                for host in self.hosts:
                    host.waitOutput()
                host_index = 0

            if int(current_mac, 16) >= int(last_mac, 16):
                current_mac = \
                    hex(int(last_mac, 16) - 0x0000ffffffff + 0x000000000001)
                # The minimum controller hard_timeout is 1 second.
                # Retransmission using the init_mac must start after the
                # minimum hard_timeout interval
                if (time.time - transmission_start) < 1:
                    time.sleep(1 - (time.time - transmission_start))
        # Cleanup hosts console outputs and write flags after finishing
        # transmission
        for host in self.hosts:
            host.waitOutput()

    def ip2long(self, ip_str):
        """
        Convert an IP string to long number
        Args:
            ip_str (str): IP address as a string
        """
        packedIP = socket.inet_aton(ip_str)
        return struct.unpack("!L", packedIP)[0]

    def long2ip(self, ip_lng):
        """
        Convert long number to IP string
        Args:
            ip_lng (long): IP address as a long number
        """
        return socket.inet_ntoa(struct.pack('!L', ip_lng))
<commit_msg>Restored functionality of dump-aggregate for dumping switches<commit_after># Copyright (c) 2015 Intracom S.A. Telecom Solutions. All rights reserved.
#
# This program and the accompanying materials are made available under the
# terms of the Eclipse Public License v1.0 which accompanies this distribution,
# and is available at http://www.eclipse.org/legal/epl-v10.html

"""
Builds on Mininet to emulate large-scale SDN networks
by creating distributed Mininet topologies
"""

import logging
import time
import mininet
import mininet.util
import mininet.net
import mininet.node
import mininet.link
import mininet.clean
import itertools
import net.topologies
import socket
import struct

logging.basicConfig(level=logging.DEBUG)

class Multinet(mininet.net.Mininet):

    """
    Superclass representing a Mininet topology that is being booted in custom
    way. Switches are being added in groups with certain delay between each
    group.
    """

    """
    name - class correspondence for the topologies
    """
    TOPOS = {
        'disconnected': net.topologies.DisconnectedTopo,
        'linear': net.topologies.LinearTopo,
        'ring': net.topologies.RingTopo,
        'mesh': net.topologies.MeshTopo
    }

    """
    name - class correspondence for the soft switches
    """
    SWITCH_CLASSES = {
        'ovsk': mininet.node.OVSKernelSwitch,
        'user': mininet.node.UserSwitch
    }

    def __init__(self, controller_ip, controller_port, switch_type, topo_type,
                 num_switches, group_size, group_delay_ms, hosts_per_switch,
                 dpid_offset, traffic_generation_duration_ms,
                 interpacket_delay_ms, auto_detect_hosts=False):
        """
        Call the super constructor and initialize any extra properties we want to user

        Args:
            controller_ip (str): The IP address of the RemoteController
            controller_port (int): The OpenFlow port of the RemoteController
            switch_type (str): The type of the soft switch to use for the emulation
            topo_type (str): The type of the topology to build
            num_switches (int): The number of the switches in the topology
            group_size (int): The number of switches in a gorup for gradual bootup
            group_delay_ms (int): The delay between the bootup of each group
            hosts_per_switch (int): The number of hosts connected to each switch
            dpid_offset (int): The dpid offset of this worker
            traffic_generation_duration_ms (int): The interval of time, during
                                                  which transmission of
                                                  Packet_IN's occures
            interpacket_delay_ms (int): The interval of time between 2
                                        Packet_IN transmissions
            auto_detect_hosts (bool): Enable or disable automatic host detection
        """
        self.__network_mask_bits = 8
        self.__base_network = '10.0.0.0'
        self.__network_ip_range = long(2 ** (32 - self.__network_mask_bits))
        self.__available_networks = long(2 ** self.__network_mask_bits -
            (self.ip2long(self.__base_network) / self.__network_ip_range))
        # Initialize the Mininet network of the worker, based on the dpid.
        # Each worker has its own network.
        if dpid_offset <= self.__available_networks:
            self.__mininet_network = self.long2ip(
                self.ip2long(self.__base_network) +
                (dpid_offset * self.__network_ip_range))
        else:
            error('Worker Mininet network is out of range.')
            raise ValueError('Worker Mininet network is out of range.')

        self._topo_type = topo_type
        self._num_switches = num_switches
        self._dpid_offset = dpid_offset
        self._group_size = group_size
        self._group_delay = float(group_delay_ms) / 1000
        self._hosts_per_switch = hosts_per_switch
        self.auto_detect_hosts = auto_detect_hosts
        self._controller_ip = controller_ip
        self._controller_port = controller_port
        self.booted_switches = 0
        self._traffic_generation_duration_ms = traffic_generation_duration_ms
        self._interpacket_delay_ms = interpacket_delay_ms

        super(
            Multinet,
            self).__init__(
            topo=self.TOPOS[
                self._topo_type](
                k=self._num_switches,
                n=self._hosts_per_switch,
                dpid=self._dpid_offset),
            switch=self.SWITCH_CLASSES[switch_type],
            host=mininet.node.Host,
            controller=mininet.node.RemoteController,
            link=mininet.link.Link,
            intf=mininet.link.Intf,
            build=False,
            xterms=False,
            cleanup=False,
            ipBase=self.__mininet_network + '/' + str(self.__network_mask_bits),
            inNamespace=False,
            autoSetMacs=False,
            autoStaticArp=False,
            autoPinCpus=False,
            listenPort=6634,
            waitConnected=False)
        self.ipBaseNum += self._dpid_offset

    def buildFromTopo(self, topo=None):
        """
        Build mininet from a topology object
        At the end of this function, everything should be connected and up.
        Use the dpid offset to distinguise the nodes between Multinet Instances
        """

        info = logging.info
        # Possibly we should clean up here and/or validate
        # the topo
        if self.cleanup:
            pass

        info('*** Creating network\n')

        if not self.controllers and self.controller:
            # Add a default controller
            info('*** Adding controller\n')
            try:
                classes = self.controller
                if not isinstance(classes, list):
                    classes = [classes]
                for i, cls in enumerate(classes):
                    # Allow Controller objects because nobody understands partial()
                    if isinstance(cls, mininet.node.Controller):
                        self.addController(controller=cls,
                                           ip=self._controller_ip,
                                           port=self._controller_port)
                    else:
                        self.addController(name='c{0}'.format(i),
                                           controller=cls,
                                           ip=self._controller_ip,
                                           port=self._controller_port)
            except:
                self.addController(name='c{0}'.format(i), controller=mininet.node.DefaultController)

        info('*** Adding hosts:\n')
        for hostName in topo.hosts():
            kwargs_host = topo.nodeInfo(hostName)
            self.addHost(hostName, **kwargs_host)
            info(hostName + ' ')

        info('\n*** Adding switches:\n')
        for switchName in topo.switches():
            # A bit ugly: add batch parameter if appropriate
            params = topo.nodeInfo(switchName)
            cls = params.get('cls', self.switch)
            params['dpid'] = None
            params['protocols'] = 'OpenFlow13'
            #if hasattr(cls, 'batchStartup'):
            #    params.setdefault('batch', True)
            self.addSwitch(switchName, **params)
            info(switchName + ' ')

        info('\n*** Adding links:\n')
        for srcName, dstName, params in topo.links(
                sort=True, withInfo=True):
            self.addLink(**params)
            info('(%s, %s) ' % (srcName, dstName))

        info('\n')

    def init_topology(self):
        """
        Init the topology
        """

        logging.info("[mininet] Initializing topology.")
        self.build()
        logging.info('[mininet] Topology initialized successfully. '
                     'Booted up {0} switches'.format(self._num_switches))

    def start_topology(self):
        """
        Start controller and switches.
        Do a gradual bootup.
        """
        info = logging.info
        if not self.built:
            self.build()
        info('*** Starting controller\n')
        for controller in self.controllers:
            info(controller.name + ' ')
            controller.start()
        info('\n')
        info('*** Starting %s switches\n' % len(self.switches))

        for ind, switch in enumerate(self.switches):
            if ind % self._group_size == 0:
                time.sleep(self._group_delay)
            logging.debug('[mininet] Starting switch with index {0}'.
                          format(ind + 1))
            info(switch.name + ' ')
            switch.start(self.controllers)
            self.booted_switches += 1

        started = {}
        for swclass, switches in itertools.groupby(
                sorted(self.switches, key=type), type):
            switches = tuple(switches)
            if hasattr(swclass, 'batchStartup'):
                success = swclass.batchStartup(switches)
                started.update({s: s for s in success})
        info('\n')
        if self.waitConn:
            self.waitConnected()

        logging.info('[mininet] Topology started successfully. '
                     'Booted up {0} switches'.format(self._num_switches))
        time.sleep(self._group_delay * 2)

        if self.auto_detect_hosts:
            self.detect_hosts(ping_cnt=50)

    def detect_hosts(self, ping_cnt=50):
        """
        Do a ping from each host to the void to send a PACKET_IN to the controller
        and enable the controller host detector

        Args:
            ping_cnt: Number of pings to send from each host
        """
        for host in self.hosts:
            # ping the void
            host.sendCmd('ping -c{0} {1}'.format(str(ping_cnt),
                                                 str(self.controllers[0].IP())))

        logging.debug('[mininet] Hosts should be visible now')

    def get_switches(self):
        """Returns the total number of switches of the topology

        Returns:
            (int): number of switches in the topology
        """
        return self.booted_switches

    def stop_topology(self):
        """
        Stops the topology
        """

        logging.info('[mininet] Halting topology. Terminating switches.')
        for h in self.hosts:
            h.sendInt()
        mininet.clean.cleanup()
        self.switches = []
        self.hosts = []
        self.links = []
        self.controllers = []
        self.built = False
        self.booted_switches = 0
        logging.info('[mininet] Topology halted successfully')

    def ping_all(self):
        """
        All-to-all host pinging used for testing.
        """
        self.pingAll(timeout=None)

    def get_flows(self):
        """
        Getting flows from switches
        """
        logging.info('[get_flows] Getting flows from switches.')
        flow_number_total = 0
        t_start = time.time()
        for switch in self.switches:
            #logging.debug('[get_flows] Raw command output' +  \
            #              switch.dpctl('-O OpenFlow13 dump-aggregate'))
            for stat_item in switch.dpctl('-O OpenFlow13 dump-aggregate').split(' '):
                if stat_item.split('=')[0] == 'flow_count' and len(stat_item.split('=')) == 2:
                    flow_number_total += int(stat_item.split('=')[-1])
            """
            flows_list = switch.dpctl('-O OpenFlow13 dump-flows').split('\n')
            flow_number = len(flows_list) - 2
            flow_number_total += flow_number
            """
            #-------------------------------------------------------------------
        logging.debug('[get_flows] number of flows: {0}'.format(flow_number_total))
        get_flow_latency = time.time() - t_start
        logging.info('[get_flows] Flow latency interval on worker: {0} [sec]]'.
                     format(get_flow_latency))
        return flow_number_total

    def generate_mac_address_pairs(self, current_mac):
        """
        Generated tuple of source/destination mac addresses

        Args:
          current_mac (str): The last generated mac used for traffic
                             generation. It is used as reference to generate
                             the next pair of source and destination mac
                             addresses.
        """

        # We place an extra 11 in front of base_mac and these digits are
        # filtered out, in order to get a full range of mac addresses. We get
        # generated mac addresses as string separated with : for every 2
        # characters.
        base_mac = 0x11000000000000
        generated_mac = hex(base_mac + int(current_mac, 16))

        source_mac = ':'.join(''.join(pair) for pair in zip(*[iter(hex(int(generated_mac, 16) + 1))]*2))[6:]
        dest_mac = ':'.join(''.join(pair) for pair in zip(*[iter(hex(int(generated_mac, 16) + 2))]*2))[6:]
        return source_mac, dest_mac

    def generate_traffic(self):
        """
        Traffic generation from switches to controller
        """

        logging.info('[mininet] Generating traffic from switches.')

        if not self._hosts_per_switch>1:
            raise AssertionError(
                '_hosts_per_switch must be at least 2 or greater.')
        traffic_transmission_delay = self._interpacket_delay_ms / 1000
        traffic_transmission_interval = \
            self._traffic_generation_duration_ms / 1000
        host_index = 0

        transmission_start = time.time()
        last_mac = hex(int(hex(self._dpid_offset) + '00000000', 16) + 0xffffffff)
        current_mac = hex(int(last_mac, 16) - 0x0000ffffffff + 0x000000000001)

        while (time.time() - transmission_start) <= traffic_transmission_interval:
            src_mac, dst_mac = self.generate_mac_address_pairs(current_mac)
            current_mac = hex(int(current_mac, 16) + 2)
            # This is the place where generation of flows happens. In order to
            # work properly we must configure the ODL controller with L2Switch
            # plugin and each switch of the topology must have at least 2
            # hosts.
            # Step1:
            # From host1 of switch1 we initially send a Gratuitous ARP Reply.
            # We encapsulate this Reply in an ethernet frame with a specific
            # src and dst MAC addresses, generated from the MAC address
            # generator in this class.
            # Step2:
            # We repeat the above steps from host2 of switch1 reversing the src
            # and dst MAC addresses of the ethernet frame.
            # The above sequence has as a result to trigger ODL controller to
            # respond with 2 FlowMod messages in order to establish a datapath
            # between the 2 hosts
            self.hosts[host_index].sendCmd(
                'sudo mz -a {0} -b {1} -t arp'.format(src_mac, dst_mac))
            # We brake transmission delay and we place a delay between the
            # transmission of the 2 Gratuitous ARP messages in order to avoid
            # bursts of messages
            time.sleep(traffic_transmission_delay/2)
            self.hosts[host_index + 1].sendCmd(
                'sudo mz -a {0} -b {1} -t arp'.format(dst_mac, src_mac))
            time.sleep(traffic_transmission_delay/2)
            host_index += self._hosts_per_switch

            if host_index >= len(self.hosts):
                for host in self.hosts:
                    host.waitOutput()
                host_index = 0

            if int(current_mac, 16) >= int(last_mac, 16):
                current_mac = \
                    hex(int(last_mac, 16) - 0x0000ffffffff + 0x000000000001)
                # The minimum controller hard_timeout is 1 second.
                # Retransmission using the init_mac must start after the
                # minimum hard_timeout interval
                if (time.time - transmission_start) < 1:
                    time.sleep(1 - (time.time - transmission_start))
        # Cleanup hosts console outputs and write flags after finishing
        # transmission
        for host in self.hosts:
            host.waitOutput()

    def ip2long(self, ip_str):
        """
        Convert an IP string to long number
        Args:
            ip_str (str): IP address as a string
        """
        packedIP = socket.inet_aton(ip_str)
        return struct.unpack("!L", packedIP)[0]

    def long2ip(self, ip_lng):
        """
        Convert long number to IP string
        Args:
            ip_lng (long): IP address as a long number
        """
        return socket.inet_ntoa(struct.pack('!L', ip_lng))
<|endoftext|>


<commit_before>from __future__ import print_function

import os

from mdtraj.utils import in_units_of
from mdtraj.formats.registry import _FormatRegistry
from six import string_types

__all__ = ['load_gromacs', 'GROMACSTopologyFile']

@_FormatRegistry.register_loader('.gro')
@_FormatRegistry.register_loader('.top')
def load_gromacstop(top_filename=None, gro_filename=None):
    """Load a GROMACS .top file and accompanying .gro from disk.

    Parameters
    ----------
    filename : str
        Path to data file.
    optional_nodes : str, optional, default=['bonds']
        Read specified nodes in file other than 'box', 'position' and 'type'.

    Returns
    -------
    compound : mb.Compound

    """
    # TODO: If only one is given, assume the other has the same name, different
    #  extension.

    # if not isinstance(filename, string_types):
    #     raise TypeError('Filename must be of type string for load_lammpstrj. '
    #                     'you supplied {0}'.format(type(filename)))
    #
    # with GROMACSTopologyFile(filename) as f:
    #     compound = f.read()
    # return compound
    raise NotImplementedError

@_FormatRegistry.register_fileobject('.gro')
@_FormatRegistry.register_fileobject('.top')
class GROMACSTopologyFile(object):
    """ """
    def __init__(self, top_filename, gro_filename, mode='r', force_overwrite=True):
        """Open a GROMACS .gro/.top file pair for reading/writing. """

        self._is_open = False
        self._top_filename = top_filename
        self._gro_filename = gro_filename
        self._mode = mode
        self._frame_index = 0
        # Track which line we're on. this is not essential, but its useful
        # when reporting errors to the user to say what line it occured on.
        # Looking at you GROMACS...
        self._line_counter = 0

        if mode == 'r':
            if not os.path.exists(top_filename):
                raise IOError("The file '{0}' doesn't exist".format(top_filename))
            if not os.path.exists(gro_filename):
                raise IOError("The file '{0}' doesn't exist".format(gro_filename))
            self._topfh = open(top_filename, 'r')
            self._grofh = open(gro_filename, 'r')
            self._is_open = True

            from mbuild.compound import Compound
            self.compound = Compound()
        elif mode == 'w':
            if os.path.exists(top_filename) and not force_overwrite:
                raise IOError("The file '{0}' already exists".format(top_filename))
            if os.path.exists(gro_filename) and not force_overwrite:
                raise IOError("The file '{0}' already exists".format(gro_filename))
            self._topfh = open(top_filename, 'w')
            self._grofh = open(gro_filename, 'w')
            self._is_open = True
        else:
            raise ValueError('mode must be one of "r" or "w". '
                             'you supplied "{0}"'.format(mode))

        self.u = dict()
        self.u['radians'] = 'radians'
        self.u['degrees'] = 'degrees'
        self.u['velocity'] = 'nanometers/picosecond'
        self.u['energy'] = 'kilojoules/mole'
        self.u['mass'] = 'grams/mole'
        self.u['charge'] = 'elementary_charge'
        self.u['mole'] = 'mole'

    def read(self):
        """ """
        raise NotImplementedError

    def _read_unitcell_vectors(self, box):
        """Parse unitcell vectors from box node.  """
        pass

    def write(self, traj):
        """Output a Trajectory as a GROMACS data file.

        Args:
            traj (md.Trajectory): The Trajectory to be output.
            filename (str, optional): Path of the output file.

        """
        basename = os.path.splitext(self._top_filename)[0]
        # .top file
        if traj.top.forcefield == 'opls-aa':
            self._topfh.write('#include "oplsaa.ff/forcefield.itp"\n\n')

        # TODO: iteration over chains
        self._topfh.write('\n[ moleculetype ]\n')
        self._topfh.write('; name nrexcl\n')
        self._topfh.write('mbuild 3\n')

        self._topfh.write('\n[ atoms ]\n')
        for i, atom in enumerate(traj.topology._atoms):
            self._topfh.write('{:8d} {:5s} {:8d} {:5s} {:5s} {:8d} {:8.4f} {:8.4f}\n'.format(
                atom.index + 1, 'opls_' + atom.atomtype, atom.residue.index,
                atom.residue.name,
                atom.name, i+1, atom.charge, atom.element.mass))

        if traj.topology._ff_bonds:
            self._topfh.write('\n[ bonds ]\n')
            for bond in traj.topology.ff_bonds:
                self._topfh.write('{:d} {:d} {:d}\n'.format(
                    bond.atom1.index + 1, bond.atom2.index + 1, 1))

        if traj.topology._ff_angles:
            self._topfh.write('\n[ angles ]\n')
            for angle in traj.topology.ff_angles:
                self._topfh.write('{:d} {:d} {:d} {:d}\n'.format(
                    angle.atom1.index + 1, angle.atom2.index + 1,
                    angle.atom3.index + 1, 1))

        if traj.topology._ff_dihedrals:
            self._topfh.write('\n[ dihedrals ]\n')
            for dihedral in traj.topology.ff_dihedrals:
                self._topfh.write('{:d} {:d} {:d} {:d} {:d}\n'.format(
                    dihedral.atom1.index + 1, dihedral.atom2.index + 1,
                    dihedral.atom3.index + 1, dihedral.atom4.index + 1, 3))

        self._topfh.write('\n[ system ]\n')
        self._topfh.write('{}\n'.format(basename))

        self._topfh.write('\n[ molecules ]\n')
        self._topfh.write('mbuild 1\n')

        # .gro file
        self._grofh.write('{}\n'.format(basename))
        self._grofh.write('{}\n'.format(traj.n_atoms))
        for atom, xyz in zip(traj.top._atoms, traj.xyz[0]):
            self._grofh.write('{:5d}{:<5s}{:5s}{:5d}'
                    '{:8.3f}{:8.3f}{:8.3f}{:8.3f}{:8.3f}{:8.3f}\n'.format(
                atom.residue.index, atom.residue.name, atom.name, atom.index + 1,
                xyz[0], xyz[1], xyz[2], 0.0, 0.0, 0.0))
        box = traj.unitcell_vectors[0]
        self._grofh.write('{:10.5f}{:10.5f}{:10.5f}'
                          '{:10.5f}{:10.5f}{:10.5f}'
                          '{:10.5f}{:10.5f}{:10.5f}'.format(
            box[0, 0], box[1, 1], box[2, 2],
            box[1, 0], box[2, 0], box[0, 1],
            box[2, 1], box[0, 2], box[1, 2]))
        self._grofh.write('\n')

    def close(self):
        """Close the GROMACS file. """
        if self._is_open:
            self._topfh.close()
            self._grofh.close()
            self._is_open = False

    def __del__(self):
        self.close()

    def __enter__(self):
        """Support the context manager protocol. """
        return self

    def __exit__(self, *exc_info):
        """Support the context manager protocol. """
        self.close()

if __name__ == "__main__":
    from mbuild.examples.ethane.ethane import Ethane

    Ethane().save('ethane.gro', forcefield='opls-aa')

<commit_msg>gromacs writer updates<commit_after>from __future__ import print_function

import os

import simtk.unit as units
#from mdtraj.utils import in_units_of
from mdtraj.formats.registry import _FormatRegistry
from six import string_types

__all__ = ['load_gromacs', 'GROMACSTopologyFile']

@_FormatRegistry.register_loader('.gro')
@_FormatRegistry.register_loader('.top')
def load_gromacstop(top_filename=None, gro_filename=None):
    """Load a GROMACS .top file and accompanying .gro from disk.

    Parameters
    ----------
    filename : str
        Path to data file.
    optional_nodes : str, optional, default=['bonds']
        Read specified nodes in file other than 'box', 'position' and 'type'.

    Returns
    -------
    compound : mb.Compound

    """
    # TODO: If only one is given, assume the other has the same name, different
    #  extension.

    # if not isinstance(filename, string_types):
    #     raise TypeError('Filename must be of type string for load_lammpstrj. '
    #                     'you supplied {0}'.format(type(filename)))
    #
    # with GROMACSTopologyFile(filename) as f:
    #     compound = f.read()
    # return compound
    raise NotImplementedError

@_FormatRegistry.register_fileobject('.gro')
@_FormatRegistry.register_fileobject('.top')
class GROMACSTopologyFile(object):
    """ """
    def __init__(self, top_filename, gro_filename, mode='r', force_overwrite=True):
        """Open a GROMACS .gro/.top file pair for reading/writing. """

        self._is_open = False
        self._top_filename = top_filename
        self._gro_filename = gro_filename
        self._mode = mode
        self._frame_index = 0
        # Track which line we're on. this is not essential, but its useful
        # when reporting errors to the user to say what line it occured on.
        # Looking at you GROMACS...
        self._line_counter = 0

        if mode == 'r':
            if not os.path.exists(top_filename):
                raise IOError("The file '{0}' doesn't exist".format(top_filename))
            if not os.path.exists(gro_filename):
                raise IOError("The file '{0}' doesn't exist".format(gro_filename))
            self._topfh = open(top_filename, 'r')
            self._grofh = open(gro_filename, 'r')
            self._is_open = True

            from mbuild.compound import Compound
            self.compound = Compound()
        elif mode == 'w':
            if os.path.exists(top_filename) and not force_overwrite:
                raise IOError("The file '{0}' already exists".format(top_filename))
            if os.path.exists(gro_filename) and not force_overwrite:
                raise IOError("The file '{0}' already exists".format(gro_filename))
            self._topfh = open(top_filename, 'w')
            self._grofh = open(gro_filename, 'w')
            self._is_open = True
        else:
            raise ValueError('mode must be one of "r" or "w". '
                             'you supplied "{0}"'.format(mode))

        self.u = dict()
        self.u['radians'] = 'radians'
        self.u['degrees'] = 'degrees'
        self.u['distance'] = 'nanometers'
        self.u['velocity'] = 'nanometers/picosecond'
        self.u['energy'] = 'kilojoules/mole'
        self.u['mass'] = 'grams/mole'
        self.u['charge'] = 'elementary_charge'
        self.u['mole'] = 'mole'

    def read(self):
        """ """
        raise NotImplementedError

    def _read_unitcell_vectors(self, box):
        """Parse unitcell vectors from box node.  """
        pass

    def write(self, traj):
        """Output a Trajectory as a GROMACS data file.

        Args:
            traj (md.Trajectory): The Trajectory to be output.
            filename (str, optional): Path of the output file.

        """
        basename = os.path.splitext(self._top_filename)[0]
        # .top file
        if traj.top.forcefield == 'opls-aa':
            self._topfh.write('#include "oplsaa.ff/forcefield.itp"\n\n')

        # TODO: iteration over chains
        self._topfh.write('\n[ moleculetype ]\n')
        self._topfh.write('; name nrexcl\n')
        self._topfh.write('mbuild 3\n')

        self._topfh.write('\n[ atoms ]\n')
        for i, atom in enumerate(traj.topology._atoms):
            self._topfh.write('{:8d} {:5s} {:8d} {:5s} {:5s} {:8d} {:8.4f} {:8.4f}\n'.format(
                atom.index + 1, atom.atomtype, atom.residue.index,
                atom.residue.name,
                atom.bondtype, i+1, atom.charge, atom.element.mass))

        if traj.topology._ff_pairs:
            self._topfh.write('\n[ pairs ]\n')
            for atom1, atom2 in traj.topology.ff_pairs:
                self._topfh.write('{:d} {:d}\n'.format(
                    atom1.index + 1, atom2.index + 1))

        if traj.topology._ff_bonds:
            self._topfh.write('\n[ bonds ]\n')
            for bond in traj.topology.ff_bonds:
                self._topfh.write('{:d} {:d} {:d}\n'.format(
                    bond.atom1.index + 1, bond.atom2.index + 1, 1))

        if traj.topology._ff_angles:
            self._topfh.write('\n[ angles ]\n')
            for angle in traj.topology.ff_angles:
                self._topfh.write('{:d} {:d} {:d} {:d}\n'.format(
                    angle.atom1.index + 1, angle.atom2.index + 1,
                    angle.atom3.index + 1, 1))

        if traj.topology._ff_dihedrals:
            self._topfh.write('\n[ dihedrals ]\n')
            for dihedral in traj.topology.ff_dihedrals:
                self._topfh.write('{:d} {:d} {:d} {:d} {:d}\n'.format(
                    dihedral.atom1.index + 1, dihedral.atom2.index + 1,
                    dihedral.atom3.index + 1, dihedral.atom4.index + 1, 3))

        self._topfh.write('\n[ system ]\n')
        self._topfh.write('{}\n'.format(basename))

        self._topfh.write('\n[ molecules ]\n')
        self._topfh.write('mbuild 1\n')

        # .gro file
        self._grofh.write('{}\n'.format(basename))
        self._grofh.write('{}\n'.format(traj.n_atoms))
        for atom, xyz in zip(traj.top._atoms, traj.xyz[0]):
            self._grofh.write('{:5d}{:<5s}{:5s}{:5d}'
                    '{:8.3f}{:8.3f}{:8.3f}{:8.3f}{:8.3f}{:8.3f}\n'.format(
                atom.residue.index, atom.residue.name, atom.bondtype, atom.index + 1,
                xyz[0], xyz[1], xyz[2], 0.0, 0.0, 0.0))
        box = traj.unitcell_vectors[0]
        self._grofh.write('{:10.5f}{:10.5f}{:10.5f}'
                          '{:10.5f}{:10.5f}{:10.5f}'
                          '{:10.5f}{:10.5f}{:10.5f}'.format(
            box[0, 0], box[1, 1], box[2, 2],
            box[1, 0], box[2, 0], box[0, 1],
            box[2, 1], box[0, 2], box[1, 2]))
        self._grofh.write('\n')

    def close(self):
        """Close the GROMACS file. """
        if self._is_open:
            self._topfh.close()
            self._grofh.close()
            self._is_open = False

    def __del__(self):
        self.close()

    def __enter__(self):
        """Support the context manager protocol. """
        return self

    def __exit__(self, *exc_info):
        """Support the context manager protocol. """
        self.close()

if __name__ == "__main__":
    from mbuild.examples.ethane.ethane import Ethane

    Ethane().save('ethane.gro', forcefield='opls-aa')

<|endoftext|>


<commit_before>"""Integer utilities and bitvectors.

This file is part of Titanic, available under the MIT license.

The BV class implements bitvectors of known size. Bitvectors
can be shifted and concatenated, and bits or sub-vectors can be
extracted. They can be compared for equality, but not order,
and interpreted as (unsigned only?) integers.

Also provides small utilities are provided for integers:
  msb(x): "most significant bit" of x, as x.bit_length()
  floorlog2(x): x.bit_length() - 1, 0 if x is 0
  ctz(x): "count trailing zeros"
"""

import typing

class BV(object):
    pass

def mask(n: int) -> int:
    """Produces a bitmask of n 1s."""
    return ~(-1 << n)

def mask_from(start: int, end: int) -> int:
    """Produces a bitmask of 1s from start to end."""
    return (~(-1 << (end - start))) << start

def msb(x: int) -> int:
    return x.bit_length()

def floorlog2(x: int) -> int:
    return max(x.bit_length() - 1, 0)

def _de_bruijn(k: int, n: int) -> typing.List[int]:
    """Generate a De Bruijn sequence.

    This is a piece of mathematical heavy machinery needed for O(1) clz on
    integers of bounded size. The algorithm is adapted from chapter 7 of
    Frank Ruskey's "Combinatorial Generation".

    Args:
        k: The number of symbols in the alphabet. Should be > 1.
        n: The length of subsequences. Should be > 1.

    Returns:
        The De Bruijn sequence, as a list of integer indices.

    >>> _de_bruijn(2, 3)
    [0, 0, 0, 1, 0, 1, 1, 1]
    >>> _de_bruijn(4, 2)
    [0, 0, 1, 0, 2, 0, 3, 1, 1, 2, 1, 3, 2, 2, 3, 3]
    >>> _de_bruijn(2, 5)
    [0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1]
    """
    a: typing.List[int] = [0] * k * n
    sequence: typing.List[int] = []

    def gen(t: int, p: int):
        if t > n:
            if n % p == 0:
                sequence.extend(a[1:p + 1])
        else:
            a[t] = a[t - p]
            gen(t + 1, p)
            for j in range(a[t - p] + 1, k):
                a[t] = j
                gen(t + 1, t)

    gen(1, 1)
    return sequence

def _init_clz_table(n: int) -> typing.Tuple[int, typing.List[int]]:
    """Setup for fast clz of ints with up to 2**n bits.

    This code initializes the De Bruijn sequence (as an integer) and
    the lookup table needed to do constant time (1 multiply and some
    bit twiddling) clz, as described in Philib Busch's
    "Computing Trailing Zeros HOWTO".

    See: http://7ooo.mooo.com/text/ComputingTrailingZerosHOWTO.pdf
    """
    nbits = 1 << n
    nmask = mask(nbits)

    db : int = 0

    sequence = _de_bruijn(2, n)
    # we want the first elements to go in the high bits, so reverse the order
    for i, bit in enumerate(reversed(sequence)):
        db |= bit << i

    table: typing.List[int] = [-1] * nbits

    shift_offset = nbits - n
    shifted_db = db
    for i in range(nbits):
        table[shifted_db >> shift_offset] = i
        # mask to nbits bits
        shifted_db = (shifted_db << 1) & nmask

    return db, table

# For small integers, it may be possible to perform ctz in constant
# time. This parameter control how small those integers have to be.
# Note that _CTZ_DB_LENGTH cannot be changed at runtime unless the
# De Bruijn sequence and table being used are also updated.
_CTZ_DB_LENGTH = 6

# These constants are used internally by the ctz algorithm. They are
# generated based on _CTZ_DB_LENGTH.
_CTZ_DB_THRESH = 1 << (1 << _CTZ_DB_LENGTH)
_CTZ_DB_SHIFT = (1 << _CTZ_DB_LENGTH) - _CTZ_DB_LENGTH
_CTZ_DB_MASK = mask(_CTZ_DB_LENGTH)
_CTZ_DB, _CTZ_DB_TABLE = _init_clz_table(_CTZ_DB_LENGTH)

# For this to work at all, the De Bruijn sequence we're using must start
# with _CTZ_DB_LENGTH zeros. All sequences returned by our generator should
# have this property, but just in case let's check.
assert(_CTZ_DB >> _CTZ_DB_SHIFT == 0)

def ctz(x: int) -> typing.Tuple[int, int]:
    """Count trailing zeros.

    Counts the zeros on the least-significant end of the binary representation
    of x.

    Args:
        x: An int.

    Returns:
        A tuple (zeros, x >> zeros) where zeros is the number of trailing
        zeros in x.

    >>> ctz(0)
    (0, 0)
    >>> ctz(1)
    (0, 1)
    >>> ctz(2)
    (1, 1)
    >>> ctz(40)
    (5, 3)
    >>> ctz(-40)
    (-5, 3)
    >>> ctz(1 << 100)
    (100, 1)

    Should be fast for all conceivable integers.
    """
    if x < _CTZ_DB_THRESH:
        zeros = _CTZ_DB_TABLE[
            (((x & -x) * _CTZ_DB) >> _CTZ_DB_SHIFT) & _CTZ_DB_MASK
        ]
        return zeros, x >> zeros
    else:
        return -1, x

def naive_ctz(x: int) -> typing.Tuple[int, int]:
    """Count trailing zeros, naively.

    The behavior should be identical to ctz(x). This is a slow reference
    implementation for checking correctness.
    """
    zeros = 0
    while x & 1 == 0 and x > 0:
        x = x >> 1
        zeros += 1
    return zeros, x
<commit_msg>improvements, still uses sqrt algo instead of log<commit_after>"""Integer utilities and bitvectors.

This file is part of Titanic, available under the MIT license.

The BV class implements bitvectors of known size. Bitvectors
can be shifted and concatenated, and bits or sub-vectors can be
extracted. They can be compared for equality, but not order,
and interpreted as (unsigned only?) integers.

Also provides small utilities are provided for integers:
  msb(x): "most significant bit" of x, as x.bit_length()
  floorlog2(x): x.bit_length() - 1, 0 if x is 0
  ctz(x): "count trailing zeros"
"""

import typing

class BV(object):
    pass

def mask(n: int) -> int:
    """Produces a bitmask of n 1s."""
    return ~(-1 << n)

def mask_from(start: int, end: int) -> int:
    """Produces a bitmask of 1s from start to end."""
    return (~(-1 << (end - start))) << start

def msb(x: int) -> int:
    return x.bit_length()

def floorlog2(x: int) -> int:
    return max(x.bit_length() - 1, 0)

def _de_bruijn(k: int, n: int) -> typing.List[int]:
    """Generate a De Bruijn sequence.

    This is a piece of mathematical heavy machinery needed for O(1) ctz on
    integers of bounded size. The algorithm is adapted from chapter 7 of
    Frank Ruskey's "Combinatorial Generation".

    Args:
        k: The number of symbols in the alphabet. Must be > 1.
        n: The length of subsequences. Should be >= 0.

    Returns:
        The De Bruijn sequence, as a list of integer indices.

    >>> _de_bruijn(2, 3)
    [0, 0, 0, 1, 0, 1, 1, 1]
    >>> _de_bruijn(4, 2)
    [0, 0, 1, 0, 2, 0, 3, 1, 1, 2, 1, 3, 2, 2, 3, 3]
    >>> _de_bruijn(2, 5)
    [0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1]
    """
    a: typing.List[int] = [0] * k * n
    sequence: typing.List[int] = []

    def gen(t: int, p: int):
        if t > n:
            if n % p == 0:
                sequence.extend(a[1:p + 1])
        else:
            a[t] = a[t - p]
            gen(t + 1, p)
            for j in range(a[t - p] + 1, k):
                a[t] = j
                gen(t + 1, t)

    gen(1, 1)
    return sequence

def _init_ctz_table(n: int) -> typing.Tuple[int, typing.List[int]]:
    """Setup for fast ctz of ints with up to 2**n bits.

    This code initializes the De Bruijn sequence (as an integer) and
    the lookup table needed to do constant time (1 multiply and some
    bit twiddling) ctz, as described in Philib Busch's
    "Computing Trailing Zeros HOWTO".

    See: http://7ooo.mooo.com/text/ComputingTrailingZerosHOWTO.pdf

    The following weird edge cases are correct, though they aren't exactly
    useful except to turn the fast ctz algorithm off:
    >>> _init_ctz_table(0)
    (0, [0])
    >>> _init_ctz_table(1)
    (1, [0, 1])

    This initialization is too small to be much use:
    >>> _init_ctz_table(4)
    (2479, [0, 1, 2, 5, 3, 9, 6, 11, 15, 4, 8, 10, 14, 7, 13, 12])
    """
    nbits = 1 << n
    nmask = mask(nbits)

    db : int = 0

    sequence = _de_bruijn(2, n)
    # we want the first elements to go in the high bits, so reverse the order
    for i, bit in enumerate(reversed(sequence)):
        db |= bit << i

    table: typing.List[int] = [-1] * nbits

    shift_offset = nbits - n
    shifted_db = db
    for i in range(nbits):
        table[shifted_db >> shift_offset] = i
        # mask to nbits bits
        shifted_db = (shifted_db << 1) & nmask

    return db, table

# For small integers, it may be possible to perform ctz in constant
# time. This parameter control how small those integers have to be.
# Note that _CTZ_DB_LENGTH cannot be changed at runtime unless the
# De Bruijn sequence and table being used are also updated. If this
# update is not atomic, everything could explode.
#
# With a bit of very quick tuning, it seems like we can set this at about
# 10 before the overhead of large multiplies starts to bite us, even for
# very small inputs.
_CTZ_DB_LENGTH = 10

# These constants are used internally by the ctz algorithm. They are
# generated based on _CTZ_DB_LENGTH.
_CTZ_DB_THRESH_BITS = 1 << _CTZ_DB_LENGTH
_CTZ_DB_SHIFT = (1 << _CTZ_DB_LENGTH) - _CTZ_DB_LENGTH
_CTZ_DB_MASK = mask(_CTZ_DB_LENGTH)
_CTZ_DB, _CTZ_DB_TABLE = _init_ctz_table(_CTZ_DB_LENGTH)

# For this to work at all, the De Bruijn sequence we're using must start
# with _CTZ_DB_LENGTH zeros. All sequences returned by our generator should
# have this property, but just in case let's check.
assert(_CTZ_DB >> _CTZ_DB_SHIFT == 0)

def ctz(x: int) -> int:
    """Count trailing zeros.

    Counts the zeros on the least-significant end of the binary representation
    of x.

    Args:
        x: An int. Must be >= 0.

    Returns:
        The number of trailing zeros.

    >>> ctz(0)
    0
    >>> ctz(1)
    0
    >>> ctz(2)
    1
    >>> ctz(40) # 2**3 * 5
    3
    >>> ctz(37 << 100)
    100

    Should be fast for all conceivable integers.
    """
    if x.bit_length() <= _CTZ_DB_THRESH_BITS:
        return _CTZ_DB_TABLE[
            (((x & -x) * _CTZ_DB) >> _CTZ_DB_SHIFT) & _CTZ_DB_MASK
        ]
    else:
        zeros = 0
        f2 = 1
        fm = 1
        # divide out larger and larger powers, until x % 2**f2 != 0
        while x & fm == 0 and x > 0:
            x >>= f2
            zeros += f2
            f2 += 1
            fm = (fm << 1) | 1
        # recover smaller powers
        for f2 in range(f2-1, 0, -1):
            fm >>= 1
            if x & fm == 0:
                x = x >> f2
                zeros += f2
        return zeros

def ctz_split(x: int) -> typing.Tuple[int, int]:
    """Count trailing zeros, and separate out the interesting bits.

    As ctz(x), but also reports the "significant" part of x, shifted over
    by the number of zeros. This is the same as factoring out all powers
    of 2 to produce z and y such that x = 2**z * y.

    Args:
        x: An int. Must be >= 0.

    Returns:
        A tuple (zeros, x >> zeros) where zeros is the number of trailing
        zeros in x.

    >>> ctz_split(0)
    (0, 0)
    >>> ctz_split(1)
    (0, 1)
    >>> ctz_split(2)
    (1, 1)
    >>> ctz_split(40) # 2**3 * 5
    (3, 5)
    >>> ctz_split(37 << 100)
    (100, 37)
    """
    if x.bit_length() <= _CTZ_DB_THRESH_BITS:
        zeros = _CTZ_DB_TABLE[
            (((x & -x) * _CTZ_DB) >> _CTZ_DB_SHIFT) & _CTZ_DB_MASK
        ]
        return zeros, x >> zeros
    else:
        zeros = 0
        f2 = 1
        fm = 1
        # divide out larger and larger powers, until x % 2**f2 != 0
        while x & fm == 0 and x > 0:
            x >>= f2
            zeros += f2
            f2 += 1
            fm = (fm << 1) | 1
        # recover smaller powers
        for f2 in range(f2-1, 0, -1):
            fm >>= 1
            if x & fm == 0:
                x = x >> f2
                zeros += f2
        return zeros, x

def naive_ctz_split(x: int) -> typing.Tuple[int, int]:
    """Count trailing zeros, naively.

    The behavior should be identical to ctz_split(x). This is a slow reference
    implementation for checking correctness. Since it never needs to check the
    bit_length of x or do any multiplication, it may be slightly faster for
    numbers that have a very small number of trailing zeros.
    """
    zeros = 0
    while x & 1 == 0 and x > 0:
        x = x >> 1
        zeros += 1
    return zeros, x

def test_range(e, mbits):
    for m in range(1 << mbits):
        for shift in range(e):
            x = m << shift
            naive_z, naive_y = naive_ctz_split(x)
            z, y = ctz_split(x)
            just_z = ctz(x)

            if not (naive_z == z == just_z):
                print(x, naive_z, z, just_z)
            if not (naive_y == y):
                print(x, naive_y, y)

def time_range(e, mbits):
    for m in range(1 << mbits):
        for shift in range(e):
            x = m << shift
            z, y = ctz_split(x)

def time_blank(e, mbits):
    for m in range(1 << mbits):
        for shift in range(e):
            x = m << shift

if __name__ == "__main__":
    import doctest
    doctest.testmod()

    test_range(100, 8)
    test_range(1, 17)

    import timeit
    reps = 3
    e = 1528
    mbits = 8
    print('ref: {:.3f}'.format(timeit.timeit(lambda: time_blank(e, mbits), number=reps)))
    print('run: {:.3f}'.format(timeit.timeit(lambda: time_range(e, mbits), number=reps)))
<|endoftext|>


<commit_before>## Right now, this is imported by test.py look for all spin states of 65Zn up to 5000keV

from decimal import *

def isNumber(inputString):
    return not any(not (char.isdigit() or char == '.') for char in inputString)

class data:##This is the main data class.
    def __init__(self,ENSDF,ISOvar,option = 'EoL',energyLimit = 999999999, maxSpin = 9):
        ###maybe get rid of option, find out how energy limit and max spin are used

        ##Initialize Parameters
        #filtDataSet=[]
            ###may not need filt data set
        self.data = []
        self.name = ISOvar
        self.op = option

        ## nucID is what is compared to the first 6 characters of the line to find the correct data
        nucID=self.name.upper()+" "
        ## This while loop makes sure that nucID has the correct leading whitespace to match the data file
        charIndex=0
        while(charIndex<3):
            if nucID[charIndex].isalpha():
                nucID=' '+nucID
            charIndex+=1
        ## if the element symbol is one letter, an additional space must be appended so len(nucID)==6
        if(len(nucID)<6):
            nucID=nucID+' '

        ##open the appropriate ensdf file
        self.f = open("Data/"+str(ENSDF),'rU')

        linecount = 0 ## printing linecount can help locate problem-causing lines in the ensdf file
        desiredData = False
        for line in self.f:
            linecount+=1
            ## The line parsing algorithm is derived from the labeling system of the ensdf files
            ## See the endsf manual, pg. 22, for more information about how the lines of data are organized

            ## the for loop must exit when the ensdf switches from evaluated data to experimental results
            ## this is indicated by an empty line in the ensdf file, which is detected her
            if (desiredData and line[0:6].strip() == ''):
                break

             ## Identifies which lines in the data file have relevant data
            if (line[6:8]==' L' and line[0:6]==nucID):

                ## set desiredData bool so the program wil exit after reading adopted data
                desiredData = True
                print(ENSDF,':',linecount)

                ## finding the energy
                energy = line[9:19].strip()

                ## check if valid energy data (i.e. no letter at beginning or end)
                if (energy[0].isalpha() or energy[-1].isalpha()):
                    continue

                if 'E' in energy: ## This will convert scientific to decimal notation if needed
                    significand = float(energy[:energy.find('E')])
                    power = 10** float(energy[energy.find('E')+1:])
                    energy = str(significand * power)
                ## discard points where energy contains letters FIXME: this should not be necessary
                #if not isNumber(energy):
                    #continue

                ## Finding the uncertainty
                uncert = line[19:21].strip()
                #print(uncert)
                ## Set unsert to 0 if no uncertainty is given.
                if (uncert == ''):
                    uncert = '0'
                ## Set uncert to 0 if not numeric
                elif (not uncert.isnumeric()):
                    uncert = '0'

                elif ('.' in energy): ## Convert uncertainty to proper magnitude
                    numberino = energy
                    topLimit = 0

                    for i in range(len(numberino)-1):

                        if (numberino[-1-i] == '.'):
                            decIndex = i
                            if (decIndex == 0):
                                ## If the '.' is the last digit in the energy, numberino[-i:] will
                                ## return the entire string, which is no bueno
                                numberino = numberino[:-1-i]
                            else:
                                numberino = numberino[:-1-i] + numberino[-i:]
                            #print(linecount,line,numberino,numberino[-i:])
                        if (i < len(uncert)):
                            #print(linecount,energy,numberino)
                            digit = int(numberino[-1-i])+int(uncert[-1-i])
                        else:
                            digit = int(numberino[-1-i])
                        topLimit = digit * (10**i) + topLimit
                    topLimit = str(topLimit)[:-decIndex]+'.'+str(topLimit)[-decIndex:]
                    uncert = str(Decimal(topLimit)-Decimal(energy))

                ## Finding ALL spin and pairity states (to be filtered later)
                jpi = line[21:39].strip()

                ## Finding stability info FIXME: this data is currently not used
                hlife = line[39:49].strip()

                if(float(energy)<=energyLimit):
                    #include the data
                    self.data.append([energy,jpi,uncert])
                    #print(str(linecount)+' :'+str(self.data[-1]))
                else:
                    break

    ## extraTitleText would be desired spin states, for example
    def export(self,fExtOption = '.dat',extraTitleText = ''):
#            if(fExtOption==".dat"or fExtOption=="_Fil.dat"):##To make data files for use in gnuplot and plt file.
            fileName=str(self.name)+extraTitleText+fExtOption##creates filename
            fileName="Output/" + "gnuPlot/"+fileName.replace('/','_')
            datFile = open(fileName,'wb')##Creates a file with a valid file name.
            for i in range(len(self.data)):
                datFile.write(str.encode(str(self.name)+';'+str(self.data[i][0])+';'+str(self.data[i][1])+';'+str(self.data[i][2])+'\n'))

    def spinMatchFinder(self,matchVal,checkVal):
        ## matchVal is the desired spin, checkVal is the spin to be checked for matchVal
        tempList = checkVal.split(',')
        ## Look for parities that need to be distributed.
        if any((')+' in value or ')-' in value) for value in tempList):
            addPlus = False
            addMinus = False
            for j in range(len(tempList)):
                if ')+' in tempList[j]:
                    addPlus = True
                    print('addPlus is True')
                    tempList[j] = tempList[j].replace('+','')
                if ')-' in tempList[j]:
                    addMinus = True
                    tempList[j] = tempList[j].replace('-','')

            if addMinus and addPlus: #FIXME check ALL isotopes in ALL datafiles
                print(self.data[i])
            ## Distribute the parities to each term
            for j in range(len(tempList)):
                if addPlus:
                    tempList[j] = tempList[j] + '+'
                if addMinus:
                    tempList[j] = tempList[j] + '-'

        ## Remove () so that spins can be identified with ==
        for j in range(len(tempList)):
            tempList[j] = tempList[j].replace('(','')
            tempList[j] = tempList[j].replace(')','')
        print(tempList)

        if (matchVal in tempList):
            return True
        else:
            ## Spin not found
            return False

    def filterData(self,userInput,UI=False):
        ## no spin input
        if (userInput == ''):
            #print(self.data)
            if (not self.data):
                if(UI):
                    ## Prints a statement telling user than no file was found
                    print("Warning:No data filtered/selected for "+ self.name +".")
                self.data=[[0.0,"--",0.0]]##Enters a dummy entry to file with something.

        #if(self.op == 'EoL'):
        ## Filter by spin states
        else:
            newData=[] ## storage for new data
            for wantedString in userInput.split(","):##adds all the strings that are included in the userInput.
                for i in range(0,len(self.data)):
                    #print(self.data[i][1])
                    ## The spinMatchFinder will identify if the state is the desired spin
                    if(self.spinMatchFinder(wantedString, self.data[i][1])):
                        newData.append(self.data[i])
            if(newData):
                self.data=newData##changes data to the new data.
            else:
                if(UI):
                    print("Warning:No data filtered/selected for "+ self.name +".")#Prints a statement telling user than no file was found
                self.data=[[0.0,"--",0.0]]##Enters a dummy entry to file with something.

<commit_msg>saving newDataClass progress<commit_after>from decimal import *
from fractions import Fraction

def getPI(JPI): ## Extract parity
        if '+' in JPI:
            return '+'
        elif '-' in JPI:
            return '-'
        else:
            return ''
def getJ(JPI): ## Extract spin w/o parity

    return Fraction(JPI.replace('+','').replace('-',''))

def getJrange(lowval,highval): ## Creates range of spins
    jRange = [lowval]
    while (jRange[-1] < highval):
        jRange.append(jRange[-1]+1)
    for jIndex in range(len(jRange)):
        jRange[jIndex] = str(jRange[jIndex])
    return jRange

class data:##This is the main data class.
    def __init__(self,ENSDF,ISOvar,option = 'EoL',energyLimit = 999999999, maxSpin = 9):
        ###maybe get rid of option, find out how energy limit and max spin are used

        ##Initialize Parameters
        self.data = []
        self.name = ISOvar
        self.op = option

        ## nucID is what is compared to the first 6 characters of the line to find the correct data
        nucID=self.name.upper()+" "
        ## This while loop makes sure that nucID has the correct leading whitespace to match the data file
        charIndex=0
        while(charIndex<3):
            if nucID[charIndex].isalpha():
                nucID=' '+nucID
            charIndex+=1
        ## if the element symbol is one letter, an additional space must be appended so len(nucID)==6
        if(len(nucID)<6):
            nucID=nucID+' '

        ##open the appropriate ensdf file
        self.f = open("Data/"+str(ENSDF),'rU')

        linecount = 0 ## printing linecount can help locate problem-causing lines in the ensdf file
        desiredData = False
        for line in self.f:
            linecount+=1
            ## The line parsing algorithm is derived from the labeling system of the ensdf files
            ## See the endsf manual, pg. 22, for more information about how the lines of data are organized

            ## the for loop must exit when the ensdf switches from evaluated data to experimental results
            ## this is indicated by an empty line in the ensdf file, which is detected her
            if (desiredData and line[0:6].strip() == ''):
                break

             ## Identifies which lines in the data file have relevant data
            if (line[6:8]==' L' and line[0:6]==nucID):

                ## set desiredData bool so the program wil exit after reading adopted data
                desiredData = True

                ## finding the energy
                energy = line[9:19].strip()

                ## check if valid energy data (i.e. no letter at beginning or end)
                if (energy[0].isalpha() or energy[-1].isalpha()):
                    continue

                ## This will handle states with deduced energies enclosed in ()
                deducedEnergy = False
                if '(' in energy:
                    deducedEnergy = True
                    energy = energy.replace('(','')
                    energy = energy.replace(')','')

                if 'E' in energy: ## This will convert scientific to decimal notation if needed
                    significand = float(energy[:energy.find('E')])
                    power = 10** float(energy[energy.find('E')+1:])
                    energy = str(significand * power)

                ## Finding the uncertainty
                uncert = line[19:21].strip()
                ## Set unsert to 0 if no uncertainty is given.
                if (uncert == ''):
                    uncert = '0'
                ## Set uncert to 0 if not numeric
                elif (not uncert.isnumeric()):
                    uncert = '0'

                elif ('.' in energy): ## Convert uncertainty to proper magnitude
                    numberino = energy
                    topLimit = 0

                    for i in range(len(numberino)-1):

                        if (numberino[-1-i] == '.'):
                            decIndex = i
                            if (decIndex == 0):
                                ## If the '.' is the last digit in the energy, numberino[-i:] will
                                ## return the entire string, which is no bueno
                                numberino = numberino[:-1-i]
                            else:
                                numberino = numberino[:-1-i] + numberino[-i:]
                        if (i < len(uncert)):
                            digit = int(numberino[-1-i])+int(uncert[-1-i])
                        else:
                            digit = int(numberino[-1-i])
                        topLimit = digit * (10**i) + topLimit
                    topLimit = str(topLimit)[:-decIndex]+'.'+str(topLimit)[-decIndex:]
                    uncert = str(Decimal(topLimit)-Decimal(energy))

                ## Finding ALL spin and pairity states (to be filtered later)
                jpi = line[21:39].strip()
                ## indicating deduced energy
                if deducedEnergy:
                    jpi = jpi + '**'

                ## Finding stability info FIXME: this data is currently not used
                hlife = line[39:49].strip()
                # if stable, set to 10**9 year

                if(float(energy)<=energyLimit):
                    #include the data
                    self.data.append([energy,jpi,uncert])
                    #print(str(linecount)+' :'+str(self.data[-1]))
                else:
                    break

    ## extraTitleText would be desired spin states, for example
    def export(self,fExtOption = '.dat',extraTitleText = ''):
#            if(fExtOption==".dat"or fExtOption=="_Fil.dat"):##To make data files for use in gnuplot and plt file.
            fileName=str(self.name)+extraTitleText+fExtOption##creates filename
            fileName="Output/" + "gnuPlot/"+fileName.replace('/','_')
            datFile = open(fileName,'wb')##Creates a file with a valid file name.
            for i in range(len(self.data)):
                datFile.write(str.encode(str(self.name)+';'+str(self.data[i][0])+';'+str(self.data[i][1])+';'+str(self.data[i][2])+'\n'))

    def spinMatchFinder(self,matchVal,checkVal):
        ## matchVal is the desired spin, checkVal is the spin to be checked for matchVal
        tempList = checkVal.replace('**','').replace('&',',')
        tempList = tempList.split(',')

        ## Look for parities that need to be distributed.
        if any((')+' in value or ')-' in value) for value in tempList):
            addPlus = False
            addMinus = False
            for j in range(len(tempList)):
                if ')+' in tempList[j]:
                    addPlus = True
                    tempList[j] = tempList[j].replace('+','')
                if ')-' in tempList[j]:
                    addMinus = True
                    tempList[j] = tempList[j].replace('-','')

            ## Distribute the parities to each term
            for j in range(len(tempList)):
                if addPlus:
                    tempList[j] = tempList[j] + '+'
                if addMinus:
                    tempList[j] = tempList[j] + '-'

        ## Remove () and whitespace so that spins can be identified with ==
        for j in range(len(tempList)):
            tempList[j] = tempList[j].replace('(','')
            tempList[j] = tempList[j].replace(')','')
            tempList[j] = tempList[j].strip()

        ## Handing of ranges of spins (eg. JPI to J'PI', see ensdf manual pg. 46 for more info)
        if any(('TO' in value or ':' in value) for value in tempList):
            ## if a range is given, len(tempList) = 1 ALWAYS
            tempList[0] = tempList[0].replace('TO',':')
            [lhs,rhs] = tempList[0].split(':')

            ## JPI TO J'PI'
            if (('+' in lhs) or ('-' in lhs)) and (('+' in rhs) or ('-' in rhs)):
                ## Note that lowJ and highJ are Fraction objects
                lowJ = getJ(lhs)
                lowPI = getPI(lhs)
                highJ = getJ(rhs)
                highPI = getPI(rhs)
                ## jRange is a list of strings
                jRange = getJrange(lowJ,highJ)

                jRange[0] = jRange[0]+lowPI
                jRange[-1] = jRange[-1]+highPI
                tempList = jRange

            ## J TO J'PI
            elif ('+' in rhs) or ('-' in rhs):
                lowJ = getJ(lhs)
                highJ = getJ(rhs)
                highPI = getPI(rhs)
                jRange = getJrange(lowJ,highJ)

                for jIndex in range(len(jRange)):
                    jRange[jIndex] = jRange[jIndex]+highPI
                tempList = jRange

            ## JPI TO J'
            elif ('+' in lhs) or ('-' in lhs):
                lowJ = getJ(lhs)
                lowPI = getPI(lhs)
                highJ = getJ(rhs)
                jRange = getJrange(lowJ,highJ)

                jRange[0] = jRange[0]+lowPI
                tempList = jRange

            else: ## J TO J'
                lowJ = getJ(lhs)
                highJ = getJ(rhs)
                jRange = getJrange(lowJ,highJ)
                tempList = jRange

        ## assign + and - to states with no indicated pairity
        if (not tempList == ['']) and any(('+' not in value and '-' not in value) for value in tempList):
            j=0
            while (j < len(tempList)):
                if ('+' not in tempList[j] and '-' not in tempList[j]):
                    tempList.insert(j,tempList[j]+'-')
                    tempList[j+1] = tempList[j+1]+'+'
                j+=1

        if (matchVal in tempList):
            return True
        else:
            ## Spin not found
            return False

    def filterData(self,userInput,UI=False):
        ## no spin input
        if (userInput == ''):
            #print(self.data)
            if (not self.data):
                if(UI):
                    ## Prints a statement telling user than no file was found
                    print("Warning:No data filtered/selected for "+ self.name +".")
                self.data=[[0.0,"--",0.0]]##Enters a dummy entry to file with something.

        #if(self.op == 'EoL'):
        ## Filter by spin states
        else:
            newData=[] ## storage for new data
            for wantedString in userInput.split(","):##adds all the strings that are included in the userInput.
                for i in range(0,len(self.data)):
                    #print(self.name,self.data[i])
                    ## The spinMatchFinder will identify if the state is the desired spin
                    if(self.spinMatchFinder(wantedString, self.data[i][1])):
                        newData.append(self.data[i])
            if(newData):
                self.data=newData##changes data to the new data.
            else:
                if(UI):
                    ## Prints a statement telling user than no file was found
                    print("Warning:No data filtered/selected for "+ self.name +".")
                self.data=[[0.0,"--",0.0]]##Enters a dummy entry to file with something.

<|endoftext|>


<commit_before>import urllib
import urllib2
import urlparse
import json

SMS_CHARS = 160

class NexmoMessage:
    def __init__(self, s, u, p):
        self.apiserver = s
        self.apiuser = u
        self.apipass = p

    def urlFix(self, s, charset = 'utf-8'):
        if isinstance(s, unicode):
            s = s.encode(charset, 'ignore')
        scheme, netloc, path, qs, anchor = urlparse.urlsplit(s)
        path = urllib.quote(path, '/%')
        qs = urllib.quote_plus(qs, ':&=')
        return urlparse.urlunsplit((scheme, netloc, path, qs, anchor))

    def textMessage(self, f, t, m):
        self.apifrom = f
        self.apito = t
        self.apimsg = m
        self.request = "%s?username=%s&password=%s&from=%s&to=%s&text=%s" % (self.apiserver, self.apiuser, self.apipass, self.apifrom, self.apito, self.apimsg)

    def sendRequest(self):
        print json.load(urllib2.urlopen(self.urlFix(self.request)))

# EOF
<commit_msg>lame fixes<commit_after>import urllib
import urllib2
import urlparse
import json

class NexmoMessage:

    def __init__(self, s, u, p):
        self.apiserver = s
        self.apiuser = u
        self.apipass = p

    def urlFix(self, s, charset = 'utf-8'):
        if isinstance(s, unicode):
            s = s.encode(charset, 'ignore')
        scheme, netloc, path, qs, anchor = urlparse.urlsplit(s)
        path = urllib.quote(path, '/%')
        qs = urllib.quote_plus(qs, ':&=')
        return urlparse.urlunsplit((scheme, netloc, path, qs, anchor))

    def textMessage(self, f, t, m):
        self.apifrom = f
        self.apito = t
        self.apimsg = m
        self.request = "%s?username=%s&password=%s&from=%s&to=%s&text=%s" %
                (self.apiserver, self.apiuser, self.apipass, self.apifrom,
                self.apito, self.apimsg)

    def sendRequest(self):
        print(json.load(urllib2.urlopen(self.urlFix(self.request))))

# EOF
<|endoftext|>