<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
test_access.py

Created by Magnus Morton on 2012-02-12.
Copyright (c) 2012 __MyCompanyName__. All rights reserved.
"""

import unittest

import nest.affine_access

class TestAccess(unittest.TestCase):
	def setUp(self):
		self.access = nest.affine_access.AffineAccess()

	def test_parameter_extracted(self):
	    self.add_params("i")
	    self.assertTrue("i" in self.access.params)

	def test_multiple_parameters_extractable(self):
	    self.add_params("i", "j")
	    self.assertTrue("i" in self.access.params and "j" in self.access.params, "i and j are not both present")

	def test_coeff_extractable(self):
	    self.add_params("i")
	    self.assertEqual(self.access.get_coeff("i"), 1, "coeffecient of i is not 1")

	def add_params(self, *params):
	    for param in params:
	        self.access.add_param(param)

if __name__ == '__main__':
	unittest.main()<commit_msg>renamed test to make explicit the implicity<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
test_access.py

Created by Magnus Morton on 2012-02-12.
Copyright (c) 2012 __MyCompanyName__. All rights reserved.
"""

import unittest

import nest.affine_access

class TestAccess(unittest.TestCase):
	def setUp(self):
		self.access = nest.affine_access.AffineAccess()

	def test_parameter_extracted(self):
	    self.add_params("i")
	    self.assertTrue("i" in self.access.params)

	def test_multiple_parameters_extractable(self):
	    self.add_params("i", "j")
	    self.assertTrue("i" in self.access.params and "j" in self.access.params, "i and j are not both present")

	def test_implicit_coeff_extractable(self):
	    self.add_params("i")
	    self.assertEqual(self.access.get_coeff("i"), 1, "coeffecient of i is not 1")

	def add_params(self, *params):
	    for param in params:
	        self.access.add_param(param)

if __name__ == '__main__':
	unittest.main()<|endoftext|>


<commit_before>#!/usr/bin/python

#----------------------------------------------------------------------
# Copyright (c) 2011 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------
"""
   Utilities to parse credentials
"""

import datetime
import dateutil.parser
import logging
import traceback
import xml.dom.minidom as md

def is_valid_v3(logger, credString):
    '''Is the given credential a valid sfa style v3 credential?'''
# 1) Call:
#    sfa.trust.credential.cred.verify(trusted_certs=None, schema=FIXME, trusted_certs_required=False)
    # Will have to check in the xsd I think

# 2)
    # also for each cert:
    #   retrieve gid from cred
    #   check not expired
    #   check version is 3
    #   get email, urn, uuid - all not empty
    #   more? CA:TRUE/FALSE? real serialNum? Has a DN?
    #    if type is slice or user, CA:FALSE. It type is authority, CA:TRUE
    #   Should this be in gid.py? or in geni.util.cert_util?
    #

    return True

def get_cred_exp(logger, cred):
    '''Parse the given credential in GENI AM API XML format to get its expiration time and return that'''

    # Don't fully parse credential: grab the expiration from the string directly
    credexp = datetime.datetime.fromordinal(1)

    credString = get_cred_xml(cred)

    if credString is None:
        # failed to get a credential string. Can't check
        return credexp

    # If credString is not a string then complain and return
    if type(credString) != type('abc'):
        if logger is None:
            level = logging.INFO
            logging.basicConfig(level=level)
            logger = logging.getLogger("omni.credparsing")
            logger.setLevel(level)
        logger.error("Cannot parse expiration date: Credential is not a string: %s", str(credString))
        return credexp

    try:
        doc = md.parseString(credString)
        signed_cred = doc.getElementsByTagName("signed-credential")

        # Is this a signed-cred or just a cred?
        if len(signed_cred) > 0:
            cred = signed_cred[0].getElementsByTagName("credential")[0]
        else:
            cred = doc.getElementsByTagName("credential")[0]
        expirnode = cred.getElementsByTagName("expires")[0]
        if len(expirnode.childNodes) > 0:
            credexp = dateutil.parser.parse(expirnode.childNodes[0].nodeValue)
    except Exception, exc:
        if logger is None:
            level = logging.INFO
            logging.basicConfig(level=level)
            logger = logging.getLogger("omni.credparsing")
            logger.setLevel(level)
        logger.error("Failed to parse credential for expiration time: %s", exc)
        logger.info("Unparsable credential: %s", credString)
        logger.debug(traceback.format_exc())

    return credexp

def is_cred_xml(cred):
    '''Is this a cred in XML format, or a struct? Return true if XML'''
    if cred is None:
        return False
    if not isinstance(cred, str):
        return False
    # Anything else? Starts with <?
    return True

def get_cred_xml(cred):
    '''Return the cred XML, from the struct if any, else None'''
    if is_cred_xml(cred):
        return cred
    # Make sure the cred is the right struct?
    # extract the real cred
#        {
#           geni_type: <string>,
#           geni_version: <string>,
#           geni_value: <the credential as a string>
#        }

    if isinstance(cred, dict) and cred.has_key("geni_value"):
        return cred["geni_value"]

    return None

<commit_msg>util to get the target urn from a cred string<commit_after>#!/usr/bin/python

#----------------------------------------------------------------------
# Copyright (c) 2011 Raytheon BBN Technologies
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and/or hardware specification (the "Work") to
# deal in the Work without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Work, and to permit persons to whom the Work
# is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Work.
#
# THE WORK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE WORK OR THE USE OR OTHER DEALINGS
# IN THE WORK.
#----------------------------------------------------------------------
"""
   Utilities to parse credentials
"""

import datetime
import dateutil.parser
import logging
import traceback
import xml.dom.minidom as md

def is_valid_v3(logger, credString):
    '''Is the given credential a valid sfa style v3 credential?'''
# 1) Call:
#    sfa.trust.credential.cred.verify(trusted_certs=None, schema=FIXME, trusted_certs_required=False)
    # Will have to check in the xsd I think

# 2)
    # also for each cert:
    #   retrieve gid from cred
    #   check not expired
    #   check version is 3
    #   get email, urn, uuid - all not empty
    #   more? CA:TRUE/FALSE? real serialNum? Has a DN?
    #    if type is slice or user, CA:FALSE. It type is authority, CA:TRUE
    #   Should this be in gid.py? or in geni.util.cert_util?
    #

    return True

def get_cred_target_urn(logger, cred):
    '''Parse the given credential to get is target URN'''
    credString = get_cred_xml(cred)
    urn = ""

    if credString is None:
        return urn

    # If credString is not a string then complain and return
    if type(credString) != type('abc'):
        if logger is None:
            level = logging.INFO
            logging.basicConfig(level=level)
            logger = logging.getLogger("omni.credparsing")
            logger.setLevel(level)
        logger.error("Cannot parse target URN: Credential is not a string: %s", str(credString))
        return credexp

    try:
        doc = md.parseString(credString)
        signed_cred = doc.getElementsByTagName("signed-credential")

        # Is this a signed-cred or just a cred?
        if len(signed_cred) > 0:
            cred = signed_cred[0].getElementsByTagName("credential")[0]
        else:
            cred = doc.getElementsByTagName("credential")[0]

        targetnode = cred.getElementsByTagName("target_urn")[0]
        if len(targetnode.childNodes) > 0:
            urn = str(targetnode.childNodes[0].nodeValue)
    except Exception, exc:
        if logger is None:
            level = logging.INFO
            logging.basicConfig(level=level)
            logger = logging.getLogger("omni.credparsing")
            logger.setLevel(level)
        logger.error("Failed to parse credential for target URN: %s", exc)
        logger.info("Unparsable credential: %s", credString)
        logger.debug(traceback.format_exc())

    return urn

def get_cred_exp(logger, cred):
    '''Parse the given credential in GENI AM API XML format to get its expiration time and return that'''

    # Don't fully parse credential: grab the expiration from the string directly
    credexp = datetime.datetime.fromordinal(1)

    credString = get_cred_xml(cred)

    if credString is None:
        # failed to get a credential string. Can't check
        return credexp

    # If credString is not a string then complain and return
    if type(credString) != type('abc'):
        if logger is None:
            level = logging.INFO
            logging.basicConfig(level=level)
            logger = logging.getLogger("omni.credparsing")
            logger.setLevel(level)
        logger.error("Cannot parse expiration date: Credential is not a string: %s", str(credString))
        return credexp

    try:
        doc = md.parseString(credString)
        signed_cred = doc.getElementsByTagName("signed-credential")

        # Is this a signed-cred or just a cred?
        if len(signed_cred) > 0:
            cred = signed_cred[0].getElementsByTagName("credential")[0]
        else:
            cred = doc.getElementsByTagName("credential")[0]
        expirnode = cred.getElementsByTagName("expires")[0]
        if len(expirnode.childNodes) > 0:
            credexp = dateutil.parser.parse(expirnode.childNodes[0].nodeValue)
    except Exception, exc:
        if logger is None:
            level = logging.INFO
            logging.basicConfig(level=level)
            logger = logging.getLogger("omni.credparsing")
            logger.setLevel(level)
        logger.error("Failed to parse credential for expiration time: %s", exc)
        logger.info("Unparsable credential: %s", credString)
        logger.debug(traceback.format_exc())

    return credexp

def is_cred_xml(cred):
    '''Is this a cred in XML format, or a struct? Return true if XML'''
    if cred is None:
        return False
    if not isinstance(cred, str):
        return False
    # Anything else? Starts with <?
    return True

def get_cred_xml(cred):
    '''Return the cred XML, from the struct if any, else None'''
    if is_cred_xml(cred):
        return cred
    # Make sure the cred is the right struct?
    # extract the real cred
#        {
#           geni_type: <string>,
#           geni_version: <string>,
#           geni_value: <the credential as a string>
#        }

    if isinstance(cred, dict) and cred.has_key("geni_value"):
        return cred["geni_value"]

    return None

<|endoftext|>


<commit_before># Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

from cloudify import ctx
from cloudify import exceptions as cfy_exc
from cloudify.decorators import operation
from vcloud_plugin_common import (with_vca_client, wait_for_task,
                                  get_vcloud_config, get_mandatory)
import collections
from network_plugin import (check_ip, is_valid_ip_range, is_separate_ranges,
                            is_ips_in_same_subnet, save_gateway_configuration,
                            get_network_name, is_network_exists)

VCLOUD_NETWORK_NAME = 'vcloud_network_name'
ADD_POOL = 1
DELETE_POOL = 2

@operation
@with_vca_client
def create(vca_client, **kwargs):
    """
        create new vcloud air network, e.g.:
        {
            'use_external_resource': False,
            'resource_id': 'secret_network',
            'network': {
                'dhcp': {
                    'dhcp_range': "10.1.1.128-10.1.1.255"
                },
                'static_range':  "10.1.1.2-10.1.1.127",
                'gateway_ip': "10.1.1.1",
                'edge_gateway': 'gateway',
                'name': 'secret_network',
                "netmask": '255.255.255.0',
                "dns": ["8.8.8.8", "4.4.4.4"]
            }
        }
    """
    vdc_name = get_vcloud_config()['vdc']
    if ctx.node.properties['use_external_resource']:
        network_name = ctx.node.properties['resource_id']
        if not is_network_exists(vca_client, network_name):
            raise cfy_exc.NonRecoverableError(
                "Can't find external resource: {0}".format(network_name))
        ctx.instance.runtime_properties[VCLOUD_NETWORK_NAME] = network_name
        ctx.logger.info(
            "External resource {0} has been used".format(network_name))
        return
    net_prop = ctx.node.properties["network"]
    network_name = get_network_name(ctx.node.properties)
    if network_name in _get_network_list(vca_client,
                                         get_vcloud_config()['vdc']):
        raise cfy_exc.NonRecoverableError(
            "Network {0} already exists, but parameter "
            "'use_external_resource' is 'false' or absent"
            .format(network_name))

    ip = _split_adresses(net_prop['static_range'])
    gateway_name = net_prop['edge_gateway']
    if not vca_client.get_gateway(vdc_name, gateway_name):
        raise cfy_exc.NonRecoverableError(
            "Gateway {0} not found".format(gateway_name))
    start_address = ip.start
    end_address = ip.end
    gateway_ip = net_prop["gateway_ip"]
    netmask = net_prop["netmask"]
    dns1 = ""
    dns2 = ""
    dns_list = net_prop.get("dns")
    if dns_list:
        dns1 = dns_list[0]
        if len(dns_list) > 1:
            dns2 = dns_list[1]
    dns_suffix = net_prop.get("dns_suffix")
    success, result = vca_client.create_vdc_network(
        vdc_name, network_name, gateway_name, start_address,
        end_address, gateway_ip, netmask, dns1, dns2, dns_suffix)
    if success:
        ctx.logger.info("Network {0} has been successfully created."
                        .format(network_name))
    else:
        raise cfy_exc.NonRecoverableError(
            "Could not create network {0}: {1}".format(network_name, result))
    wait_for_task(vca_client, result)
    ctx.instance.runtime_properties[VCLOUD_NETWORK_NAME] = network_name
    _dhcp_operation(vca_client, network_name, ADD_POOL)

@operation
@with_vca_client
def delete(vca_client, **kwargs):
    """
        delete vcloud air network
    """
    if ctx.node.properties['use_external_resource'] is True:
        del ctx.instance.runtime_properties[VCLOUD_NETWORK_NAME]
        ctx.logger.info("Network was not deleted - external resource has"
                        " been used")
        return
    network_name = get_network_name(ctx.node.properties)
    _dhcp_operation(vca_client, network_name, DELETE_POOL)
    success, task = vca_client.delete_vdc_network(
        get_vcloud_config()['vdc'], network_name)
    if success:
        ctx.logger.info(
            "Network {0} has been successful deleted.".format(network_name))
    else:
        raise cfy_exc.NonRecoverableError(
            "Could not delete network {0}".format(network_name))
    wait_for_task(vca_client, task)

@operation
@with_vca_client
def creation_validation(vca_client, **kwargs):
    """
        check network description from node description
    """
    network_name = get_network_name(ctx.node.properties)
    ctx.logger.info("Validation cloudify.vcloud.nodes.Network node: {0}"
                    .format(network_name))
    if is_network_exists(vca_client, network_name):
        if ctx.node.properties.get('use_external_resource'):
            # TODO: check: default gateway must exists
            return
        else:
            raise cfy_exc.NonRecoverableError(
                "Network already exsists: {0}".format(network_name))

    net_prop = get_mandatory(ctx.node.properties, "network")
    gateway_name = get_mandatory(net_prop, 'edge_gateway')
    if not vca_client.get_gateway(get_vcloud_config()['vdc'], gateway_name):
        raise cfy_exc.NonRecoverableError(
            "Gateway {0} not found".format(gateway_name))

    static_ip = _split_adresses(get_mandatory(net_prop, 'static_range'))
    check_ip(static_ip.start)
    check_ip(static_ip.end)
    dns_list = net_prop.get("dns")
    if dns_list:
        for ip in dns_list:
            check_ip(ip)
    gateway_ip = check_ip(get_mandatory(net_prop, "gateway_ip"))
    netmask = check_ip(get_mandatory(net_prop, "netmask"))

    ips = [gateway_ip, static_ip.start, static_ip.end]
    dhcp = net_prop.get("dhcp")
    if dhcp:
        dhcp_range = get_mandatory(net_prop["dhcp"], "dhcp_range")
        dhcp_ip = _split_adresses(dhcp_range)
        if not is_separate_ranges(static_ip, dhcp_ip):
            raise cfy_exc.NonRecoverableError(
                "Static_range and dhcp_range is overlapped.")
        ips.extend([dhcp_ip.start, dhcp_ip.end])
    if not is_ips_in_same_subnet(ips, netmask):
            raise cfy_exc.NonRecoverableError(
                "IP addresses in different subnets.")

def _dhcp_operation(vca_client, network_name, operation):
    """
        update dhcp setting for network
    """
    dhcp_settings = ctx.node.properties['network'].get('dhcp')
    if dhcp_settings is None:
        return
    gateway_name = ctx.node.properties["network"]['edge_gateway']
    gateway = vca_client.get_gateway(get_vcloud_config()['vdc'], gateway_name)
    if not gateway:
        raise cfy_exc.NonRecoverableError(
            "Gateway {0} not found!".format(gateway_name))

    if operation == ADD_POOL:
        ip = _split_adresses(dhcp_settings['dhcp_range'])
        low_ip_address = check_ip(ip.start)
        hight_ip_address = check_ip(ip.end)
        default_lease = dhcp_settings.get('default_lease')
        max_lease = dhcp_settings.get('max_lease')
        gateway.add_dhcp_pool(network_name, low_ip_address, hight_ip_address,
                              default_lease, max_lease)
        ctx.logger.info("DHCP rule successful created for network {0}"
                        .format(network_name))

    if operation == DELETE_POOL:
        gateway.delete_dhcp_pool(network_name)
        ctx.logger.info("DHCP rule successful deleted for network {0}"
                        .format(network_name))

    if not save_gateway_configuration(gateway, vca_client):
        return ctx.operation.retry(message='Waiting for gateway.',
                                   retry_after=10)

def _split_adresses(address_range):
    """
        split network addresses from 1.1.1.1-2.2.2.2 representation to
        separate (start,end) tuple
    """
    adresses = [ip.strip() for ip in address_range.split('-')]
    IPRange = collections.namedtuple('IPRange', 'start end')
    try:
        start = check_ip(adresses[0])
        end = check_ip(adresses[1])
        if not is_valid_ip_range(start, end):
            raise cfy_exc.NonRecoverableError(
                "Start address {0} is greater than end address: {1}"
                .format(start, end))
        return IPRange(start=start, end=end)
    except IndexError:
        raise cfy_exc.NonRecoverableError("Can't parse IP range:{0}".
                                          format(address_range))

def _get_network_list(vca_client, vdc_name):
    """
        list all avable network for current vdc
    """
    vdc = vca_client.get_vdc(vdc_name)
    if not vdc:
        raise cfy_exc.NonRecoverableError(
            "Vdc {0} not found.".format(vdc_name))
    return [net.name for net in vdc.AvailableNetworks.Network]
<commit_msg>Add error message, if network not deleted.<commit_after># Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.

from cloudify import ctx
from cloudify import exceptions as cfy_exc
from cloudify.decorators import operation
from vcloud_plugin_common import (with_vca_client, wait_for_task,
                                  get_vcloud_config, get_mandatory)
import collections
from network_plugin import (check_ip, is_valid_ip_range, is_separate_ranges,
                            is_ips_in_same_subnet, save_gateway_configuration,
                            get_network_name, is_network_exists)

VCLOUD_NETWORK_NAME = 'vcloud_network_name'
ADD_POOL = 1
DELETE_POOL = 2

@operation
@with_vca_client
def create(vca_client, **kwargs):
    """
        create new vcloud air network, e.g.:
        {
            'use_external_resource': False,
            'resource_id': 'secret_network',
            'network': {
                'dhcp': {
                    'dhcp_range': "10.1.1.128-10.1.1.255"
                },
                'static_range':  "10.1.1.2-10.1.1.127",
                'gateway_ip': "10.1.1.1",
                'edge_gateway': 'gateway',
                'name': 'secret_network',
                "netmask": '255.255.255.0',
                "dns": ["8.8.8.8", "4.4.4.4"]
            }
        }
    """
    vdc_name = get_vcloud_config()['vdc']
    if ctx.node.properties['use_external_resource']:
        network_name = ctx.node.properties['resource_id']
        if not is_network_exists(vca_client, network_name):
            raise cfy_exc.NonRecoverableError(
                "Can't find external resource: {0}".format(network_name))
        ctx.instance.runtime_properties[VCLOUD_NETWORK_NAME] = network_name
        ctx.logger.info(
            "External resource {0} has been used".format(network_name))
        return
    net_prop = ctx.node.properties["network"]
    network_name = get_network_name(ctx.node.properties)
    if network_name in _get_network_list(vca_client,
                                         get_vcloud_config()['vdc']):
        raise cfy_exc.NonRecoverableError(
            "Network {0} already exists, but parameter "
            "'use_external_resource' is 'false' or absent"
            .format(network_name))

    ip = _split_adresses(net_prop['static_range'])
    gateway_name = net_prop['edge_gateway']
    if not vca_client.get_gateway(vdc_name, gateway_name):
        raise cfy_exc.NonRecoverableError(
            "Gateway {0} not found".format(gateway_name))
    start_address = ip.start
    end_address = ip.end
    gateway_ip = net_prop["gateway_ip"]
    netmask = net_prop["netmask"]
    dns1 = ""
    dns2 = ""
    dns_list = net_prop.get("dns")
    if dns_list:
        dns1 = dns_list[0]
        if len(dns_list) > 1:
            dns2 = dns_list[1]
    dns_suffix = net_prop.get("dns_suffix")
    success, result = vca_client.create_vdc_network(
        vdc_name, network_name, gateway_name, start_address,
        end_address, gateway_ip, netmask, dns1, dns2, dns_suffix)
    if success:
        ctx.logger.info("Network {0} has been successfully created."
                        .format(network_name))
    else:
        raise cfy_exc.NonRecoverableError(
            "Could not create network {0}: {1}".format(network_name, result))
    wait_for_task(vca_client, result)
    ctx.instance.runtime_properties[VCLOUD_NETWORK_NAME] = network_name
    _dhcp_operation(vca_client, network_name, ADD_POOL)

@operation
@with_vca_client
def delete(vca_client, **kwargs):
    """
        delete vcloud air network
    """
    if ctx.node.properties['use_external_resource'] is True:
        del ctx.instance.runtime_properties[VCLOUD_NETWORK_NAME]
        ctx.logger.info("Network was not deleted - external resource has"
                        " been used")
        return
    network_name = get_network_name(ctx.node.properties)
    _dhcp_operation(vca_client, network_name, DELETE_POOL)
    success, task = vca_client.delete_vdc_network(
        get_vcloud_config()['vdc'], network_name)
    if success:
        ctx.logger.info(
            "Network {0} has been successful deleted.".format(network_name))
    else:
        raise cfy_exc.NonRecoverableError(
            "Could not delete network {0}: {1}".format(network_name, task))
    wait_for_task(vca_client, task)

@operation
@with_vca_client
def creation_validation(vca_client, **kwargs):
    """
        check network description from node description
    """
    network_name = get_network_name(ctx.node.properties)
    ctx.logger.info("Validation cloudify.vcloud.nodes.Network node: {0}"
                    .format(network_name))
    if is_network_exists(vca_client, network_name):
        if ctx.node.properties.get('use_external_resource'):
            # TODO: check: default gateway must exists
            return
        else:
            raise cfy_exc.NonRecoverableError(
                "Network already exsists: {0}".format(network_name))

    net_prop = get_mandatory(ctx.node.properties, "network")
    gateway_name = get_mandatory(net_prop, 'edge_gateway')
    if not vca_client.get_gateway(get_vcloud_config()['vdc'], gateway_name):
        raise cfy_exc.NonRecoverableError(
            "Gateway {0} not found".format(gateway_name))

    static_ip = _split_adresses(get_mandatory(net_prop, 'static_range'))
    check_ip(static_ip.start)
    check_ip(static_ip.end)
    dns_list = net_prop.get("dns")
    if dns_list:
        for ip in dns_list:
            check_ip(ip)
    gateway_ip = check_ip(get_mandatory(net_prop, "gateway_ip"))
    netmask = check_ip(get_mandatory(net_prop, "netmask"))

    ips = [gateway_ip, static_ip.start, static_ip.end]
    dhcp = net_prop.get("dhcp")
    if dhcp:
        dhcp_range = get_mandatory(net_prop["dhcp"], "dhcp_range")
        dhcp_ip = _split_adresses(dhcp_range)
        if not is_separate_ranges(static_ip, dhcp_ip):
            raise cfy_exc.NonRecoverableError(
                "Static_range and dhcp_range is overlapped.")
        ips.extend([dhcp_ip.start, dhcp_ip.end])
    if not is_ips_in_same_subnet(ips, netmask):
            raise cfy_exc.NonRecoverableError(
                "IP addresses in different subnets.")

def _dhcp_operation(vca_client, network_name, operation):
    """
        update dhcp setting for network
    """
    dhcp_settings = ctx.node.properties['network'].get('dhcp')
    if dhcp_settings is None:
        return
    gateway_name = ctx.node.properties["network"]['edge_gateway']
    gateway = vca_client.get_gateway(get_vcloud_config()['vdc'], gateway_name)
    if not gateway:
        raise cfy_exc.NonRecoverableError(
            "Gateway {0} not found!".format(gateway_name))

    if operation == ADD_POOL:
        ip = _split_adresses(dhcp_settings['dhcp_range'])
        low_ip_address = check_ip(ip.start)
        hight_ip_address = check_ip(ip.end)
        default_lease = dhcp_settings.get('default_lease')
        max_lease = dhcp_settings.get('max_lease')
        gateway.add_dhcp_pool(network_name, low_ip_address, hight_ip_address,
                              default_lease, max_lease)
        ctx.logger.info("DHCP rule successful created for network {0}"
                        .format(network_name))

    if operation == DELETE_POOL:
        gateway.delete_dhcp_pool(network_name)
        ctx.logger.info("DHCP rule successful deleted for network {0}"
                        .format(network_name))

    if not save_gateway_configuration(gateway, vca_client):
        return ctx.operation.retry(message='Waiting for gateway.',
                                   retry_after=10)

def _split_adresses(address_range):
    """
        split network addresses from 1.1.1.1-2.2.2.2 representation to
        separate (start,end) tuple
    """
    adresses = [ip.strip() for ip in address_range.split('-')]
    IPRange = collections.namedtuple('IPRange', 'start end')
    try:
        start = check_ip(adresses[0])
        end = check_ip(adresses[1])
        if not is_valid_ip_range(start, end):
            raise cfy_exc.NonRecoverableError(
                "Start address {0} is greater than end address: {1}"
                .format(start, end))
        return IPRange(start=start, end=end)
    except IndexError:
        raise cfy_exc.NonRecoverableError("Can't parse IP range:{0}".
                                          format(address_range))

def _get_network_list(vca_client, vdc_name):
    """
        list all avable network for current vdc
    """
    vdc = vca_client.get_vdc(vdc_name)
    if not vdc:
        raise cfy_exc.NonRecoverableError(
            "Vdc {0} not found.".format(vdc_name))
    return [net.name for net in vdc.AvailableNetworks.Network]
<|endoftext|>


<commit_before># encoding: utf-8
"""
Read graphs in GML format.

"GML, the G>raph Modelling Language, is our proposal for a portable
file format for graphs. GML's key features are portability, simple
syntax, extensibility and flexibility. A GML file consists of a
hierarchical key-value lists. Graphs can be annotated with arbitrary
data structures. The idea for a common file format was born at the
GD'95; this proposal is the outcome of many discussions. GML is the
standard file format in the Graphlet graph editor system. It has been
overtaken and adapted by several other systems for drawing graphs."

See http://www.infosun.fim.uni-passau.de/Graphlet/GML/gml-tr.html

Requires pyparsing: http://pyparsing.wikispaces.com/

Format
------
See http://www.infosun.fim.uni-passau.de/Graphlet/GML/gml-tr.html
for format specification.

Example graphs in GML format:
http://www-personal.umich.edu/~mejn/netdata/

"""
from __future__ import unicode_literals

__author__ = """Aric Hagberg (hagberg@lanl.gov)"""
#    Copyright (C) 2008-2010 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.

__all__ = ['read_gml', 'parse_gml', 'generate_gml', 'write_gml']

from cgi import escape

import networkx as nx
from networkx.exception import NetworkXError
from networkx.utils import is_string_like, open_file

##
# Removes HTML or XML character references and entities from a text string.
#
# @param text The HTML (or XML) source text (as a unicode object)
# @return The plain text, as a Unicode string, if necessary.
#
# Source: http://effbot.org/zone/re-sub.htm#unescape-html
#
import re
try:
    import htmlentitydefs
except ImportError:
    # Python 3.x
    import html.entities as htmlentitydefs

def unescape(text):
    def fixup(m):
        text = m.group(0)
        if text[:2] == "&#":
            # character reference
            try:
                if text[:3] == "&#x":
                    return unichr(int(text[3:-1], 16))
                else:
                    return unichr(int(text[2:-1]))
            except ValueError:
                pass
        else:
            # named entity
            try:
                text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
            except KeyError:
                pass
        return text # leave as is
    return re.sub("&#?\w+;", fixup, text)

@open_file(0, mode='rb')
def read_gml(path, relabel=False):
    """Read graph in GML format from path.

    Parameters
    ----------
    path : filename or filehandle
       The filename or filehandle to read from.

    relabel : bool, optional
       If True use the GML node label attribute for node names otherwise use
       the node id.

    Returns
    -------
    G : MultiGraph or MultiDiGraph

    Raises
    ------
    ImportError
        If the pyparsing module is not available.

    See Also
    --------
    write_gml, parse_gml

    Notes
    -----
    Requires pyparsing: http://pyparsing.wikispaces.com/
    The GML specification says that files should be ASCII encoded, with any
    extended ASCII characters (iso8859-1) appearing as HTML character entities.

    References
    ----------
    GML specification:
    http://www.infosun.fim.uni-passau.de/Graphlet/GML/gml-tr.html

    Examples
    --------
    >>> G=nx.path_graph(4)
    >>> nx.write_gml(G,'test.gml')
    >>> H=nx.read_gml('test.gml')
    """
    lines = (unescape(line.decode('ascii')) for line in path)
    G = parse_gml(lines, relabel=relabel)
    return G

def parse_gml(lines, relabel=True):
    """Parse GML graph from a string or iterable.

    Parameters
    ----------
    lines : string or iterable
       Data in GML format.

    relabel : bool, optional
       If True use the GML node label attribute for node names otherwise use
       the node id.

    Returns
    -------
    G : MultiGraph or MultiDiGraph

    Raises
    ------
    ImportError
        If the pyparsing module is not available.

    See Also
    --------
    write_gml, read_gml

    Notes
    -----
    This stores nested GML attributes as dictionaries in the
    NetworkX graph, node, and edge attribute structures.

    Requires pyparsing: http://pyparsing.wikispaces.com/

    References
    ----------
    GML specification:
    http://www.infosun.fim.uni-passau.de/Graphlet/GML/gml-tr.html
    """
    try:
        from pyparsing import ParseException
    except ImportError:
        try:
            from matplotlib.pyparsing import ParseException
        except:
            raise ImportError('Import Error: not able to import pyparsing:',
                              'http://pyparsing.wikispaces.com/')
    try:
        data = "".join(lines)
        gml = pyparse_gml()
        tokens = gml.parseString(data)
    except ParseException as err:
        print((err.line))
        print((" "*(err.column-1) + "^"))
        print(err)
        raise

    # function to recursively make dicts of key/value pairs
    def wrap(tok):
        listtype = type(tok)
        result={}
        for k,v in tok:
            if type(v) == listtype:
                result[str(k)] = wrap(v)
            else:
                result[str(k)] = v
        return result

    # Set flag
    multigraph = False
    # but assume multigraphs to start
    if tokens.directed == 1:
        G=nx.MultiDiGraph()
    else:
        G=nx.MultiGraph()

    for k,v in tokens.asList():
        if k=="node":
            vdict=wrap(v)
            node=vdict['id']
            G.add_node(node,attr_dict=vdict)
        elif k=="edge":
            vdict=wrap(v)
            source=vdict.pop('source')
            target=vdict.pop('target')
            if G.has_edge(source,target):
                multigraph=True
            G.add_edge(source,target,attr_dict=vdict)
        else:
            G.graph[k]=v

    # switch to Graph or DiGraph if no parallel edges were found.
    if not multigraph:
        if G.is_directed():
            G=nx.DiGraph(G)
        else:
            G=nx.Graph(G)

    if relabel:
        # relabel, but check for duplicate labels first
        mapping=[(n,d['label']) for n,d in G.node.items()]
        x,y=zip(*mapping)
        if len(set(y))!=len(G):
            raise NetworkXError('Failed to relabel nodes: '
                                'duplicate node labels found. '
                                'Use relabel=False.')
        G=nx.relabel_nodes(G,dict(mapping))
    return G

def pyparse_gml():
    """A pyparsing tokenizer for GML graph format.

    This is not intended to be called directly.

    See Also
    --------
    write_gml, read_gml, parse_gml
    """
    try:
        from pyparsing import \
             Literal, CaselessLiteral, Word, Forward,\
             ZeroOrMore, Group, Dict, Optional, Combine,\
             ParseException, restOfLine, White, alphas, alphanums, nums,\
             OneOrMore,quotedString,removeQuotes,dblQuotedString, Regex
    except ImportError:
        try:
            from matplotlib.pyparsing import \
             Literal, CaselessLiteral, Word, Forward,\
             ZeroOrMore, Group, Dict, Optional, Combine,\
             ParseException, restOfLine, White, alphas, alphanums, nums,\
             OneOrMore,quotedString,removeQuotes,dblQuotedString, Regex
        except:
            raise ImportError('pyparsing not found',
                              'http://pyparsing.wikispaces.com/')

    lbrack = Literal("[").suppress()
    rbrack = Literal("]").suppress()
    pound = ("#")
    comment = pound + Optional( restOfLine )
    integer = Word(nums+'-').setParseAction(lambda s,l,t:[ int(t[0])])
    real = Regex(r"[+-]?\d+\.\d*([eE][+-]?\d+)?").setParseAction(
        lambda s,l,t:[ float(t[0]) ])
    dblQuotedString.setParseAction( removeQuotes )
    key = Word(alphas,alphanums+'_')
    value_atom = (real | integer | Word(alphanums) | dblQuotedString)
    value = Forward()   # to be defined later with << operator
    keyvalue = Group(key+value)
    value << (value_atom | Group( lbrack + ZeroOrMore(keyvalue) + rbrack ))
    node = Group(Literal("node") + lbrack + Group(OneOrMore(keyvalue)) + rbrack)
    edge = Group(Literal("edge") + lbrack + Group(OneOrMore(keyvalue)) + rbrack)

    creator = Group(Literal("Creator")+ Optional( restOfLine ))
    version = Group(Literal("Version")+ Optional( restOfLine ))
    graphkey = Literal("graph").suppress()

    graph = Dict (Optional(creator)+Optional(version)+\
        graphkey + lbrack + ZeroOrMore( (node|edge|keyvalue) ) + rbrack )
    graph.ignore(comment)

    return graph

def generate_gml(G):
    """Generate a single entry of the graph G in GML format.

    Parameters
    ----------
    G : NetworkX graph

    Returns
    -------
    lines: string
       Lines in GML format.

    Notes
    -----
    This implementation does not support all Python data types as GML
    data.  Nodes, node attributes, edge attributes, and graph
    attributes must be either dictionaries or single stings or
    numbers.  If they are not an attempt is made to represent them as
    strings.  For example, a list as edge data
    G[1][2]['somedata']=[1,2,3], will be represented in the GML file
    as::

       edge [
         source 1
         target 2
         somedata "[1, 2, 3]"
       ]
    """
    # recursively make dicts into gml brackets
    def listify(d,indent,indentlevel):
        result='[ \n'
        for k,v in d.items():
            result += (indentlevel+1)*indent + \
                string_item(k,v,indentlevel*indent)+'\n'
        return result+indentlevel*indent+"]"

    def string_item(k,v,indent):
        # try to make a string of the data
        if type(v) == dict:
            v = listify(v, indent, 2)
        elif is_string_like(v):
            v = '"{0}"'.format(escape(v, quote=True))
            #v = '"{0}"'.format(v, quote=True)
        elif type(v) == bool:
            v = int(v)
        return "{0} {1}".format(k,v)

    # check for attributes or assign empty dict
    if hasattr(G,'graph_attr'):
        graph_attr = G.graph_attr
    else:
        graph_attr = {}
    if hasattr(G,'node_attr'):
        node_attr = G.node_attr
    else:
        node_attr = {}

    indent = 2 * ' '
    count = iter(range(len(G)))
    node_id = {}

    yield "graph ["
    if G.is_directed():
        yield indent + "directed 1"
    # write graph attributes
    for k,v in G.graph.items():
        if k == 'directed':
            continue
        yield indent + string_item(k,v,indent)
    # write nodes
    for n in G:
        yield indent + "node ["
        # get id or assign number
        nid = G.node[n].get('id',next(count))
        node_id[n] = nid
        yield 2 * indent + "id {0}".format(nid)
        # Uses customized __str__, if implemented.
        label = str(G.node[n].get('label',n))
        # Need to escape & and " with HTML entities
        label = escape(label, quote=True)
        yield 2 * indent + 'label "{0}"'.format(label)
        if n in G:
          for k,v in G.node[n].items():
              if k == 'id' or k == 'label': continue
              yield 2 * indent + string_item(k,v,indent)
        yield indent + "]"
    # write edges
    for u,v,edgedata in G.edges_iter(data=True):
        yield indent + "edge ["
        yield 2 * indent + "source {0}".format(node_id[u])
        yield 2 * indent + "target {0}".format(node_id[v])
        for k, v in edgedata.items():
            if k == 'source': continue
            if k == 'target': continue
            yield 2 * indent + string_item(k, v, indent)
        yield indent + "]"
    yield "]"

@open_file(1, mode='wb')
def write_gml(G, path):
    """
    Write the graph G in GML format to the file or file handle path.

    Parameters
    ----------
    path : filename or filehandle
       The filename or filehandle to write.  Filenames ending in
       .gz or .gz2 will be compressed.

    See Also
    --------
    read_gml, parse_gml

    Notes
    -----
    GML specifications indicate that the file should only use
    7bit ASCII text encoding.iso8859-1 (latin-1).

    This implementation does not support all Python data types as GML
    data.  Nodes, node attributes, edge attributes, and graph
    attributes must be either dictionaries or single stings or
    numbers.  If they are not an attempt is made to represent them as
    strings.  For example, a list as edge data
    G[1][2]['somedata']=[1,2,3], will be represented in the GML file
    as::

       edge [
         source 1
         target 2
         somedata "[1, 2, 3]"
       ]

    Examples
    ---------
    >>> G=nx.path_graph(4)
    >>> nx.write_gml(G,"test.gml")

    Filenames ending in .gz or .bz2 will be compressed.

    >>> nx.write_gml(G,"test.gml.gz")
    """
    for line in generate_gml(G):
        line += '\n'
        path.write(line.encode('ascii', 'xmlcharrefreplace'))

# fixture for nose tests
def setup_module(module):
    from nose import SkipTest
    try:
        import pyparsing
    except:
        try:
            import matplotlib.pyparsing
        except:
            raise SkipTest("pyparsing not available")

# fixture for nose tests
def teardown_module(module):
    import os
    os.unlink('test.gml')
    os.unlink('test.gml.gz')
<commit_msg>Fixed warnings with "<<" operator when loading<commit_after># encoding: utf-8
"""
Read graphs in GML format.

"GML, the G>raph Modelling Language, is our proposal for a portable
file format for graphs. GML's key features are portability, simple
syntax, extensibility and flexibility. A GML file consists of a
hierarchical key-value lists. Graphs can be annotated with arbitrary
data structures. The idea for a common file format was born at the
GD'95; this proposal is the outcome of many discussions. GML is the
standard file format in the Graphlet graph editor system. It has been
overtaken and adapted by several other systems for drawing graphs."

See http://www.infosun.fim.uni-passau.de/Graphlet/GML/gml-tr.html

Requires pyparsing: http://pyparsing.wikispaces.com/

Format
------
See http://www.infosun.fim.uni-passau.de/Graphlet/GML/gml-tr.html
for format specification.

Example graphs in GML format:
http://www-personal.umich.edu/~mejn/netdata/

"""
from __future__ import unicode_literals

__author__ = """Aric Hagberg (hagberg@lanl.gov)"""
#    Copyright (C) 2008-2010 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.

__all__ = ['read_gml', 'parse_gml', 'generate_gml', 'write_gml']

from cgi import escape

import networkx as nx
from networkx.exception import NetworkXError
from networkx.utils import is_string_like, open_file

##
# Removes HTML or XML character references and entities from a text string.
#
# @param text The HTML (or XML) source text (as a unicode object)
# @return The plain text, as a Unicode string, if necessary.
#
# Source: http://effbot.org/zone/re-sub.htm#unescape-html
#
import re
try:
    import htmlentitydefs
except ImportError:
    # Python 3.x
    import html.entities as htmlentitydefs

def unescape(text):
    def fixup(m):
        text = m.group(0)
        if text[:2] == "&#":
            # character reference
            try:
                if text[:3] == "&#x":
                    return unichr(int(text[3:-1], 16))
                else:
                    return unichr(int(text[2:-1]))
            except ValueError:
                pass
        else:
            # named entity
            try:
                text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
            except KeyError:
                pass
        return text # leave as is
    return re.sub("&#?\w+;", fixup, text)

@open_file(0, mode='rb')
def read_gml(path, relabel=False):
    """Read graph in GML format from path.

    Parameters
    ----------
    path : filename or filehandle
       The filename or filehandle to read from.

    relabel : bool, optional
       If True use the GML node label attribute for node names otherwise use
       the node id.

    Returns
    -------
    G : MultiGraph or MultiDiGraph

    Raises
    ------
    ImportError
        If the pyparsing module is not available.

    See Also
    --------
    write_gml, parse_gml

    Notes
    -----
    Requires pyparsing: http://pyparsing.wikispaces.com/
    The GML specification says that files should be ASCII encoded, with any
    extended ASCII characters (iso8859-1) appearing as HTML character entities.

    References
    ----------
    GML specification:
    http://www.infosun.fim.uni-passau.de/Graphlet/GML/gml-tr.html

    Examples
    --------
    >>> G=nx.path_graph(4)
    >>> nx.write_gml(G,'test.gml')
    >>> H=nx.read_gml('test.gml')
    """
    lines = (unescape(line.decode('ascii')) for line in path)
    G = parse_gml(lines, relabel=relabel)
    return G

def parse_gml(lines, relabel=True):
    """Parse GML graph from a string or iterable.

    Parameters
    ----------
    lines : string or iterable
       Data in GML format.

    relabel : bool, optional
       If True use the GML node label attribute for node names otherwise use
       the node id.

    Returns
    -------
    G : MultiGraph or MultiDiGraph

    Raises
    ------
    ImportError
        If the pyparsing module is not available.

    See Also
    --------
    write_gml, read_gml

    Notes
    -----
    This stores nested GML attributes as dictionaries in the
    NetworkX graph, node, and edge attribute structures.

    Requires pyparsing: http://pyparsing.wikispaces.com/

    References
    ----------
    GML specification:
    http://www.infosun.fim.uni-passau.de/Graphlet/GML/gml-tr.html
    """
    try:
        from pyparsing import ParseException
    except ImportError:
        try:
            from matplotlib.pyparsing import ParseException
        except:
            raise ImportError('Import Error: not able to import pyparsing:',
                              'http://pyparsing.wikispaces.com/')
    try:
        data = "".join(lines)
        gml = pyparse_gml()
        tokens = gml.parseString(data)
    except ParseException as err:
        print((err.line))
        print((" "*(err.column-1) + "^"))
        print(err)
        raise

    # function to recursively make dicts of key/value pairs
    def wrap(tok):
        listtype = type(tok)
        result={}
        for k,v in tok:
            if type(v) == listtype:
                result[str(k)] = wrap(v)
            else:
                result[str(k)] = v
        return result

    # Set flag
    multigraph = False
    # but assume multigraphs to start
    if tokens.directed == 1:
        G=nx.MultiDiGraph()
    else:
        G=nx.MultiGraph()

    for k,v in tokens.asList():
        if k=="node":
            vdict=wrap(v)
            node=vdict['id']
            G.add_node(node,attr_dict=vdict)
        elif k=="edge":
            vdict=wrap(v)
            source=vdict.pop('source')
            target=vdict.pop('target')
            if G.has_edge(source,target):
                multigraph=True
            G.add_edge(source,target,attr_dict=vdict)
        else:
            G.graph[k]=v

    # switch to Graph or DiGraph if no parallel edges were found.
    if not multigraph:
        if G.is_directed():
            G=nx.DiGraph(G)
        else:
            G=nx.Graph(G)

    if relabel:
        # relabel, but check for duplicate labels first
        mapping=[(n,d['label']) for n,d in G.node.items()]
        x,y=zip(*mapping)
        if len(set(y))!=len(G):
            raise NetworkXError('Failed to relabel nodes: '
                                'duplicate node labels found. '
                                'Use relabel=False.')
        G=nx.relabel_nodes(G,dict(mapping))
    return G

def pyparse_gml():
    """A pyparsing tokenizer for GML graph format.

    This is not intended to be called directly.

    See Also
    --------
    write_gml, read_gml, parse_gml
    """
    try:
        from pyparsing import \
             Literal, CaselessLiteral, Word, Forward,\
             ZeroOrMore, Group, Dict, Optional, Combine,\
             ParseException, restOfLine, White, alphas, alphanums, nums,\
             OneOrMore,quotedString,removeQuotes,dblQuotedString, Regex
    except ImportError:
        try:
            from matplotlib.pyparsing import \
             Literal, CaselessLiteral, Word, Forward,\
             ZeroOrMore, Group, Dict, Optional, Combine,\
             ParseException, restOfLine, White, alphas, alphanums, nums,\
             OneOrMore,quotedString,removeQuotes,dblQuotedString, Regex
        except:
            raise ImportError('pyparsing not found',
                              'http://pyparsing.wikispaces.com/')

    lbrack = Literal("[").suppress()
    rbrack = Literal("]").suppress()
    pound = ("#")
    comment = pound + Optional( restOfLine )
    integer = Word(nums+'-').setParseAction(lambda s,l,t:[ int(t[0])])
    real = Regex(r"[+-]?\d+\.\d*([eE][+-]?\d+)?").setParseAction(
        lambda s,l,t:[ float(t[0]) ])
    dblQuotedString.setParseAction( removeQuotes )
    key = Word(alphas,alphanums+'_')
    value_atom = (real | integer | Word(alphanums) | dblQuotedString)
    value = Forward()   # to be defined later with <<= operator
    keyvalue = Group(key+value)
    value <<= (value_atom | Group( lbrack + ZeroOrMore(keyvalue) + rbrack ))
    node = Group(Literal("node") + lbrack + Group(OneOrMore(keyvalue)) + rbrack)
    edge = Group(Literal("edge") + lbrack + Group(OneOrMore(keyvalue)) + rbrack)

    creator = Group(Literal("Creator")+ Optional( restOfLine ))
    version = Group(Literal("Version")+ Optional( restOfLine ))
    graphkey = Literal("graph").suppress()

    graph = Dict (Optional(creator)+Optional(version)+\
        graphkey + lbrack + ZeroOrMore( (node|edge|keyvalue) ) + rbrack )
    graph.ignore(comment)

    return graph

def generate_gml(G):
    """Generate a single entry of the graph G in GML format.

    Parameters
    ----------
    G : NetworkX graph

    Returns
    -------
    lines: string
       Lines in GML format.

    Notes
    -----
    This implementation does not support all Python data types as GML
    data.  Nodes, node attributes, edge attributes, and graph
    attributes must be either dictionaries or single stings or
    numbers.  If they are not an attempt is made to represent them as
    strings.  For example, a list as edge data
    G[1][2]['somedata']=[1,2,3], will be represented in the GML file
    as::

       edge [
         source 1
         target 2
         somedata "[1, 2, 3]"
       ]
    """
    # recursively make dicts into gml brackets
    def listify(d,indent,indentlevel):
        result='[ \n'
        for k,v in d.items():
            result += (indentlevel+1)*indent + \
                string_item(k,v,indentlevel*indent)+'\n'
        return result+indentlevel*indent+"]"

    def string_item(k,v,indent):
        # try to make a string of the data
        if type(v) == dict:
            v = listify(v, indent, 2)
        elif is_string_like(v):
            v = '"{0}"'.format(escape(v, quote=True))
            #v = '"{0}"'.format(v, quote=True)
        elif type(v) == bool:
            v = int(v)
        return "{0} {1}".format(k,v)

    # check for attributes or assign empty dict
    if hasattr(G,'graph_attr'):
        graph_attr = G.graph_attr
    else:
        graph_attr = {}
    if hasattr(G,'node_attr'):
        node_attr = G.node_attr
    else:
        node_attr = {}

    indent = 2 * ' '
    count = iter(range(len(G)))
    node_id = {}

    yield "graph ["
    if G.is_directed():
        yield indent + "directed 1"
    # write graph attributes
    for k,v in G.graph.items():
        if k == 'directed':
            continue
        yield indent + string_item(k,v,indent)
    # write nodes
    for n in G:
        yield indent + "node ["
        # get id or assign number
        nid = G.node[n].get('id',next(count))
        node_id[n] = nid
        yield 2 * indent + "id {0}".format(nid)
        # Uses customized __str__, if implemented.
        label = str(G.node[n].get('label',n))
        # Need to escape & and " with HTML entities
        label = escape(label, quote=True)
        yield 2 * indent + 'label "{0}"'.format(label)
        if n in G:
          for k,v in G.node[n].items():
              if k == 'id' or k == 'label': continue
              yield 2 * indent + string_item(k,v,indent)
        yield indent + "]"
    # write edges
    for u,v,edgedata in G.edges_iter(data=True):
        yield indent + "edge ["
        yield 2 * indent + "source {0}".format(node_id[u])
        yield 2 * indent + "target {0}".format(node_id[v])
        for k, v in edgedata.items():
            if k == 'source': continue
            if k == 'target': continue
            yield 2 * indent + string_item(k, v, indent)
        yield indent + "]"
    yield "]"

@open_file(1, mode='wb')
def write_gml(G, path):
    """
    Write the graph G in GML format to the file or file handle path.

    Parameters
    ----------
    path : filename or filehandle
       The filename or filehandle to write.  Filenames ending in
       .gz or .gz2 will be compressed.

    See Also
    --------
    read_gml, parse_gml

    Notes
    -----
    GML specifications indicate that the file should only use
    7bit ASCII text encoding.iso8859-1 (latin-1).

    This implementation does not support all Python data types as GML
    data.  Nodes, node attributes, edge attributes, and graph
    attributes must be either dictionaries or single stings or
    numbers.  If they are not an attempt is made to represent them as
    strings.  For example, a list as edge data
    G[1][2]['somedata']=[1,2,3], will be represented in the GML file
    as::

       edge [
         source 1
         target 2
         somedata "[1, 2, 3]"
       ]

    Examples
    ---------
    >>> G=nx.path_graph(4)
    >>> nx.write_gml(G,"test.gml")

    Filenames ending in .gz or .bz2 will be compressed.

    >>> nx.write_gml(G,"test.gml.gz")
    """
    for line in generate_gml(G):
        line += '\n'
        path.write(line.encode('ascii', 'xmlcharrefreplace'))

# fixture for nose tests
def setup_module(module):
    from nose import SkipTest
    try:
        import pyparsing
    except:
        try:
            import matplotlib.pyparsing
        except:
            raise SkipTest("pyparsing not available")

# fixture for nose tests
def teardown_module(module):
    import os
    os.unlink('test.gml')
    os.unlink('test.gml.gz')
<|endoftext|>


<commit_before>import os
import json
import random
import time

from boto.s3.connection import S3Connection
from boto.exception import S3ResponseError
from boto.s3.key import Key

from constants import *
from cachingHelper import getCache
from dbhelper import *
from cluster import Cluster
from clusterTableManager import ClusterTableManager
from loggingHelper import *
from clusterJobManager import ClusterJobManager
from workerJob import WorkerJob

#constants
NEW_CLUSTER_FOLDER = "new/"
DOCLIST_FILE = "docList"
CLUSTERS_FILE = "clusters"
METADATA_FILE = "metadata"

METADATA_NAME_STATE = 'state'

class ClusterManager:
    """
    Manage clusters stored in cloud.
    """

    def __init__(self):
        """
        Instantiates a new instance of ClusterManager class

        'bucketConnString' : connection string of s3 bucket in which clusters
        are stored.
        """

        self.bucketConnString = os.environ['CLUSTERSBUCKET_CONNECTIONSTRING'];
        self.clusterTableManager = ClusterTableManager()
        self.cache = getCache()

    def __getBucket(self):
        bucketConnParams = parseConnectionString(self.bucketConnString);

        conn = S3Connection(
            bucketConnParams['accessKeyId'],
            bucketConnParams['secretAccessKey']);

        return conn.get_bucket(bucketConnParams['bucketName']);

    def __putObject(self, key, content):
	k = Key(self.__getBucket());
        k.key = key;
        k.set_contents_from_string(content)

    def __getObject(self, docKey):
        k = Key(self.__getBucket());
        k.key = docKey;
        return k.get_contents_as_string()

    def __copyObject(self, fromKey, toKey):
        bucket = self.__getBucket()
        source_key = bucket.get_key(fromKey)

        source_key.copy(bucket, toKey)

    def __archiveCluster(self):
        """
        Save a copy of current clusters in a new folder.
        """

        archiveFolderName = str(int(time.time())) + '/'

        self.__copyObject(
            NEW_CLUSTER_FOLDER + DOCLIST_FILE,
            archiveFolderName + DOCLIST_FILE)
        self.__copyObject(
            NEW_CLUSTER_FOLDER + CLUSTERS_FILE,
            archiveFolderName + CLUSTERS_FILE)
        self.__copyObject(
            NEW_CLUSTER_FOLDER + METADATA_FILE,
            archiveFolderName + METADATA_FILE)

    def __cleanupOldDocsFromCluster(self, expiredDocs):
        clusters = self.getClusters()

        newClusters = [Cluster(list(cluster - expiredDocs)) for cluster in clusters
                    if (cluster - expiredDocs)]
        self.putClusters(newClusters)

    def __addNewDocsToCluster(self, newDocs):
        clusters = self.getClusters() + [Cluster([docId]) for docId in newDocs]
        self.putClusters(clusters)

    def __getMetadata(self, key):
        metadata = eval(self.__getObject(NEW_CLUSTER_FOLDER + METADATA_FILE))

        try:
          return metadata[key]
        except KeyError:
          return None;

    def __setMetadata(self, key, value):
        try:
            metadata = eval(self.__getObject(NEW_CLUSTER_FOLDER + METADATA_FILE))
        except S3ResponseError:
            metadata = {}

        metadata[key] = value
        self.__putObject(NEW_CLUSTER_FOLDER + METADATA_FILE, str(metadata))

    def initNewClusters(self, docList):
        # initialize doc list
        self.putDocList(docList)

        # initizalize each point as separate cluster
        clusters = [Cluster([docId]) for docId in docList]
        self.putClusters(clusters)

        # set state
        self.setState(CLUSTER_STATE_INITIALIZED)

    def initNewIncrementalCluster(self, docList):
        oldDocList = self.getDocList()

        docSet = set(docList)
        oldDocSet = set(oldDocList)

        newDocs = docSet - oldDocSet
        retainedDocs = docSet.intersection(oldDocSet)
        expiredDocs = oldDocSet - docSet

        self.__archiveCluster()

        self.putDocList(docList)
        self.__cleanupOldDocsFromCluster(expiredDocs)
        self.__addNewDocsToCluster(newDocs)
        self.setState(CLUSTER_STATE_INITIALIZED)

        return (list(newDocs), list(retainedDocs), list(expiredDocs))

    def putDocList(self, docList):
        self.__putObject(
            NEW_CLUSTER_FOLDER + DOCLIST_FILE,
            json.dumps(docList))

    def getDocList(self):
        return json.loads(self.__getObject(NEW_CLUSTER_FOLDER + DOCLIST_FILE))

    def processNewCluster(self, cluster):
        cluster.process()
        self.clusterTableManager.addCluster(cluster)

    def getProcessedCluster(self, clusterId):
        cluster = self.clusterTableManager.getCluster(clusterId);
        cluster.process();
        return cluster.articles;

    def __computeClusterRankingScore(self, cluster):
        return len(cluster) - len(cluster.duplicates) + \
            (0.1 * len(cluster.publishers))

    def __constructQueryResponse(self, clusters, skip, top):
        response = []
        clusterList = list(clusters)
        clusterList.sort(key = self.__computeClusterRankingScore, reverse=True)

        for cluster in clusterList[skip:(skip + top)]:
            try:
                response.append(self.getProcessedCluster(cluster.id))
            except:
                continue

        return response;

    def queryByCategoryAndCountry(self, category, country, skip, top):
        clusters = self.clusterTableManager.queryByCategoryAndCountry(
            category,
            country)
        return self.__constructQueryResponse(clusters, skip, top)

    def queryByLocale(self, locale, skip, top):
        clusters = self.clusterTableManager.queryByLocale(locale)
        response = []

        return self.__constructQueryResponse(clusters, skip, top)

    def putClusters(self, clusters):
        jobManager = ClusterJobManager()

        existingClusters = self.getClusters()
        newClusters = [cluster for cluster in clusters
            if cluster not in existingClusters]
        expiredClusters = [cluster for cluster in existingClusters
            if cluster not in clusters]

        for cluster in newClusters:
            job = WorkerJob(
                JOB_PROCESSNEWCLUSTER,
                { JOBARG_PROCESSNEWCLUSTER_CLUSTER : str(cluster)})
            jobManager.enqueueJob(job)
            logging.info(
                "Put process new cluster job. Cluster id: %s.",
                cluster.id)

        self.clusterTableManager.deleteClusters(expiredClusters)

        self.__putObject(NEW_CLUSTER_FOLDER + CLUSTERS_FILE, str(clusters))

    def getClusters(self):
        try:
            return eval(self.__getObject(NEW_CLUSTER_FOLDER + CLUSTERS_FILE))
        except S3ResponseError:
            return []

    def setState(self, state):
        self.__setMetadata(METADATA_NAME_STATE, state)

    def getState(self):
        return self.__getMetadata(METADATA_NAME_STATE)
<commit_msg>Adjusting the story ranking algo<commit_after>import os
import json
import random
import time

from boto.s3.connection import S3Connection
from boto.exception import S3ResponseError
from boto.s3.key import Key

from constants import *
from cachingHelper import getCache
from dbhelper import *
from cluster import Cluster
from clusterTableManager import ClusterTableManager
from loggingHelper import *
from clusterJobManager import ClusterJobManager
from workerJob import WorkerJob

#constants
NEW_CLUSTER_FOLDER = "new/"
DOCLIST_FILE = "docList"
CLUSTERS_FILE = "clusters"
METADATA_FILE = "metadata"

METADATA_NAME_STATE = 'state'

class ClusterManager:
    """
    Manage clusters stored in cloud.
    """

    def __init__(self):
        """
        Instantiates a new instance of ClusterManager class

        'bucketConnString' : connection string of s3 bucket in which clusters
        are stored.
        """

        self.bucketConnString = os.environ['CLUSTERSBUCKET_CONNECTIONSTRING'];
        self.clusterTableManager = ClusterTableManager()
        self.cache = getCache()

    def __getBucket(self):
        bucketConnParams = parseConnectionString(self.bucketConnString);

        conn = S3Connection(
            bucketConnParams['accessKeyId'],
            bucketConnParams['secretAccessKey']);

        return conn.get_bucket(bucketConnParams['bucketName']);

    def __putObject(self, key, content):
	k = Key(self.__getBucket());
        k.key = key;
        k.set_contents_from_string(content)

    def __getObject(self, docKey):
        k = Key(self.__getBucket());
        k.key = docKey;
        return k.get_contents_as_string()

    def __copyObject(self, fromKey, toKey):
        bucket = self.__getBucket()
        source_key = bucket.get_key(fromKey)

        source_key.copy(bucket, toKey)

    def __archiveCluster(self):
        """
        Save a copy of current clusters in a new folder.
        """

        archiveFolderName = str(int(time.time())) + '/'

        self.__copyObject(
            NEW_CLUSTER_FOLDER + DOCLIST_FILE,
            archiveFolderName + DOCLIST_FILE)
        self.__copyObject(
            NEW_CLUSTER_FOLDER + CLUSTERS_FILE,
            archiveFolderName + CLUSTERS_FILE)
        self.__copyObject(
            NEW_CLUSTER_FOLDER + METADATA_FILE,
            archiveFolderName + METADATA_FILE)

    def __cleanupOldDocsFromCluster(self, expiredDocs):
        clusters = self.getClusters()

        newClusters = [Cluster(list(cluster - expiredDocs)) for cluster in clusters
                    if (cluster - expiredDocs)]
        self.putClusters(newClusters)

    def __addNewDocsToCluster(self, newDocs):
        clusters = self.getClusters() + [Cluster([docId]) for docId in newDocs]
        self.putClusters(clusters)

    def __getMetadata(self, key):
        metadata = eval(self.__getObject(NEW_CLUSTER_FOLDER + METADATA_FILE))

        try:
          return metadata[key]
        except KeyError:
          return None;

    def __setMetadata(self, key, value):
        try:
            metadata = eval(self.__getObject(NEW_CLUSTER_FOLDER + METADATA_FILE))
        except S3ResponseError:
            metadata = {}

        metadata[key] = value
        self.__putObject(NEW_CLUSTER_FOLDER + METADATA_FILE, str(metadata))

    def initNewClusters(self, docList):
        # initialize doc list
        self.putDocList(docList)

        # initizalize each point as separate cluster
        clusters = [Cluster([docId]) for docId in docList]
        self.putClusters(clusters)

        # set state
        self.setState(CLUSTER_STATE_INITIALIZED)

    def initNewIncrementalCluster(self, docList):
        oldDocList = self.getDocList()

        docSet = set(docList)
        oldDocSet = set(oldDocList)

        newDocs = docSet - oldDocSet
        retainedDocs = docSet.intersection(oldDocSet)
        expiredDocs = oldDocSet - docSet

        self.__archiveCluster()

        self.putDocList(docList)
        self.__cleanupOldDocsFromCluster(expiredDocs)
        self.__addNewDocsToCluster(newDocs)
        self.setState(CLUSTER_STATE_INITIALIZED)

        return (list(newDocs), list(retainedDocs), list(expiredDocs))

    def putDocList(self, docList):
        self.__putObject(
            NEW_CLUSTER_FOLDER + DOCLIST_FILE,
            json.dumps(docList))

    def getDocList(self):
        return json.loads(self.__getObject(NEW_CLUSTER_FOLDER + DOCLIST_FILE))

    def processNewCluster(self, cluster):
        cluster.process()
        self.clusterTableManager.addCluster(cluster)

    def getProcessedCluster(self, clusterId):
        cluster = self.clusterTableManager.getCluster(clusterId);
        cluster.process();
        return cluster.articles;

    def __computeClusterRankingScore(self, cluster):
        return (0.4 * (len(cluster) - len(cluster.duplicates))) + \
            (0.6 * len(cluster.publishers))

    def __constructQueryResponse(self, clusters, skip, top):
        response = []
        clusterList = list(clusters)
        clusterList.sort(key = self.__computeClusterRankingScore, reverse=True)

        for cluster in clusterList[skip:(skip + top)]:
            try:
                response.append(self.getProcessedCluster(cluster.id))
            except:
                continue

        return response;

    def queryByCategoryAndCountry(self, category, country, skip, top):
        clusters = self.clusterTableManager.queryByCategoryAndCountry(
            category,
            country)
        return self.__constructQueryResponse(clusters, skip, top)

    def queryByLocale(self, locale, skip, top):
        clusters = self.clusterTableManager.queryByLocale(locale)
        response = []

        return self.__constructQueryResponse(clusters, skip, top)

    def putClusters(self, clusters):
        jobManager = ClusterJobManager()

        existingClusters = self.getClusters()
        newClusters = [cluster for cluster in clusters
            if cluster not in existingClusters]
        expiredClusters = [cluster for cluster in existingClusters
            if cluster not in clusters]

        for cluster in newClusters:
            job = WorkerJob(
                JOB_PROCESSNEWCLUSTER,
                { JOBARG_PROCESSNEWCLUSTER_CLUSTER : str(cluster)})
            jobManager.enqueueJob(job)
            logging.info(
                "Put process new cluster job. Cluster id: %s.",
                cluster.id)

        self.clusterTableManager.deleteClusters(expiredClusters)

        self.__putObject(NEW_CLUSTER_FOLDER + CLUSTERS_FILE, str(clusters))

    def getClusters(self):
        try:
            return eval(self.__getObject(NEW_CLUSTER_FOLDER + CLUSTERS_FILE))
        except S3ResponseError:
            return []

    def setState(self, state):
        self.__setMetadata(METADATA_NAME_STATE, state)

    def getState(self):
        return self.__getMetadata(METADATA_NAME_STATE)
<|endoftext|>


<commit_before>from flask import Blueprint, render_template, flash, request, redirect, url_for, jsonify
from flask.ext.login import login_user, logout_user, login_required, current_user
from flask.ext.restful import abort, Api, Resource
from next.api import api_util
from next.api.keychain import KeyChain
from next.api.targetmapper import TargetMapper

keychain = KeyChain()
targetmapper = TargetMapper()
# Initialize flask.Flask application and restful.api objects
api = Blueprint('api', __name__)
api_interface = api_util.NextBackendApi(api)

# Format: Resource Class, get url, post url (when applicable)
from next.api.resources.experiment import Experiment
api_interface.add_resource(Experiment, '/experiment', '/experiment/<string:exp_uid>/<string:exp_key>')

from next.api.resources.get_query import getQuery
api_interface.add_resource(getQuery, '/experiment/<string:exp_uid>/<string:exp_key>/getQuery', '/experiment/getQuery')

from next.api.resources.process_answer import processAnswer
api_interface.add_resource(processAnswer, '/experiment/processAnswer')

from next.api.resources.stats import Stats
api_interface.add_resource(Stats, '/experiment/stats')

from next.api.resources.predict import Predict
api_interface.add_resource(Predict, '/experiment/predict')

from next.api.resources.about import About
api_interface.add_resource(About, '/about')

from next.api.resources.logs import Logs
api_interface.add_resource(Logs, '/experiment/<string:exp_uid>/<string:exp_key>/logs')

from next.api.resources.participants import Participants
api_interface.add_resource(Participants, '/experiment/<string:exp_uid>/<string:exp_key>/participants')

from next.api.resources.targets import Targets

api_interface.add_resource(Targets,'/targets/<string:exp_uid>/<string:exp_key>', '/targets/createtargetmapping')

from next.api.resources.widgets import Widgets
api_interface.add_resource(Widgets,'/widgets/getwidget')

from next.api.resources.widgets import WidgetKeys
api_interface.add_resource(WidgetKeys,'/widgets/temp-widget-keys')

from next.api.resources.database import DabaseBackup
api_interface.add_resource(DatabaseBackup,'/databasebackup')
<commit_msg>Database backup python files.<commit_after>from flask import Blueprint, render_template, flash, request, redirect, url_for, jsonify
from flask.ext.login import login_user, logout_user, login_required, current_user
from flask.ext.restful import abort, Api, Resource
from next.api import api_util
from next.api.keychain import KeyChain
from next.api.targetmapper import TargetMapper

keychain = KeyChain()
targetmapper = TargetMapper()
# Initialize flask.Flask application and restful.api objects
api = Blueprint('api', __name__)
api_interface = api_util.NextBackendApi(api)

# Format: Resource Class, get url, post url (when applicable)
from next.api.resources.experiment import Experiment
api_interface.add_resource(Experiment, '/experiment', '/experiment/<string:exp_uid>/<string:exp_key>')

from next.api.resources.get_query import getQuery
api_interface.add_resource(getQuery, '/experiment/<string:exp_uid>/<string:exp_key>/getQuery', '/experiment/getQuery')

from next.api.resources.process_answer import processAnswer
api_interface.add_resource(processAnswer, '/experiment/processAnswer')

from next.api.resources.stats import Stats
api_interface.add_resource(Stats, '/experiment/stats')

from next.api.resources.predict import Predict
api_interface.add_resource(Predict, '/experiment/predict')

from next.api.resources.about import About
api_interface.add_resource(About, '/about')

from next.api.resources.logs import Logs
api_interface.add_resource(Logs, '/experiment/<string:exp_uid>/<string:exp_key>/logs')

from next.api.resources.participants import Participants
api_interface.add_resource(Participants, '/experiment/<string:exp_uid>/<string:exp_key>/participants')

from next.api.resources.targets import Targets

api_interface.add_resource(Targets,'/targets/<string:exp_uid>/<string:exp_key>', '/targets/createtargetmapping')

from next.api.resources.widgets import Widgets
api_interface.add_resource(Widgets,'/widgets/getwidget')

from next.api.resources.widgets import WidgetKeys
api_interface.add_resource(WidgetKeys,'/widgets/temp-widget-keys')

from next.api.resources.database import DatabaseBackup
api_interface.add_resource(DatabaseBackup,'/databasebackup')
<|endoftext|>


<commit_before>import json
import numpy
import numpy.random
from datetime import datetime
from datetime import timedelta
from next.utils import utils

import matplotlib
matplotlib.use('Agg')

MAX_SAMPLES_PER_PLOT = 100

class AppDashboard(object):

  def __init__(self,db,ell):
    self.db = db
    self.ell = ell

  def api_activity_histogram(self,app_id,exp_uid,task):
    """
    Description: returns the data to plot all API activity (for all algorithms) in a histogram with respect to time for any task in {getQuery,processAnswer,predict}

    Expected input:
      (string) task :  must be in {'getQuery','processAnswer','predict'}

    Expected output (in dict):
      (dict) MPLD3 plot dictionary
    """

    list_of_log_dict,didSucceed,message = self.ell.get_logs_with_filter(app_id+':APP-CALL',{'exp_uid':exp_uid,'task':task})

    from datetime import datetime
    from datetime import timedelta
    start_date_str,didSucceed,message = self.db.get('experiments_admin',exp_uid,'start_date')
    start_date = utils.str2datetime(start_date_str)
    numerical_timestamps = [ ( utils.str2datetime(item['timestamp'])-start_date).total_seconds() for item in list_of_log_dict]

    import matplotlib.pyplot as plt
    import mpld3
    fig, ax = plt.subplots(subplot_kw=dict(axisbg='#FFFFFF'),figsize=(12,1.5))
    ax.hist(numerical_timestamps,min(int(1+4*numpy.sqrt(len(numerical_timestamps))),300),alpha=0.5,color='black')
    ax.set_frame_on(False)
    ax.get_xaxis().set_ticks([])
    ax.get_yaxis().set_ticks([])
    ax.get_yaxis().set_visible(False)
    ax.set_xlim(0, max(numerical_timestamps))
    plot_dict = mpld3.fig_to_dict(fig)
    plt.close()

    return plot_dict

  def compute_duration_multiline_plot(self,app_id,exp_uid,task):
    """
    Description: Returns multiline plot where there is a one-to-one mapping lines to
    algorithms and each line indicates the durations to complete the task (wrt to the api call)

    Expected input:
      (string) task :  must be in {'getQuery','processAnswer','predict'}

    Expected output (in dict):
      (dict) MPLD3 plot dictionary
    """
    alg_list = self.db.get(app_id+':experiments',exp_uid,'args')[0]['alg_list']
    x_min = numpy.float('inf')
    x_max = -numpy.float('inf')
    y_min = numpy.float('inf')
    y_max = -numpy.float('inf')
    list_of_alg_dicts = []

    for algorithm in alg_list:
      alg_label = algorithm['alg_label']

      list_of_log_dict,didSucceed,message = self.ell.get_logs_with_filter(app_id+':ALG-DURATION',{'alg_label':alg_label,'task':task})
      list_of_log_dict = sorted(list_of_log_dict, key=lambda item: utils.str2datetime(item['timestamp']) )

      x = []
      y = []
      t = []
      k=0
      for item in list_of_log_dict:
        k+=1
        x.append(k)
        y.append( item.get('app_duration',0.) + item.get('duration_enqueued',0.) )
        t.append(str(item['timestamp'])[:-3])

      x = numpy.array(x)
      y = numpy.array(y)
      t = numpy.array(t)
      num_items = len(list_of_log_dict)
      multiplier = min(num_items,MAX_SAMPLES_PER_PLOT)
      incr_inds = [ k*num_items/multiplier for k in range(multiplier)]
      max_inds = list(numpy.argsort(-y)[0:multiplier])
      final_inds = sorted(set(incr_inds + max_inds))
      x = list(x[final_inds])
      y = list(y[final_inds])
      t = list(t[final_inds])

      alg_dict = {}
      alg_dict['legend_label'] = alg_label
      alg_dict['x'] = x
      alg_dict['y'] = y
      alg_dict['t'] = t
      try:
        x_min = min(x_min,min(x))
        x_max = max(x_max,max(x))
        y_min = min(y_min,min(y))
        y_max = max(y_max,max(y))
      except:
        pass

      list_of_alg_dicts.append(alg_dict)

    return_dict = {}
    return_dict['data'] = list_of_alg_dicts
    return_dict['plot_type'] = 'multi_line_plot'
    return_dict['x_label'] = 'API Call'
    return_dict['x_min'] = x_min
    return_dict['x_max'] = x_max
    return_dict['y_label'] = 'Duration (s)'
    return_dict['y_min'] = y_min
    return_dict['y_max'] = y_max

    import matplotlib.pyplot as plt
    import mpld3
    fig, ax = plt.subplots(subplot_kw=dict(axisbg='#EEEEEE'))
    for alg_dict in list_of_alg_dicts:
        ax.plot(alg_dict['x'],alg_dict['y'],label=alg_dict['legend_label'])
    ax.set_xlabel('API Call')
    ax.set_ylabel('Duration (s)')
    ax.set_xlim([x_min,x_max])
    ax.set_ylim([y_min,y_max])
    ax.grid(color='white', linestyle='solid')
    ax.set_title(task, size=14)
    legend = ax.legend(loc=2,ncol=3,mode="expand")
    for label in legend.get_texts():
      label.set_fontsize('small')
    plot_dict = mpld3.fig_to_dict(fig)
    plt.close()

    return plot_dict

  def compute_duration_detailed_stacked_area_plot(self,app_id,exp_uid,task,alg_label,detailedDB=False):
    """
    Description: Returns stacked area plot for a particular algorithm and task where the durations
    are broken down into compute,db_set,db_get (for cpu, database_set, database_get)

    Expected input:
      (string) task :  must be in {'getQuery','processAnswer','predict'}
      (string) alg_label : must be a valid alg_label contained in alg_list list of dicts

    Expected output (in dict):
      (dict) MPLD3 plot dictionary
    """
    list_of_log_dict,didSucceed,message = self.ell.get_logs_with_filter(app_id+':ALG-DURATION',{'alg_label':alg_label,'task':task})
    list_of_log_dict = sorted(list_of_log_dict, key=lambda item: utils.str2datetime(item['timestamp']) )

    y = []
    for item in list_of_log_dict:
      y.append( item.get('app_duration',0.) + item.get('duration_enqueued',0.) )
    y = numpy.array(y)
    num_items = len(list_of_log_dict)
    multiplier = min(num_items,MAX_SAMPLES_PER_PLOT)
    incr_inds = [ k*num_items/multiplier for k in range(multiplier)]
    max_inds = list(numpy.argsort(-y)[0:multiplier])
    final_inds = sorted(set(incr_inds + max_inds))

    x = []
    t = []
    enqueued = []
    admin = []
    dbOverhead = []
    dbGet = []
    dbSet = []
    compute = []

    max_y_value = 0.
    min_y_value = float('inf')
    for idx in final_inds:
      item = list_of_log_dict[idx]
      x.append(idx+1)
      t.append(str(item.get('timestamp','')))

      _alg_duration = item.get('duration',0.)
      _alg_duration_dbGet = item.get('duration_dbGet',0.)
      _alg_duration_dbSet = item.get('duration_dbSet',0.)
      _duration_enqueued = item.get('duration_enqueued',0.)
      _app_duration = item.get('app_duration',0.)

      if (_app_duration+_duration_enqueued) > max_y_value:
        max_y_value = _app_duration + _duration_enqueued
      if (_app_duration+_duration_enqueued) < min_y_value:
        min_y_value = _app_duration + _duration_enqueued

      enqueued.append(_duration_enqueued)
      admin.append(_app_duration-_alg_duration)
      dbSet.append(_alg_duration_dbSet)
      dbGet.append(_alg_duration_dbGet)
      compute.append( _alg_duration - _alg_duration_dbSet - _alg_duration_dbGet )

    try:
      min_x = min(x)
      max_x = max(x)
    except:
      min_x = 0.
      max_x = 0.

    import matplotlib.pyplot as plt
    import mpld3
    fig, ax = plt.subplots(subplot_kw=dict(axisbg='#EEEEEE'))
    stack_coll = ax.stackplot(x,compute,dbGet,dbSet,admin,enqueued, alpha=.5)
    ax.set_xlabel('API Call')
    ax.set_ylabel('Duration (s)')
    ax.set_xlim([min_x,max_x])
    ax.set_ylim([0.,max_y_value])
    ax.grid(color='white', linestyle='solid')
    ax.set_title(alg_label+' - '+task, size=14)
    proxy_rects = [plt.Rectangle((0, 0), 1, 1, alpha=.5,fc=pc.get_facecolor()[0]) for pc in stack_coll]
    legend = ax.legend(proxy_rects, ['compute','dbGet','dbSet','admin','enqueued'],loc=2,ncol=3,mode="expand")
    for label in legend.get_texts():
      label.set_fontsize('small')
    plot_dict = mpld3.fig_to_dict(fig)
    plt.close()

    return plot_dict

  def response_time_histogram(self,app_id,exp_uid,alg_label):
    """
    Description: returns the data to plot response time histogram of processAnswer for each algorithm

    Expected input:
      (string) alg_label : must be a valid alg_label contained in alg_list list of dicts

    Expected output (in dict):
      (dict) MPLD3 plot dictionary
    """
    alg_list = self.db.get(app_id+':experiments',exp_uid,'args')[0]['alg_list']

    list_of_query_dict,didSucceed,message = self.db.get_docs_with_filter(app_id+':queries',{'exp_uid':exp_uid,'alg_label':alg_label})

    t = []
    for item in list_of_query_dict:
      try:
        t.append(item['response_time'])
      except:
        pass

    import matplotlib.pyplot as plt
    import mpld3
    fig, ax = plt.subplots(subplot_kw=dict(axisbg='#FFFFFF'))
    ax.hist(t,MAX_SAMPLES_PER_PLOT,range=(0,30),alpha=0.5,color='black')
    ax.set_xlim(0, 30)
    ax.set_axis_off()
    ax.set_xlabel('Durations (s)')
    ax.set_ylabel('Count')
    ax.set_title(alg_label + " - response time", size=14)
    plot_dict = mpld3.fig_to_dict(fig)
    plt.close()

    return plot_dict

  def network_delay_histogram(self,app_id,exp_uid,alg_label):
    """
    Description: returns the data to network delay histogram of the time it takes to getQuery+processAnswer for each algorithm

    Expected input:
      (string) alg_label : must be a valid alg_label contained in alg_list list of dicts

    Expected output (in dict):
      (dict) MPLD3 plot dictionary
    """
    alg_list = self.db.get(app_id+':experiments',exp_uid,'args')[0]['alg_list']

    list_of_query_dict,didSucceed,message = self.db.get_docs_with_filter(app_id+':queries',{'exp_uid':exp_uid,'alg_label':alg_label})

    t = []
    for item in list_of_query_dict:
      try:
        t.append(item['network_delay'])
      except:
        pass

    import matplotlib.pyplot as plt
    import mpld3
    fig, ax = plt.subplots(subplot_kw=dict(axisbg='#FFFFFF'))
    ax.hist(t,MAX_SAMPLES_PER_PLOT,range=(0,5),alpha=0.5,color='black')
    ax.set_xlim(0, 5)
    ax.set_axis_off()
    ax.set_xlabel('Durations (s)')
    ax.set_ylabel('Count')
    ax.set_title(alg_label + " - network delay", size=14)
    plot_dict = mpld3.fig_to_dict(fig)
    plt.close()

    return plot_dict

<commit_msg>print timestamps<commit_after>import json
import numpy
import numpy.random
from datetime import datetime
from datetime import timedelta
from next.utils import utils

import matplotlib
matplotlib.use('Agg')

MAX_SAMPLES_PER_PLOT = 100

class AppDashboard(object):

  def __init__(self,db,ell):
    self.db = db
    self.ell = ell

  def api_activity_histogram(self,app_id,exp_uid,task):
    """
    Description: returns the data to plot all API activity (for all algorithms) in a histogram with respect to time for any task in {getQuery,processAnswer,predict}

    Expected input:
      (string) task :  must be in {'getQuery','processAnswer','predict'}

    Expected output (in dict):
      (dict) MPLD3 plot dictionary
    """

    list_of_log_dict,didSucceed,message = self.ell.get_logs_with_filter(app_id+':APP-CALL',{'exp_uid':exp_uid,'task':task})

    from datetime import datetime
    from datetime import timedelta
    start_date_str,didSucceed,message = self.db.get('experiments_admin',exp_uid,'start_date')
    start_date = utils.str2datetime(start_date_str)
    numerical_timestamps = [ ( utils.str2datetime(item['timestamp'])-start_date).total_seconds() for item in list_of_log_dict]

    print numerical_timestamps
    import matplotlib.pyplot as plt
    import mpld3
    fig, ax = plt.subplots(subplot_kw=dict(axisbg='#FFFFFF'),figsize=(12,1.5))
    ax.hist(numerical_timestamps,min(int(1+4*numpy.sqrt(len(numerical_timestamps))),300),alpha=0.5,color='black')
    ax.set_frame_on(False)
    ax.get_xaxis().set_ticks([])
    ax.get_yaxis().set_ticks([])
    ax.get_yaxis().set_visible(False)
    ax.set_xlim(0, max(numerical_timestamps))
    plot_dict = mpld3.fig_to_dict(fig)
    plt.close()

    return plot_dict

  def compute_duration_multiline_plot(self,app_id,exp_uid,task):
    """
    Description: Returns multiline plot where there is a one-to-one mapping lines to
    algorithms and each line indicates the durations to complete the task (wrt to the api call)

    Expected input:
      (string) task :  must be in {'getQuery','processAnswer','predict'}

    Expected output (in dict):
      (dict) MPLD3 plot dictionary
    """
    alg_list = self.db.get(app_id+':experiments',exp_uid,'args')[0]['alg_list']
    x_min = numpy.float('inf')
    x_max = -numpy.float('inf')
    y_min = numpy.float('inf')
    y_max = -numpy.float('inf')
    list_of_alg_dicts = []

    for algorithm in alg_list:
      alg_label = algorithm['alg_label']

      list_of_log_dict,didSucceed,message = self.ell.get_logs_with_filter(app_id+':ALG-DURATION',{'alg_label':alg_label,'task':task})
      list_of_log_dict = sorted(list_of_log_dict, key=lambda item: utils.str2datetime(item['timestamp']) )

      x = []
      y = []
      t = []
      k=0
      for item in list_of_log_dict:
        k+=1
        x.append(k)
        y.append( item.get('app_duration',0.) + item.get('duration_enqueued',0.) )
        t.append(str(item['timestamp'])[:-3])

      x = numpy.array(x)
      y = numpy.array(y)
      t = numpy.array(t)
      num_items = len(list_of_log_dict)
      multiplier = min(num_items,MAX_SAMPLES_PER_PLOT)
      incr_inds = [ k*num_items/multiplier for k in range(multiplier)]
      max_inds = list(numpy.argsort(-y)[0:multiplier])
      final_inds = sorted(set(incr_inds + max_inds))
      x = list(x[final_inds])
      y = list(y[final_inds])
      t = list(t[final_inds])

      alg_dict = {}
      alg_dict['legend_label'] = alg_label
      alg_dict['x'] = x
      alg_dict['y'] = y
      alg_dict['t'] = t
      try:
        x_min = min(x_min,min(x))
        x_max = max(x_max,max(x))
        y_min = min(y_min,min(y))
        y_max = max(y_max,max(y))
      except:
        pass

      list_of_alg_dicts.append(alg_dict)

    return_dict = {}
    return_dict['data'] = list_of_alg_dicts
    return_dict['plot_type'] = 'multi_line_plot'
    return_dict['x_label'] = 'API Call'
    return_dict['x_min'] = x_min
    return_dict['x_max'] = x_max
    return_dict['y_label'] = 'Duration (s)'
    return_dict['y_min'] = y_min
    return_dict['y_max'] = y_max

    import matplotlib.pyplot as plt
    import mpld3
    fig, ax = plt.subplots(subplot_kw=dict(axisbg='#EEEEEE'))
    for alg_dict in list_of_alg_dicts:
        ax.plot(alg_dict['x'],alg_dict['y'],label=alg_dict['legend_label'])
    ax.set_xlabel('API Call')
    ax.set_ylabel('Duration (s)')
    ax.set_xlim([x_min,x_max])
    ax.set_ylim([y_min,y_max])
    ax.grid(color='white', linestyle='solid')
    ax.set_title(task, size=14)
    legend = ax.legend(loc=2,ncol=3,mode="expand")
    for label in legend.get_texts():
      label.set_fontsize('small')
    plot_dict = mpld3.fig_to_dict(fig)
    plt.close()

    return plot_dict

  def compute_duration_detailed_stacked_area_plot(self,app_id,exp_uid,task,alg_label,detailedDB=False):
    """
    Description: Returns stacked area plot for a particular algorithm and task where the durations
    are broken down into compute,db_set,db_get (for cpu, database_set, database_get)

    Expected input:
      (string) task :  must be in {'getQuery','processAnswer','predict'}
      (string) alg_label : must be a valid alg_label contained in alg_list list of dicts

    Expected output (in dict):
      (dict) MPLD3 plot dictionary
    """
    list_of_log_dict,didSucceed,message = self.ell.get_logs_with_filter(app_id+':ALG-DURATION',{'alg_label':alg_label,'task':task})
    list_of_log_dict = sorted(list_of_log_dict, key=lambda item: utils.str2datetime(item['timestamp']) )

    y = []
    for item in list_of_log_dict:
      y.append( item.get('app_duration',0.) + item.get('duration_enqueued',0.) )
    y = numpy.array(y)
    num_items = len(list_of_log_dict)
    multiplier = min(num_items,MAX_SAMPLES_PER_PLOT)
    incr_inds = [ k*num_items/multiplier for k in range(multiplier)]
    max_inds = list(numpy.argsort(-y)[0:multiplier])
    final_inds = sorted(set(incr_inds + max_inds))

    x = []
    t = []
    enqueued = []
    admin = []
    dbOverhead = []
    dbGet = []
    dbSet = []
    compute = []

    max_y_value = 0.
    min_y_value = float('inf')
    for idx in final_inds:
      item = list_of_log_dict[idx]
      x.append(idx+1)
      t.append(str(item.get('timestamp','')))

      _alg_duration = item.get('duration',0.)
      _alg_duration_dbGet = item.get('duration_dbGet',0.)
      _alg_duration_dbSet = item.get('duration_dbSet',0.)
      _duration_enqueued = item.get('duration_enqueued',0.)
      _app_duration = item.get('app_duration',0.)

      if (_app_duration+_duration_enqueued) > max_y_value:
        max_y_value = _app_duration + _duration_enqueued
      if (_app_duration+_duration_enqueued) < min_y_value:
        min_y_value = _app_duration + _duration_enqueued

      enqueued.append(_duration_enqueued)
      admin.append(_app_duration-_alg_duration)
      dbSet.append(_alg_duration_dbSet)
      dbGet.append(_alg_duration_dbGet)
      compute.append( _alg_duration - _alg_duration_dbSet - _alg_duration_dbGet )

    try:
      min_x = min(x)
      max_x = max(x)
    except:
      min_x = 0.
      max_x = 0.

    import matplotlib.pyplot as plt
    import mpld3
    fig, ax = plt.subplots(subplot_kw=dict(axisbg='#EEEEEE'))
    stack_coll = ax.stackplot(x,compute,dbGet,dbSet,admin,enqueued, alpha=.5)
    ax.set_xlabel('API Call')
    ax.set_ylabel('Duration (s)')
    ax.set_xlim([min_x,max_x])
    ax.set_ylim([0.,max_y_value])
    ax.grid(color='white', linestyle='solid')
    ax.set_title(alg_label+' - '+task, size=14)
    proxy_rects = [plt.Rectangle((0, 0), 1, 1, alpha=.5,fc=pc.get_facecolor()[0]) for pc in stack_coll]
    legend = ax.legend(proxy_rects, ['compute','dbGet','dbSet','admin','enqueued'],loc=2,ncol=3,mode="expand")
    for label in legend.get_texts():
      label.set_fontsize('small')
    plot_dict = mpld3.fig_to_dict(fig)
    plt.close()

    return plot_dict

  def response_time_histogram(self,app_id,exp_uid,alg_label):
    """
    Description: returns the data to plot response time histogram of processAnswer for each algorithm

    Expected input:
      (string) alg_label : must be a valid alg_label contained in alg_list list of dicts

    Expected output (in dict):
      (dict) MPLD3 plot dictionary
    """
    alg_list = self.db.get(app_id+':experiments',exp_uid,'args')[0]['alg_list']

    list_of_query_dict,didSucceed,message = self.db.get_docs_with_filter(app_id+':queries',{'exp_uid':exp_uid,'alg_label':alg_label})

    t = []
    for item in list_of_query_dict:
      try:
        t.append(item['response_time'])
      except:
        pass

    import matplotlib.pyplot as plt
    import mpld3
    fig, ax = plt.subplots(subplot_kw=dict(axisbg='#FFFFFF'))
    ax.hist(t,MAX_SAMPLES_PER_PLOT,range=(0,30),alpha=0.5,color='black')
    ax.set_xlim(0, 30)
    ax.set_axis_off()
    ax.set_xlabel('Durations (s)')
    ax.set_ylabel('Count')
    ax.set_title(alg_label + " - response time", size=14)
    plot_dict = mpld3.fig_to_dict(fig)
    plt.close()

    return plot_dict

  def network_delay_histogram(self,app_id,exp_uid,alg_label):
    """
    Description: returns the data to network delay histogram of the time it takes to getQuery+processAnswer for each algorithm

    Expected input:
      (string) alg_label : must be a valid alg_label contained in alg_list list of dicts

    Expected output (in dict):
      (dict) MPLD3 plot dictionary
    """
    alg_list = self.db.get(app_id+':experiments',exp_uid,'args')[0]['alg_list']

    list_of_query_dict,didSucceed,message = self.db.get_docs_with_filter(app_id+':queries',{'exp_uid':exp_uid,'alg_label':alg_label})

    t = []
    for item in list_of_query_dict:
      try:
        t.append(item['network_delay'])
      except:
        pass

    import matplotlib.pyplot as plt
    import mpld3
    fig, ax = plt.subplots(subplot_kw=dict(axisbg='#FFFFFF'))
    ax.hist(t,MAX_SAMPLES_PER_PLOT,range=(0,5),alpha=0.5,color='black')
    ax.set_xlim(0, 5)
    ax.set_axis_off()
    ax.set_xlabel('Durations (s)')
    ax.set_ylabel('Count')
    ax.set_title(alg_label + " - network delay", size=14)
    plot_dict = mpld3.fig_to_dict(fig)
    plt.close()

    return plot_dict

<|endoftext|>