<commit_before># coding: utf-8
#
# Parts shamelessly stolen from:
#
#   http://www.technobabelfish.com/2013/08/boto-and-cloudformation.html
#
#
import os
import sys
import time
import yaml
import json
import collections
import shutil
import tempfile
from termcolor import colored

import boto
import boto.cloudformation
import boto.s3.connection
import boto.iam.connection

from ast import literal_eval

import nepho

class AWSProvider(nepho.core.provider.AbstractProvider):
    """
    An infrastructure provider class for AWS CloudFormation

    Note: if you create an output named SSHEndpoint, then you can
      use the "nepho stack access" command to connect directly to the stack.

    """

    PROVIDER_ID = "aws"
    TEMPLATE_FILENAME = "cf.json"

    def __init__(self, config, scenario=None):
        nepho.core.provider.AbstractProvider.__init__(self, config, scenario)
        self.params = {
            'AWSAccessKeyID': None,
            'AWSSecretAccessKey': None,
            'AWSRegion': None,
            'KeyName': None,
        }
        self._connection = None
        self._s3_conn = None
        self._iam_conn = None

    @property
    def connection(self):
        """
        Lazy-load connection when needed. This allows for including the provider
        in context creation even though the provider connection requires a valid
        context.
        """
        if self._connection is None:
            (access_key, secret_key, region) = self._load_aws_connection_settings()
            self._connection = boto.cloudformation.connect_to_region(
                region, aws_access_key_id = access_key, aws_secret_access_key = secret_key)
            if self._connection is None:
                print "Boto connection to CloudFormation failed. Please check your credentials."
                exit(1)
        return self._connection

    @property
    def s3_conn(self):
        if self._s3_conn is None:
            (access_key, secret_key, region) = self._load_aws_connection_settings()
            self._s3_conn = boto.s3.connection.S3Connection(
                aws_access_key_id = access_key, aws_secret_access_key = secret_key)
            if self._s3_conn is None:
                print "Boto connection to S3 failed. Please check your credentials."
                exit(1)
        return self._s3_conn

    @property
    def iam_conn(self):
        if self._iam_conn is None:
            (access_key, secret_key, region) = self._load_aws_connection_settings()
            self._iam_conn = boto.iam.connection.IAMConnection(
                aws_access_key_id = access_key, aws_secret_access_key = secret_key)
            if self._iam_conn is None:
                print "Boto connection to IAM failed. Please check your credentials."
                exit(1)
        return self._iam_conn

    def validate_template(self, template_str):

        # Attempt to validate that template is valid JSON, and clean it up
        try:
            template_dict = json.loads(template_str)
            template_str  = json.dumps(template_dict, indent=2, separators=(',', ': '))
        except Exception as e:
            ret = "Validation:\n  Template JSON is not valid!\n"
            ret += "Error:\n  %s\n" % (e)
            return ret

        try:
            compact_template = json.dumps(json.loads(template_str), separators=(',', ':'))
            t = self.connection.validate_template(template_body=compact_template)

            ret = "Validation:\n  Template is valid\n"
            ret += "Description:\n %s\n" % t.description
            ret += "Parameters:"
            for p in t.template_parameters:
                ret += "\n  %s" % (p.parameter_key)
        except boto.exception.BotoServerError as e:
            ret = "Boto Server Error:\n  Template is not valid!\n"
            ret += "Error:\n  %s\n" % (e.error_code)
            ret += "Description:\n  %s" % (e.body)
        return ret

    def format_template(self, raw_template):
        """Pretty formats a CF template"""

        cf_dict = parse_cf_json(raw_template)
        return get_cf_json(cf_dict, pretty=True)

    def deploy(self, debug=None):
        """Deploy a given pattern."""
        if debug is True:
            boto.set_stream_logger('aws_provider')

        context = self.scenario.context

        stack_name = create_stack_name(context)

        raw_template = self.scenario.template
        template_json = self.format_template(raw_template)

        params = list()
        for item in context['parameters'].items():
            params.append(item)

        print " - Determining owner information"
        try:
            iam_user = self.iam_conn.get_user()
            params.append(('OwnerId', iam_user.user_id))
            params.append(('OwnerName', iam_user.user_name))
        except:
            print colored("Error: ", "red") + "Unable to get IAM username"
            exit(1)

        print " - Archiving payload for upload to S3"
        try:
            tmp_dir = tempfile.mkdtemp()
            payload = shutil.make_archive(
                os.path.join(tmp_dir, 'payload'), 'zip',
                os.path.join(context['cloudlet']['path'], 'payload'))
        except:
            print colored("Error: ", "red") + "Unable to create payload file"
            shutil.rmtree(tmp_dir)
            exit(1)

        # Upload payload to a private S3 bucket
        print " - Creating S3 bucket for payload"
        (access_key, secret_key, region) = self._load_aws_connection_settings()
        try:
            payload_bucket = self.s3_conn.create_bucket(
                'nepho-payloads-' + access_key.lower(), policy='private')

            payload_name = '%s/%s/payload.zip' % (context['cloudlet']['name'], context['blueprint']['name'])
            payload_key = boto.s3.key.Key(payload_bucket)
            payload_key.key = payload_name
        except Exception as e:
            print colored("Error: ", "red") + "Unable to create S3 bucket"
            print "(%s) %s " % (e.error_code, e.body)
            shutil.rmtree(tmp_dir)
            exit(1)

        print " - Uploading payload archive to S3"
        try:
            payload_key.set_contents_from_file(open(payload, 'rb'))
            params.append(('PayloadURL', payload_key.generate_url(3600)))
        except Exception as e:
            print colored("Error: ", "red") + "Unable to upload payload to S3"
            print "(%s) %s " % (e.error_code, e.body)
            exit(1)
        finally:
            shutil.rmtree(tmp_dir)

        try:
            # Use minimal size representation of this JSON string
            compact_template_json = json.dumps(json.loads(template_json), separators=(',', ':'))

            print " - Creating CloudFormation stack"
            stack_id = self.connection.create_stack(
                stack_name,
                template_body = compact_template_json,
                parameters = params,
                capabilities = ['CAPABILITY_IAM'],
                disable_rollback = True
            )
            self._show_status(stack_id)
        except boto.exception.BotoServerError as e:
            print colored("Error: ", "red") + "Problem communicating with CloudFormation"
            # Use e.message instead of e.body as per: https://github.com/boto/boto/issues/1658
            msg = literal_eval(e.message)
            print "(%s) %s " % (msg["Error"]["Code"], msg["Error"]["Message"])
            exit(1)
        return stack_id

    def status(self):
        """Check on the status of a stack within CloudFormation."""

        context = self.scenario.context
        stack_name = create_stack_name(context)
        self._show_status(stack_name)

    def access(self):
        """Check on the status of a stack within CloudFormation."""

        context = self.scenario.context
        stack_name = create_stack_name(context)

        # Return object of type boto.cloudformation.stack.Stack
        try:
            stack = self.connection.describe_stacks(stack_name_or_id=stack_name)

            # this will need to be improved ... basically a stub for now ...
            outputs = stack.outputs
            access_hostname = outputs['SSHEndpoint']
            return "ssh %s@%s" % ("ec2-user", access_hostname)
        except boto.exception.BotoServerError as be:
            # Actually ,this may just mean that there's no stack by that name ...
            print "Error communication with the CloudFormation service: %s" % (be)
            exit(1)

        # Just for now ...
        print_stack(stack[0])
        return stack[0]

    def destroy(self):
        """Delete a CloudFormation stack."""
        context = self.scenario.context
        stack_name = create_stack_name(context)
        self.connection.delete_stack(stack_name_or_id=stack_name)
        self._show_status(stack_name)

    #===========================================================================
    # These are example helper functions for capabilities not yet needed.
    #===========================================================================

    def _get_stacks(self):
        """Return a list of CF stacks."""
        return self.connection.list_stack()

    def _get_stack(self, stack):
        context = self.scenario.context
        stack_name = create_stack_name(context)

        stacks = self.connection.describe_stacks(stack_name)
        if not stacks:
            raise Exception(stack)

        return stacks[0]

    def _list_stacks(self):
        stacks = self.get_stacks()
        for stackSumm in stacks:
            print_stack(self._get_stack(stackSumm.stack_id))

    def _load_aws_connection_settings(self):
        context = self.scenario.context
        required_params = ('AWSAccessKeyID', 'AWSSecretAccessKey', 'AWSRegion', 'KeyName')
        for param in required_params:
            if context['parameters'][param] is None:
                print colored("Error: ", "red"), "In order to use the AWS provider you must provide valid credentials."
                print "The following parameters are required:"
                print " ", "\n  ".join(required_params)
                print "Use \"nepho parameter set <key> <value>\" to set a parameter."
                exit(1)
        return (
            context['parameters']['AWSAccessKeyID'],
            context['parameters']['AWSSecretAccessKey'],
            context['parameters']['AWSRegion']
        )

    def _show_status(self, stack):
        while True:
            try:
                stack_update = self.connection.describe_stacks(stack_name_or_id=stack)
                resources = self.connection.list_stack_resources(stack)
            except KeyboardInterrupt:
                exit()
            except boto.exception.BotoServerError as e:
                print colored("Error: ", "red") + e.message
                exit(1)

            try:
                os.system('cls' if os.name == 'nt' else 'clear')

                su = stack_update[0]
                print "Name:    %s" % su.stack_name
                print "ID:      %s" % stack.stack_id
                print "Created: %s" % su.creation_time
                print "Status:  %s" % self._colorize_status(su.stack_status)
                print
                print "+------------------------+----------------------------------+------------------+"
                print "|%-24s|%-34s|%-18s|" % ("Resource", "Type", "Status")
                print "+------------------------+----------------------------------+------------------+"
                for r in resources:
                    print "|%-24s|%-34s|%-27s|" % (r.logical_resource_id[0:24],
                                                   r.resource_type[0:34],
                                                   self._colorize_status(r.resource_status)[0:27])
                print "+------------------------+----------------------------------+------------------+"
                if su.stack_status == "CREATE_COMPLETE" or su.stack_status == "UPDATE_COMPLETE":
                    print
                    print "+------------------------+-----------------------------------------------------+"
                    print "|%-24s|%-53s|" % ("Output", "Value")
                    print "+------------------------+-----------------------------------------------------+"
                    for o in su.outputs:
                        print "|%-24s|%-53s|" % (o.key[0:24], o.value)
                    print "+------------------------+-----------------------------------------------------+"
                print "\nMore information: https://console.aws.amazon.com/cloudformation/home"
                if su.stack_status.endswith("FAILED") or su.stack_status.endswith("COMPLETE"):
                    break
                else:
                    print "Updating in ",
                    for i in xrange(9, -1, -1):
                        sys.stdout.write(str(i))
                        sys.stdout.flush()
                        time.sleep(1)
                        sys.stdout.write('\b')
            except KeyboardInterrupt:
                exit()

    def _colorize_status(self, status):
        if status.endswith("COMPLETE"):
            return colored(status, "green")
        elif status.endswith("IN_PROGRESS"):
            return colored(status, "yellow")
        elif status.endswith("FAILED"):
            return colored(status, "red")
        else:
            return status

def print_stack(stack):
    print "Name:            %s" % stack.stack_name
    print "ID:              %s" % stack.stack_id
    print "Status:          %s" % stack.stack_status
    print "Creation Time:   %s" % stack.creation_time
    print "Outputs:         %s" % stack.outputs
    print "Parameters:      %s" % stack.parameters
    print "Tags:            %s" % stack.tags
    print "Capabilities:    %s" % stack.capabilities

def create_stack_name(context):
    return "%s-%s" % (context['cloudlet']['name'], context['blueprint']['name'])

def parse_cf_json(str):
    try:
        cf_dict = json.loads(str, object_pairs_hook=collections.OrderedDict)
    except ValueError as e:
        print "Error loading JSON template file."
        print e
        exit(1)
    else:
        return cf_dict

def get_cf_json(order_dict, pretty=False):
    outstr = None
    if pretty:
        outstr = json.dumps(order_dict, indent=2, separators=(',', ': '))
    else:
        outstr = json.dumps(order_dict)
    return outstr
<commit_msg>Cement-style signal handling<commit_after># coding: utf-8
#
# Parts shamelessly stolen from:
#
#   http://www.technobabelfish.com/2013/08/boto-and-cloudformation.html
#
#
import os
import sys
import time
import yaml
import json
import collections
import shutil
import tempfile
from termcolor import colored

import boto
import boto.cloudformation
import boto.s3.connection
import boto.iam.connection

from ast import literal_eval

import signal
from cement.core import exc

import nepho

class AWSProvider(nepho.core.provider.AbstractProvider):
    """
    An infrastructure provider class for AWS CloudFormation

    Note: if you create an output named SSHEndpoint, then you can
      use the "nepho stack access" command to connect directly to the stack.

    """

    PROVIDER_ID = "aws"
    TEMPLATE_FILENAME = "cf.json"

    def __init__(self, config, scenario=None):
        nepho.core.provider.AbstractProvider.__init__(self, config, scenario)
        self.params = {
            'AWSAccessKeyID': None,
            'AWSSecretAccessKey': None,
            'AWSRegion': None,
            'KeyName': None,
        }
        self._connection = None
        self._s3_conn = None
        self._iam_conn = None

    @property
    def connection(self):
        """
        Lazy-load connection when needed. This allows for including the provider
        in context creation even though the provider connection requires a valid
        context.
        """
        if self._connection is None:
            (access_key, secret_key, region) = self._load_aws_connection_settings()
            self._connection = boto.cloudformation.connect_to_region(
                region, aws_access_key_id = access_key, aws_secret_access_key = secret_key)
            if self._connection is None:
                print "Boto connection to CloudFormation failed. Please check your credentials."
                exit(1)
        return self._connection

    @property
    def s3_conn(self):
        if self._s3_conn is None:
            (access_key, secret_key, region) = self._load_aws_connection_settings()
            self._s3_conn = boto.s3.connection.S3Connection(
                aws_access_key_id = access_key, aws_secret_access_key = secret_key)
            if self._s3_conn is None:
                print "Boto connection to S3 failed. Please check your credentials."
                exit(1)
        return self._s3_conn

    @property
    def iam_conn(self):
        if self._iam_conn is None:
            (access_key, secret_key, region) = self._load_aws_connection_settings()
            self._iam_conn = boto.iam.connection.IAMConnection(
                aws_access_key_id = access_key, aws_secret_access_key = secret_key)
            if self._iam_conn is None:
                print "Boto connection to IAM failed. Please check your credentials."
                exit(1)
        return self._iam_conn

    def validate_template(self, template_str):

        # Attempt to validate that template is valid JSON, and clean it up
        try:
            template_dict = json.loads(template_str)
            template_str  = json.dumps(template_dict, indent=2, separators=(',', ': '))
        except Exception as e:
            ret = "Validation:\n  Template JSON is not valid!\n"
            ret += "Error:\n  %s\n" % (e)
            return ret

        try:
            compact_template = json.dumps(json.loads(template_str), separators=(',', ':'))
            t = self.connection.validate_template(template_body=compact_template)

            ret = "Validation:\n  Template is valid\n"
            ret += "Description:\n %s\n" % t.description
            ret += "Parameters:"
            for p in t.template_parameters:
                ret += "\n  %s" % (p.parameter_key)
        except boto.exception.BotoServerError as e:
            ret = "Boto Server Error:\n  Template is not valid!\n"
            ret += "Error:\n  %s\n" % (e.error_code)
            ret += "Description:\n  %s" % (e.body)
        return ret

    def format_template(self, raw_template):
        """Pretty formats a CF template"""

        cf_dict = parse_cf_json(raw_template)
        return get_cf_json(cf_dict, pretty=True)

    def deploy(self, debug=None):
        """Deploy a given pattern."""
        if debug is True:
            boto.set_stream_logger('aws_provider')

        context = self.scenario.context

        stack_name = create_stack_name(context)

        raw_template = self.scenario.template
        template_json = self.format_template(raw_template)

        params = list()
        for item in context['parameters'].items():
            params.append(item)

        print " - Determining owner information"
        try:
            iam_user = self.iam_conn.get_user()
            params.append(('OwnerId', iam_user.user_id))
            params.append(('OwnerName', iam_user.user_name))
        except:
            print colored("Error: ", "red") + "Unable to get IAM username"
            exit(1)

        print " - Archiving payload for upload to S3"
        try:
            tmp_dir = tempfile.mkdtemp()
            payload = shutil.make_archive(
                os.path.join(tmp_dir, 'payload'), 'zip',
                os.path.join(context['cloudlet']['path'], 'payload'))
        except:
            print colored("Error: ", "red") + "Unable to create payload file"
            shutil.rmtree(tmp_dir)
            exit(1)

        # Upload payload to a private S3 bucket
        print " - Creating S3 bucket for payload"
        (access_key, secret_key, region) = self._load_aws_connection_settings()
        try:
            payload_bucket = self.s3_conn.create_bucket(
                'nepho-payloads-' + access_key.lower(), policy='private')

            payload_name = '%s/%s/payload.zip' % (context['cloudlet']['name'], context['blueprint']['name'])
            payload_key = boto.s3.key.Key(payload_bucket)
            payload_key.key = payload_name
        except Exception as e:
            print colored("Error: ", "red") + "Unable to create S3 bucket"
            print "(%s) %s " % (e.error_code, e.body)
            shutil.rmtree(tmp_dir)
            exit(1)

        print " - Uploading payload archive to S3"
        try:
            payload_key.set_contents_from_file(open(payload, 'rb'))
            params.append(('PayloadURL', payload_key.generate_url(3600)))
        except Exception as e:
            print colored("Error: ", "red") + "Unable to upload payload to S3"
            print "(%s) %s " % (e.error_code, e.body)
            exit(1)
        finally:
            shutil.rmtree(tmp_dir)

        try:
            # Use minimal size representation of this JSON string
            compact_template_json = json.dumps(json.loads(template_json), separators=(',', ':'))

            print " - Creating CloudFormation stack"
            stack_id = self.connection.create_stack(
                stack_name,
                template_body = compact_template_json,
                parameters = params,
                capabilities = ['CAPABILITY_IAM'],
                disable_rollback = True
            )

            try:
                self._show_status(stack_name)
            except exc.CaughtSignal:
                exit()
        except boto.exception.BotoServerError as e:
            print colored("Error: ", "red") + "Problem communicating with CloudFormation"
            # Use e.message instead of e.body as per: https://github.com/boto/boto/issues/1658
            msg = literal_eval(e.message)
            print "(%s) %s " % (msg["Error"]["Code"], msg["Error"]["Message"])
            exit(1)
        return stack_id

    def status(self):
        """Check on the status of a stack within CloudFormation."""

        context = self.scenario.context
        stack_name = create_stack_name(context)

        try:
            self._show_status(stack_name)
        except exc.CaughtSignal:
            exit()

    def access(self):
        """Check on the status of a stack within CloudFormation."""

        context = self.scenario.context
        stack_name = create_stack_name(context)

        # Return object of type boto.cloudformation.stack.Stack
        try:
            stack = self.connection.describe_stacks(stack_name_or_id=stack_name)

            # this will need to be improved ... basically a stub for now ...
            outputs = stack.outputs
            access_hostname = outputs['SSHEndpoint']
            return "ssh %s@%s" % ("ec2-user", access_hostname)
        except boto.exception.BotoServerError as be:
            # Actually ,this may just mean that there's no stack by that name ...
            print "Error communication with the CloudFormation service: %s" % (be)
            exit(1)

        # Just for now ...
        print_stack(stack[0])
        return stack[0]

    def destroy(self):
        """Delete a CloudFormation stack."""
        context = self.scenario.context
        stack_name = create_stack_name(context)
        self.connection.delete_stack(stack_name_or_id=stack_name)

        try:
            self._show_status(stack_name)
        except exc.CaughtSignal:
            exit()

    #===========================================================================
    # These are example helper functions for capabilities not yet needed.
    #===========================================================================

    def _get_stacks(self):
        """Return a list of CF stacks."""
        return self.connection.list_stack()

    def _get_stack(self, stack):
        context = self.scenario.context
        stack_name = create_stack_name(context)

        stacks = self.connection.describe_stacks(stack_name)
        if not stacks:
            raise Exception(stack)

        return stacks[0]

    def _list_stacks(self):
        stacks = self.get_stacks()
        for stackSumm in stacks:
            print_stack(self._get_stack(stackSumm.stack_id))

    def _load_aws_connection_settings(self):
        context = self.scenario.context
        required_params = ('AWSAccessKeyID', 'AWSSecretAccessKey', 'AWSRegion', 'KeyName')
        for param in required_params:
            if context['parameters'][param] is None:
                print colored("Error: ", "red"), "In order to use the AWS provider you must provide valid credentials."
                print "The following parameters are required:"
                print " ", "\n  ".join(required_params)
                print "Use \"nepho parameter set <key> <value>\" to set a parameter."
                exit(1)
        return (
            context['parameters']['AWSAccessKeyID'],
            context['parameters']['AWSSecretAccessKey'],
            context['parameters']['AWSRegion']
        )

    def _show_status(self, stack):
        while True:
            try:
                stack_update = self.connection.describe_stacks(stack_name_or_id=stack)
                resources = self.connection.list_stack_resources(stack)
            except boto.exception.BotoServerError as e:
                print colored("Error: ", "red") + e.message
                exit(1)
            os.system('cls' if os.name == 'nt' else 'clear')

            su = stack_update[0]
            print "Name:    %s" % su.stack_name
            print "Created: %s" % su.creation_time
            print "Status:  %s" % self._colorize_status(su.stack_status)
            print
            print "+------------------------+----------------------------------+------------------+"
            print "|%-24s|%-34s|%-18s|" % ("Resource", "Type", "Status")
            print "+------------------------+----------------------------------+------------------+"
            for r in resources:
                print "|%-24s|%-34s|%-27s|" % (r.logical_resource_id[0:24],
                                               r.resource_type[0:34],
                                               self._colorize_status(r.resource_status)[0:27])
            print "+------------------------+----------------------------------+------------------+"
            if su.stack_status == "CREATE_COMPLETE" or su.stack_status == "UPDATE_COMPLETE":
                print
                print "+------------------------+-----------------------------------------------------+"
                print "|%-24s|%-53s|" % ("Output", "Value")
                print "+------------------------+-----------------------------------------------------+"
                for o in su.outputs:
                    print "|%-24s|%-53s|" % (o.key[0:24], o.value)
                print "+------------------------+-----------------------------------------------------+"
            print "\nMore information: https://console.aws.amazon.com/cloudformation/home"
            if su.stack_status.endswith("FAILED") or su.stack_status.endswith("COMPLETE"):
                break
            else:
                print "Updating in ",
                for i in xrange(9, -1, -1):
                    sys.stdout.write(str(i))
                    sys.stdout.flush()
                    time.sleep(1)
                    sys.stdout.write('\b')

    def _colorize_status(self, status):
        if status.endswith("COMPLETE"):
            return colored(status, "green")
        elif status.endswith("IN_PROGRESS"):
            return colored(status, "yellow")
        elif status.endswith("FAILED"):
            return colored(status, "red")
        else:
            return status

def print_stack(stack):
    print "Name:            %s" % stack.stack_name
    print "ID:              %s" % stack.stack_id
    print "Status:          %s" % stack.stack_status
    print "Creation Time:   %s" % stack.creation_time
    print "Outputs:         %s" % stack.outputs
    print "Parameters:      %s" % stack.parameters
    print "Tags:            %s" % stack.tags
    print "Capabilities:    %s" % stack.capabilities

def create_stack_name(context):
    return "%s-%s" % (context['cloudlet']['name'], context['blueprint']['name'])

def parse_cf_json(str):
    try:
        cf_dict = json.loads(str, object_pairs_hook=collections.OrderedDict)
    except ValueError as e:
        print "Error loading JSON template file."
        print e
        exit(1)
    else:
        return cf_dict

def get_cf_json(order_dict, pretty=False):
    outstr = None
    if pretty:
        outstr = json.dumps(order_dict, indent=2, separators=(',', ': '))
    else:
        outstr = json.dumps(order_dict)
    return outstr
<|endoftext|>


<commit_before>"""
*****
Pydot
*****

Import and export NetworkX graphs in Graphviz dot format using pydot.

Either this module or nx_pygraphviz can be used to interface with graphviz.

See Also
--------
Pydot: http://www.dkbza.org/pydot.html
Graphviz:	   http://www.research.att.com/sw/tools/graphviz/
DOT Language:  http://www.graphviz.org/doc/info/lang.html

"""
__author__ = """Aric Hagberg (hagberg@lanl.gov)"""
#    Copyright (C) 2004-2008 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.

__all__ = ['write_dot', 'read_dot', 'graphviz_layout', 'pydot_layout',
           'to_pydot', 'from_pydot']

import sys
from networkx.utils import _get_fh
import networkx

def write_dot(G,path):
    """Write NetworkX graph G to Graphviz dot format on path.

    Path can be a string or a file handle.
    """
    fh=_get_fh(path,'w')
    P=to_pydot(G)
    fh.write(P.to_string())
    fh.flush() # might be a user filehandle so leave open (but flush)
    return

def read_dot(path):
    """Return a NetworkX Graph or DiGraph from a dot file on path.

    Path can be a string or a file handle.

    """
    try:
        import pydot
    except ImportError:
        raise ImportError, \
          "read_dot() requires pydot http://dkbza.org/pydot.html/"

    fh=_get_fh(path,'r')
    data=fh.read()
    P=pydot.graph_from_dot_data(data)
    return from_pydot(P)

def from_pydot(P):
    """Return a NetworkX graph from a Pydot graph.

    Parameters
    ----------
    P : Pydot graph
      A graph created with Pydot

    Examples
    --------
    >>> K5=nx.complete_graph(5)
    >>> A=nx.to_pydot(K5)
    >>> G=nx.from_pydot(A)

    """

    if P.get_strict(None): # pydot bug: get_strict() shouldn't take argument
        multiedges=False
    else:
        multiedges=True

    if P.get_type()=='graph': # undirected
        if multiedges:
            create_using=networkx.MultiGraph()
        else:
            create_using=networkx.Graph()
    else:
        if multiedges:
            create_using=networkx.MultiDiGraph()
        else:
            create_using=networkx.DiGraph()

    # assign defaults
    N=networkx.empty_graph(0,create_using)
    N.name=P.get_name()
    node_attr={}

    # add nodes, attributes to N.node_attr
    for p in P.get_node_list():
        n=p.get_name()
        if n.startswith('"'):
            n=n[1:-1]
        if n in ('node','graph','edge'):
            continue
        N.add_node(n,**p.get_attributes())

    # add edges
    for e in P.get_edge_list():
        u=e.get_source()
        v=e.get_destination()
        attr=e.get_attributes()
        N.add_edge(u,v,**attr)

    # add default attributes for graph, nodes, edges
    N.graph['graph']=P.get_attributes()
    # get atributes not working for these?
    # get_node_defaults()
    N.graph['node']={}
    if 'node' in P.obj_dict['nodes']:
        N.graph['node']=P.obj_dict['nodes']['node'][0]['attributes']
    # get_edge_defaults()
    N.graph['edge']={}
    if 'edge' in P.obj_dict['nodes']:
        N.graph['edge']=P.obj_dict['nodes']['edge'][0]['attributes']
    N.node_attr=node_attr

    return N

def to_pydot(N, strict=True):
    """Return a pydot graph from a NetworkX graph N.

    Parameters
    ----------
    N : NetworkX graph
      A graph created with NetworkX

    Examples
    --------
    >>> K5=nx.complete_graph(5)
    >>> P=nx.to_pydot(K5)

    Notes
    -----

    """
    try:
        import pydot
    except ImportError:
        raise ImportError, \
          "to_pydot() requires pydot http://dkbza.org/pydot.html/"

    # set Graphviz graph type
    if N.is_directed():
        graph_type='digraph'
    else:
        graph_type='graph'
    strict=N.number_of_selfloops()==0 and not N.is_multigraph()

    P = pydot.Dot(graph_type=graph_type,strict=strict)

    for n,nodedata in N.nodes_iter(data=True):
        str_nodedata=dict((k,str(v)) for k,v in nodedata.iteritems())
        p=pydot.Node(str(n),**str_nodedata)
        P.add_node(p)

    if N.is_multigraph():
        for u,v,key,edgedata in N.edges_iter(data=True,keys=True):
            str_edgedata=dict((k,str(v)) for k,v in edgedata.iteritems())
            edge=pydot.Edge(str(u),str(v),key=str(key),**str_edgedata)
            P.add_edge(edge)

    else:
        for u,v,edgedata in N.edges_iter(data=True):
            str_edgedata=dict((k,str(v)) for k,v in edgedata.iteritems())
            edge=pydot.Edge(str(u),str(v),**str_edgedata)
            P.add_edge(edge)

    try:
        P.obj_dict['attributes'].update(N.graph.get('graph',{}))
    except:
        pass
    try:
        P.obj_dict['nodes']['node'][0]['attributes'].update(N.graph.get('node',{}))
    except:
        pass
    try:
        P.obj_dict['nodes']['edge'][0]['attributes'].update(N.graph.get('edge',{}))
    except:
        pass

    return P

def pydot_from_networkx(N):
	"""Create a Pydot graph from a NetworkX graph."""
        from warnings import warn
        warn('pydot_from_networkx is replaced by to_pydot', DeprecationWarning)
        return to_pydot(N)

def networkx_from_pydot(D, create_using=None):
	"""Create a NetworkX graph from a Pydot graph."""
        from warnings import warn
        warn('networkx_from_pydot is replaced by from_pydot',
             DeprecationWarning)
        return from_pydot(D)

def graphviz_layout(G,prog='neato',root=None, **kwds):
    """Create node positions using Pydot and Graphviz.

    Returns a dictionary of positions keyed by node.

    Examples
    --------
    >>> G=nx.complete_graph(4)
    >>> pos=nx.graphviz_layout(G)
    >>> pos=nx.graphviz_layout(G,prog='dot')

    Notes
    -----
    This is a wrapper for pydot_layout.

    """
    return pydot_layout(G=G,prog=prog,root=root,**kwds)

def pydot_layout(G,prog='neato',root=None, **kwds):
    """Create node positions using Pydot and Graphviz.

    Returns a dictionary of positions keyed by node.

    Examples
    --------
    >>> G=nx.complete_graph(4)
    >>> pos=nx.pydot_layout(G)
    >>> pos=nx.pydot_layout(G,prog='dot')

    """
    try:
        import pydot
    except ImportError:
        raise ImportError, \
          "pydot_layout() requires pydot http://dkbza.org/pydot.html/"

    P=to_pydot(G)
    if root is not None :
        P.set("root",str(root))

    D=P.create_dot(prog=prog)

    if D=="":  # no data returned
        print "Graphviz layout with %s failed"%(prog)
        print
        print "To debug what happened try:"
        print "P=pydot_from_networkx(G)"
        print "P.write_dot(\"file.dot\")"
        print "And then run %s on file.dot"%(prog)
        return

    Q=pydot.graph_from_dot_data(D)

    node_pos={}
    for n in G.nodes():
        node=Q.get_node(str(n))
        pos=node.get_pos()[1:-1] # strip leading and trailing double quotes
        if pos != None:
            xx,yy=pos.split(",")
            node_pos[n]=(float(xx),float(yy))
    return node_pos

<commit_msg>Handle escaped node names in pydot_layout<commit_after>"""
*****
Pydot
*****

Import and export NetworkX graphs in Graphviz dot format using pydot.

Either this module or nx_pygraphviz can be used to interface with graphviz.

See Also
--------
Pydot: http://www.dkbza.org/pydot.html
Graphviz:	   http://www.research.att.com/sw/tools/graphviz/
DOT Language:  http://www.graphviz.org/doc/info/lang.html

"""
__author__ = """Aric Hagberg (hagberg@lanl.gov)"""
#    Copyright (C) 2004-2008 by
#    Aric Hagberg <hagberg@lanl.gov>
#    Dan Schult <dschult@colgate.edu>
#    Pieter Swart <swart@lanl.gov>
#    All rights reserved.
#    BSD license.

__all__ = ['write_dot', 'read_dot', 'graphviz_layout', 'pydot_layout',
           'to_pydot', 'from_pydot']

import sys
from networkx.utils import _get_fh
import networkx

def write_dot(G,path):
    """Write NetworkX graph G to Graphviz dot format on path.

    Path can be a string or a file handle.
    """
    fh=_get_fh(path,'w')
    P=to_pydot(G)
    fh.write(P.to_string())
    fh.flush() # might be a user filehandle so leave open (but flush)
    return

def read_dot(path):
    """Return a NetworkX Graph or DiGraph from a dot file on path.

    Path can be a string or a file handle.

    """
    try:
        import pydot
    except ImportError:
        raise ImportError, \
          "read_dot() requires pydot http://dkbza.org/pydot.html/"

    fh=_get_fh(path,'r')
    data=fh.read()
    P=pydot.graph_from_dot_data(data)
    return from_pydot(P)

def from_pydot(P):
    """Return a NetworkX graph from a Pydot graph.

    Parameters
    ----------
    P : Pydot graph
      A graph created with Pydot

    Examples
    --------
    >>> K5=nx.complete_graph(5)
    >>> A=nx.to_pydot(K5)
    >>> G=nx.from_pydot(A)

    """

    if P.get_strict(None): # pydot bug: get_strict() shouldn't take argument
        multiedges=False
    else:
        multiedges=True

    if P.get_type()=='graph': # undirected
        if multiedges:
            create_using=networkx.MultiGraph()
        else:
            create_using=networkx.Graph()
    else:
        if multiedges:
            create_using=networkx.MultiDiGraph()
        else:
            create_using=networkx.DiGraph()

    # assign defaults
    N=networkx.empty_graph(0,create_using)
    N.name=P.get_name()
    node_attr={}

    # add nodes, attributes to N.node_attr
    for p in P.get_node_list():
        n=p.get_name()
        if n.startswith('"'):
            n=n[1:-1]
        if n in ('node','graph','edge'):
            continue
        N.add_node(n,**p.get_attributes())

    # add edges
    for e in P.get_edge_list():
        u=e.get_source()
        v=e.get_destination()
        attr=e.get_attributes()
        N.add_edge(u,v,**attr)

    # add default attributes for graph, nodes, edges
    N.graph['graph']=P.get_attributes()
    # get atributes not working for these?
    # get_node_defaults()
    N.graph['node']={}
    if 'node' in P.obj_dict['nodes']:
        N.graph['node']=P.obj_dict['nodes']['node'][0]['attributes']
    # get_edge_defaults()
    N.graph['edge']={}
    if 'edge' in P.obj_dict['nodes']:
        N.graph['edge']=P.obj_dict['nodes']['edge'][0]['attributes']
    N.node_attr=node_attr

    return N

def to_pydot(N, strict=True):
    """Return a pydot graph from a NetworkX graph N.

    Parameters
    ----------
    N : NetworkX graph
      A graph created with NetworkX

    Examples
    --------
    >>> K5=nx.complete_graph(5)
    >>> P=nx.to_pydot(K5)

    Notes
    -----

    """
    try:
        import pydot
    except ImportError:
        raise ImportError, \
          "to_pydot() requires pydot http://dkbza.org/pydot.html/"

    # set Graphviz graph type
    if N.is_directed():
        graph_type='digraph'
    else:
        graph_type='graph'
    strict=N.number_of_selfloops()==0 and not N.is_multigraph()

    P = pydot.Dot(graph_type=graph_type,strict=strict)

    for n,nodedata in N.nodes_iter(data=True):
        str_nodedata=dict((k,str(v)) for k,v in nodedata.iteritems())
        p=pydot.Node(str(n),**str_nodedata)
        P.add_node(p)

    if N.is_multigraph():
        for u,v,key,edgedata in N.edges_iter(data=True,keys=True):
            str_edgedata=dict((k,str(v)) for k,v in edgedata.iteritems())
            edge=pydot.Edge(str(u),str(v),key=str(key),**str_edgedata)
            P.add_edge(edge)

    else:
        for u,v,edgedata in N.edges_iter(data=True):
            str_edgedata=dict((k,str(v)) for k,v in edgedata.iteritems())
            edge=pydot.Edge(str(u),str(v),**str_edgedata)
            P.add_edge(edge)

    try:
        P.obj_dict['attributes'].update(N.graph.get('graph',{}))
    except:
        pass
    try:
        P.obj_dict['nodes']['node'][0]['attributes'].update(N.graph.get('node',{}))
    except:
        pass
    try:
        P.obj_dict['nodes']['edge'][0]['attributes'].update(N.graph.get('edge',{}))
    except:
        pass

    return P

def pydot_from_networkx(N):
	"""Create a Pydot graph from a NetworkX graph."""
        from warnings import warn
        warn('pydot_from_networkx is replaced by to_pydot', DeprecationWarning)
        return to_pydot(N)

def networkx_from_pydot(D, create_using=None):
	"""Create a NetworkX graph from a Pydot graph."""
        from warnings import warn
        warn('networkx_from_pydot is replaced by from_pydot',
             DeprecationWarning)
        return from_pydot(D)

def graphviz_layout(G,prog='neato',root=None, **kwds):
    """Create node positions using Pydot and Graphviz.

    Returns a dictionary of positions keyed by node.

    Examples
    --------
    >>> G=nx.complete_graph(4)
    >>> pos=nx.graphviz_layout(G)
    >>> pos=nx.graphviz_layout(G,prog='dot')

    Notes
    -----
    This is a wrapper for pydot_layout.

    """
    return pydot_layout(G=G,prog=prog,root=root,**kwds)

def pydot_layout(G,prog='neato',root=None, **kwds):
    """Create node positions using Pydot and Graphviz.

    Returns a dictionary of positions keyed by node.

    Examples
    --------
    >>> G=nx.complete_graph(4)
    >>> pos=nx.pydot_layout(G)
    >>> pos=nx.pydot_layout(G,prog='dot')

    """
    try:
        import pydot
    except ImportError:
        raise ImportError, \
          "pydot_layout() requires pydot http://dkbza.org/pydot.html/"

    P=to_pydot(G)
    if root is not None :
        P.set("root",str(root))

    D=P.create_dot(prog=prog)

    if D=="":  # no data returned
        print "Graphviz layout with %s failed"%(prog)
        print
        print "To debug what happened try:"
        print "P=pydot_from_networkx(G)"
        print "P.write_dot(\"file.dot\")"
        print "And then run %s on file.dot"%(prog)
        return

    Q=pydot.graph_from_dot_data(D)

    node_pos={}
    for n in G.nodes():
        node=Q.get_node(pydot.Node(n).get_name())
        pos=node.get_pos()[1:-1] # strip leading and trailing double quotes
        if pos != None:
            xx,yy=pos.split(",")
            node_pos[n]=(float(xx),float(yy))
    return node_pos

<|endoftext|>


<commit_before>"""Provides programs to process and analyse NoRH lightcurve data."""

from __future__ import absolute_import

import datetime
import os
from astropy.io import fits
import pandas
import sunpy
from sunpy.lightcurve import LightCurve
from sunpy.time import parse_time

__all__ = ['NoRHLightCurve']

class NoRHLightCurve(LightCurve):

    @classmethod
    def _get_url_for_date(cls,date):
        """This method retrieves the url for NoRH correlation data for the given date."""
        baseurl='ftp://solar-pub.nao.ac.jp/pub/nsro/norh/data/tcx/'
        #date is a datetime object
        year=date.strftime('%Y')
        year_trim=date.strftime('%y')
        mon=date.strftime('%m')
        day=date.strftime('%d')
        final_url_17=os.path.join(baseurl,year,mon,'tca'+year_trim+mon+day)
        final_url_34=os.path.join(baseurl,year,mon,'tcz'+year_trim+mon+day)

        return final_url_17

    #now want to download from these urls using the sunpy net functions
    @staticmethod
    def _parse_fits(filepath):
        """This method parses NoRH tca and tcz correlation files."""
        hdulist=fits.open(filepath)
        header=hdulist[0].header
        #for these NoRH files, the time series data is recorded in the primary HDU
        data=hdulist[0].data

        #No explicit time array in FITS file, so construct the time array from the FITS header
        obs_start_time=parse_time(header['date-obs'] + 'T' + header['crval1'])
        length=len(data)
        cadence=np.float(header['cdelt1'])
        sec_array=np.linspace(0,length-1,(length/cadence))

        deltas=[]
        for s in sec_array:
            deltas.append(datetime.timedelta(0,s))

        norh_time=[]
        for d in deltas:
            norh_time.append(obs_start_time+d)

        return header, pandas.DataFrame(data,index=norh_time)

<commit_msg>bug fixes for NoRH lightcurves<commit_after>"""Provides programs to process and analyse NoRH lightcurve data."""

from __future__ import absolute_import

import datetime
import os
from astropy.io import fits
import pandas
import sunpy
from sunpy.lightcurve import LightCurve
from sunpy.time import parse_time
import numpy as np

__all__ = ['NoRHLightCurve']

class NoRHLightCurve(LightCurve):

    @classmethod
    def _get_url_for_date(cls,date):
        """This method retrieves the url for NoRH correlation data for the given date."""
        baseurl='ftp://solar-pub.nao.ac.jp/pub/nsro/norh/data/tcx/'
        #date is a datetime object
        year=date.strftime('%Y')
        year_trim=date.strftime('%y')
        mon=date.strftime('%m')
        day=date.strftime('%d')
        final_url_17=os.path.join(baseurl,year,mon,'tca'+year_trim+mon+day)
        final_url_34=os.path.join(baseurl,year,mon,'tcz'+year_trim+mon+day)

        return final_url_34

    #now want to download from these urls using the sunpy net functions
    @staticmethod
    def _parse_fits(filepath):
        """This method parses NoRH tca and tcz correlation files."""
        hdulist=fits.open(filepath)
        header=hdulist[0].header
        #for these NoRH files, the time series data is recorded in the primary HDU
        data=hdulist[0].data

        #No explicit time array in FITS file, so construct the time array from the FITS header
        obs_start_time=parse_time(header['date-obs'] + 'T' + header['crval1'])
        length=len(data)
        cadence=np.float(header['cdelt1'])
        sec_array=np.linspace(0,length-1,(length/cadence))

        deltas=[]
        for s in sec_array:
            deltas.append(datetime.timedelta(0,s))

        norh_time=[]
        for d in deltas:
            norh_time.append(obs_start_time+d)

        return header, pandas.DataFrame(data,index=norh_time)

<|endoftext|>


<commit_before>try:
    from cStringIO import StringIO as BytesIO
except ImportError:
    from io import BytesIO

from chardet import detect
from lxml.html import parse, HTMLParser
from libextract.coretools import highest_scoring, memoized

NODES_WITH_TEXT = '//*[not(self::script or self::style)]/text()/..'
TEXT_IN_NODE = './/text()[normalize-space()]'

FILTER_TEXT = './/*[not(self::script or self::style or \
        self::figure or self::span or self::time)]/\
        text()[normalize-space()]'

def get_xpath_finder(etree):
    if hasattr(etree, 'getroot'):
        etree = etree.getroot()
    return etree.getroottree().getpath

def parent_finder(etree):
    xpath_for = get_xpath_finder(etree)
    getter = memoized(lambda k: etree.xpath(k)[0])

    def finder(node):
        parent = xpath_for(node).rsplit('/', 1)[0]
        return getter(parent)

    return finder

def node_text_length(node):
    return len(''.join(node.xpath(TEXT_IN_NODE)))

def get_etree(document):
    return parse(BytesIO(document),
                 HTMLParser(encoding=detect(document)['encoding'],
                            remove_blank_text=True))

def get_pairs(etree):
    parent_of = parent_finder(etree)
    for node in etree.xpath(NODES_WITH_TEXT):
        yield parent_of(node), node_text_length(node)

def get_final_text(node):
    return ' '.join(node.xpath(FILTER_TEXT))

STRATEGY = (get_etree, get_pairs, highest_scoring, get_final_text)
<commit_msg>removed need for memoizing<commit_after>try:
    from cStringIO import StringIO as BytesIO
except ImportError:
    from io import BytesIO

from chardet import detect
from lxml.html import parse, HTMLParser
from libextract.coretools import highest_scoring

NODES_WITH_TEXT = '//*[not(self::script or self::style)]/text()/..'
TEXT_IN_NODE = './/text()[normalize-space()]'

FILTER_TEXT = './/*[not(self::script or self::style or \
        self::figure or self::span or self::time)]/\
        text()[normalize-space()]'

def node_text_length(node):
    return len(''.join(node.xpath(TEXT_IN_NODE)))

def get_etree(document):
    return parse(BytesIO(document),
                 HTMLParser(encoding=detect(document)['encoding'],
                            remove_blank_text=True))

def get_pairs(etree):
    for node in etree.xpath(NODES_WITH_TEXT):
        yield node.getparent(), node_text_length(node)

def get_final_text(node):
    return ' '.join(node.xpath(FILTER_TEXT))

STRATEGY = (get_etree, get_pairs, highest_scoring, get_final_text)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# pyroTorrent documentation build configuration file, created by
# sphinx-quickstart on Sun Jan 23 16:32:39 2011.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
# XXX:  Change this to model path later.
sys.path.insert(0, os.path.abspath('..'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'pyroTorrent'
copyright = u'2011, Merlijn Wajer, Bas Weelinck'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.01'
# The full version, including alpha/beta/rc tags.
release = '0.01'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'pyroTorrentdoc'

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'pyroTorrent.tex', u'pyroTorrent Documentation',
   u'Merlijn Wajer, Bas Weelinck', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'pyrotorrent', u'pyroTorrent Documentation',
     [u'Merlijn Wajer, Bas Weelinck'], 1)
]
<commit_msg>Documentation: Update the path to the models.<commit_after># -*- coding: utf-8 -*-
#
# pyroTorrent documentation build configuration file, created by
# sphinx-quickstart on Sun Jan 23 16:32:39 2011.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
# XXX:  Change this to model path later.
sys.path.insert(0, os.path.abspath('../pyroTorrent/pyrotorrent/model'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'pyroTorrent'
copyright = u'2011, Merlijn Wajer, Bas Weelinck'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.01'
# The full version, including alpha/beta/rc tags.
release = '0.01'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'pyroTorrentdoc'

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'pyroTorrent.tex', u'pyroTorrent Documentation',
   u'Merlijn Wajer, Bas Weelinck', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Additional stuff for the LaTeX preamble.
#latex_preamble = ''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'pyrotorrent', u'pyroTorrent Documentation',
     [u'Merlijn Wajer, Bas Weelinck'], 1)
]
<|endoftext|>


<commit_before>from matplotlib import pyplot as plt
from matplotlib import animation
import random
import numpy as np

Birds = 50 #number of birds in the simulation

def new_flock(count, lower_limits, upper_limits):
    #returns two colums of random numbers between the limits
    #for ll=[a,b] & ul=[c,d], colum1 is between a-c and 2 is b-d
    width=upper_limits-lower_limits
    return (lower_limits[:,np.newaxis] + np.random.rand(2, count)*width[:,np.newaxis])

positions = new_flock(Birds, np.array([-450, 300.0]), np.array([50.0, 600.0]))
velocities = new_flock(Birds,np.array([0, -20.0]), np.array([10.0, 20.0]))

boids=(positions[0], positions[1], velocities[0],velocities[1])

def update_boids(boids, positions, velocities):
    xs,ys,xvs,yvs=boids
    # Fly towards the middle
    strength_of_atraction = 0.01
    middle_of_flock = np.mean(positions, 1)
    direction_to_middle = positions - middle_of_flock[:, np.newaxis]
    velocities -= direction_to_middle*strength_of_atraction

    for i in range(Birds):
        for j in range(Birds):
            yvs[i]=yvs[i]+(ys[j]-ys[i])*0.01/len(xs)
    # Fly away from nearby boids
    for i in range(Birds):
        for j in range(Birds):
            if (xs[j]-xs[i])**2 + (ys[j]-ys[i])**2 < 100:
                xvs[i]=xvs[i]+(xs[i]-xs[j])
                yvs[i]=yvs[i]+(ys[i]-ys[j])
    # Try to match speed with nearby boids
    for i in range(Birds):
        for j in range(Birds):
            if (xs[j]-xs[i])**2 + (ys[j]-ys[i])**2 < 10000:
                xvs[i]=xvs[i]+(xvs[j]-xvs[i])*0.125/len(xs)
                yvs[i]=yvs[i]+(yvs[j]-yvs[i])*0.125/len(xs)
    # Move according to velocities
    positions += velocities

figure=plt.figure()
axes=plt.axes(xlim=(-500,1500), ylim=(-500,1500))
scatter=axes.scatter(boids[0],boids[1])

def animate(frame):
    update_boids(boids, positions, velocities)
    scatter.set_offsets(zip(boids[0],boids[1]))

anim = animation.FuncAnimation(figure, animate, frames=50, interval=50)

if __name__ == "__main__":
    plt.show()

<commit_msg>Delete left over code V1.1.3<commit_after>from matplotlib import pyplot as plt
from matplotlib import animation
import random
import numpy as np

Birds = 50 #number of birds in the simulation

def new_flock(count, lower_limits, upper_limits):
    #returns two colums of random numbers between the limits
    #for ll=[a,b] & ul=[c,d], colum1 is between a-c and 2 is b-d
    width=upper_limits-lower_limits
    return (lower_limits[:,np.newaxis] + np.random.rand(2, count)*width[:,np.newaxis])

positions = new_flock(Birds, np.array([-450, 300.0]), np.array([50.0, 600.0]))
velocities = new_flock(Birds,np.array([0, -20.0]), np.array([10.0, 20.0]))

boids=(positions[0], positions[1], velocities[0],velocities[1])

def update_boids(boids, positions, velocities):
    xs,ys,xvs,yvs=boids
    # Fly towards the middle
    strength_of_atraction = 0.01
    middle_of_flock = np.mean(positions, 1)
    direction_to_middle = positions - middle_of_flock[:, np.newaxis]
    velocities -= direction_to_middle*strength_of_atraction

    # Fly away from nearby boids
    for i in range(Birds):
        for j in range(Birds):
            if (xs[j]-xs[i])**2 + (ys[j]-ys[i])**2 < 100:
                xvs[i]=xvs[i]+(xs[i]-xs[j])
                yvs[i]=yvs[i]+(ys[i]-ys[j])
    # Try to match speed with nearby boids
    for i in range(Birds):
        for j in range(Birds):
            if (xs[j]-xs[i])**2 + (ys[j]-ys[i])**2 < 10000:
                xvs[i]=xvs[i]+(xvs[j]-xvs[i])*0.125/len(xs)
                yvs[i]=yvs[i]+(yvs[j]-yvs[i])*0.125/len(xs)
    # Move according to velocities
    positions += velocities

figure=plt.figure()
axes=plt.axes(xlim=(-500,1500), ylim=(-500,1500))
scatter=axes.scatter(boids[0],boids[1])

def animate(frame):
    update_boids(boids, positions, velocities)
    scatter.set_offsets(zip(boids[0],boids[1]))

anim = animation.FuncAnimation(figure, animate, frames=50, interval=50)

if __name__ == "__main__":
    plt.show()

<|endoftext|>


<commit_before>#!/usr/bin/python

#from PIL import Image, ImageFilter

#with Image.open("forum.jpg") as img:
#    print(img.size)
#    blurred = img.filter(ImageFilter.GaussianBlur(96))
#    blurred.save("blurred.jpg", "JPEG")

import sqlite3
import os
from flask import Flask, json, send_file, g, request, redirect, abort
from werkzeug.utils import secure_filename

DATABASE = "database.db"
UPLOAD_FOLDER = "./images/"

app = Flask(__name__)
app.config.from_object(__name__)

@app.route("/")
def index():
    return "Wall Art"

def connect_db():
    return sqlite3.connect(app.config["DATABASE"])

def get_db():
    db = getattr(g, "_database", None)
    if db is None:
        db = g._database = connect_db()
    return db

@app.teardown_appcontext
def close_db(exception):
    db = getattr(g, "_database", None)
    if db is not None:
        db.close()

def query_db(query, args=(), one=False):
    cur = get_db().execute(query, args)
    rv = cur.fetchall()
    cur.close()
    return (rv[0] if rv else None) if one else rv

# Build a list of all images as an array of dicts, with one dict per image
# to be easily returned as a JSON string
def build_image_dict(query_result):
    images = []
    for img in query_result:
        images.append({"id": img[0], "title": img[1], "artist": img[2],
            "work_type": img[3], "culture": img[4], "has_nudity": img[5],
            "filename": img[6], "blurred_filename": img[7]})
    return images

# API endpoint to get list of all images on server
@app.route("/api/image", methods=["GET"])
def get_images():
    images = build_image_dict(query_db("select * from images"))
    return json.dumps(images, ensure_ascii=False).encode("utf8")

# API endpoint to get information about image <id>
@app.route("/api/image/<int:image_id>", methods=["GET"])
def get_image_info(image_id):
    img = query_db("select * from images where id = ?", [image_id], one=True)
    if img is None:
        abort(404)
    image_info = {"id": img[0], "title": img[1], "artist": img[2],
            "work_type": img[3], "culture": img[4], "has_nudity": img[5],
            "filename": img[6], "blurred_filename": img[7]}
    return json.dumps(image_info, ensure_ascii=False).encode("utf8")

# API endpoint to get the original image for image <id>
@app.route("/api/image/<int:image_id>/original", methods=["GET"])
def get_original_image(image_id):
    img = query_db("select * from images where id = ?", [image_id], one=True)
    if img is None:
        abort(404)
    return send_file(img[6], mimetype="image/jpeg")

# API endpoint to get blurred image with some id
@app.route("/api/image/<int:image_id>/blurred", methods=["GET"])
def get_blurred_image(image_id):
    img = query_db("select * from images where id = ?", [image_id], one=True)
    if img is None:
        abort(404)
    return send_file(img[7], mimetype="image/jpeg")

@app.route("/api/artist/<artist_name>")
def get_artist_paintings(artist_name):
    artist = query_db("select * from images where artist = ?", [artist_name])
    if artist is None:
        abort(404)
    images = build_image_dict(artist)
    return json.dumps(images, ensure_ascii=False).encode("utf8")

@app.route("/upload", methods=["GET", "POST"])
def upload_image():
    if request.method == "POST":
        f = request.files["file"]
        bf = request.files["blurred_file"]
        if f and bf:
            filename = os.path.join(app.config["UPLOAD_FOLDER"],
                    secure_filename(f.filename))
            blurred_filename = os.path.join(app.config["UPLOAD_FOLDER"],
					secure_filename(bf.filename))
            f.save(filename)
            bf.save(blurred_filename)
            if query_db("select * from images where title = ?",
                    [request.form["title"]], one=True) is None:
                db = get_db()
                # TODO: We should actually have a check box for has nudity and
                # also accept a second file upload for the blurred file
                db.execute("""insert into images (title, artist, work_type,
                    culture, has_nudity, filename, blurred_filename)
                    values (?, ?, ?, ?, ?, ?, ?)""", [request.form["title"],
                        request.form["artist"], request.form["work_type"],
                        request.form["culture"], False, filename, blurred_filename])
                db.commit()
        return redirect("/upload")

    return """
    <!DOCTYPE html5>
    <title>Upload image</title>

Upload File

    <form action="" method=post enctype=multipart/form-data>

		   Original Image
           <input type=file name=file>

		   Blurred Image
           <input type=file name=blurred_file>

           Title
           <input type=text name=title>

           Artist
           <input type=text name=artist>

           Work Type
           <input type=text name=work_type>

           Culture
           <input type=text name=culture>

           <input type=submit value=Upload>

    </form>
    """

if __name__ == "__main__":
    app.run(debug=True)

<commit_msg>Fix some mixed indentation<commit_after>#!/usr/bin/python

#from PIL import Image, ImageFilter

#with Image.open("forum.jpg") as img:
#    print(img.size)
#    blurred = img.filter(ImageFilter.GaussianBlur(96))
#    blurred.save("blurred.jpg", "JPEG")

import sqlite3
import os
from flask import Flask, json, send_file, g, request, redirect, abort
from werkzeug.utils import secure_filename

DATABASE = "database.db"
UPLOAD_FOLDER = "./images/"

app = Flask(__name__)
app.config.from_object(__name__)

@app.route("/")
def index():
    return "Wall Art"

def connect_db():
    return sqlite3.connect(app.config["DATABASE"])

def get_db():
    db = getattr(g, "_database", None)
    if db is None:
        db = g._database = connect_db()
    return db

@app.teardown_appcontext
def close_db(exception):
    db = getattr(g, "_database", None)
    if db is not None:
        db.close()

def query_db(query, args=(), one=False):
    cur = get_db().execute(query, args)
    rv = cur.fetchall()
    cur.close()
    return (rv[0] if rv else None) if one else rv

# Build a list of all images as an array of dicts, with one dict per image
# to be easily returned as a JSON string
def build_image_dict(query_result):
    images = []
    for img in query_result:
        images.append({"id": img[0], "title": img[1], "artist": img[2],
            "work_type": img[3], "culture": img[4], "has_nudity": img[5],
            "filename": img[6], "blurred_filename": img[7]})
    return images

# API endpoint to get list of all images on server
@app.route("/api/image", methods=["GET"])
def get_images():
    images = build_image_dict(query_db("select * from images"))
    return json.dumps(images, ensure_ascii=False).encode("utf8")

# API endpoint to get information about image <id>
@app.route("/api/image/<int:image_id>", methods=["GET"])
def get_image_info(image_id):
    img = query_db("select * from images where id = ?", [image_id], one=True)
    if img is None:
        abort(404)
    image_info = {"id": img[0], "title": img[1], "artist": img[2],
            "work_type": img[3], "culture": img[4], "has_nudity": img[5],
            "filename": img[6], "blurred_filename": img[7]}
    return json.dumps(image_info, ensure_ascii=False).encode("utf8")

# API endpoint to get the original image for image <id>
@app.route("/api/image/<int:image_id>/original", methods=["GET"])
def get_original_image(image_id):
    img = query_db("select * from images where id = ?", [image_id], one=True)
    if img is None:
        abort(404)
    return send_file(img[6], mimetype="image/jpeg")

# API endpoint to get blurred image with some id
@app.route("/api/image/<int:image_id>/blurred", methods=["GET"])
def get_blurred_image(image_id):
    img = query_db("select * from images where id = ?", [image_id], one=True)
    if img is None:
        abort(404)
    return send_file(img[7], mimetype="image/jpeg")

@app.route("/api/artist/<artist_name>")
def get_artist_paintings(artist_name):
    artist = query_db("select * from images where artist = ?", [artist_name])
    if artist is None:
        abort(404)
    images = build_image_dict(artist)
    return json.dumps(images, ensure_ascii=False).encode("utf8")

@app.route("/upload", methods=["GET", "POST"])
def upload_image():
    if request.method == "POST":
        f = request.files["file"]
        bf = request.files["blurred_file"]
        if f and bf:
            filename = os.path.join(app.config["UPLOAD_FOLDER"],
                    secure_filename(f.filename))
            blurred_filename = os.path.join(app.config["UPLOAD_FOLDER"],
					secure_filename(bf.filename))
            f.save(filename)
            bf.save(blurred_filename)
            if query_db("select * from images where title = ?",
                    [request.form["title"]], one=True) is None:
                db = get_db()
                # TODO: We should actually have a check box for has nudity and
                # also accept a second file upload for the blurred file
                db.execute("""insert into images (title, artist, work_type,
                    culture, has_nudity, filename, blurred_filename)
                    values (?, ?, ?, ?, ?, ?, ?)""", [request.form["title"],
                        request.form["artist"], request.form["work_type"],
                        request.form["culture"], False, filename, blurred_filename])
                db.commit()
        return redirect("/upload")

    return """
    <!DOCTYPE html5>
    <title>Upload image</title>

Upload File

    <form action="" method=post enctype=multipart/form-data>

           Original Image
           <input type=file name=file>

           Blurred Image
           <input type=file name=blurred_file>

           Title
           <input type=text name=title>

           Artist
           <input type=text name=artist>

           Work Type
           <input type=text name=work_type>

           Culture
           <input type=text name=culture>

           <input type=submit value=Upload>

    </form>
    """

if __name__ == "__main__":
    app.run(debug=True)

<|endoftext|>


<commit_before>
"""
    Copyright (C) 2017 SunSpec Alliance

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included
    in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
    IN THE SOFTWARE.
"""

import sys
import os
import unittest

import sunspec.core.client as client
import sunspec.core.device as device
import sunspec.core.util as util
import sunspec.core.util

class TestClientDevice(unittest.TestCase):
    def setUp(self):
        path = os.path.abspath(__file__)
        self.pathlist = util.PathList(['.',
                                       os.path.join(os.path.dirname(path),
                                                    'devices')])

        sunspec.core.util.check_for_models(pathlist=self.pathlist)

    def test_client_device(self):
        d = client.ClientDevice(client.MAPPED, slave_id=1,
                                name='mbmap_test_device_1.xml',
                                pathlist=self.pathlist)
        d.scan()
        d.read_points()

        dp = device.Device()
        dp.from_pics(filename='pics_test_device_1.xml',
                        pathlist=self.pathlist)
        not_equal = dp.not_equal(d)
        if not_equal:
            raise Exception(not_equal)

    def test_sunspec_client_device_1(self):
        d = client.SunSpecClientDevice(client.MAPPED, slave_id=1,
                                        name='mbmap_test_device_1.xml',
                                        pathlist=self.pathlist)
        d.read()

        dp = device.Device()
        dp.from_pics(filename='pics_test_device_1.xml',
                        pathlist=self.pathlist)
        not_equal = dp.not_equal(d.device)
        if not_equal:
            raise Exception(not_equal)

        expected = 'SunSpecTest'
        if d.common.Mn != expected:
            raise Exception("'common.Mn' point mismatch: %s %s" % (d.common.Mn, expected))

        expected = 'sn-123456789'
        if d.common.SN != expected:
            raise Exception("'common.SN' point mismatch: %s %s" % (d.common.SN, expected))

        # int16 read and write
        d.model_63001.read()
        expected = -20
        if d.model_63001.int16_4 != expected:
            raise Exception("'model_63001.int16_4' point mismatch: %s %s" % (d.model_63001.int16_4, expected))
        value = 330
        d.model_63001.int16_4 = value
        d.model_63001.write()
        d.model_63001.read()
        value = d.model_63001.int16_4
        if d.model_63001.int16_4 != value:
            raise Exception("'model_63001.int16_4' write failure: %s %s" % (d.model_63001.int16_4, value))

        # string read and write
        expected = '12345678'
        if d.model_63001.string != expected:
            raise Exception("'model_63001.string' point mismatch: %s %s" % (d.model_63001.string, expected))

        value = 'abcdefg'
        d.model_63001.string = value
        d.model_63001.write()
        d.model_63001.read()
        if d.model_63001.string != value:
            raise Exception("'model_63001.string' write failure: %s %s" % (d.model_63001.string, value))

        # write multiple
        d.model_63001.read()
        expected = 65524
        if d.model_63001.uint16_3 != expected:
            raise Exception("'model_63001.uint16_3' point mismatch: %s %s" % (d.model_63001.uint16_3, expected))
        expected = 60
        if d.model_63001.uint16_4 != expected:
            raise Exception("'model_63001.uint16_4' point mismatch: %s %s" % (d.model_63001.uint16_4, expected))
        expected = 7
        if d.model_63001.uint16_5 != expected:
            raise Exception("'model_63001.uint16_5' point mismatch: %s %s" % (d.model_63001.uint16_5, expected))
        value_3 = 65525
        value_4 = 70
        value_5 = 8
        d.model_63001.uint16_3 = value_3
        d.model_63001.uint16_4 = value_4
        d.model_63001.uint16_5 = value_5
        d.model_63001.write()
        d.model_63001.read()
        value = d.model_63001.uint16_3
        if d.model_63001.uint16_3 != value_3:
            raise Exception("'model_63001.int16_3' write failure: %s %s" % (d.model_63001.uint16_3, value_3))
        value = d.model_63001.uint16_4
        if d.model_63001.uint16_4 != value_4:
            raise Exception("'model_63001.int16_4' write failure: %s %s" % (d.model_63001.uint16_4, value_4))
        value = d.model_63001.uint16_5
        if d.model_63001.uint16_5 != value_5:
            raise Exception("'model_63001.int16_5' write failure: %s %s" % (d.model_63001.uint16_5, value_5))

        # write multiple
        d.model_63001.read()
        expected = value_3
        if d.model_63001.uint16_3 != expected:
            raise Exception("'model_63001.uint16_3' point mismatch: %s %s" % (d.model_63001.uint16_3, expected))
        expected = value_4
        if d.model_63001.uint16_4 != expected:
            raise Exception("'model_63001.uint16_4' point mismatch: %s %s" % (d.model_63001.uint16_4, expected))
        expected = value_5
        if d.model_63001.uint16_5 != expected:
            raise Exception("'model_63001.uint16_5' point mismatch: %s %s" % (d.model_63001.uint16_5, expected))
        value_3 = 65524
        value_5 = 7
        d.model_63001.uint16_3 = value_3
        d.model_63001.uint16_5 = value_5
        d.model_63001.write()
        d.model_63001.read()
        value = d.model_63001.uint16_3
        if d.model_63001.uint16_3 != value_3:
            raise Exception("'model_63001.int16_3' write failure: %s %s" % (d.model_63001.uint16_3, value_3))
        value = d.model_63001.uint16_4
        if d.model_63001.uint16_4 != value_4:
            raise Exception("'model_63001.int16_4' write failure: %s %s" % (d.model_63001.uint16_4, value_4))
        value = d.model_63001.uint16_5
        if d.model_63001.uint16_5 != value_5:
            raise Exception("'model_63001.int16_5' write failure: %s %s" % (d.model_63001.uint16_5, value_5))

        d.close()

    def test_sunspec_client_device_3(self):
        d = client.SunSpecClientDevice(client.MAPPED, slave_id=1, name='mbmap_test_device_3.xml', pathlist=self.pathlist)

        # int16 read and write
        d.model_63002.read()
        expected = 1111
        value = int(d.model_63002.repeating[1].int16_1 * 10)
        if  value != expected:
            raise Exception("'model_63002.int16_1' point mismatch: %s %s" % (value, expected))

        d.model_63002.repeating[1].int16_1 = 333.3
        d.model_63002.write()
        d.model_63002.read()
        expected = 3333
        value = int(d.model_63002.repeating[1].int16_1 * 10)
        if value != expected:
            raise Exception("'model_63002.int16_2' write failure: %s %s" % (value, expected))

        expected = 2222
        value = int(d.model_63002.repeating[1].int16_2 * 100)
        if  value != expected:
            raise Exception("'model_63002.int16_1' point mismatch: %s %s" % (value, expected))

        d.close()

if __name__ == "__main__":

    unittest.main()
<commit_msg>Tidy duplicated import of sunspec.core.util<commit_after>
"""
    Copyright (C) 2017 SunSpec Alliance

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included
    in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
    IN THE SOFTWARE.
"""

import sys
import os
import unittest

import sunspec.core.client as client
import sunspec.core.device as device
import sunspec.core.util as util

class TestClientDevice(unittest.TestCase):
    def setUp(self):
        path = os.path.abspath(__file__)
        self.pathlist = util.PathList(['.',
                                       os.path.join(os.path.dirname(path),
                                                    'devices')])

        util.check_for_models(pathlist=self.pathlist)

    def test_client_device(self):
        d = client.ClientDevice(client.MAPPED, slave_id=1,
                                name='mbmap_test_device_1.xml',
                                pathlist=self.pathlist)
        d.scan()
        d.read_points()

        dp = device.Device()
        dp.from_pics(filename='pics_test_device_1.xml',
                        pathlist=self.pathlist)
        not_equal = dp.not_equal(d)
        if not_equal:
            raise Exception(not_equal)

    def test_sunspec_client_device_1(self):
        d = client.SunSpecClientDevice(client.MAPPED, slave_id=1,
                                        name='mbmap_test_device_1.xml',
                                        pathlist=self.pathlist)
        d.read()

        dp = device.Device()
        dp.from_pics(filename='pics_test_device_1.xml',
                        pathlist=self.pathlist)
        not_equal = dp.not_equal(d.device)
        if not_equal:
            raise Exception(not_equal)

        expected = 'SunSpecTest'
        if d.common.Mn != expected:
            raise Exception("'common.Mn' point mismatch: %s %s" % (d.common.Mn, expected))

        expected = 'sn-123456789'
        if d.common.SN != expected:
            raise Exception("'common.SN' point mismatch: %s %s" % (d.common.SN, expected))

        # int16 read and write
        d.model_63001.read()
        expected = -20
        if d.model_63001.int16_4 != expected:
            raise Exception("'model_63001.int16_4' point mismatch: %s %s" % (d.model_63001.int16_4, expected))
        value = 330
        d.model_63001.int16_4 = value
        d.model_63001.write()
        d.model_63001.read()
        value = d.model_63001.int16_4
        if d.model_63001.int16_4 != value:
            raise Exception("'model_63001.int16_4' write failure: %s %s" % (d.model_63001.int16_4, value))

        # string read and write
        expected = '12345678'
        if d.model_63001.string != expected:
            raise Exception("'model_63001.string' point mismatch: %s %s" % (d.model_63001.string, expected))

        value = 'abcdefg'
        d.model_63001.string = value
        d.model_63001.write()
        d.model_63001.read()
        if d.model_63001.string != value:
            raise Exception("'model_63001.string' write failure: %s %s" % (d.model_63001.string, value))

        # write multiple
        d.model_63001.read()
        expected = 65524
        if d.model_63001.uint16_3 != expected:
            raise Exception("'model_63001.uint16_3' point mismatch: %s %s" % (d.model_63001.uint16_3, expected))
        expected = 60
        if d.model_63001.uint16_4 != expected:
            raise Exception("'model_63001.uint16_4' point mismatch: %s %s" % (d.model_63001.uint16_4, expected))
        expected = 7
        if d.model_63001.uint16_5 != expected:
            raise Exception("'model_63001.uint16_5' point mismatch: %s %s" % (d.model_63001.uint16_5, expected))
        value_3 = 65525
        value_4 = 70
        value_5 = 8
        d.model_63001.uint16_3 = value_3
        d.model_63001.uint16_4 = value_4
        d.model_63001.uint16_5 = value_5
        d.model_63001.write()
        d.model_63001.read()
        value = d.model_63001.uint16_3
        if d.model_63001.uint16_3 != value_3:
            raise Exception("'model_63001.int16_3' write failure: %s %s" % (d.model_63001.uint16_3, value_3))
        value = d.model_63001.uint16_4
        if d.model_63001.uint16_4 != value_4:
            raise Exception("'model_63001.int16_4' write failure: %s %s" % (d.model_63001.uint16_4, value_4))
        value = d.model_63001.uint16_5
        if d.model_63001.uint16_5 != value_5:
            raise Exception("'model_63001.int16_5' write failure: %s %s" % (d.model_63001.uint16_5, value_5))

        # write multiple
        d.model_63001.read()
        expected = value_3
        if d.model_63001.uint16_3 != expected:
            raise Exception("'model_63001.uint16_3' point mismatch: %s %s" % (d.model_63001.uint16_3, expected))
        expected = value_4
        if d.model_63001.uint16_4 != expected:
            raise Exception("'model_63001.uint16_4' point mismatch: %s %s" % (d.model_63001.uint16_4, expected))
        expected = value_5
        if d.model_63001.uint16_5 != expected:
            raise Exception("'model_63001.uint16_5' point mismatch: %s %s" % (d.model_63001.uint16_5, expected))
        value_3 = 65524
        value_5 = 7
        d.model_63001.uint16_3 = value_3
        d.model_63001.uint16_5 = value_5
        d.model_63001.write()
        d.model_63001.read()
        value = d.model_63001.uint16_3
        if d.model_63001.uint16_3 != value_3:
            raise Exception("'model_63001.int16_3' write failure: %s %s" % (d.model_63001.uint16_3, value_3))
        value = d.model_63001.uint16_4
        if d.model_63001.uint16_4 != value_4:
            raise Exception("'model_63001.int16_4' write failure: %s %s" % (d.model_63001.uint16_4, value_4))
        value = d.model_63001.uint16_5
        if d.model_63001.uint16_5 != value_5:
            raise Exception("'model_63001.int16_5' write failure: %s %s" % (d.model_63001.uint16_5, value_5))

        d.close()

    def test_sunspec_client_device_3(self):
        d = client.SunSpecClientDevice(client.MAPPED, slave_id=1, name='mbmap_test_device_3.xml', pathlist=self.pathlist)

        # int16 read and write
        d.model_63002.read()
        expected = 1111
        value = int(d.model_63002.repeating[1].int16_1 * 10)
        if  value != expected:
            raise Exception("'model_63002.int16_1' point mismatch: %s %s" % (value, expected))

        d.model_63002.repeating[1].int16_1 = 333.3
        d.model_63002.write()
        d.model_63002.read()
        expected = 3333
        value = int(d.model_63002.repeating[1].int16_1 * 10)
        if value != expected:
            raise Exception("'model_63002.int16_2' write failure: %s %s" % (value, expected))

        expected = 2222
        value = int(d.model_63002.repeating[1].int16_2 * 100)
        if  value != expected:
            raise Exception("'model_63002.int16_1' point mismatch: %s %s" % (value, expected))

        d.close()

if __name__ == "__main__":

    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
scrape is a task to scrape rss resources
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Aug. 14, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
# sys.path.append('/home/work/newsman/newsman')
# sys.path.append('/home/users/jinyuan/newsman/newsman')
# sys.path.append('/home/ubuntu/newsman/newsman')
sys.path.append('/home/jinyuan/Downloads/newsman/newsman')

from config.settings import Collection, db
from config.settings import logger
import Queue
from spider import scraper
from threading import Thread

# CONSTANTS
from config.settings import FEED_REGISTRAR
queue = Queue.Queue()

class UpdateThread(Thread):

    """
    Update news rss/twitter ...
    """

    def __init__(self, queue):
        Thread.__init__(self)
        self.queue = queue

    def run(self):
        while True:
            try:
                feed_id = self.queue.get()
                updated_feed = scraper.update(feed_id=feed_id)
                if updated_feed:
                    logger.error(
                        '--------------- %s got updated! ---------------' % updated_feed)
                self.queue.task_done()
            except Exception as k:
                logger.error(str(k))
                self.queue.task_done()
                continue

def _update(feed_ids):
    """
    update links find in feeds
    """
    if not feed_ids:
        logger.error("No feed found!")
        return None

    thread_limit = min(len(feed_ids) / 2, 20)
    for i in range(thread_limit):
        thread = UpdateThread(queue)
        #thread.setDaemon(True)
        thread.start()

    # populate queue with feeds
    for feed_id in feed_ids:
        queue.put(feed_id)

    queue.join()

def _read_feeds(language='en', country='US'):
    """
    read feed information from database feeds
    """
    try:
        db_feeds = Collection(db, FEED_REGISTRAR)
        items = db_feeds.find({'language': language, 'countries': country})
        if items:
            return [str(item['_id']) for item in items]
        else:
            logger.error("Cannot find any feeds for language %s!" % language)
            return None
    except Exception as k:
        logger.error(str(k))
        return None

def _scrape(language='en', country='US'):
    """
    update news from stored feeds
    """
    logger.error('############### scraping ###############')
    _update(_read_feeds(language, country))
    logger.error(
        "############### Feeds of %s.%s all got updated! ###############" %
        (country, language))

if __name__ == "__main__":
    language = sys.argv[1]
    country = sys.argv[2]
    _scrape(language, country)
<commit_msg>Copy code from production server<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
scrape is a task to scrape rss resources
"""
# @author chengdujin
# @contact chengdujin@gmail.com
# @created Aug. 14, 2013

import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
# sys.path.append('/home/work/newsman/newsman')
# sys.path.append('/home/users/jinyuan/newsman/newsman')
# sys.path.append('/home/ubuntu/newsman/newsman')
# sys.path.append('/home/jinyuan/Downloads/newsman/newsman')

from config.settings import Collection, db
from config.settings import logger
import Queue
from spider import scraper
import threading

# CONSTANTS
from config.settings import FEED_REGISTRAR
queue = Queue.Queue()

class UpdateThread(threading.Thread):

    """
    Update news rss/twitter ...
    """

    def __init__(self, name):
        threading.Thread.__init__(self)
        self._name = name

    def run(self):
        while True:
            try:
                feed_id = queue.get()
                updated_feed = scraper.update(feed_id=feed_id)
                if updated_feed:
                    logger.error(
                        '--------------- %s: %s[%s] is successfully updated! ---------------' % (self._name, feed_id, updated_feed))
                else:
                    logger.error(
                        '--------------- %s: %s receives no update! ---------------' % (self._name, feed_id))
                queue.task_done()
            except Exception as k:
                logger.error(
                    '+++++++++++++++ %s: [%s] has no update but exception +++++++++++++++' % (self._name, str(k)))
                queue.task_done()

def _update(feed_ids):
    """
    update links find in feeds
    """
    if not feed_ids:
        logger.error("No feed found!")
        return None

    # populate queue with feeds
    for feed_id in feed_ids:
        queue.put(feed_id)

    thread_limit = min(len(feed_ids) / 2, 25)
    for i in range(thread_limit):
        thread = UpdateThread('thread-%i' % i)
        thread.setDaemon(True)
        thread.start()

    queue.join()

def _read_feeds(language='en', country='US'):
    """
    read feed information from database feeds
    """
    try:
        db_feeds = Collection(db, FEED_REGISTRAR)
        items = db_feeds.find({'language': language, 'countries': country})
        if items:
            return [str(item['_id']) for item in items]
        else:
            logger.error("Cannot find any feeds for language %s!" % language)
            return None
    except Exception as k:
        logger.error(str(k))
        return None

def _scrape(language='en', country='US'):
    """
    update news from stored feeds
    """
    logger.error('############### scraping ###############')
    _update(_read_feeds(language, country))
    logger.error(
        "############### Feeds of %s.%s all got updated! ###############" %
        (country, language))

if __name__ == "__main__":
    language = sys.argv[1]
    country = sys.argv[2]
    _scrape(language, country)
<|endoftext|>