<commit_before>#!/usr/bin/env python

# Copyright 2011 OpenStack, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""vm_vdi_cleaner.py - List or clean orphaned VDIs/instances on XenServer."""

import doctest
import optparse
import sys
import XenAPI

from nova import context
from nova import db
from nova import exception
from nova import flags
from nova import utils

from nova.virt.xenapi import vmops

flags.DECLARE("resize_confirm_window", "nova.compute.manager")
flags.DECLARE("xenapi_connection_url", "nova.virt.xenapi_conn")
flags.DECLARE("xenapi_connection_username", "nova.virt.xenapi_conn")
flags.DECLARE("xenapi_connection_password", "nova.virt.xenapi_conn")

FLAGS = flags.FLAGS
# NOTE(sirp): Nova futzs with the sys.argv in order to provide default
# flagfile. To isolate this awful practice, we're supplying a dummy
# argument list.
dummy = ["fakearg"]
utils.default_flagfile(args=dummy)
FLAGS(dummy)

class UnrecognizedNameLabel(Exception):
    pass

def parse_options():
    """Generate command line options."""

    ALLOWED_COMMANDS = ["list-vdis", "clean-vdis", "list-instances",
                        "clean-instances", "test"]
    arg_str = "|".join(ALLOWED_COMMANDS)
    parser = optparse.OptionParser("%prog [options] [" + arg_str + "]")
    parser.add_option("--verbose", action="store_true")

    options, args = parser.parse_args()

    if not args:
        parser.print_usage()
        sys.exit(1)

    return options, args

def get_instance_id_from_name_label(name_label, template):
    """In order to derive the instance_id from the name label on the VM, we
    take the following steps:

        1. We substitute a dummy value in to the instance_name_template so we
           can figure out the prefix and the suffix of the template (the
           instance_id is between the two)

        2. We delete the prefix and suffix from the name_label.

        3. What's left *should* be the instance_id which we cast to an int
           and return.

    >>> get_instance_id_from_name_label("", "instance-%08x")
    Traceback (most recent call last):
        ...
    UnrecognizedNameLabel

    >>> get_instance_id_from_name_label("instance-00000001", "instance-%08x")
    1

    >>> get_instance_id_from_name_label("instance-0000000A", "instance-%08x")
    10

    >>> get_instance_id_from_name_label("instance-42-suffix", \
            "instance-%d-suffix")
    42
    """

    # Interpolate template to figure out where to extract the instance_id from.
    # The instance_id may be in hex "%x" or decimal "%d", so try decimal first
    # then fall back to hex.
    fake_instance_id = 123456789
    result = template % fake_instance_id
    in_hex = False
    base_10 = "%d" % fake_instance_id

    try:
        prefix, suffix = result.split(base_10)
    except ValueError:
        base_16 = "%x" % fake_instance_id
        prefix, suffix = result.split(base_16)
        in_hex = True

    if prefix:
        name_label = name_label.replace(prefix, '')

    if suffix:
        name_label = name_label.replace(suffix, '')

    try:
        if in_hex:
            instance_id = int(name_label, 16)
        else:
            instance_id = int(name_label)
    except ValueError:
        raise UnrecognizedNameLabel(name_label)

    return instance_id

def find_orphaned_instances(session, verbose=False):
    """Find and return a list of orphaned instances."""
    ctxt = context.get_admin_context()
    orphaned_instances = []

    for vm_rec in _get_applicable_vm_recs(session):
        try:
            instance_id = get_instance_id_from_name_label(
                vm_rec["name_label"], FLAGS.instance_name_template)
        except UnrecognizedNameLabel, exc:
            print_xen_object("WARNING: Unrecognized VM", vm_rec,
                    indent_level=0, verbose=verbose)
            continue

        try:
            instance = db.api.instance_get(ctxt, instance_id)
        except exception.InstanceNotFound:
            # NOTE(jk0): Err on the side of caution here. If we don't know
            # anything about the particular instance, print a warning and let
            # the operator handle it manually.
            print >> sys.stderr, "Instance %s not found" % instance_id

        # NOTE(jk0): A zombie VM is an instance that is not active and hasn't
        # been updated in over the specified period.
        is_zombie_vm = (instance.vm_state != "active"
                and utils.is_older_than(instance.updated_at,
                        FLAGS.zombie_instance_updated_at_window))
        if is_zombie_vm:
            orphaned_instances.append(instance)

    return orphaned_instances

def cleanup_instance(session, instance):
    """Delete orphaned instances."""
    vmops = VMOps(session)
    network_info = None
    vmops.destroy(instance, network_info)

def _get_applicable_vm_recs(session):
    """An 'applicable' VM is one that is not a template and not the control
    domain.
    """
    for vm_ref in session.xenapi.VM.get_all():
        vm_rec = session.xenapi.VM.get_record(vm_ref)

        if vm_rec["is_a_template"] or vm_rec["is_control_domain"]:
            continue
        yield vm_rec

def print_xen_object(obj_type, obj, indent_level=0, spaces_per_indent=4,
                     verbose=False):
    """Pretty-print a Xen object.

    Looks like:

        VM (abcd-abcd-abcd): 'name label here'
    """
    if not verbose:
        return
    uuid = obj["uuid"]
    try:
        name_label = obj["name_label"]
    except KeyError:
        name_label = ""
    msg = "%(obj_type)s (%(uuid)s) '%(name_label)s'" % locals()
    indent = " " * spaces_per_indent * indent_level
    print "".join([indent, msg])

def _find_vdis_connected_to_vm(session, connected_vdi_uuids, verbose=False):
    """Find VDIs which are connected to VBDs which are connected to VMs."""
    def _is_null_ref(ref):
        return ref == "OpaqueRef:NULL"

    def _add_vdi_and_parents_to_connected(vdi_rec, indent_level):
        indent_level += 1

        vdi_and_parent_uuids = []
        cur_vdi_rec = vdi_rec
        while True:
            cur_vdi_uuid = cur_vdi_rec["uuid"]
            print_xen_object("VDI", vdi_rec, indent_level=indent_level,
                             verbose=verbose)
            connected_vdi_uuids.add(cur_vdi_uuid)
            vdi_and_parent_uuids.append(cur_vdi_uuid)

            try:
                parent_vdi_uuid = vdi_rec["sm_config"]["vhd-parent"]
            except KeyError:
                parent_vdi_uuid = None

            # NOTE(sirp): VDI's can have themselves as a parent?!
            if parent_vdi_uuid and parent_vdi_uuid != cur_vdi_uuid:
                indent_level += 1
                cur_vdi_ref = session.xenapi.VDI.get_by_uuid(
                    parent_vdi_uuid)
                cur_vdi_rec = session.xenapi.VDI.get_record(
                    cur_vdi_ref)
            else:
                break

    for vm_rec in _get_applicable_vm_recs(session):
        indent_level = 0
        print_xen_object("VM", vm_rec, indent_level=indent_level,
                         verbose=verbose)

        vbd_refs = vm_rec["VBDs"]
        for vbd_ref in vbd_refs:
            vbd_rec = session.xenapi.VBD.get_record(vbd_ref)

            indent_level = 1
            print_xen_object("VBD", vbd_rec, indent_level=indent_level,
                             verbose=verbose)

            vbd_vdi_ref = vbd_rec["VDI"]

            if _is_null_ref(vbd_vdi_ref):
                continue

            vdi_rec = session.xenapi.VDI.get_record(vbd_vdi_ref)

            _add_vdi_and_parents_to_connected(vdi_rec, indent_level)

def _find_all_vdis_and_system_vdis(session, all_vdi_uuids, connected_vdi_uuids,
                                   verbose=False):
    """Collects all VDIs and adds system VDIs to the connected set."""
    def _system_owned(vdi_rec):
        vdi_name = vdi_rec["name_label"]
        return (vdi_name.startswith("USB") or
                vdi_name.endswith(".iso") or
                vdi_rec["type"] == "system")

    for vdi_ref in session.xenapi.VDI.get_all():
        vdi_rec = session.xenapi.VDI.get_record(vdi_ref)
        vdi_uuid = vdi_rec["uuid"]
        all_vdi_uuids.add(vdi_uuid)

        # System owned and non-managed VDIs should be considered 'connected'
        # for our purposes.
        if _system_owned(vdi_rec):
            print_xen_object("SYSTEM VDI", vdi_rec, indent_level=0,
                             verbose=verbose)
            connected_vdi_uuids.add(vdi_uuid)
        elif not vdi_rec["managed"]:
            print_xen_object("UNMANAGED VDI", vdi_rec, indent_level=0,
                             verbose=verbose)
            connected_vdi_uuids.add(vdi_uuid)

def find_orphaned_vdi_uuids(session, verbose=False):
    """Walk VM -> VBD -> VDI change and accumulate connected VDIs."""
    connected_vdi_uuids = set()

    _find_vdis_connected_to_vm(session, connected_vdi_uuids, verbose=verbose)

    all_vdi_uuids = set()
    _find_all_vdis_and_system_vdis(session, all_vdi_uuids, connected_vdi_uuids,
                                   verbose=verbose)

    orphaned_vdi_uuids = all_vdi_uuids - connected_vdi_uuids
    return orphaned_vdi_uuids

def list_orphaned_vdis(vdi_uuids, verbose=False):
    """List orphaned VDIs."""
    for vdi_uuid in vdi_uuids:
        if verbose:
            print "ORPHANED VDI (%s)" % vdi_uuid
        else:
            print vdi_uuid

def clean_orphaned_vdis(session, vdi_uuids, verbose=False):
    """Clean orphaned VDIs."""
    for vdi_uuid in vdi_uuids:
        if verbose:
            print "CLEANING VDI (%s)" % vdi_uuid

        vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid)
        try:
            session.xenapi.VDI.destroy(vdi_ref)
        except XenAPI.Failure, exc:
            print >> sys.stderr, "Skipping %s: %s" % (vdi_uuid, exc)

def list_orphaned_instances(orphaned_instances, verbose=False):
    """List orphaned instances."""
    for orphaned_instance in orphaned_instances:
        if verbose:
            print "ORPHANED INSTANCE (%s)" % instance.name
        else:
            print instance.name

def clean_orphaned_instances(session, orphaned_instances, verbose=False):
    """Clean orphaned instances."""
    for instance in orphaned_instances:
        if verbose:
            print "CLEANING INSTANCE (%s)" % instance.name

        cleanup_instance(session, instance)

def main():
    """Main loop."""
    options, args = parse_options()
    verbose = options.verbose
    command = args[0]

    if FLAGS.zombie_instance_updated_at_window < FLAGS.resize_confirm_window:
        raise Exception("`zombie_instance_updated_at_window` has to be longer"
                " than `resize_confirm_window`.")

    session = XenAPI.Session(FLAGS.xenapi_connection_url)
    session.xenapi.login_with_password(FLAGS.xenapi_connection_username,
            FLAGS.xenapi_connection_password)

    if command == "list-vdis":
        if verbose:
            print "Connected VDIs:\n"
        orphaned_vdi_uuids = find_orphaned_vdi_uuids(session, verbose=verbose)
        if verbose:
            print "\nOprhaned VDIs:\n"
        list_orphaned_vdis(orphaned_vdi_uuids, verbose=verbose)
    elif command == "clean-vdis":
        orphaned_vdi_uuids = find_orphaned_vdi_uuids(session, verbose=verbose)
        clean_orphaned_vdis(session, orphaned_vdi_uuids, verbose=verbose)
    elif command == "list-instances":
        orphaned_instances = find_orphaned_instances(session, verbose=verbose)
        list_orphaned_instances(orphaned_instances, verbose=verbose)
    elif command == "clean-instances":
        orphaned_instances = find_orphaned_instances(session, verbose=verbose)
        clean_orphaned_instances(session, orphaned_instances,
                verbose=verbose)
    elif command == "test":
        doctest.testmod()
    else:
        print "Unknown command '%s'" % command
        sys.exit(1)

if __name__ == "__main__":
    main()
<commit_msg>Continue to the next iteration of the loop if an instance is not found.<commit_after>#!/usr/bin/env python

# Copyright 2011 OpenStack, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""vm_vdi_cleaner.py - List or clean orphaned VDIs/instances on XenServer."""

import doctest
import optparse
import sys
import XenAPI

from nova import context
from nova import db
from nova import exception
from nova import flags
from nova import utils

from nova.virt.xenapi import vmops

flags.DECLARE("resize_confirm_window", "nova.compute.manager")
flags.DECLARE("xenapi_connection_url", "nova.virt.xenapi_conn")
flags.DECLARE("xenapi_connection_username", "nova.virt.xenapi_conn")
flags.DECLARE("xenapi_connection_password", "nova.virt.xenapi_conn")

FLAGS = flags.FLAGS
# NOTE(sirp): Nova futzs with the sys.argv in order to provide default
# flagfile. To isolate this awful practice, we're supplying a dummy
# argument list.
dummy = ["fakearg"]
utils.default_flagfile(args=dummy)
FLAGS(dummy)

class UnrecognizedNameLabel(Exception):
    pass

def parse_options():
    """Generate command line options."""

    ALLOWED_COMMANDS = ["list-vdis", "clean-vdis", "list-instances",
                        "clean-instances", "test"]
    arg_str = "|".join(ALLOWED_COMMANDS)
    parser = optparse.OptionParser("%prog [options] [" + arg_str + "]")
    parser.add_option("--verbose", action="store_true")

    options, args = parser.parse_args()

    if not args:
        parser.print_usage()
        sys.exit(1)

    return options, args

def get_instance_id_from_name_label(name_label, template):
    """In order to derive the instance_id from the name label on the VM, we
    take the following steps:

        1. We substitute a dummy value in to the instance_name_template so we
           can figure out the prefix and the suffix of the template (the
           instance_id is between the two)

        2. We delete the prefix and suffix from the name_label.

        3. What's left *should* be the instance_id which we cast to an int
           and return.

    >>> get_instance_id_from_name_label("", "instance-%08x")
    Traceback (most recent call last):
        ...
    UnrecognizedNameLabel

    >>> get_instance_id_from_name_label("instance-00000001", "instance-%08x")
    1

    >>> get_instance_id_from_name_label("instance-0000000A", "instance-%08x")
    10

    >>> get_instance_id_from_name_label("instance-42-suffix", \
            "instance-%d-suffix")
    42
    """

    # Interpolate template to figure out where to extract the instance_id from.
    # The instance_id may be in hex "%x" or decimal "%d", so try decimal first
    # then fall back to hex.
    fake_instance_id = 123456789
    result = template % fake_instance_id
    in_hex = False
    base_10 = "%d" % fake_instance_id

    try:
        prefix, suffix = result.split(base_10)
    except ValueError:
        base_16 = "%x" % fake_instance_id
        prefix, suffix = result.split(base_16)
        in_hex = True

    if prefix:
        name_label = name_label.replace(prefix, '')

    if suffix:
        name_label = name_label.replace(suffix, '')

    try:
        if in_hex:
            instance_id = int(name_label, 16)
        else:
            instance_id = int(name_label)
    except ValueError:
        raise UnrecognizedNameLabel(name_label)

    return instance_id

def find_orphaned_instances(session, verbose=False):
    """Find and return a list of orphaned instances."""
    ctxt = context.get_admin_context()
    orphaned_instances = []

    for vm_rec in _get_applicable_vm_recs(session):
        try:
            instance_id = get_instance_id_from_name_label(
                vm_rec["name_label"], FLAGS.instance_name_template)
        except UnrecognizedNameLabel, exc:
            print_xen_object("WARNING: Unrecognized VM", vm_rec,
                    indent_level=0, verbose=verbose)
            continue

        try:
            instance = db.api.instance_get(ctxt, instance_id)
        except exception.InstanceNotFound:
            # NOTE(jk0): Err on the side of caution here. If we don't know
            # anything about the particular instance, print a warning and let
            # the operator handle it manually.
            print >> sys.stderr, "Instance %s not found" % instance_id
            continue

        # NOTE(jk0): A zombie VM is an instance that is not active and hasn't
        # been updated in over the specified period.
        is_zombie_vm = (instance.vm_state != "active"
                and utils.is_older_than(instance.updated_at,
                        FLAGS.zombie_instance_updated_at_window))
        if is_zombie_vm:
            orphaned_instances.append(instance)

    return orphaned_instances

def cleanup_instance(session, instance):
    """Delete orphaned instances."""
    vmops = VMOps(session)
    network_info = None
    vmops.destroy(instance, network_info)

def _get_applicable_vm_recs(session):
    """An 'applicable' VM is one that is not a template and not the control
    domain.
    """
    for vm_ref in session.xenapi.VM.get_all():
        vm_rec = session.xenapi.VM.get_record(vm_ref)

        if vm_rec["is_a_template"] or vm_rec["is_control_domain"]:
            continue
        yield vm_rec

def print_xen_object(obj_type, obj, indent_level=0, spaces_per_indent=4,
                     verbose=False):
    """Pretty-print a Xen object.

    Looks like:

        VM (abcd-abcd-abcd): 'name label here'
    """
    if not verbose:
        return
    uuid = obj["uuid"]
    try:
        name_label = obj["name_label"]
    except KeyError:
        name_label = ""
    msg = "%(obj_type)s (%(uuid)s) '%(name_label)s'" % locals()
    indent = " " * spaces_per_indent * indent_level
    print "".join([indent, msg])

def _find_vdis_connected_to_vm(session, connected_vdi_uuids, verbose=False):
    """Find VDIs which are connected to VBDs which are connected to VMs."""
    def _is_null_ref(ref):
        return ref == "OpaqueRef:NULL"

    def _add_vdi_and_parents_to_connected(vdi_rec, indent_level):
        indent_level += 1

        vdi_and_parent_uuids = []
        cur_vdi_rec = vdi_rec
        while True:
            cur_vdi_uuid = cur_vdi_rec["uuid"]
            print_xen_object("VDI", vdi_rec, indent_level=indent_level,
                             verbose=verbose)
            connected_vdi_uuids.add(cur_vdi_uuid)
            vdi_and_parent_uuids.append(cur_vdi_uuid)

            try:
                parent_vdi_uuid = vdi_rec["sm_config"]["vhd-parent"]
            except KeyError:
                parent_vdi_uuid = None

            # NOTE(sirp): VDI's can have themselves as a parent?!
            if parent_vdi_uuid and parent_vdi_uuid != cur_vdi_uuid:
                indent_level += 1
                cur_vdi_ref = session.xenapi.VDI.get_by_uuid(
                    parent_vdi_uuid)
                cur_vdi_rec = session.xenapi.VDI.get_record(
                    cur_vdi_ref)
            else:
                break

    for vm_rec in _get_applicable_vm_recs(session):
        indent_level = 0
        print_xen_object("VM", vm_rec, indent_level=indent_level,
                         verbose=verbose)

        vbd_refs = vm_rec["VBDs"]
        for vbd_ref in vbd_refs:
            vbd_rec = session.xenapi.VBD.get_record(vbd_ref)

            indent_level = 1
            print_xen_object("VBD", vbd_rec, indent_level=indent_level,
                             verbose=verbose)

            vbd_vdi_ref = vbd_rec["VDI"]

            if _is_null_ref(vbd_vdi_ref):
                continue

            vdi_rec = session.xenapi.VDI.get_record(vbd_vdi_ref)

            _add_vdi_and_parents_to_connected(vdi_rec, indent_level)

def _find_all_vdis_and_system_vdis(session, all_vdi_uuids, connected_vdi_uuids,
                                   verbose=False):
    """Collects all VDIs and adds system VDIs to the connected set."""
    def _system_owned(vdi_rec):
        vdi_name = vdi_rec["name_label"]
        return (vdi_name.startswith("USB") or
                vdi_name.endswith(".iso") or
                vdi_rec["type"] == "system")

    for vdi_ref in session.xenapi.VDI.get_all():
        vdi_rec = session.xenapi.VDI.get_record(vdi_ref)
        vdi_uuid = vdi_rec["uuid"]
        all_vdi_uuids.add(vdi_uuid)

        # System owned and non-managed VDIs should be considered 'connected'
        # for our purposes.
        if _system_owned(vdi_rec):
            print_xen_object("SYSTEM VDI", vdi_rec, indent_level=0,
                             verbose=verbose)
            connected_vdi_uuids.add(vdi_uuid)
        elif not vdi_rec["managed"]:
            print_xen_object("UNMANAGED VDI", vdi_rec, indent_level=0,
                             verbose=verbose)
            connected_vdi_uuids.add(vdi_uuid)

def find_orphaned_vdi_uuids(session, verbose=False):
    """Walk VM -> VBD -> VDI change and accumulate connected VDIs."""
    connected_vdi_uuids = set()

    _find_vdis_connected_to_vm(session, connected_vdi_uuids, verbose=verbose)

    all_vdi_uuids = set()
    _find_all_vdis_and_system_vdis(session, all_vdi_uuids, connected_vdi_uuids,
                                   verbose=verbose)

    orphaned_vdi_uuids = all_vdi_uuids - connected_vdi_uuids
    return orphaned_vdi_uuids

def list_orphaned_vdis(vdi_uuids, verbose=False):
    """List orphaned VDIs."""
    for vdi_uuid in vdi_uuids:
        if verbose:
            print "ORPHANED VDI (%s)" % vdi_uuid
        else:
            print vdi_uuid

def clean_orphaned_vdis(session, vdi_uuids, verbose=False):
    """Clean orphaned VDIs."""
    for vdi_uuid in vdi_uuids:
        if verbose:
            print "CLEANING VDI (%s)" % vdi_uuid

        vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid)
        try:
            session.xenapi.VDI.destroy(vdi_ref)
        except XenAPI.Failure, exc:
            print >> sys.stderr, "Skipping %s: %s" % (vdi_uuid, exc)

def list_orphaned_instances(orphaned_instances, verbose=False):
    """List orphaned instances."""
    for orphaned_instance in orphaned_instances:
        if verbose:
            print "ORPHANED INSTANCE (%s)" % instance.name
        else:
            print instance.name

def clean_orphaned_instances(session, orphaned_instances, verbose=False):
    """Clean orphaned instances."""
    for instance in orphaned_instances:
        if verbose:
            print "CLEANING INSTANCE (%s)" % instance.name

        cleanup_instance(session, instance)

def main():
    """Main loop."""
    options, args = parse_options()
    verbose = options.verbose
    command = args[0]

    if FLAGS.zombie_instance_updated_at_window < FLAGS.resize_confirm_window:
        raise Exception("`zombie_instance_updated_at_window` has to be longer"
                " than `resize_confirm_window`.")

    session = XenAPI.Session(FLAGS.xenapi_connection_url)
    session.xenapi.login_with_password(FLAGS.xenapi_connection_username,
            FLAGS.xenapi_connection_password)

    if command == "list-vdis":
        if verbose:
            print "Connected VDIs:\n"
        orphaned_vdi_uuids = find_orphaned_vdi_uuids(session, verbose=verbose)
        if verbose:
            print "\nOprhaned VDIs:\n"
        list_orphaned_vdis(orphaned_vdi_uuids, verbose=verbose)
    elif command == "clean-vdis":
        orphaned_vdi_uuids = find_orphaned_vdi_uuids(session, verbose=verbose)
        clean_orphaned_vdis(session, orphaned_vdi_uuids, verbose=verbose)
    elif command == "list-instances":
        orphaned_instances = find_orphaned_instances(session, verbose=verbose)
        list_orphaned_instances(orphaned_instances, verbose=verbose)
    elif command == "clean-instances":
        orphaned_instances = find_orphaned_instances(session, verbose=verbose)
        clean_orphaned_instances(session, orphaned_instances,
                verbose=verbose)
    elif command == "test":
        doctest.testmod()
    else:
        print "Unknown command '%s'" % command
        sys.exit(1)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

import json
import logging
import os
import shutil
import zipfile

import stoneridge

class StoneRidgeArchiver(object):
    """A class to zip up all the results and logging from a stone ridge
    run, and put the results with the stone ridge archvies.
    """
    def run(self):
        logging.debug('archiver running')
        outdir = stoneridge.get_config('run', 'out')
        archivedir = stoneridge.get_config('stoneridge', 'archives')
        srid = stoneridge.get_config('run', 'srid')
        netconfig = stoneridge.get_config('run', 'netconfig')
        infofile = stoneridge.get_config('run', 'info')
        metadata = stoneridge.get_config('run', 'metadata')

        with file(infofile, 'rb') as f:
            info = json.load(f)
            logging.debug('loaded info %s' % (info,))

        arcname = 'stoneridge_%s_%s_%s_%s_%s' % (info['date'],
                                                 info['test_machine']['name'],
                                                 info['test_build']['revision'],
                                                 srid,
                                                 netconfig)
        logging.debug('archive name %s.zip' % (arcname,))

        filename = os.path.join(archivedir, '%s.zip' % (arcname,))
        if not os.path.exists(archivedir):
            logging.debug('making archive directory %s' % (archivedir,))
            os.mkdir(archivedir)

        logging.debug('opening zip file for writing')
        zfile = zipfile.ZipFile(filename, mode='w')

        # Put all the files under a directory in the zip named for the zip
        # file itself, for easy separation when unzipping multiple archives
        # in the same place
        logging.debug('adding files to zip')
        for dirpath, dirs, files in os.walk(outdir):
            dirname = dirpath.replace(outdir, arcname, 1)
            logging.debug('directory %s -> %s' % (dirpath, dirname))
            # Add the directories to the zip
            for d in dirs:
                logging.debug('subdirectory %s' % (d,))
                zfile.write(os.path.join(dirpath, d),
                        arcname=os.path.join(dirname, d))
            # Add the files to the zip
            for f in files:
                logging.debug('file %s' % (f,))
                zfile.write(os.path.join(dirpath, f),
                        arcname=os.path.join(dirname, f))

        logging.debug('closing zip file')
        zfile.close()

        # Make a copy where the uploader will find it
        shutil.copyfile(filename, metadata)

@stoneridge.main
def main():
    parser = stoneridge.TestRunArgumentParser()
    parser.parse_args()

    archiver = StoneRidgeArchiver()
    archiver.run()
<commit_msg>Fix lint warnings in archiver<commit_after>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.

import json
import logging
import os
import shutil
import zipfile

import stoneridge

class StoneRidgeArchiver(object):
    """A class to zip up all the results and logging from a stone ridge
    run, and put the results with the stone ridge archvies.
    """
    def run(self):
        logging.debug('archiver running')
        outdir = stoneridge.get_config('run', 'out')
        archivedir = stoneridge.get_config('stoneridge', 'archives')
        srid = stoneridge.get_config('run', 'srid')
        netconfig = stoneridge.get_config('run', 'netconfig')
        infofile = stoneridge.get_config('run', 'info')
        metadata = stoneridge.get_config('run', 'metadata')

        with file(infofile, 'rb') as f:
            info = json.load(f)
            logging.debug('loaded info %s' % (info,))

        arcname = 'stoneridge_%s_%s_%s_%s_%s' % (info['date'],
                                                 info['test_machine']['name'],
                                                 info['test_build']['revision'],
                                                 srid,
                                                 netconfig)
        logging.debug('archive name %s.zip' % (arcname,))

        filename = os.path.join(archivedir, '%s.zip' % (arcname,))
        if not os.path.exists(archivedir):
            logging.debug('making archive directory %s' % (archivedir,))
            os.mkdir(archivedir)

        logging.debug('opening zip file for writing')
        zfile = zipfile.ZipFile(filename, mode='w')

        # Put all the files under a directory in the zip named for the zip
        # file itself, for easy separation when unzipping multiple archives
        # in the same place
        logging.debug('adding files to zip')
        for dirpath, dirs, files in os.walk(outdir):
            dirname = dirpath.replace(outdir, arcname, 1)
            logging.debug('directory %s -> %s' % (dirpath, dirname))
            # Add the directories to the zip
            for d in dirs:
                logging.debug('subdirectory %s' % (d,))
                zfile.write(os.path.join(dirpath, d),
                            arcname=os.path.join(dirname, d))
            # Add the files to the zip
            for f in files:
                logging.debug('file %s' % (f,))
                zfile.write(os.path.join(dirpath, f),
                            arcname=os.path.join(dirname, f))

        logging.debug('closing zip file')
        zfile.close()

        # Make a copy where the uploader will find it
        shutil.copyfile(filename, metadata)

@stoneridge.main
def main():
    parser = stoneridge.TestRunArgumentParser()
    parser.parse_args()

    archiver = StoneRidgeArchiver()
    archiver.run()
<|endoftext|>


<commit_before>#
# Copyright (c) 2014 QuarksLab.
# This file is part of IRMA project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the top-level directory
# of this distribution and at:
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# No part of the project, including this file, may be copied,
# modified, propagated, or distributed except according to the
# terms contained in the LICENSE file.

import logging
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, scoped_session
from irma.common.exceptions import IrmaDatabaseError

DEBUG = False
if DEBUG:
    logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
else:
    logging.getLogger('sqlalchemy.engine').setLevel(logging.ERROR)

class SQLDatabase(object):
    """Internal database.
    This class handles the creation of the internal database.
    """

    __engine = None
    __Session = None

    def __init__(self):
        raise Exception('This class must not be instantiated')

    @classmethod
    def connect(cls, dbms, dialect, username, passwd, host, dbname):
        """Create a connexion to the db
        :param dbms: the database management system (ex: postgresql)
        :param dialect: the dialect to use (ex: psycopg2 (for postgre))
        :param username: the username for the connexion
        :param passwd: the password for the connexion
        :param host: the host (ex: localhost or localhost:port)
        :param dbname: the name of the database (has to exist)
        """
        if cls.__engine is not None:
            raise IrmaDatabaseError('the engine already exist')

        if dialect:
            dbms = "{0}+{1}".format(dbms, dialect)
        host_and_id = ''
        if host and username:
            if passwd:
                host_and_id = "{0}:{1}@{2}".format(username, passwd, host)
            else:
                host_and_id = "{0}@{1}".format(username, host)
        url = "{0}://{1}/{2}".format(dbms, host_and_id, dbname)
        cls.__engine = create_engine(url, echo=True)

        session_factory = sessionmaker(bind=cls.__engine)
        cls.__Session = scoped_session(session_factory)

    @classmethod
    def get_engine(cls):
        """Return the engine
        :rtype: engine
        :raise IrmaDatabaseError: if the engine is None
        """
        if cls.__engine is None:
            raise IrmaDatabaseError('the engine has to be connected first')
        return cls.__engine

    @classmethod
    def get_session(cls):
        """Return a session
        :rtype: scoped_session
        :raise IrmaDatabaseError: if the engine is None
        """
        if cls.__engine is None:
            raise IrmaDatabaseError('the engine has to be connected first')
        return cls.__Session()<commit_msg>SQL connexion issue fixed<commit_after>#
# Copyright (c) 2014 QuarksLab.
# This file is part of IRMA project.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License in the top-level directory
# of this distribution and at:
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# No part of the project, including this file, may be copied,
# modified, propagated, or distributed except according to the
# terms contained in the LICENSE file.

import logging
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, scoped_session
from irma.common.exceptions import IrmaDatabaseError

DEBUG = False
if DEBUG:
    logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
else:
    logging.getLogger('sqlalchemy.engine').setLevel(logging.ERROR)

class SQLDatabase(object):
    """Internal database.
    This class handles the creation of the internal database.
    """

    __engine = None
    __Session = None

    def __init__(self):
        raise Exception('This class must not be instantiated')

    @classmethod
    def connect(cls, dbms, dialect, username, passwd, host, dbname):
        """Create a connexion to the db
        :param dbms: the database management system (ex: postgresql)
        :param dialect: the dialect to use (ex: psycopg2 (for postgre))
        :param username: the username for the connexion
        :param passwd: the password for the connexion
        :param host: the host (ex: localhost or localhost:port)
        :param dbname: the name of the database (has to exist)
        """
        if cls.__engine is None:
            if dialect:
                dbms = "{0}+{1}".format(dbms, dialect)
            host_and_id = ''
            if host and username:
                if passwd:
                    host_and_id = "{0}:{1}@{2}".format(username, passwd, host)
                else:
                    host_and_id = "{0}@{1}".format(username, host)
            url = "{0}://{1}/{2}".format(dbms, host_and_id, dbname)
            cls.__engine = create_engine(url, echo=True)

            session_factory = sessionmaker(bind=cls.__engine)
            cls.__Session = scoped_session(session_factory)
        else:
            logging.info('engine already connected, nothing to do')

    @classmethod
    def get_engine(cls):
        """Return the engine
        :rtype: engine
        :raise IrmaDatabaseError: if the engine is None
        """
        if cls.__engine is None:
            raise IrmaDatabaseError('the engine has to be connected first')
        return cls.__engine

    @classmethod
    def get_session(cls):
        """Return a session
        :rtype: scoped_session
        :raise IrmaDatabaseError: if the engine is None
        """
        if cls.__engine is None:
            raise IrmaDatabaseError('the engine has to be connected first')
        return cls.__Session()<|endoftext|>


<commit_before>"""
Testing of the speeches app.
Testing documentation is at https://docs.djangoproject.com/en/1.4/topics/testing/
"""

from selenium import webdriver

from django.test import TestCase, LiveServerTestCase

from speeches.models import Speech, Speaker

class SpeechTest(TestCase):
    def test_add_speech(self):
        # Test that the page exists and has the right title
        resp = self.client.get('/speech/add')
        self.assertEqual(resp.status_code, 200)
        self.assertTrue('add a new speech' in resp.content)

        # Test that the form won't submit if empty
        resp = self.client.post('/speech/add')
        self.assertFormError(resp, 'form', None, 'You must provide either text or some audio')

        # Test form without speaker
        resp = self.client.post('/speech/add', {
            'text': 'This is a speech'
        })
        self.assertRedirects(resp, '/speech/1')
        # Check in db
        speech = Speech.objects.get(id=1)
        self.assertEqual(speech.text, 'This is a speech')

        # Test form with speaker, we need to add a speaker first
        steve = Speaker.objects.create(popit_id='abcd', name='Steve')
        resp = self.client.post('/speech/add', {
            'text': 'This is a Steve speech',
            'speaker': steve.id
        })
        self.assertRedirects(resp, '/speech/2')
        # Check in db
        speech = Speech.objects.get(speaker=steve.id)
        self.assertEqual(speech.text, 'This is a Steve speech')

        # Test file upload, audio
        # Use LiveServerTestCase

class SeleniumTests(LiveServerTestCase):
    @classmethod
    def setUpClass(cls):
        cls.selenium = webdriver.Firefox()
        super(SeleniumTests, cls).setUpClass()

    @classmethod
    def tearDownClass(cls):
        cls.selenium.quit()
        super(SeleniumTests, cls).tearDownClass()

    def test_add_speech(self):
        self.selenium.get('%s%s' % (self.live_server_url, '/speech/add'))
        text_input = self.selenium.find_element_by_name('text')
        text_input.send_keys('This is a speech')
        self.selenium.find_element_by_xpath('//input[@value="Add speech"]').click()
        self.assertIn('/speech/1', self.selenium.current_url)

<commit_msg>Refactor tests to separate assertions out a bit<commit_after>"""
Testing of the speeches app.
Testing documentation is at https://docs.djangoproject.com/en/1.4/topics/testing/
"""

from selenium import webdriver

from django.test import TestCase, LiveServerTestCase

from speeches.models import Speech, Speaker

class SpeechTest(TestCase):
    def test_add_speech_page_exists(self):
        # Test that the page exists and has the right title
        resp = self.client.get('/speech/add')
        self.assertEqual(resp.status_code, 200)
        self.assertTrue('add a new speech' in resp.content)

    def test_add_speech_fails_on_empty_form(self):
        # Test that the form won't submit if empty
        resp = self.client.post('/speech/add')
        self.assertFormError(resp, 'form', None, 'You must provide either text or some audio')

    def test_add_speech_without_speaker(self):
        # Test form without speaker
        resp = self.client.post('/speech/add', {
            'text': 'This is a speech'
        })
        self.assertRedirects(resp, '/speech/1')
        # Check in db
        speech = Speech.objects.get(id=1)
        self.assertEqual(speech.text, 'This is a speech')

    def test_add_speech_with_speaker(self):
        # Test form with speaker, we need to add a speaker first
        speaker = Speaker.objects.create(popit_id='abcd', name='Steve')
        resp = self.client.post('/speech/add', {
            'text': 'This is a Steve speech',
            'speaker': speaker.id
        })
        self.assertRedirects(resp, '/speech/1')
        # Check in db
        speech = Speech.objects.get(speaker=speaker.id)
        self.assertEqual(speech.text, 'This is a Steve speech')

        # Test file upload, audio
        # Use LiveServerTestCase

class SeleniumTests(LiveServerTestCase):
    @classmethod
    def setUpClass(cls):
        cls.selenium = webdriver.Firefox()
        super(SeleniumTests, cls).setUpClass()

    @classmethod
    def tearDownClass(cls):
        cls.selenium.quit()
        super(SeleniumTests, cls).tearDownClass()

    def test_add_speech(self):
        self.selenium.get('%s%s' % (self.live_server_url, '/speech/add'))
        text_input = self.selenium.find_element_by_name('text')
        text_input.send_keys('This is a speech')
        self.selenium.find_element_by_xpath('//input[@value="Add speech"]').click()
        self.assertIn('/speech/1', self.selenium.current_url)

<|endoftext|>


<commit_before>import distutils.ccompiler as cc
import os, os.path
import platform
import ctypes
import distutils.cmd
from distutils.core import setup #, Extension

class build_iio(distutils.cmd.Command):
    description = """build iio to a shared library"""

    user_options = [('compiler=', 'c', 'specify the compiler type. It must understand GCC arguments')
                    ,('release', 'r', 'build iio without debug asserts')
                    ]

    boolean_options = ['release']

    help_options = [
        ('help-compiler', None, "list available compilers", cc.show_compilers)
        ]

    compiler = None

      ### libraries for IIO
    libraries = ['png','jpeg','tiff']

    def initialize_options (self):
        self.compiler = None
        self.release = True

    def finalize_options (self):
        pass

    def compile(self):
        if self.release:
            print("compiling iio in Release mode (No debug output or asserts)" )
        else:
            print("compiling iio in Debug mode (Defualt, prints debug output and asserts)")

        compiler = cc.new_compiler(compiler=self.compiler)

         # sources for IIO
        source_folders = ['./']
        sources = ['iio.c','freemem.c']

       ### removed the sources are just a couple of files
        #sources = []
        #for folder in source_folders:
        #    for fn in os.listdir(folder):
        #        fn_path = os.path.join(folder, fn)
        #        if fn_path[-1] == 'c':
        #            sources.append(fn_path)
        #        elif fn_path[-1] == 'o':
        #            os.remove(fn_path)

        include_folders = source_folders

        compiler_preargs = ['-std=gnu99']

        if self.release:
            compiler_preargs.append('-DNDEBUG')

        # check if we are on a 64bit python
        arch = ctypes.sizeof(ctypes.c_voidp) * 8

        if arch == 64 and platform.system() == 'Linux':
            compiler_preargs += ['-m64', '-O3', '-fPIC']
        elif arch == 32 and platform.system() == 'Linux':
            compiler_preargs += ['-m32', '-O3']
        elif platform.system() == 'Darwin':
            #No -O3 on OSX. There's a bug in the clang compiler when using O3.
            compiler_preargs += ['-O3', '-arch', 'i386', '-arch', 'x86_64']

        if platform.system() in ('Windows', 'Microsoft'):
            # Compile with stddecl instead of cdecl (-mrtd).
            # Using cdecl cause a missing bytes issue in some cases
            # Because -mrtd and -fomit-frame-pointer (which is included in -O)
            # gives problem with function pointer to the sdtlib free function
            # we also have to use -fno-omit-frame-pointer
            compiler_preargs += ['-mrtd', '-O3', '-shared', '-fno-omit-frame-pointer']

        if arch == 64 and platform.system() in ('Windows', 'Microsoft'):
            compiler_preargs += ['-m64']
        if arch == 32 and platform.system() in ('Windows', 'Microsoft'):
            compiler_preargs += ['-m32']

        for x in compiler.executables:
            args = getattr(compiler, x)
            try:
                args.remove('-mno-cygwin') #Not available on newer versions of gcc
                args.remove('-mdll')
            except:
                pass

        objs = compiler.compile(sources, include_dirs=include_folders, extra_preargs=compiler_preargs)

        libname = 'iio'
        if arch == 64 and platform.system() != 'Darwin':
            libname += ''
        if platform.system() == 'Darwin':
            libname = compiler.library_filename(libname, lib_type='shared')
            compiler.set_executable('linker_so', ['cc', '-dynamiclib', '-arch', 'i386', '-arch', 'x86_64'])
        else:
            libname = compiler.library_filename(libname, lib_type='shared')
        linker_preargs = [ '-lpng', '-ljpeg', '-ltiff']
        if platform.system() == 'Linux' and platform.machine() == 'x86_64':
            linker_preargs += ['-fPIC']
        if platform.system() in ('Windows', 'Microsoft'):
            # link with stddecl instead of cdecl
            linker_preargs += ['-mrtd']
            # remove link against msvcr*. this is a bit ugly maybe.. :)
            compiler.dll_libraries = [lib for lib in compiler.dll_libraries if not lib.startswith("msvcr")]
        compiler.link(cc.CCompiler.SHARED_LIBRARY, objs, libname, output_dir = './', extra_preargs=linker_preargs)

    def run(self):
        self.compile()

setup(
    name="piio",
    version="0.1.0",
    description="python iio wrapper",
    author="Gabriele Facciolo",
    author_email="contact@example.com",
    license='BSD',
    url='http://github.com//',
    packages=['.'],
#    ext_package='piio',
#    ext_modules = [iiomodule],
    cmdclass={'build':build_iio}
   )
<commit_msg>fix compiler invocation string for piio<commit_after>import distutils.ccompiler as cc
import os, os.path
import platform
import ctypes
import distutils.cmd
from distutils.core import setup #, Extension

class build_iio(distutils.cmd.Command):
    description = """build iio to a shared library"""

    user_options = [('compiler=', 'c', 'specify the compiler type. It must understand GCC arguments')
                    ,('release', 'r', 'build iio without debug asserts')
                    ]

    boolean_options = ['release']

    help_options = [
        ('help-compiler', None, "list available compilers", cc.show_compilers)
        ]

    compiler = None

      ### libraries for IIO
    libraries = ['png','jpeg','tiff']

    def initialize_options (self):
        self.compiler = None
        self.release = True

    def finalize_options (self):
        pass

    def compile(self):
        if self.release:
            print("compiling iio in Release mode (No debug output or asserts)" )
        else:
            print("compiling iio in Debug mode (Defualt, prints debug output and asserts)")

        compiler = cc.new_compiler(compiler=self.compiler)

         # sources for IIO
        source_folders = ['./']
        sources = ['iio.c','freemem.c']

       ### removed the sources are just a couple of files
        #sources = []
        #for folder in source_folders:
        #    for fn in os.listdir(folder):
        #        fn_path = os.path.join(folder, fn)
        #        if fn_path[-1] == 'c':
        #            sources.append(fn_path)
        #        elif fn_path[-1] == 'o':
        #            os.remove(fn_path)

        include_folders = source_folders

        compiler_preargs = ['-std=gnu99']

        if self.release:
            compiler_preargs.append('-DNDEBUG')

        # check if we are on a 64bit python
        arch = ctypes.sizeof(ctypes.c_voidp) * 8

        if arch == 64 and platform.system() == 'Linux':
            compiler_preargs += ['-m64', '-O3', '-fPIC']
        elif arch == 32 and platform.system() == 'Linux':
            compiler_preargs += ['-m32', '-O3']
        elif platform.system() == 'Darwin':
            #No -O3 on OSX. There's a bug in the clang compiler when using O3.
            compiler_preargs += ['-O3', '-arch', 'i386', '-arch', 'x86_64']

        if platform.system() in ('Windows', 'Microsoft'):
            # Compile with stddecl instead of cdecl (-mrtd).
            # Using cdecl cause a missing bytes issue in some cases
            # Because -mrtd and -fomit-frame-pointer (which is included in -O)
            # gives problem with function pointer to the sdtlib free function
            # we also have to use -fno-omit-frame-pointer
            compiler_preargs += ['-mrtd', '-O3', '-shared', '-fno-omit-frame-pointer']

        if arch == 64 and platform.system() in ('Windows', 'Microsoft'):
            compiler_preargs += ['-m64']
        if arch == 32 and platform.system() in ('Windows', 'Microsoft'):
            compiler_preargs += ['-m32']

        for x in compiler.executables:
            args = getattr(compiler, x)
            try:
                args.remove('-mno-cygwin') #Not available on newer versions of gcc
                args.remove('-mdll')
            except:
                pass

        objs = compiler.compile(sources, include_dirs=include_folders, extra_preargs=compiler_preargs)

        libname = 'iio'
        if arch == 64 and platform.system() != 'Darwin':
            libname += ''
        if platform.system() == 'Darwin':
            libname = compiler.library_filename(libname, lib_type='shared')
            compiler.set_executable('linker_so', ['cc', '-dynamiclib', '-arch', 'i386', '-arch', 'x86_64'])
        else:
            libname = compiler.library_filename(libname, lib_type='shared')
        linker_preargs = [ '-lpng', '-ljpeg', '-ltiff']
        if platform.system() == 'Linux' and platform.machine() == 'x86_64':
            linker_preargs += ['-fPIC']
        if platform.system() in ('Windows', 'Microsoft'):
            # link with stddecl instead of cdecl
            linker_preargs += ['-mrtd']
            # remove link against msvcr*. this is a bit ugly maybe.. :)
            compiler.dll_libraries = [lib for lib in compiler.dll_libraries if not lib.startswith("msvcr")]
        compiler.link(cc.CCompiler.SHARED_LIBRARY, objs, libname, output_dir = './', extra_postargs=linker_preargs)

    def run(self):
        self.compile()

setup(
    name="piio",
    version="0.1.0",
    description="python iio wrapper",
    author="Gabriele Facciolo",
    author_email="contact@example.com",
    license='BSD',
    url='http://github.com//',
    packages=['.'],
#    ext_package='piio',
#    ext_modules = [iiomodule],
    cmdclass={'build':build_iio}
   )
<|endoftext|>


<commit_before>#  Copyright (c) 2015 SONATA-NFV, UBIWHERE
# ALL RIGHTS RESERVED.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Neither the name of the SONATA-NFV, UBIWHERE
# nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# This work has been performed in the framework of the SONATA project,
# funded by the European Commission under Grant number 671517 through
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).

import os
import yaml

from son.workspace.workspace import Workspace, Project

class UndirectedGraph(object):

    def __init__(self):
        pass

class Validator(object):

    def __init__(self, project):
        self._project = project

    def validate(self):
        self._build_service_graph()

    def _build_service_graph(self):

        # load project service descriptor
        nsd_file = self._project.get_ns_descriptor()
        with open(nsd_file) as _file:
            nsd = yaml.load(_file)
            assert nsd is not None

        # load all project function descriptors
        prj_vnfds = {}
        vnfd_files = self._project.get_vnf_descriptors()
        for vnfd_file in vnfd_files:
            with open(vnfd_file) as _file:
                vnfd = yaml.load(_file)
                assert vnfd is not None
                vnf_combo_id = vnfd['vendor'] + '.' + vnfd['name'] + '.' + \
                               vnfd['version']
                prj_vnfds[vnf_combo_id] = vnfd

        # assign vnf descriptors referenced in the service descriptor
        ref_vnfds = {}
        for func in nsd['network_functions']:
            vnf_combo_id = func['vnd_vendor'] + '.' + func['vnf_name'] + \
                           '.' + func['vnf_version']
            if vnf_combo_id in prj_vnfds.keys():
                ref_vnfds[func['vnf_id']] = prj_vnfds[vnf_combo_id]

def main():
    import argparse

    parser = argparse.ArgumentParser(
        description="Validate a SONATA project service")

    parser.add_argument(
        "--workspace",
        help="Specify workspace. If not specified "
             "will assume '{}'".format(Workspace.DEFAULT_WORKSPACE_DIR),
        required=False)

    parser.add_argument(
        "--project",
        help="create a new package based on the project at the specified "
             "location. If not specified will assume current directory '{}'"
             .format(os.getcwd()),
        required=False)

<commit_msg>- build graph from connection points of the service and functions<commit_after>#  Copyright (c) 2015 SONATA-NFV, UBIWHERE
# ALL RIGHTS RESERVED.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Neither the name of the SONATA-NFV, UBIWHERE
# nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written
# permission.
#
# This work has been performed in the framework of the SONATA project,
# funded by the European Commission under Grant number 671517 through
# the Horizon 2020 and 5G-PPP programmes. The authors would like to
# acknowledge the contributions of their colleagues of the SONATA
# partner consortium (www.sonata-nfv.eu).

import os
import yaml
import networkx as nx

from son.workspace.workspace import Workspace, Project

class UndirectedGraph(object):

    def __init__(self):
        pass

class Validator(object):

    def __init__(self, project):
        self._project = project

    def validate(self):
        self._build_service_graph()

    def _build_service_graph(self):

        # init service network graph
        sg = nx.Graph()

        # load project service descriptor
        nsd_file = self._project.get_ns_descriptor()
        with open(nsd_file, 'r') as _file:
            nsd = yaml.load(_file)
            assert nsd is not None

        # load all project function descriptors
        prj_vnfds = {}
        vnfd_files = self._project.get_vnf_descriptors()
        for vnfd_file in vnfd_files:
            with open(vnfd_file, 'r') as _file:
                vnfd = yaml.load(_file)
                assert vnfd is not None
                vnf_combo_id = vnfd['vendor'] + '.' + vnfd['name'] + '.' + \
                               vnfd['version']
                prj_vnfds[vnf_combo_id] = vnfd

        # assign vnf descriptors referenced in the service descriptor
        ref_vnfds = {}
        for func in nsd['network_functions']:
            vnf_combo_id = func['vnd_vendor'] + '.' + func['vnf_name'] + \
                           '.' + func['vnf_version']
            if vnf_combo_id in prj_vnfds.keys():
                ref_vnfds[func['vnf_id']] = prj_vnfds[vnf_combo_id]

        # add connection points as nodes to the service graph
        for cp in nsd['connection_points']:
            if cp['type'] is 'interface':
                sg.add_node(cp['id'])
        for vnf_id in ref_vnfds.keys():
            for cp in ref_vnfds[vnf_id]['connection_points']:
                if cp['type'] is 'interface':
                    sg.add_node(vnf_id + ':' + cp['id'])

        # temp
        print(sg.nodes())

def main():
    import argparse

    parser = argparse.ArgumentParser(
        description="Validate a SONATA project service")

    parser.add_argument(
        "--workspace",
        help="Specify workspace. If not specified "
             "will assume '{}'".format(Workspace.DEFAULT_WORKSPACE_DIR),
        required=False)

    parser.add_argument(
        "--project",
        help="Validate the project at the specified location. "
             "If not specified will assume current directory '{}'"
             .format(os.getcwd()),
        required=False)

    args = parser.parse_args()

    if args.workspace:
        ws_root = args.workspace
    else:
        ws_root = Workspace.DEFAULT_WORKSPACE_DIR

    prj_root = args.project if args.project else os.getcwd()

    # Obtain Workspace object
    workspace = Workspace.__create_from_descriptor__(ws_root)
    project = Project.__create_from_descriptor__(workspace, prj_root)

    val = Validator(project)
    val.validate()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2005 onwards University of Deusto
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.
#
# This software consists of contributions made by many individuals,
# listed below:
#
# Author: Luis Rodriguez-Gil <luis.rodriguezgil@deusto.es>
#
import base64
import json
import unittest
import threading
import time
from mock import patch, PropertyMock, MagicMock
from experiments.archimedes import Archimedes

from voodoo.configuration import ConfigurationManager
from voodoo.sessions.session_id import SessionId

class TestArchimedes(unittest.TestCase):

    def setUp(self):
        # Initialize the experiment for testing.
        # We set the archimedes_real_device setting to False so that
        # it doesn't attempt to contact the real ip.
        self.cfg_manager = ConfigurationManager()
        self.cfg_manager._set_value("archimedes_real_device", False)
        self.cfg_manager._set_value("archimedes_instances", {"default": "http://localhost:8000", "second": "http://localhost:8001"})
        self.experiment = Archimedes(None, None, self.cfg_manager)
        self.lab_session_id = SessionId('my-session-id')

    def tearDown(self):
        self.experiment.do_dispose()

    def test_start(self):
        #m = MagicMock(return_value="")
        self.experiment._send = m
        start = self.experiment.do_start_experiment("{}", "{}")

        #assert m.mock_calls[0][1][1] == "up"
        #assert m.mock_calls[1][1][1] == "up"

    def test_unknown_instance(self):
        """
        Check that it replies an error if the instance isn't in the config.
        """
        self.cfg_manager = ConfigurationManager()
        self.cfg_manager._set_value("archimedes_instances", {"first": "http://localhost:8000", "second": "http://localhost:8001"})
        self.experiment = Archimedes(None, None, self.cfg_manager)
        start = self.experiment.do_start_experiment("{}", "{}")

        up_resp = self.experiment.do_send_command_to_device("UP")
        assert up_resp.startswith("ERROR:")

        up_resp = self.experiment.do_send_command_to_device("default:UP")
        assert up_resp.startswith("ERROR:")

    def test_control_ball_commands(self):
        start = self.experiment.do_start_experiment("{}", "{}")
        up_resp = self.experiment.do_send_command_to_device("UP")
        down_resp = self.experiment.do_send_command_to_device("DOWN")
        slow_resp = self.experiment.do_send_command_to_device("SLOW")

    def test_basic_data_commands(self):
        start = self.experiment.do_start_experiment("{}", "{}")
        level_resp = self.experiment.do_send_command_to_device("LEVEL")
        assert float(level_resp) == 1200

        load_resp = self.experiment.do_send_command_to_device("LOAD")
        assert float(load_resp) == 1300

    def test_advanced_data_commands(self):
        start = self.experiment.do_start_experiment("{}", "{}")
        image_resp = self.experiment.do_send_command_to_device("IMAGE")
        dec = base64.b64decode(image_resp)
        assert len(dec) > 100

        plot_resp = self.experiment.do_send_command_to_device("PLOT")

        f = file("/tmp/img.html", "w+")
        f.write("""

            """ % (image_resp)
        )
        f.close()

    def test_explicit_instance_commands(self):
        """
        Test that commands can be sent to a specific instance.
        """
        start = self.experiment.do_start_experiment("{}", "{}")
        up_resp = self.experiment.do_send_command_to_device("default:UP")
        down_resp = self.experiment.do_send_command_to_device("default:DOWN")
        slow_resp = self.experiment.do_send_command_to_device("default:SLOW")
        level_resp = self.experiment.do_send_command_to_device("default:LEVEL")
        assert float(level_resp) == 1200
        load_resp = self.experiment.do_send_command_to_device("default:LOAD")
        assert float(load_resp) == 1300

    def test_allinfo_command(self):
        start = self.experiment.do_start_experiment("{}", "{}")
        resp = self.experiment.do_send_command_to_device("ALLINFO:default")
        r = json.loads(resp)
        assert float(r["default"]["level"]) == 1200
        assert float(r["default"]["load"]) == 1300

    def test_allinfo_command_multiple(self):
        start = self.experiment.do_start_experiment("{}", "{}")
        resp = self.experiment.do_send_command_to_device("ALLINFO:default:second")
        r = json.loads(resp)
        assert float(r["default"]["level"]) == 1200
        assert float(r["default"]["load"]) == 1300

        assert float(r["second"]["level"]) == 1200
        assert float(r["second"]["load"]) == 1300

    def test_dispose_cleans_threads(self):
        """
        Archimedes Start creates a thread pool. Ensure that calling Dispose clears every thread.
        """
        initialThreads = len(threading.enumerate())
        self.experiment.do_start_experiment("{}", "{}")
        self.experiment.do_dispose()
        time.sleep(0.5)
        afterThreads = len(threading.enumerate())

        self.assertEqual(initialThreads, afterThreads, "all threads destroyed")

def suite():
    return unittest.TestSuite(
        (
            unittest.makeSuite(TestArchimedes)
        )
    )

if __name__ == '__main__':
    unittest.main()
<commit_msg>[archimedes]: Fixed test. Parts were commented out for some reason<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2005 onwards University of Deusto
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.
#
# This software consists of contributions made by many individuals,
# listed below:
#
# Author: Luis Rodriguez-Gil <luis.rodriguezgil@deusto.es>
#
import base64
import json
import unittest
import threading
import time
from mock import patch, PropertyMock, MagicMock
from experiments.archimedes import Archimedes

from voodoo.configuration import ConfigurationManager
from voodoo.sessions.session_id import SessionId

class TestArchimedes(unittest.TestCase):

    def setUp(self):
        # Initialize the experiment for testing.
        # We set the archimedes_real_device setting to False so that
        # it doesn't attempt to contact the real ip.
        self.cfg_manager = ConfigurationManager()
        self.cfg_manager._set_value("archimedes_real_device", False)
        self.cfg_manager._set_value("archimedes_instances", {"default": "http://localhost:8000", "second": "http://localhost:8001"})
        self.experiment = Archimedes(None, None, self.cfg_manager)
        self.lab_session_id = SessionId('my-session-id')

    def tearDown(self):
        self.experiment.do_dispose()

    def test_start(self):
        m = MagicMock(return_value="")
        self.experiment._send = m
        start = self.experiment.do_start_experiment("{}", "{}")

        assert m.mock_calls[0][1][1] == "up"
        assert m.mock_calls[1][1][1] == "up"

    def test_unknown_instance(self):
        """
        Check that it replies an error if the instance isn't in the config.
        """
        self.cfg_manager = ConfigurationManager()
        self.cfg_manager._set_value("archimedes_instances", {"first": "http://localhost:8000", "second": "http://localhost:8001"})
        self.experiment = Archimedes(None, None, self.cfg_manager)
        start = self.experiment.do_start_experiment("{}", "{}")

        up_resp = self.experiment.do_send_command_to_device("UP")
        assert up_resp.startswith("ERROR:")

        up_resp = self.experiment.do_send_command_to_device("default:UP")
        assert up_resp.startswith("ERROR:")

    def test_control_ball_commands(self):
        start = self.experiment.do_start_experiment("{}", "{}")
        up_resp = self.experiment.do_send_command_to_device("UP")
        down_resp = self.experiment.do_send_command_to_device("DOWN")
        slow_resp = self.experiment.do_send_command_to_device("SLOW")

    def test_basic_data_commands(self):
        start = self.experiment.do_start_experiment("{}", "{}")
        level_resp = self.experiment.do_send_command_to_device("LEVEL")
        assert float(level_resp) == 1200

        load_resp = self.experiment.do_send_command_to_device("LOAD")
        assert float(load_resp) == 1300

    def test_advanced_data_commands(self):
        start = self.experiment.do_start_experiment("{}", "{}")
        image_resp = self.experiment.do_send_command_to_device("IMAGE")
        dec = base64.b64decode(image_resp)
        assert len(dec) > 100

        plot_resp = self.experiment.do_send_command_to_device("PLOT")

        f = file("/tmp/img.html", "w+")
        f.write("""

            """ % (image_resp)
        )
        f.close()

    def test_explicit_instance_commands(self):
        """
        Test that commands can be sent to a specific instance.
        """
        start = self.experiment.do_start_experiment("{}", "{}")
        up_resp = self.experiment.do_send_command_to_device("default:UP")
        down_resp = self.experiment.do_send_command_to_device("default:DOWN")
        slow_resp = self.experiment.do_send_command_to_device("default:SLOW")
        level_resp = self.experiment.do_send_command_to_device("default:LEVEL")
        assert float(level_resp) == 1200
        load_resp = self.experiment.do_send_command_to_device("default:LOAD")
        assert float(load_resp) == 1300

    def test_allinfo_command(self):
        start = self.experiment.do_start_experiment("{}", "{}")
        resp = self.experiment.do_send_command_to_device("ALLINFO:default")
        r = json.loads(resp)
        assert float(r["default"]["level"]) == 1200
        assert float(r["default"]["load"]) == 1300

    def test_allinfo_command_multiple(self):
        start = self.experiment.do_start_experiment("{}", "{}")
        resp = self.experiment.do_send_command_to_device("ALLINFO:default:second")
        r = json.loads(resp)
        assert float(r["default"]["level"]) == 1200
        assert float(r["default"]["load"]) == 1300

        assert float(r["second"]["level"]) == 1200
        assert float(r["second"]["load"]) == 1300

    def test_dispose_cleans_threads(self):
        """
        Archimedes Start creates a thread pool. Ensure that calling Dispose clears every thread.
        """
        initialThreads = len(threading.enumerate())
        self.experiment.do_start_experiment("{}", "{}")
        self.experiment.do_dispose()
        time.sleep(0.5)
        afterThreads = len(threading.enumerate())

        self.assertEqual(initialThreads, afterThreads, "all threads destroyed")

def suite():
    return unittest.TestSuite(
        (
            unittest.makeSuite(TestArchimedes)
        )
    )

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># coding=utf-8
# Filename: tohdf5.py
"""
Send a push message to a device using Pushover.net API.

Usage:
    pushover MESSAGE...
    pushover (-h | --help)
    pushover --version

Options:
    MESSAGE     The message to send.
    -h --help   Show this screen.

"""
from __future__ import division, absolute_import, print_function

from km3pipe.config import Config
from km3pipe import version

try:
    import http.client as httplib
    from urllib.parse import urlencode
except ImportError:
    import httplib
    from urllib import urlencode

__author__ = "Tamas Gal"
__copyright__ = "Copyright 2017, Tamas Gal and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Tamas Gal"
__email__ = "tgal@km3net.de"
__status__ = "Development"

def main():
    from docopt import docopt
    args = docopt(__doc__, version=version)

    config = Config()

    token = config.get("Pushover", "token")
    user_key = config.get("Pushover", "user_key")
    if token is None or user_key is None:
        log.critical("Please define your 'token' and 'user_key' in the "
                     "'Pushover' section of your ~/.km3net configuration.")
        exit(1)

    conn = httplib.HTTPSConnection("api.pushover.net:443")
    conn.request("POST", "/1/messages.json",
      urlencode({
        "token": token,
        "user": user_key,
        "message": ' '.join(args["MESSAGE"]),
      }), { "Content-type": "application/x-www-form-urlencoded" })

    conn.getresponse()
<commit_msg>Print error message<commit_after># coding=utf-8
# Filename: tohdf5.py
"""
Send a push message to a device using Pushover.net API.

Usage:
    pushover MESSAGE...
    pushover (-h | --help)
    pushover --version

Options:
    MESSAGE     The message to send.
    -h --help   Show this screen.

"""
from __future__ import division, absolute_import, print_function

from km3pipe.config import Config
from km3pipe import version

try:
    import http.client as httplib
    from urllib.parse import urlencode
except ImportError:
    import httplib
    from urllib import urlencode

__author__ = "Tamas Gal"
__copyright__ = "Copyright 2017, Tamas Gal and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Tamas Gal"
__email__ = "tgal@km3net.de"
__status__ = "Development"

def main():
    from docopt import docopt
    args = docopt(__doc__, version=version)

    config = Config()

    token = config.get("Pushover", "token")
    user_key = config.get("Pushover", "user_key")
    if token is None or user_key is None:
        print("Please define your 'token' and 'user_key' in the "
              "'Pushover' section of your ~/.km3net configuration.")
        exit(1)

    conn = httplib.HTTPSConnection("api.pushover.net:443")
    conn.request("POST", "/1/messages.json",
      urlencode({
        "token": token,
        "user": user_key,
        "message": ' '.join(args["MESSAGE"]),
      }), { "Content-type": "application/x-www-form-urlencoded" })

    conn.getresponse()
<|endoftext|>


<commit_before>import collections

from vj4 import app
from vj4.model import builtin
from vj4.model import domain
from vj4.model.adaptor import discussion
from vj4.handler import base

@app.route('/', 'domain_main')
class DomainMainHandler(base.Handler):
  async def get(self):
    self.render('domain_main.html', discussion_nodes=await discussion.get_nodes(self.domain_id))

@app.route('/manage', 'domain_manage')
class DomainMainHandler(base.Handler):
  async def get(self):
    self.render('domain_manage.html')

@app.route('/domain/edit', 'domain_edit')
class DomainEditHandler(base.Handler):
  @base.require_perm(builtin.PERM_EDIT_DESCRIPTION)
  async def get(self):
    self.render('domain_edit.html')

  @base.require_perm(builtin.PERM_EDIT_DESCRIPTION)
  @base.post_argument
  @base.require_csrf_token
  @base.sanitize
  async def post(self, *, name: str, gravatar: str):
    ddoc = await domain.edit(self.domain_id, name=name, gravatar=gravatar)
    # TODO(iceboy): FIXME!!! THIS IS DISASTER!!!
    if ddoc:
      self.domain = ddoc
    self.render('domain_edit.html')

@app.route('/domain/user', 'domain_user')
class DomainUserHandler(base.Handler):
  async def get(self):
    self.render('domain_user.html')

@app.route('/domain/permission', 'domain_permission')
class DomainPermissionHandler(base.Handler):
  async def get(self):
    self.render('domain_permission.html')

@app.route('/domain/role', 'domain_role')
class DomainRoleHandler(base.OperationHandler):
  async def get(self):
    rudocs = collections.defaultdict(list)
    async for uddoc in domain.get_multi_user(domain_id=self.domain_id,
                                             fields={'uid': 1, 'role': 1}):
      if 'role' in uddoc:
        rudocs[uddoc['role']].append(uddoc)
    self.render('domain_role.html', rudocs=rudocs)

  @base.require_perm(builtin.PERM_EDIT_PERM)
  @base.require_csrf_token
  @base.sanitize
  async def post_set(self, *, role: str, perm: int):
    await domain.set_role(self.domain_id, role, perm)
    self.json_or_redirect(self.referer_or_main)

  @base.require_perm(builtin.PERM_EDIT_PERM)
  @base.require_csrf_token
  @base.sanitize
  async def post_delete(self, *, role: str, perm: int=None):
    await domain.delete_role(self.domain_id, role)
    self.json_or_redirect(self.referer_or_main)

  @base.require_perm(builtin.PERM_EDIT_PERM)
  @base.require_csrf_token
  @base.sanitize
  async def post_set_user(self, *, uid: int, role: str):
    await domain.set_user_role(self.domain_id, uid, role)
    self.json_or_redirect(self.referer_or_main)

  @base.require_perm(builtin.PERM_EDIT_PERM)
  @base.require_csrf_token
  @base.sanitize
  async def post_unset_user(self, *, uid: int):
    await domain.unset_user_role(self.domain_id, uid)
    self.json_or_redirect(self.referer_or_main)
<commit_msg>Fix: typo (#71)<commit_after>import collections

from vj4 import app
from vj4.model import builtin
from vj4.model import domain
from vj4.model.adaptor import discussion
from vj4.handler import base

@app.route('/', 'domain_main')
class DomainMainHandler(base.Handler):
  async def get(self):
    self.render('domain_main.html', discussion_nodes=await discussion.get_nodes(self.domain_id))

@app.route('/manage', 'domain_manage')
class DomainMainHandler(base.Handler):
  async def get(self):
    self.render('domain_manage.html')

@app.route('/domain/edit', 'domain_edit')
class DomainEditHandler(base.Handler):
  @base.require_perm(builtin.PERM_EDIT_DESCRIPTION)
  async def get(self):
    self.render('domain_edit.html')

  @base.require_perm(builtin.PERM_EDIT_DESCRIPTION)
  @base.post_argument
  @base.require_csrf_token
  @base.sanitize
  async def post(self, *, name: str, gravatar: str):
    ddoc = await domain.edit(self.domain_id, name=name, gravatar=gravatar)
    # TODO(iceboy): FIXME!!! THIS IS A DISASTER!!!
    if ddoc:
      self.domain = ddoc
    self.render('domain_edit.html')

@app.route('/domain/user', 'domain_user')
class DomainUserHandler(base.Handler):
  async def get(self):
    self.render('domain_user.html')

@app.route('/domain/permission', 'domain_permission')
class DomainPermissionHandler(base.Handler):
  async def get(self):
    self.render('domain_permission.html')

@app.route('/domain/role', 'domain_role')
class DomainRoleHandler(base.OperationHandler):
  async def get(self):
    rudocs = collections.defaultdict(list)
    async for uddoc in domain.get_multi_user(domain_id=self.domain_id,
                                             fields={'uid': 1, 'role': 1}):
      if 'role' in uddoc:
        rudocs[uddoc['role']].append(uddoc)
    self.render('domain_role.html', rudocs=rudocs)

  @base.require_perm(builtin.PERM_EDIT_PERM)
  @base.require_csrf_token
  @base.sanitize
  async def post_set(self, *, role: str, perm: int):
    await domain.set_role(self.domain_id, role, perm)
    self.json_or_redirect(self.referer_or_main)

  @base.require_perm(builtin.PERM_EDIT_PERM)
  @base.require_csrf_token
  @base.sanitize
  async def post_delete(self, *, role: str, perm: int=None):
    await domain.delete_role(self.domain_id, role)
    self.json_or_redirect(self.referer_or_main)

  @base.require_perm(builtin.PERM_EDIT_PERM)
  @base.require_csrf_token
  @base.sanitize
  async def post_set_user(self, *, uid: int, role: str):
    await domain.set_user_role(self.domain_id, uid, role)
    self.json_or_redirect(self.referer_or_main)

  @base.require_perm(builtin.PERM_EDIT_PERM)
  @base.require_csrf_token
  @base.sanitize
  async def post_unset_user(self, *, uid: int):
    await domain.unset_user_role(self.domain_id, uid)
    self.json_or_redirect(self.referer_or_main)
<|endoftext|>


<commit_before>import bpy
from bpy.types import Panel, UIList

class SprytileValidateGridList(bpy.types.Operator):
    bl_idname = "sprytile.validate_grids"
    bl_label = "Sprytile Validate Grids"

    @classmethod
    def poll(cls,context):
        return True

    def execute(self, context):
        """
        """
        grids = context.scene.sprytile_grids
        grids.clear()
        mats_valid = []
        print("Material count: %d" % len(bpy.data.materials))
        # Loop through available materials, checking if grids has
        # at least one entry with the id
        for mat_id, mat in enumerate(bpy.data.materials):
            is_mat_valid = False
            for grid in grids:
                if grid.mat_id == mat_id:
                    is_mat_valid = True
                    break
            mats_valid.append(is_mat_valid)

        # Check the mats valid list and add a new grid for any invalid setting
        for mat_id, mat_valid in enumerate(mats_valid):
            if mat_valid is True:
                continue
            grid_setting = grids.add()
            grid_setting.mat_id = mat_id
            grid_setting.is_main = True

        # Remove any grids with invalid material ids
        invalid_id = []
        mat_size = len(bpy.data.materials)
        for idx, grid in enumerate(grids):
            if grid.mat_id >= mat_size:
                invalid_id.append(idx)
                print("Remove index ", idx)
        invalid_id.reverse()
        for idx in invalid_id:
            grids.remove(idx)

        print(context.scene.sprytile_grids)

        return self.invoke(context, None)

    def invoke(self, context, event):
        return {'FINISHED'}

class SprytileMaterialGridList(bpy.types.UIList):
    def draw_item(self, context, layout, data, item, icon, active_data, active_propname, index):
        """
        """
        if item.mat_id >= len(bpy.data.materials):
            layout.label("Invalid Material")
        elif self.layout_type in {'DEFAULT', 'COMPACT'}:
            material = bpy.data.materials[item.mat_id]
            split = layout.split(0.6)
            split.prop(material, "name", text="", emboss=False)
            split.label("%dx%d" % (item.grid_x, item.grid_y))
        elif self.layout_type in {'GRID'}:
            layout.alignment = 'CENTER'
            layout.label(text="", icon_value=icon)

class SprytilePanel(bpy.types.Panel):
    bl_label = "Sprytile Painter"
    bl_idname = "sprytile.panel"
    bl_space_type = "VIEW_3D"
    bl_region_type = "TOOLS"
    bl_category = "Sprytile"

    # Only show panel when selected object is a mesh and in edit mode
    @classmethod
    def poll(self, context):
        if context.object and context.object.type == 'MESH':
            return context.object.mode == 'EDIT'

    def draw(self, context):
        layout = self.layout
        scene = context.scene
        obj = context.object

        layout.operator("sprytile.modal_tool", icon='BRUSH_DATA')

        layout.prop(context.scene.sprytile_data, "paint_mode", expand=True)

        layout.label("Face Normal")
        row = layout.row(align=True)
        row.prop(context.scene.sprytile_data, "normal_mode", expand=True)
        row.prop(context.scene.sprytile_data, "lock_normal", toggle=True)

        layout.prop(context.scene.sprytile_data, "world_pixels")

        layout.label("Select Material", icon='MATERIAL_DATA')
        layout.template_list("SprytileMaterialGridList", "", scene, "sprytile_grids", obj, "sprytile_gridid", rows=3)

        if obj.sprytile_gridid in {None, -1}:
            return

        selected_grid = scene.sprytile_grids[obj.sprytile_gridid]

        layout.label("Grid Settings", icon='GRID')
        row = layout.row(align=True)
        row.prop(selected_grid, "grid_x")
        row.prop(selected_grid, "grid_y")

        layout.prop(selected_grid, "offset")

class SprytileWorkflowPanel(bpy.types.Panel):
    bl_label = "Sprytile Workflow"
    bl_idname = "sprytile.panel_workflow"
    bl_space_type = "VIEW_3D"
    bl_region_type = "TOOLS"
    bl_category = "Sprytile"

    @classmethod
    def poll(self, context):
        if context.object and context.object.type == 'MESH':
            return context.object.mode == 'EDIT'

    def draw(self, context):
        layout = self.layout
        layout.operator("sprytile.validate_grids")

def register():
    bpy.utils.register_module(__name__)

def unregister():
    bpy.utils.unregister_module(__name__)

if __name__ == '__main__':
    register()
<commit_msg>Fixed validate grid operator<commit_after>import bpy
from bpy.types import Panel, UIList

class SprytileValidateGridList(bpy.types.Operator):
    bl_idname = "sprytile.validate_grids"
    bl_label = "Sprytile Validate Grids"

    @classmethod
    def poll(cls,context):
        return True

    def validate_grids(self, context):
        grids = context.scene.sprytile_grids
        mats_valid = []
        print("Material count: %d" % len(bpy.data.materials))
        # Loop through available materials, checking if grids has
        # at least one entry with the id
        for mat_id, mat in enumerate(bpy.data.materials):
            is_mat_valid = False
            for grid in grids:
                if grid.mat_id == mat_id:
                    is_mat_valid = True
                    break
            mats_valid.append(is_mat_valid)

        # Check the mats valid list and add a new grid for any invalid setting
        for mat_id, mat_valid in enumerate(mats_valid):
            if mat_valid is True:
                continue
            grid_setting = grids.add()
            grid_setting.mat_id = mat_id
            grid_setting.is_main = True

        # Remove any grids with invalid material ids
        invalid_id = []
        mat_size = len(bpy.data.materials)
        for idx, grid in enumerate(grids):
            if grid.mat_id >= mat_size:
                invalid_id.append(idx)
                print("Remove index ", idx)
        invalid_id.reverse()
        for idx in invalid_id:
            grids.remove(idx)

        print(context.scene.sprytile_grids)

    def execute(self, context):
        self.validate_grids(context)
        return self.invoke(context, None)

    def invoke(self, context, event):
        if event is not None:
            self.validate_grids(context)
        return {'FINISHED'}

class SprytileMaterialGridList(bpy.types.UIList):
    def draw_item(self, context, layout, data, item, icon, active_data, active_propname, index):
        """
        """
        if item.mat_id >= len(bpy.data.materials):
            layout.label("Invalid Material")
        elif self.layout_type in {'DEFAULT', 'COMPACT'}:
            material = bpy.data.materials[item.mat_id]
            split = layout.split(0.6)
            split.prop(material, "name", text="", emboss=False)
            split.label("%dx%d" % (item.grid_x, item.grid_y))
        elif self.layout_type in {'GRID'}:
            layout.alignment = 'CENTER'
            layout.label(text="", icon_value=icon)

class SprytilePanel(bpy.types.Panel):
    bl_label = "Sprytile Painter"
    bl_idname = "sprytile.panel"
    bl_space_type = "VIEW_3D"
    bl_region_type = "TOOLS"
    bl_category = "Sprytile"

    # Only show panel when selected object is a mesh and in edit mode
    @classmethod
    def poll(self, context):
        if context.object and context.object.type == 'MESH':
            return context.object.mode == 'EDIT'

    def draw(self, context):
        layout = self.layout
        scene = context.scene
        obj = context.object

        layout.operator("sprytile.modal_tool", icon='BRUSH_DATA')

        layout.prop(context.scene.sprytile_data, "paint_mode", expand=True)

        layout.label("Face Normal")
        row = layout.row(align=True)
        row.prop(context.scene.sprytile_data, "normal_mode", expand=True)
        row.prop(context.scene.sprytile_data, "lock_normal", toggle=True)

        layout.prop(context.scene.sprytile_data, "world_pixels")

        layout.label("Select Material", icon='MATERIAL_DATA')
        layout.template_list("SprytileMaterialGridList", "", scene, "sprytile_grids", obj, "sprytile_gridid", rows=3)

        if obj.sprytile_gridid in {None, -1}:
            return

        selected_grid = scene.sprytile_grids[obj.sprytile_gridid]

        layout.label("Grid Settings", icon='GRID')
        row = layout.row(align=True)
        row.prop(selected_grid, "grid_x")
        row.prop(selected_grid, "grid_y")

        layout.prop(selected_grid, "offset")

class SprytileWorkflowPanel(bpy.types.Panel):
    bl_label = "Sprytile Workflow"
    bl_idname = "sprytile.panel_workflow"
    bl_space_type = "VIEW_3D"
    bl_region_type = "TOOLS"
    bl_category = "Sprytile"

    @classmethod
    def poll(self, context):
        if context.object and context.object.type == 'MESH':
            return context.object.mode == 'EDIT'

    def draw(self, context):
        layout = self.layout
        layout.operator("sprytile.validate_grids")

def register():
    bpy.utils.register_module(__name__)

def unregister():
    bpy.utils.unregister_module(__name__)

if __name__ == '__main__':
    register()
<|endoftext|>


<commit_before><commit_msg>added cli option for --support<commit_after><|endoftext|>


<commit_before>'''
Parse CLI options
'''
# Import python libs
import os
import copy
import argparse
# Import pkgcmp libs
import pkgcmp.scan
# Import third party libs
import yaml

DEFAULTS = {'cachedir': '/var/cache/pkgcmp'}

def parse():
    '''
    Parse!!
    '''
    parser = argparse.ArgumentParser(description='The pkgcmp map generator')
    parser.add_argument(
            '--cachedir',
            dest='cachedir',
            default=None,
            help='The location to store all the files while working')
    parser.add_argument(
            '--config',
            dest='config',
            default='/etc/pkgcmp/pkgcmp',
            help='The location of the pkgcmp config file')
    opts = parser.parse_args().__dict__
    conf = config(opts['config'])
    for key in opts:
        if opts[key] is not None:
            conf[key] = opts[key]
    return conf

def config(cfn):
    '''
    Read in the config file
    '''
    ret = copy.copy(DEFAULTS)
    if os.path.isfile(cfn):
        with open(cfn, 'r') as cfp:
            conf = yaml.safe_load(cfp)
            if isinstance(conf, dict):
                ret.update(conf)
    return ret

class PkgCmp:
    '''
    Build and run the application
    '''
    def __init__(self):
        self.opts = parse()
        self.scan = pkgcmp.scan.Scanner(self.opts)

    def run(self):
        self.scan.run()
<commit_msg>Add extension_modueles to the default configuration<commit_after>'''
Parse CLI options
'''
# Import python libs
import os
import copy
import argparse
# Import pkgcmp libs
import pkgcmp.scan
# Import third party libs
import yaml

DEFAULTS = {'cachedir': '/var/cache/pkgcmp',
            'extension_modules': ''}

def parse():
    '''
    Parse!!
    '''
    parser = argparse.ArgumentParser(description='The pkgcmp map generator')
    parser.add_argument(
            '--cachedir',
            dest='cachedir',
            default=None,
            help='The location to store all the files while working')
    parser.add_argument(
            '--config',
            dest='config',
            default='/etc/pkgcmp/pkgcmp',
            help='The location of the pkgcmp config file')
    opts = parser.parse_args().__dict__
    conf = config(opts['config'])
    for key in opts:
        if opts[key] is not None:
            conf[key] = opts[key]
    return conf

def config(cfn):
    '''
    Read in the config file
    '''
    ret = copy.copy(DEFAULTS)
    if os.path.isfile(cfn):
        with open(cfn, 'r') as cfp:
            conf = yaml.safe_load(cfp)
            if isinstance(conf, dict):
                ret.update(conf)
    return ret

class PkgCmp:
    '''
    Build and run the application
    '''
    def __init__(self):
        self.opts = parse()
        self.scan = pkgcmp.scan.Scanner(self.opts)

    def run(self):
        self.scan.run()
<|endoftext|>