<commit_before>from telephus.cassandra.ttypes import *
from telephus.protocol import ManagedThriftRequest
from collections import defaultdict
import time

class CassandraClient(object):
    def __init__(self, manager, consistency=ConsistencyLevel.ONE):
        self.manager = manager
        self.consistency = consistency

    def _time(self):
        return int(time.time() * 1000000)

    def _getparent(self, columnParentOrCF, super_column=None):
        if isinstance(columnParentOrCF, str):
            return ColumnParent(columnParentOrCF, super_column=super_column)
        else:
            return columnParentOrCF

    def _getpath(self, columnPathOrCF, col, super_column=None):
        if isinstance(columnPathOrCF, str):
            return ColumnPath(columnPathOrCF, super_column=super_column, column=col)
        else:
            return columnPathOrCF

    def _mkpred(self, names, start, finish, reverse, count):
        if names:
            srange = None
        else:
            srange = SliceRange(start, finish, reverse, count)
        return SlicePredicate(names, srange)

    def get(self, key, columnPath, column=None, super_column=None, consistency=None,
            retries=None):
        cp = self._getpath(columnPath, column, super_column)
        consistency = consistency or self.consistency
        req = ManagedThriftRequest('get', key, cp, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def get_slice(self, key, columnParent, names=None, start='', finish='',
                  reverse=False, count=100, consistency=None, super_column=None,
                  retries=None):
        cp = self._getparent(columnParent, super_column)
        consistency = consistency or self.consistency
        pred = self._mkpred(names, start, finish, reverse, count)
        req = ManagedThriftRequest('get_slice', key, cp, pred, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def multiget(self, keys, columnParent, column=None, super_column=None,
                  consistency=None, retries=None):
        return self.multiget_slice(keys, columnParent, names=[column], count=100,
                   consistency=consistency, retries=retries, super_column=super_column)

    def multiget_slice(self, keys, columnParent, names=None, start='', finish='',
                  reverse=False, count=100, consistency=None, super_column=None,
                  retries=None):
        cp = self._getparent(columnParent, super_column)
        consistency = consistency or self.consistency
        pred = self._mkpred(names, start, finish, reverse, count)
        req = ManagedThriftRequest('multiget_slice', keys, cp, pred, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def multiget_count(self, keys, columnParent, super_column=None, start='', finish='',
                  consistency=None, retries=None):
        cp = self._getparent(columnParent, super_column)
        pred = self._mkpred(None, start, finish, False, 2147483647)
        consistency = consistency or self.consistency
        req = ManagedThriftRequest('multiget_count', keys, cp, pred, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def get_count(self, key, columnParent, super_column=None, start='', finish='',
                  consistency=None, retries=None):
        cp = self._getparent(columnParent, super_column)
        pred = self._mkpred(None, start, finish, False, 2147483647)
        consistency = consistency or self.consistency
        req = ManagedThriftRequest('get_count', key, cp, pred, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def get_key_range(self, columnParent, **kwargs):
        return self.get_range_slices(columnParent, **kwargs)

    def get_range_slice(self, columnParent, **kwargs):
        return self.get_range_slices(columnParent, **kwargs)

    def get_range_slices(self, columnParent, start='', finish='', column_start='',
            column_finish='', names=None, count=100, column_count=100,
            reverse=False, use_tokens=False, consistency=None, super_column=None,
            retries=None):
        cp = self._getparent(columnParent, super_column)
        consistency = consistency or self.consistency
        if not use_tokens:
            krange = KeyRange(start_key=start, end_key=finish, count=count)
        else:
            krange = KeyRange(start_token=start, end_token=finish, count=count)
        pred = self._mkpred(names, column_start, column_finish, reverse, column_count)
        req = ManagedThriftRequest('get_range_slices', cp, pred, krange, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def insert(self, key, columnParent, value, column=None, super_column=None,
               timestamp=None, consistency=None, retries=None):
        timestamp = timestamp or self._time()
        cp = self._getparent(columnParent, super_column)
        consistency = consistency or self.consistency
        req = ManagedThriftRequest('insert', key, cp, Column(column, value, timestamp), consistency)
        return self.manager.pushRequest(req, retries=retries)

    def remove(self, key, columnPath, column=None, super_column=None,
               timestamp=None, consistency=None, retries=None):
        cp = self._getpath(columnPath, column, super_column)
        timestamp = timestamp or self._time()
        consistency = consistency or self.consistency
        req = ManagedThriftRequest('remove', key, cp, timestamp, self.consistency)
        return self.manager.pushRequest(req, retries=retries)

    def batch_insert(self, key, columnFamily, mapping, timestamp=None,
                     consistency=None, retries=None):
        if isinstance(mapping, list) and timestamp is not None:
            raise RuntimeError('Timestamp cannot be specified with a list of Mutations')
        timestamp = timestamp or self._time()
        consistency = consistency or self.consistency
        mutmap = {key: {columnFamily: self._mk_cols_or_supers(mapping, timestamp)}}
        return self.batch_mutate(mutmap, timestamp=timestamp, consistency=consistency,
                                 retries=retries)

    def batch_remove(self, cfmap, start='', finish='', count=100, names=None,
                     reverse=False, consistency=None, timestamp=None, supercolumn=None,
                     retries=None):
        timestamp = timestamp or self._time()
        consistency = consistency or self.consistency
        mutmap = defaultdict(dict)
        for cf, keys in cfmap.iteritems():
            pred = self._mkpred(names, start, finish, reverse, count)
            for key in keys:
                mutmap[key][cf] = [Mutation(deletion=Deletion(timestamp, supercolumn, pred))]
        req = ManagedThriftRequest('batch_mutate', mutmap, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def batch_mutate(self, mutationmap, timestamp=None, consistency=None, retries=None):
        timestamp = timestamp or self._time()
        consistency = consistency or self.consistency
        mutmap = defaultdict(dict)
        for key, cfmap in mutationmap.iteritems():
            for cf, colmap in cfmap.iteritems():
                cols_or_supers_or_deletions = self._mk_cols_or_supers(colmap, timestamp, make_deletions=True)
                muts = []
                for c in cols_or_supers_or_deletions:
                    if isinstance(c, SuperColumn):
                        muts.append(Mutation(ColumnOrSuperColumn(super_column=c)))
                    elif isinstance(c, Column):
                        muts.append(Mutation(ColumnOrSuperColumn(column=c)))
                    elif isinstance(c, Deletion):
                        muts.append(Mutation(deletion=c))
                    else:
                        muts.append(c)
                mutmap[key][cf] = muts
        req = ManagedThriftRequest('batch_mutate', mutmap, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def _mk_cols_or_supers(self, mapping, timestamp, make_deletions=False):
        if isinstance(mapping, list):
            return mapping
        colsorsupers = []
        if isinstance(mapping, dict):
            first = mapping.keys()[0]
            if isinstance(mapping[first], dict):
                for name in mapping:
                    cols = []
                    for col,val in mapping[name].iteritems():
                        cols.append(Column(col, val, timestamp))
                    colsorsupers.append(SuperColumn(name=name, columns=cols))
            else:
                cols2delete = []
                for col, val in mapping.iteritems():
                    if val is None and make_deletions:
                        cols2delete.append(col)
                    else:
                        colsorsupers.append(Column(col, val, timestamp))
                if cols2delete:
                    colsorsupers.append(Deletion(timestamp, None, SlicePredicate(column_names=cols2delete)))
        else:
            raise TypeError('dict (of dicts) or list of Columns/SuperColumns expected')
        return colsorsupers

    def set_keyspace(self, keyspace):
        return self.manager.set_keyspace(keyspace)

    def login(self, credentials):
        return self.manager.login(credentials)

    def describe_keyspaces(self, retries=None):
        req = ManagedThriftRequest('describe_keyspaces')
        return self.manager.pushRequest(req, retries=retries)

    def describe_keyspace(self, keyspace, retries=None):
        req = ManagedThriftRequest('describe_keyspace', keyspace)
        return self.manager.pushRequest(req, retries=retries)

    def describe_cluster_name(self, retries=None):
        req = ManagedThriftRequest('describe_cluster_name')
        return self.manager.pushRequest(req, retries=retries)

    def describe_partitioner(self, retries=None):
        req = ManagedThriftRequest('describe_partitioner')
        return self.manager.pushRequest(req, retries=retries)

    def describe_snitch(self, retries=None):
        req = ManagedThriftRequest('describe_snitch')
        return self.manager.pushRequest(req, retries=retries)

    def describe_ring(self, keyspace, retries=None):
        req = ManagedThriftRequest('describe_ring', keyspace)
        return self.manager.pushRequest(req, retries=retries)

    def describe_splits(self, cfName, start_token, end_token, keys_per_split,
                        retries=None):
        req = ManagedThriftRequest('describe_splits', cfName, start_token,
                                   end_token, keys_per_split)
        return self.manager.pushRequest(req, retries=retries)

    def truncate(self, cfName, retries=None):
        req = ManagedThriftRequest('truncate', cfName)
        return self.manager.pushRequest(req, retries=retries)

    def describe_schema_versions(self, retries=None):
        req = ManagedThriftRequest('describe_schema_versions')
        return self.manager.pushRequest(req, retries=retries)

    def system_drop_column_family(self, cfName, retries=None):
        req = ManagedThriftRequest('system_drop_column_family', cfName)
        return self.manager.pushRequest(req, retries=retries)

    def system_drop_keyspace(self, keyspace, retries=None):
        req = ManagedThriftRequest('system_drop_keyspace', keyspace)
        return self.manager.pushRequest(req, retries=retries)

    def system_rename_column_family(self, oldname, newname, retries=None):
        req = ManagedThriftRequest('system_rename_column_family', oldname, newname)
        return self.manager.pushRequest(req, retries=retries)

    def system_rename_keyspace(self, oldname, newname, retries=None):
        req = ManagedThriftRequest('system_rename_keyspace', oldname, newname)
        return self.manager.pushRequest(req, retries=retries)

    # TODO: make friendly
    def system_add_column_family(self, cfDef, retries=None):
        req = ManagedThriftRequest('system_add_column_family', cfDef)
        return self.manager.pushRequest(req, retries=retries)

    def system_update_column_family(self, cfDef, retries=None):
        req = ManagedThriftRequest('system_update_column_family', cfDef)
        return self.manager.pushRequest(req, retries=retries)

    def system_add_keyspace(self, ksDef, retries=None):
        req = ManagedThriftRequest('system_add_keyspace', ksDef)
        return self.manager.pushRequest(req, retries=retries)

    def system_update_keyspace(self, ksDef, retries=None):
        req = ManagedThriftRequest('system_update_keyspace', ksDef)
        return self.manager.pushRequest(req, retries=retries)

    def describe_version(self, retries=None):
        req = ManagedThriftRequest('describe_version')
        return self.manager.pushRequest(req, retries=retries)
<commit_msg>added schema agreement check to system_* calls<commit_after>from telephus.cassandra.ttypes import *
from telephus.protocol import ManagedThriftRequest
from collections import defaultdict
import time

class CassandraClient(object):
    def __init__(self, manager, consistency=ConsistencyLevel.ONE):
        self.manager = manager
        self.consistency = consistency

    def _time(self):
        return int(time.time() * 1000000)

    def _getparent(self, columnParentOrCF, super_column=None):
        if isinstance(columnParentOrCF, str):
            return ColumnParent(columnParentOrCF, super_column=super_column)
        else:
            return columnParentOrCF

    def _getpath(self, columnPathOrCF, col, super_column=None):
        if isinstance(columnPathOrCF, str):
            return ColumnPath(columnPathOrCF, super_column=super_column, column=col)
        else:
            return columnPathOrCF

    def _mkpred(self, names, start, finish, reverse, count):
        if names:
            srange = None
        else:
            srange = SliceRange(start, finish, reverse, count)
        return SlicePredicate(names, srange)

    def _wait_for_schema_agreement(self):
        agreement = False
        while not agreement:
            result = yield self.describe_schema_versions
            agreement = len(result) == 1

    def _push_system_request(self,req,retries=None,block=True):
        result = self.manager.pushRequest(req, retries=retries)
        if block: self._wait_for_schema_agreement()
        return result

    def get(self, key, columnPath, column=None, super_column=None, consistency=None,
            retries=None):
        cp = self._getpath(columnPath, column, super_column)
        consistency = consistency or self.consistency
        req = ManagedThriftRequest('get', key, cp, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def get_slice(self, key, columnParent, names=None, start='', finish='',
                  reverse=False, count=100, consistency=None, super_column=None,
                  retries=None):
        cp = self._getparent(columnParent, super_column)
        consistency = consistency or self.consistency
        pred = self._mkpred(names, start, finish, reverse, count)
        req = ManagedThriftRequest('get_slice', key, cp, pred, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def multiget(self, keys, columnParent, column=None, super_column=None,
                  consistency=None, retries=None):
        return self.multiget_slice(keys, columnParent, names=[column], count=100,
                   consistency=consistency, retries=retries, super_column=super_column)

    def multiget_slice(self, keys, columnParent, names=None, start='', finish='',
                  reverse=False, count=100, consistency=None, super_column=None,
                  retries=None):
        cp = self._getparent(columnParent, super_column)
        consistency = consistency or self.consistency
        pred = self._mkpred(names, start, finish, reverse, count)
        req = ManagedThriftRequest('multiget_slice', keys, cp, pred, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def multiget_count(self, keys, columnParent, super_column=None, start='', finish='',
                  consistency=None, retries=None):
        cp = self._getparent(columnParent, super_column)
        pred = self._mkpred(None, start, finish, False, 2147483647)
        consistency = consistency or self.consistency
        req = ManagedThriftRequest('multiget_count', keys, cp, pred, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def get_count(self, key, columnParent, super_column=None, start='', finish='',
                  consistency=None, retries=None):
        cp = self._getparent(columnParent, super_column)
        pred = self._mkpred(None, start, finish, False, 2147483647)
        consistency = consistency or self.consistency
        req = ManagedThriftRequest('get_count', key, cp, pred, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def get_key_range(self, columnParent, **kwargs):
        return self.get_range_slices(columnParent, **kwargs)

    def get_range_slice(self, columnParent, **kwargs):
        return self.get_range_slices(columnParent, **kwargs)

    def get_range_slices(self, columnParent, start='', finish='', column_start='',
            column_finish='', names=None, count=100, column_count=100,
            reverse=False, use_tokens=False, consistency=None, super_column=None,
            retries=None):
        cp = self._getparent(columnParent, super_column)
        consistency = consistency or self.consistency
        if not use_tokens:
            krange = KeyRange(start_key=start, end_key=finish, count=count)
        else:
            krange = KeyRange(start_token=start, end_token=finish, count=count)
        pred = self._mkpred(names, column_start, column_finish, reverse, column_count)
        req = ManagedThriftRequest('get_range_slices', cp, pred, krange, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def insert(self, key, columnParent, value, column=None, super_column=None,
               timestamp=None, consistency=None, retries=None):
        timestamp = timestamp or self._time()
        cp = self._getparent(columnParent, super_column)
        consistency = consistency or self.consistency
        req = ManagedThriftRequest('insert', key, cp, Column(column, value, timestamp), consistency)
        return self.manager.pushRequest(req, retries=retries)

    def remove(self, key, columnPath, column=None, super_column=None,
               timestamp=None, consistency=None, retries=None):
        cp = self._getpath(columnPath, column, super_column)
        timestamp = timestamp or self._time()
        consistency = consistency or self.consistency
        req = ManagedThriftRequest('remove', key, cp, timestamp, self.consistency)
        return self.manager.pushRequest(req, retries=retries)

    def batch_insert(self, key, columnFamily, mapping, timestamp=None,
                     consistency=None, retries=None):
        if isinstance(mapping, list) and timestamp is not None:
            raise RuntimeError('Timestamp cannot be specified with a list of Mutations')
        timestamp = timestamp or self._time()
        consistency = consistency or self.consistency
        mutmap = {key: {columnFamily: self._mk_cols_or_supers(mapping, timestamp)}}
        return self.batch_mutate(mutmap, timestamp=timestamp, consistency=consistency,
                                 retries=retries)

    def batch_remove(self, cfmap, start='', finish='', count=100, names=None,
                     reverse=False, consistency=None, timestamp=None, supercolumn=None,
                     retries=None):
        timestamp = timestamp or self._time()
        consistency = consistency or self.consistency
        mutmap = defaultdict(dict)
        for cf, keys in cfmap.iteritems():
            pred = self._mkpred(names, start, finish, reverse, count)
            for key in keys:
                mutmap[key][cf] = [Mutation(deletion=Deletion(timestamp, supercolumn, pred))]
        req = ManagedThriftRequest('batch_mutate', mutmap, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def batch_mutate(self, mutationmap, timestamp=None, consistency=None, retries=None):
        timestamp = timestamp or self._time()
        consistency = consistency or self.consistency
        mutmap = defaultdict(dict)
        for key, cfmap in mutationmap.iteritems():
            for cf, colmap in cfmap.iteritems():
                cols_or_supers_or_deletions = self._mk_cols_or_supers(colmap, timestamp, make_deletions=True)
                muts = []
                for c in cols_or_supers_or_deletions:
                    if isinstance(c, SuperColumn):
                        muts.append(Mutation(ColumnOrSuperColumn(super_column=c)))
                    elif isinstance(c, Column):
                        muts.append(Mutation(ColumnOrSuperColumn(column=c)))
                    elif isinstance(c, Deletion):
                        muts.append(Mutation(deletion=c))
                    else:
                        muts.append(c)
                mutmap[key][cf] = muts
        req = ManagedThriftRequest('batch_mutate', mutmap, consistency)
        return self.manager.pushRequest(req, retries=retries)

    def _mk_cols_or_supers(self, mapping, timestamp, make_deletions=False):
        if isinstance(mapping, list):
            return mapping
        colsorsupers = []
        if isinstance(mapping, dict):
            first = mapping.keys()[0]
            if isinstance(mapping[first], dict):
                for name in mapping:
                    cols = []
                    for col,val in mapping[name].iteritems():
                        cols.append(Column(col, val, timestamp))
                    colsorsupers.append(SuperColumn(name=name, columns=cols))
            else:
                cols2delete = []
                for col, val in mapping.iteritems():
                    if val is None and make_deletions:
                        cols2delete.append(col)
                    else:
                        colsorsupers.append(Column(col, val, timestamp))
                if cols2delete:
                    colsorsupers.append(Deletion(timestamp, None, SlicePredicate(column_names=cols2delete)))
        else:
            raise TypeError('dict (of dicts) or list of Columns/SuperColumns expected')
        return colsorsupers

    def set_keyspace(self, keyspace):
        return self.manager.set_keyspace(keyspace)

    def login(self, credentials):
        return self.manager.login(credentials)

    def describe_keyspaces(self, retries=None):
        req = ManagedThriftRequest('describe_keyspaces')
        return self.manager.pushRequest(req, retries=retries)

    def describe_keyspace(self, keyspace, retries=None):
        req = ManagedThriftRequest('describe_keyspace', keyspace)
        return self.manager.pushRequest(req, retries=retries)

    def describe_cluster_name(self, retries=None):
        req = ManagedThriftRequest('describe_cluster_name')
        return self.manager.pushRequest(req, retries=retries)

    def describe_partitioner(self, retries=None):
        req = ManagedThriftRequest('describe_partitioner')
        return self.manager.pushRequest(req, retries=retries)

    def describe_snitch(self, retries=None):
        req = ManagedThriftRequest('describe_snitch')
        return self.manager.pushRequest(req, retries=retries)

    def describe_ring(self, keyspace, retries=None):
        req = ManagedThriftRequest('describe_ring', keyspace)
        return self.manager.pushRequest(req, retries=retries)

    def describe_splits(self, cfName, start_token, end_token, keys_per_split,
                        retries=None):
        req = ManagedThriftRequest('describe_splits', cfName, start_token,
                                   end_token, keys_per_split)
        return self.manager.pushRequest(req, retries=retries)

    def truncate(self, cfName, retries=None):
        req = ManagedThriftRequest('truncate', cfName)
        return self.manager.pushRequest(req, retries=retries)

    def describe_schema_versions(self, retries=None):
        req = ManagedThriftRequest('describe_schema_versions')
        return self.manager.pushRequest(req, retries=retries)

    def system_drop_column_family(self, cfName, retries=None, block=True):
        req = ManagedThriftRequest('system_drop_column_family', cfName)
        return self._push_system_request(req,retries=retries,block=block)

    def system_drop_keyspace(self, keyspace, retries=None, block=True):
        req = ManagedThriftRequest('system_drop_keyspace', keyspace)
        return self._push_system_request(req,retries=retries,block=block)

    def system_rename_column_family(self, oldname, newname, retries=None, block=True):
        if block: self._wait_for_schema_agreement()
        req = ManagedThriftRequest('system_rename_column_family', oldname, newname)
        return self._push_system_request(req,retries=retries,block=block)

    def system_rename_keyspace(self, oldname, newname, retries=None, block=True):
        req = ManagedThriftRequest('system_rename_keyspace', oldname, newname)
        return self._push_system_request(req,retries=retries,block=block)

    # TODO: make friendly
    def system_add_column_family(self, cfDef, retries=None, block=True):
        req = ManagedThriftRequest('system_add_column_family', cfDef)
        return self._push_system_request(req,retries=retries,block=block)

    def system_update_column_family(self, cfDef, retries=None, block=True):
        req = ManagedThriftRequest('system_update_column_family', cfDef)
        return self._push_system_request(req,retries=retries,block=block)

    def system_add_keyspace(self, ksDef, retries=None, block=True):
        req = ManagedThriftRequest('system_add_keyspace', ksDef)
        return self._push_system_request(req,retries=retries,block=block)

    def system_update_keyspace(self, ksDef, retries=None, block=True):
        req = ManagedThriftRequest('system_update_keyspace', ksDef)
        return self._push_system_request(req,retries=retries,block=block)

    def describe_version(self, retries=None):
        req = ManagedThriftRequest('describe_version')
        return self.manager.pushRequest(req, retries=retries)
<|endoftext|>


<commit_before><commit_msg>Add nan / inf tests to the federated_sample aggregation. This aggregation should preserve the percentage of both in the final sample.<commit_after><|endoftext|>


<commit_before>from clint.textui import colored

class DatacatsError(Exception):

    def __init__(self, message, format_args=(), parent_exception=None):
        try:
            self.message = message.format(*format_args)
        except (ValueError, KeyError):
            self.message = message
        if parent_exception and hasattr(parent_exception, 'user_description'):
            vals = {
                "original": self.message,
                "type_description": parent_exception.user_description,
                "message": str(parent_exception),
            }
            self.message = "".join([str(colored.blue("{original}\n\n")),
                                    "~" * 30,
                                    "\n{type_description}:\n",
                                    str(colored.yellow("{message}\n"))]
                                    ).format(**vals)

        self.format_args = format_args
        super(DatacatsError, self).__init__(message, format_args)

    def __str__(self):
        return self.message

    def pretty_print(self):
        """
        Print the error message to stdout with colors and borders
        """
        print colored.blue("-" * 40)
        print colored.red("datacats: problem was encountered:")
        print self.message
        print colored.blue("-" * 40)

class WebCommandError(Exception):
    user_description = "Docker container \"/web\" command failed"

    def __init__(self, command, container_id, logs):
        # pylint: disable=unused-argument
        super(WebCommandError, self).__init__()
        self.command = command
        self.logs = logs

    def __str__(self):
        serialized_command = " ".join(self.command) \
                if type(self.command) is list else \
                str(self.command)

        return ('    Command: {0}\n'
                '    Docker Error Log:\n'
                '    {1}\n'
                ).format(serialized_command, self.logs)

class PortAllocatedError(Exception):
    user_description = "Unable to allocate port"

class UndocumentedError(Exception):
    user_description = "Please quote this traceback when reporting this error"
<commit_msg>Use isinstance instead of type() for type checking<commit_after>from clint.textui import colored

class DatacatsError(Exception):

    def __init__(self, message, format_args=(), parent_exception=None):
        try:
            self.message = message.format(*format_args)
        except (ValueError, KeyError):
            self.message = message
        if parent_exception and hasattr(parent_exception, 'user_description'):
            vals = {
                "original": self.message,
                "type_description": parent_exception.user_description,
                "message": str(parent_exception),
            }
            self.message = "".join([str(colored.blue("{original}\n\n")),
                                    "~" * 30,
                                    "\n{type_description}:\n",
                                    str(colored.yellow("{message}\n"))]
                                    ).format(**vals)

        self.format_args = format_args
        super(DatacatsError, self).__init__(message, format_args)

    def __str__(self):
        return self.message

    def pretty_print(self):
        """
        Print the error message to stdout with colors and borders
        """
        print colored.blue("-" * 40)
        print colored.red("datacats: problem was encountered:")
        print self.message
        print colored.blue("-" * 40)

class WebCommandError(Exception):
    user_description = "Docker container \"/web\" command failed"

    def __init__(self, command, container_id, logs):
        # pylint: disable=unused-argument
        super(WebCommandError, self).__init__()
        self.command = command
        self.logs = logs

    def __str__(self):
        serialized_command = " ".join(self.command) \
                if isinstance(self.command, list) else \
                str(self.command)

        return ('    Command: {0}\n'
                '    Docker Error Log:\n'
                '    {1}\n'
                ).format(serialized_command, self.logs)

class PortAllocatedError(Exception):
    user_description = "Unable to allocate port"

class UndocumentedError(Exception):
    user_description = "Please quote this traceback when reporting this error"
<|endoftext|>


<commit_before># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import mock

import inquirer
from st2actions.container import service
from st2common.constants.action import LIVEACTION_STATUS_PENDING
from st2common.constants.action import LIVEACTION_STATUS_FAILED
from st2common.constants.pack import SYSTEM_PACK_NAME
from st2tests.base import RunnerTestCase

mock_liveaction_db = mock.Mock()
mock_liveaction_db.result = {"response_data": {}}

mock_action_utils = mock.Mock()
mock_action_utils.get_liveaction_by_id.return_value = mock_liveaction_db

mock_trigger_dispatcher = mock.Mock()

class InquiryTestCase(RunnerTestCase):

    def test_runner_creation(self):
        runner = inquirer.get_runner()
        self.assertTrue(runner is not None, 'Creation failed. No instance.')
        self.assertEqual(type(runner), inquirer.Inquirer, 'Creation failed. No instance.')

    @mock.patch('inquirer.TriggerDispatcher', mock_trigger_dispatcher)
    @mock.patch('inquirer.action_utils', mock_action_utils)
    def test_simple_inquiry(self):
        runner = inquirer.get_runner()
        runner.context = {
            'user': 'st2admin'
        }
        runner.action = self._get_mock_action_obj()
        runner.runner_parameters = {}
        runner.container_service = service.RunnerContainerService()
        runner.pre_run()
        (status, output, _) = runner.run({})
        self.assertEqual(status, LIVEACTION_STATUS_PENDING)
        self.assertTrue(output is not None)
        self.assertEqual(output, {"response_data": {}})

    def _get_mock_action_obj(self):
        action = mock.Mock()
        action.pack = SYSTEM_PACK_NAME
        return action
<commit_msg>Added test for failure on no parent<commit_after># Licensed to the StackStorm, Inc ('StackStorm') under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import mock

import inquirer
from st2actions.container import service
from st2common.constants.action import LIVEACTION_STATUS_PENDING
from st2common.constants.action import LIVEACTION_STATUS_FAILED
from st2common.constants.pack import SYSTEM_PACK_NAME
from st2tests.base import RunnerTestCase

mock_liveaction_db = mock.Mock()
mock_liveaction_db.result = {"response_data": {}}

mock_action_utils = mock.Mock()
mock_action_utils.get_liveaction_by_id.return_value = mock_liveaction_db

mock_trigger_dispatcher = mock.Mock()

class InquiryTestCase(RunnerTestCase):

    def test_runner_creation(self):
        runner = inquirer.get_runner()
        self.assertTrue(runner is not None, 'Creation failed. No instance.')
        self.assertEqual(type(runner), inquirer.Inquirer, 'Creation failed. No instance.')

    @mock.patch('inquirer.TriggerDispatcher', mock_trigger_dispatcher)
    @mock.patch('inquirer.action_utils', mock_action_utils)
    def test_simple_inquiry(self):
        runner = inquirer.get_runner()
        runner.context = {
            'user': 'st2admin'
        }
        runner.action = self._get_mock_action_obj()
        runner.runner_parameters = {}
        runner.container_service = service.RunnerContainerService()
        runner.pre_run()
        mock_liveaction_db.context = {
            "parent": "1234567890"
        }
        (status, output, _) = runner.run({})
        self.assertEqual(status, LIVEACTION_STATUS_PENDING)
        self.assertTrue(output is not None)
        self.assertEqual(output, {"response_data": {}})
        mock_trigger_dispatcher.return_value.dispatch.assert_called_once()

    @mock.patch('inquirer.TriggerDispatcher', mock_trigger_dispatcher)
    @mock.patch('inquirer.action_utils', mock_action_utils)
    def test_inquiry_failed_no_parent(self):
        runner = inquirer.get_runner()
        runner.context = {
            'user': 'st2admin'
        }
        runner.action = self._get_mock_action_obj()
        runner.runner_parameters = {}
        runner.container_service = service.RunnerContainerService()
        runner.pre_run()
        mock_liveaction_db.context = {
            "parent": None
        }
        (status, output, _) = runner.run({})
        self.assertEqual(status, LIVEACTION_STATUS_FAILED)
        self.assertTrue(output is not None)
        self.assertEqual(output, {"response_data": {}})
        mock_trigger_dispatcher.return_value.dispatch.assert_not_called()

    def _get_mock_action_obj(self):
        action = mock.Mock()
        action.pack = SYSTEM_PACK_NAME
        return action
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from collections import namedtuple
import json

import requests
try:
    from urlparse import urljoin
    from urllib import urlencode
except ImportError:
    from urllib.parse import urljoin, urlencode

from django.conf import settings
from django.contrib.auth import login as django_login
from django.contrib.auth import logout as django_logout
from django.contrib.auth.backends import ModelBackend
from django.contrib.auth.decorators import login_required
from django.http import (
    HttpResponseBadRequest,
    HttpResponseRedirect,
    HttpResponsePermanentRedirect,
)
from django.http import HttpResponse
from django.views.generic.base import View
from django.utils.decorators import method_decorator

from itsdangerous import URLSafeTimedSerializer

from lizard_auth_client import client

# used so we can login User objects we instantiated ourselves
BACKEND = ModelBackend()

class HttpResponseServiceUnavailable(HttpResponse):
    status_code = 503

class TestHomeView(View):
    '''
    Test view, only used in standalone mode.
    Display a minimal HTML page with some URLS.
    '''
    def get(self, request, *args, **kwargs):
        user = request.user
        return HttpResponse(
            'home | protected'
            '| logout | user={} | home @ client'
            ''.format(user)
        )

class TestProtectedView(View):
    '''
    Test view, only used in standalone mode.
    Display a minimal HTML page with some URLS, but is "protected" by
    a login_required decorator.
    '''
    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(TestProtectedView, self).dispatch(
            request, *args, **kwargs)

    def get(self, request, *args, **kwargs):
        user = request.user
        return HttpResponse(
            'home | protected'
            '| logout '
            '| user={} | protected @ client'
            ''.format(user)
        )

class LoginView(View):
    '''
    View that redirects the user to the SSO server.

    Requests a Request Token and then redirects the User to the the SSO Server.
    '''
    def get(self, request, *args, **kwargs):
        # store the GET parameter named 'next', so we can redirect the user
        # to the requested page, after SSO login.
        next = get_next(request)
        request.session['sso_after_login_next'] = next

        wrapped_response = get_request_token_and_determine_response()

        if (issubclass(wrapped_response.http_response, HttpResponseRedirect) or
            issubclass(wrapped_response.http_response,
                       HttpResponsePermanentRedirect)):
            return wrapped_response.http_response(
                wrapped_response.redirect_url
            )
        else:
            return wrapped_response.http_response(
                wrapped_response.message
            )

class LocalLoginView(View):
    '''
    Verifies the user token with the SSO server, and logs the user in.
    '''
    def get(self, request, *args, **kwargs):
        # verify the authentication token and
        # retrieve the User instance from the SSO server
        user = verify_auth_token(request.GET['message'])
        if not user:
            return HttpResponseBadRequest('Verification failed')

        # link the user instance to the default database backend
        # and call django-auth's login function
        user.backend = "%s.%s" % (BACKEND.__module__,
                                  BACKEND.__class__.__name__)
        django_login(request, user)
        # redirect the user to the stored "next" url, which is probably a
        # protected page
        if 'sso_after_login_next' in request.session:
            sso_after_login_next = request.session['sso_after_login_next']
            del request.session['sso_after_login_next']
        else:
            sso_after_login_next = '/'

        return HttpResponseRedirect(sso_after_login_next)

class LogoutView(View):
    '''
    Redirect user to SSO server, to log out there.
    '''
    def get(self, request, *args, **kwargs):
        # store the 'next' parameter in the session so we can
        # redirect the user afterwards
        next = get_next(request)
        request.session['sso_after_logout_next'] = next

        url = build_sso_portal_action_url('logout')
        # send the redirect response
        return HttpResponseRedirect(url)

class LocalLogoutView(View):
    '''
    Log out locally. Mostly the user is redirected here by the SSO server.
    '''
    def get(self, request, *args, **kwargs):
        # get the redirect url
        next = request.session.get('sso_after_logout_next', '/')

        # django_logout also calls session.flush()
        django_logout(request)

        return HttpResponseRedirect(next)

def get_request_token():
    '''
    Requests a Request Token from the SSO Server. Returns False if the request
    failed.
    '''
    # construct a signed message containing the portal key
    params = {
        'key': settings.SSO_KEY
    }

    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    url = urljoin(settings.SSO_SERVER_PRIVATE_URL,
                  'sso/api/request_token') + '/'

    # send the message to the SSO server
    response = requests.get(
        url,
        params={
            'key': settings.SSO_KEY,
            'message': message
        },
        timeout=10
    )
    if response.status_code != 200:
        return False

    # grab the token from the response
    data = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        response.content, max_age=300)
    return data['request_token']

def verify_auth_token(untrusted_message):
    '''
    Verifies a Auth Token. Returns a
    django.contrib.auth.models.User instance if successful or False.
    '''
    # decrypt the message
    untrusted = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        untrusted_message, max_age=300)

    # do some extra validation
    if 'auth_token' not in untrusted:
        return False
    if 'request_token' not in untrusted:
        return False

    # call the SSO server to verify the token
    params = {
        'auth_token': untrusted['auth_token'],
        'key': settings.SSO_KEY
    }
    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    url = urljoin(settings.SSO_SERVER_PRIVATE_URL, 'sso/api/verify') + '/'
    response = requests.get(
        url,
        params={
            'key': settings.SSO_KEY,
            'message': message
        },
        timeout=10
    )

    # ensure the response is sane
    if response.status_code != 200:
        return False

    # build a User object from the message
    data = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        response.content, max_age=300)
    user_data = json.loads(data['user'])

    user = client.construct_user(user_data)

    if 'roles' in data:
        role_data = json.loads(data['roles'])
        client.synchronize_roles(user, role_data)

    return user

def get_next(request):
    '''
    Given a request, returns the URL where a user should be redirected to
    after login. Defaults to '/'.
    '''
    next = request.GET.get('next', None)
    if not next:
        return '/'
    return next

def get_request_token_and_determine_response():
    '''
    Retrieve a Request token from the SSO server, and determine the proper
    HttpResponse to send to the user.

    When logging using via the REST API, the response is wrapped in JSON,
    because the redirection takes place in a client-side script.
    '''
    WrappedResponse = namedtuple(
        'WrappedResponse', 'http_response, message, redirect_url')

    # get a request token, which is used by the SSO server to verify
    # that the user is allowed to make a login request
    request_token = get_request_token()
    if not request_token:
        # Status code 503 service (sso server) unavailable
        return WrappedResponse(HttpResponseServiceUnavailable,
                               'Unable to obtain token', None)

    # construct a (signed) set of GET parameters which are used to
    # redirect the user to the SSO server
    params = {
        'request_token': request_token,
        'key': settings.SSO_KEY,
    }
    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    query_string = urlencode([('message', message),
                              ('key', settings.SSO_KEY)])
    # build an absolute URL pointing to the SSO server out of it
    url = urljoin(settings.SSO_SERVER_PUBLIC_URL, 'sso/authorize') + '/'
    url = '%s?%s' % (url, query_string)

    return WrappedResponse(HttpResponseRedirect, 'OK', url)

def build_sso_portal_action_url(action):
    '''
    Constructs and signs a message containing the specified action parameter,
    and returns a URL which can be used to redirect the user.

    For example, with action='logout', this can be used to logout the user
    on the SSO server.
    '''
    params = {
        'action': action,
        'key': settings.SSO_KEY,
    }
    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    query_string = urlencode([('message', message),
                              ('key', settings.SSO_KEY)])
    url = urljoin(settings.SSO_SERVER_PUBLIC_URL, 'sso/portal_action') + '/'
    url = '%s?%s' % (url, query_string)
    return url
<commit_msg>Comment tweaks<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from collections import namedtuple
import json

import requests
try:
    from urlparse import urljoin
    from urllib import urlencode
except ImportError:
    from urllib.parse import urljoin, urlencode

from django.conf import settings
from django.contrib.auth import login as django_login
from django.contrib.auth import logout as django_logout
from django.contrib.auth.backends import ModelBackend
from django.contrib.auth.decorators import login_required
from django.http import (
    HttpResponseBadRequest,
    HttpResponseRedirect,
    HttpResponsePermanentRedirect,
)
from django.http import HttpResponse
from django.views.generic.base import View
from django.utils.decorators import method_decorator

from itsdangerous import URLSafeTimedSerializer

from lizard_auth_client import client

# used so we can login User objects we instantiated ourselves
BACKEND = ModelBackend()

class HttpResponseServiceUnavailable(HttpResponse):
    status_code = 503

class TestHomeView(View):
    '''
    Test view, only used in standalone mode.
    Display a minimal HTML page with some URLS.
    '''
    def get(self, request, *args, **kwargs):
        user = request.user
        return HttpResponse(
            'home | protected'
            '| logout | user={} | home @ client'
            ''.format(user)
        )

class TestProtectedView(View):
    '''
    Test view, only used in standalone mode.
    Display a minimal HTML page with some URLS, but is "protected" by
    a login_required decorator.
    '''
    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(TestProtectedView, self).dispatch(
            request, *args, **kwargs)

    def get(self, request, *args, **kwargs):
        user = request.user
        return HttpResponse(
            'home | protected'
            '| logout '
            '| user={} | protected @ client'
            ''.format(user)
        )

class LoginView(View):
    '''
    View that redirects the user to the SSO server.

    Requests a Request Token and then redirects the User to the the SSO Server.
    '''
    def get(self, request, *args, **kwargs):
        # store the GET parameter named 'next', so we can redirect the user
        # to the requested page, after SSO login.
        next = get_next(request)
        request.session['sso_after_login_next'] = next

        wrapped_response = get_request_token_and_determine_response()

        if (issubclass(wrapped_response.http_response, HttpResponseRedirect) or
            issubclass(wrapped_response.http_response,
                       HttpResponsePermanentRedirect)):
            return wrapped_response.http_response(
                wrapped_response.redirect_url
            )
        else:
            return wrapped_response.http_response(
                wrapped_response.message
            )

class LocalLoginView(View):
    '''
    Verifies the user token with the SSO server, and logs the user in.
    '''
    def get(self, request, *args, **kwargs):
        # verify the authentication token and
        # retrieve the User instance from the SSO server
        user = verify_auth_token(request.GET['message'])
        if not user:
            return HttpResponseBadRequest('Verification failed')

        # link the user instance to the default database backend
        # and call django-auth's login function
        user.backend = "%s.%s" % (BACKEND.__module__,
                                  BACKEND.__class__.__name__)
        django_login(request, user)
        # redirect the user to the stored "next" url, which is probably a
        # protected page
        if 'sso_after_login_next' in request.session:
            sso_after_login_next = request.session['sso_after_login_next']
            del request.session['sso_after_login_next']
        else:
            sso_after_login_next = '/'

        return HttpResponseRedirect(sso_after_login_next)

class LogoutView(View):
    '''
    Redirect user to SSO server, to log out there.
    '''
    def get(self, request, *args, **kwargs):
        # store the 'next' parameter in the session so we can
        # redirect the user afterwards
        next = get_next(request)
        request.session['sso_after_logout_next'] = next

        url = build_sso_portal_action_url('logout')
        # send the redirect response
        return HttpResponseRedirect(url)

class LocalLogoutView(View):
    '''
    Log out locally. Mostly the user is redirected here by the SSO server.
    '''
    def get(self, request, *args, **kwargs):
        # get the redirect url
        next = request.session.get('sso_after_logout_next', '/')

        # django_logout also calls session.flush()
        django_logout(request)

        return HttpResponseRedirect(next)

def get_request_token():
    '''
    Requests a Request Token from the SSO Server. Returns False if the request
    failed.
    '''
    # construct a signed message containing the portal key
    params = {
        'key': settings.SSO_KEY
    }

    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    url = urljoin(settings.SSO_SERVER_PRIVATE_URL,
                  'sso/api/request_token') + '/'

    # send the message to the SSO server
    response = requests.get(
        url,
        params={
            'key': settings.SSO_KEY,
            'message': message
        },
        timeout=10
    )
    if response.status_code != 200:
        return False

    # grab the token from the response
    data = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        response.content, max_age=300)
    return data['request_token']

def verify_auth_token(untrusted_message):
    '''
    Verifies a Auth Token. Returns a
    django.contrib.auth.models.User instance if successful or False.
    '''
    # decrypt the message
    untrusted = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        untrusted_message, max_age=300)

    # do some extra validation
    if 'auth_token' not in untrusted:
        return False
    if 'request_token' not in untrusted:
        return False

    # call the SSO server to verify the token
    params = {
        'auth_token': untrusted['auth_token'],
        'key': settings.SSO_KEY
    }
    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    url = urljoin(settings.SSO_SERVER_PRIVATE_URL, 'sso/api/verify') + '/'
    response = requests.get(
        url,
        params={
            'key': settings.SSO_KEY,
            'message': message
        },
        timeout=10
    )

    # ensure the response is sane
    if response.status_code != 200:
        return False

    # build a User object from the message
    data = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        response.content, max_age=300)
    user_data = json.loads(data['user'])

    user = client.construct_user(user_data)

    if 'roles' in data:
        role_data = json.loads(data['roles'])
        client.synchronize_roles(user, role_data)

    return user

def get_next(request):
    '''
    Given a request, returns the URL where a user should be redirected to
    after login. Defaults to '/'.
    '''
    next = request.GET.get('next', None)
    if not next:
        return '/'
    return next

def get_request_token_and_determine_response():
    '''
    Retrieve a Request token from the SSO server, and determine the proper
    HttpResponse to send to the user.

    When logging using via the REST API, the response is wrapped in JSON,
    because the redirection takes place in a client-side script.
    '''
    WrappedResponse = namedtuple(
        'WrappedResponse', 'http_response, message, redirect_url')

    # Get a request token, which is used by the SSO server to verify
    # that the user is allowed to make a login request
    request_token = get_request_token()
    if not request_token:
        # Status code 503 service (sso server) unavailable
        return WrappedResponse(HttpResponseServiceUnavailable,
                               'Unable to obtain token', None)

    # Construct a (signed) set of GET parameters which are passed in the
    # redirect URL that goes to the SSO server.
    params = {
        'request_token': request_token,
        'key': settings.SSO_KEY,
    }
    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    query_string = urlencode([('message', message),
                              ('key', settings.SSO_KEY)])
    # Build an absolute URL pointing to the SSO server out of it.
    url = urljoin(settings.SSO_SERVER_PUBLIC_URL, 'sso/authorize') + '/'
    url = '%s?%s' % (url, query_string)

    return WrappedResponse(HttpResponseRedirect, 'OK', url)

def build_sso_portal_action_url(action):
    '''
    Constructs and signs a message containing the specified action parameter,
    and returns a URL which can be used to redirect the user.

    For example, with action='logout', this can be used to logout the user
    on the SSO server.
    '''
    params = {
        'action': action,
        'key': settings.SSO_KEY,
    }
    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    query_string = urlencode([('message', message),
                              ('key', settings.SSO_KEY)])
    url = urljoin(settings.SSO_SERVER_PUBLIC_URL, 'sso/portal_action') + '/'
    url = '%s?%s' % (url, query_string)
    return url
<|endoftext|>


<commit_before>from bs4 import BeautifulSoup
import urllib.request

def read(page):
    """
    Reads the main content from a Reuters page
    """
    soup = BeautifulSoup(page, "html.parser")
    text = soup.find(id = "articleText")
    return " ".join(text.strings)

def main():
    page = urllib.request.urlopen("http://www.reuters.com/article/us-usa-election-democrats-idUSKCN0UV058?feedType=RSS&feedName=topNews")
    read(page)

if __name__ == "__main__":
    main()
<commit_msg>IO for python Reuters<commit_after>from bs4 import BeautifulSoup
import urllib.request

def read():
    """
    Reads the main content from a Reuters page
    """
    url = input()
    page = urllib.request.urlopen(url)
    soup = BeautifulSoup(page, "html.parser")
    text = soup.find(id = "articleText")
    print(" ".join(text.strings))

def main():
    read()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import cherrypy

from girder.api import access
from girder.api.v1.item import Item
from girder.api.describe import Description
from girder.api.rest import loadmodel, RestException
from girder.models.model_base import AccessType

from .tilesource import TestTileSource, TiffGirderTileSource, TileSourceException

class TilesItemResource(Item):

    def __init__(self, apiRoot):
        self.resourceName = 'item'
        apiRoot.item.route('GET', (':itemId', 'tiles'), self.getTilesInfo)
        apiRoot.item.route('POST', (':itemId', 'tiles'), self.createTiles)
        apiRoot.item.route('DELETE', (':itemId', 'tiles'), self.deleteTiles)
        apiRoot.item.route('GET', (':itemId', 'tiles', 'zxy', ':z', ':x', ':y'),
                           self.getTile)

    def _loadTileSource(self, itemId):
        try:
            if itemId == 'test':
                tileSource = TestTileSource(256)
            else:
                item = self.model('item').load(id=itemId, level=AccessType.READ,
                                               user=self.getCurrentUser(), exc=True)
                tileSource = TiffGirderTileSource(item)
            return tileSource
        except TileSourceException as e:
            # TODO: sometimes this could be 400
            raise RestException(e.message, code=500)

    @access.public
    def getTilesInfo(self, itemId, params):
        tileSource = self._loadTileSource(itemId)
        return tileSource.getMetadata()

    getTilesInfo.description = (
        Description('Get multiresolution tiles metadata.')
        .param('itemId', 'The ID of the item or "test".', paramType='path')
        .errorResponse('ID was invalid.')
        .errorResponse('Read access was denied for the item.', 403)
    )

    @access.user
    @loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.WRITE)
    def createTiles(self, item, params):
        largeImageFileId = params.get('fileId')
        if not largeImageFileId:
            raise RestException('Missing "fileId" parameter.')
        if largeImageFileId != 'test':
            largeImageFile = self.model('file').load(largeImageFileId,
                                                     force=True, exc=True)
            if largeImageFile['itemId'] != item['_id']:
                raise RestException('"fileId" must be a file on the same item as "itemId".')

            item['largeImage'] = largeImageFile['_id']
        else:
            item['largeImage'] = largeImageFileId
        self.model('item').save(item)

        # TODO: a better response
        return {
            'created': True
        }

    createTiles.description = (
        Description('Create a multiresolution image for this item.')
        .param('itemId', 'The ID of the item.', paramType='path')
        .param('fileId', 'The ID of the source file containing the image or '
               '"test".')
    )

    @access.user
    @loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.WRITE)
    def deleteTiles(self, item, params):
        deleted = False
        if 'largeImage' in item:
            del item['largeImage']
            self.model('item').save(item)
            deleted = True
        # TODO: a better response
        return {
            'deleted': deleted
        }

    deleteTiles.description = (
        Description('Remove a multiresolution image from this item.')
        .param('itemId', 'The ID of the item.', paramType='path')
    )

    @access.public
    def getTile(self, itemId, z, x, y, params):
        try:
            x, y, z = int(x), int(y), int(z)
        except ValueError:
            raise RestException('x, y, and z must be integers', code=400)

        tileSource = self._loadTileSource(itemId)
        try:
            tileData = tileSource.getTile(x, y, z)
        except TileSourceException as e:
            raise RestException(e.message, code=404)

        cherrypy.response.headers['Content-Type'] = 'image/jpeg'
        return lambda: tileData

    getTile.cookieAuth = True
    getTile.description = (
        Description('Get an image tile.')
        .param('itemId', 'The ID of the item or "test".', paramType='path')
        .param('z', 'The layer number of the tile (0 is the most zoomed-out layer).', paramType='path')
        .param('x', 'The X coordinate of the tile (0 is the left side).', paramType='path')
        .param('y', 'The Y coordinate of the tile (0 is the top).', paramType='path')
        .errorResponse('ID was invalid.')
        .errorResponse('Read access was denied for the item.', 403)
    )
<commit_msg>Call the Resource constructor<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import cherrypy

from girder.api import access
from girder.api.v1.item import Item
from girder.api.describe import Description
from girder.api.rest import loadmodel, RestException
from girder.models.model_base import AccessType

from .tilesource import TestTileSource, TiffGirderTileSource, TileSourceException

class TilesItemResource(Item):

    def __init__(self, apiRoot):
        # Don't call the parent (Item) constructor, to avoid redefining routes,
        # but do call the grandparent (Resource) constructor
        super(Item, self).__init__()

        self.resourceName = 'item'
        apiRoot.item.route('GET', (':itemId', 'tiles'), self.getTilesInfo)
        apiRoot.item.route('POST', (':itemId', 'tiles'), self.createTiles)
        apiRoot.item.route('DELETE', (':itemId', 'tiles'), self.deleteTiles)
        apiRoot.item.route('GET', (':itemId', 'tiles', 'zxy', ':z', ':x', ':y'),
                           self.getTile)

    def _loadTileSource(self, itemId):
        try:
            if itemId == 'test':
                tileSource = TestTileSource(256)
            else:
                item = self.model('item').load(id=itemId, level=AccessType.READ,
                                               user=self.getCurrentUser(), exc=True)
                tileSource = TiffGirderTileSource(item)
            return tileSource
        except TileSourceException as e:
            # TODO: sometimes this could be 400
            raise RestException(e.message, code=500)

    @access.public
    def getTilesInfo(self, itemId, params):
        tileSource = self._loadTileSource(itemId)
        return tileSource.getMetadata()

    getTilesInfo.description = (
        Description('Get multiresolution tiles metadata.')
        .param('itemId', 'The ID of the item or "test".', paramType='path')
        .errorResponse('ID was invalid.')
        .errorResponse('Read access was denied for the item.', 403)
    )

    @access.user
    @loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.WRITE)
    def createTiles(self, item, params):
        largeImageFileId = params.get('fileId')
        if not largeImageFileId:
            raise RestException('Missing "fileId" parameter.')
        if largeImageFileId != 'test':
            largeImageFile = self.model('file').load(largeImageFileId,
                                                     force=True, exc=True)
            if largeImageFile['itemId'] != item['_id']:
                raise RestException('"fileId" must be a file on the same item as "itemId".')

            item['largeImage'] = largeImageFile['_id']
        else:
            item['largeImage'] = largeImageFileId
        self.model('item').save(item)

        # TODO: a better response
        return {
            'created': True
        }

    createTiles.description = (
        Description('Create a multiresolution image for this item.')
        .param('itemId', 'The ID of the item.', paramType='path')
        .param('fileId', 'The ID of the source file containing the image or '
               '"test".')
    )

    @access.user
    @loadmodel(model='item', map={'itemId': 'item'}, level=AccessType.WRITE)
    def deleteTiles(self, item, params):
        deleted = False
        if 'largeImage' in item:
            del item['largeImage']
            self.model('item').save(item)
            deleted = True
        # TODO: a better response
        return {
            'deleted': deleted
        }

    deleteTiles.description = (
        Description('Remove a multiresolution image from this item.')
        .param('itemId', 'The ID of the item.', paramType='path')
    )

    @access.public
    def getTile(self, itemId, z, x, y, params):
        try:
            x, y, z = int(x), int(y), int(z)
        except ValueError:
            raise RestException('x, y, and z must be integers', code=400)

        tileSource = self._loadTileSource(itemId)
        try:
            tileData = tileSource.getTile(x, y, z)
        except TileSourceException as e:
            raise RestException(e.message, code=404)

        cherrypy.response.headers['Content-Type'] = 'image/jpeg'
        return lambda: tileData

    getTile.cookieAuth = True
    getTile.description = (
        Description('Get an image tile.')
        .param('itemId', 'The ID of the item or "test".', paramType='path')
        .param('z', 'The layer number of the tile (0 is the most zoomed-out layer).', paramType='path')
        .param('x', 'The X coordinate of the tile (0 is the left side).', paramType='path')
        .param('y', 'The Y coordinate of the tile (0 is the top).', paramType='path')
        .errorResponse('ID was invalid.')
        .errorResponse('Read access was denied for the item.', 403)
    )
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2011 Nicira Networks, Inc
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from nova import db
from nova import exception
from nova import flags
from nova import log as logging
from nova import manager
from nova import utils
from nova.network import manager
from nova.network.quantum import quantum_connection
from nova.network.quantum import fake

LOG = logging.getLogger("quantum_manager")

FLAGS = flags.FLAGS

flags.DEFINE_string('quantum_ipam_lib',
                    'nova.network.quantum.nova_ipam_lib',
                    "Indicates underlying IP address management library")

class QuantumManager(manager.FlatManager):

    def __init__(self, ipam_lib=None, *args, **kwargs):

        if FLAGS.fake_network:
            self.q_conn = fake.FakeQuantumClientConnection()
        else:
            self.q_conn = quantum_connection.QuantumClientConnection()

        if not ipam_lib:
            ipam_lib = FLAGS.quantum_ipam_lib
        self.ipam = utils.import_object(ipam_lib).get_ipam_lib(self)

        super(QuantumManager, self).__init__(*args, **kwargs)

    def create_networks(self, context, label, cidr, multi_host, num_networks,
                        network_size, cidr_v6, gateway_v6, bridge,
                        bridge_interface, dns1=None, dns2=None, **kwargs):
        if num_networks != 1:
            raise Exception("QuantumManager requires that only one"
                            " network is created per call")
        q_tenant_id = kwargs["project_id"] or \
                            FLAGS.quantum_default_tenant_id
        quantum_net_id = bridge
        if quantum_net_id:
            if not q_conn.network_exists(q_tenant_id, quantum_net_id):
                    raise Exception("Unable to find existing quantum " \
                        " network for tenant '%s' with net-id '%s'" % \
                         (q_tenant_id, quantum_net_id))
        else:
            # otherwise, create network from default quantum pool
            quantum_net_id = self.q_conn.create_network(q_tenant_id, label)

        ipam_tenant_id = kwargs.get("project_id", None)
        self.ipam.create_subnet(context, label, ipam_tenant_id, quantum_net_id,
                            cidr, gateway_v6, cidr_v6, dns1, dns2)

    def delete_network(self, context, fixed_range):
            project_id = context.project_id
            quantum_net_id = self.ipam.get_network_id_by_cidr(
                                    context, fixed_range, project_id)
            self.ipam.delete_subnets_by_net_id(context, quantum_net_id,
                                                            project_id)
            try:
                q_tenant_id = project_id or FLAGS.quantum_default_tenant_id
                self.q_conn.delete_network(q_tenant_id, quantum_net_id)
            except Exception, e:
                raise Exception("Unable to delete Quantum Network with "
                    "fixed_range = %s (ports still in use?)." % fixed_range)

    def allocate_for_instance(self, context, **kwargs):
        instance_id = kwargs.pop('instance_id')
        instance_type_id = kwargs['instance_type_id']
        host = kwargs.pop('host')
        project_id = kwargs.pop('project_id')
        LOG.debug(_("network allocations for instance %s"), instance_id)

        # if using the create-server-networks extension, 'requested_networks'
        # will be defined, otherwise, just grab relevant nets from IPAM
        requested_networks = kwargs.get('requested_networks')

        if requested_networks:
            net_proj_pairs = [(net_id, project_id) \
                for (net_id, _i) in requested_networks]
        else:
            net_proj_pairs = self.ipam.get_project_and_global_net_ids(context,
                                                                project_id)

        # Create a port via quantum and attach the vif
        for (net_id, project_id) in net_proj_pairs:
            vif_rec = manager.FlatManager.add_virtual_interface(self,
                                      context, instance, None)

            q_tenant_id = project_id or FLAGS.quantum_default_tenant_id
            self.q_conn.create_and_attach_port(q_tenant_id, net_id,
                                               vif_rec['uuid'])
            self.ipam.allocate_fixed_ip(context, project_id, net_id, vif_rec)

        return self.get_instance_nw_info(context, instance_id,
                                            instance_type_id, host)

    def get_instance_nw_info(self, context, instance_id,
                                instance_type_id, host):
        network_info = []
        project_id = context.project_id

        admin_context = context.elevated()
        vifs = db.virtual_interface_get_by_instance(admin_context,
                                                    instance_id)
        for vif in vifs:
            q_tenant_id = project_id
            ipam_tenant_id = project_id
            net_id, port_id = self.q_conn.get_port_by_attachment(q_tenant_id,
                                                                 vif['uuid'])
            if not net_id:
                q_tenant_id = FLAGS.quantum_default_tenant_id
                ipam_tenant_id = None
                net_id, port_id = self.q_conn.get_port_by_attachment(
                                             q_tenant_id, vif['uuid'])
            if not net_id:
                raise Exception(_("No network for for virtual interface %s") %\
                                        vif['uuid'])
            (v4_subnet, v6_subnet) = self.ipam.get_subnets_by_net_id(context,
                                        ipam_tenant_id, net_id)
            v4_ips = self.ipam.get_v4_ips_by_interface(context,
                                        net_id, vif['uuid'],
                                        project_id=ipam_tenant_id)
            v6_ips = self.ipam.get_v6_ips_by_interface(context,
                                        net_id, vif['uuid'],
                                        project_id=ipam_tenant_id)

            quantum_net_id = v4_subnet['network_id'] or v6_subnet['network_id']

            def ip_dict(ip, subnet):
                return {
                    "ip": ip,
                    "netmask": subnet["netmask"],
                    "enabled": "1"}

            network_dict = {
                'cidr': v4_subnet['cidr'],
                'injected': True,
                'multi_host': False}

            info = {
                'gateway': v4_subnet['gateway'],
                'dhcp_server': v4_subnet['gateway'],
                'broadcast': v4_subnet['broadcast'],
                'mac': vif['address'],
                'vif_uuid': vif['uuid'],
                'dns': [],
                'ips': [ip_dict(ip, v4_subnet) for ip in v4_ips]}

            if v6_subnet['cidr']:
                network_dict['cidr_v6'] = v6_subnet['cidr']
                info['ip6s'] = [ip_dict(ip, v6_subnet) for ip in v6_ips]
            # TODO(tr3buchet): handle ip6 routes here as well
            if v6_subnet['gateway']:
                info['gateway6'] = v6_subnet['gateway']

            dns_dict = {}
            for s in [v4_subnet, v6_subnet]:
                for k in ['dns1', 'dns2']:
                    if s[k]:
                        dns_dict[s[k]] = None
            info['dns'] = [d for d in dns_dict.keys()]

            network_info.append((network_dict, info))
        return network_info

    def deallocate_for_instance(self, context, **kwargs):
        instance_id = kwargs.get('instance_id')
        project_id = kwargs.pop('project_id', None)

        admin_context = context.elevated()
        vifs = db.virtual_interface_get_by_instance(admin_context,
                                                    instance_id)
        for vif_ref in vifs:
            interface_id = vif_ref['uuid']
            q_tenant_id = project_id
            ipam_tenant_id = project_id
            (net_id, port_id) = self.q_conn.get_port_by_attachment(q_tenant_id,
                                            interface_id)
            if not net_id:
                q_tenant_id = FLAGS.quantum_default_tenant_id
                ipam_tenant_id = None
                (net_id, port_id) = self.q_conn.get_port_by_attachment(
                                        q_tenant_id, interface_id)
            if not net_id:
                LOG.error("Unable to find port with attachment: %s" % \
                                                        (interface_id))
                continue
            self.q_conn.detach_and_delete_port(q_tenant_id,
                                                   net_id, port_id)

            self.ipam.deallocate_ips_by_vif(context, ipam_tenant_id,
                                                    net_id, vif_ref)

        self.net_manager.db.virtual_interface_delete_by_instance(admin_context,
                                                            instance_id)
        self._do_trigger_security_group_members_refresh_for_instance(
                                                                   instance_id)

    # validates that this tenant has quantum networks with the associated
    # UUIDs.  This is called by the 'os-create-server-ext' API extension
    # code so that we can return an API error code to the caller if they
    # request an invalid network.
    def validate_networks(self, context, networks):
        if networks is None:
            return

        project_id = context.project_id
        for (net_id, _i) in networks:
            self.ipam.verify_subnet_exists(context, project_id, net_id)
            if not self.q_conn.network_exists(project_id, net_id):
                raise exception.NetworkNotFound(network_id=net_id)
<commit_msg>Minor fixes<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2011 Nicira Networks, Inc
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from nova import db
from nova import exception
from nova import flags
from nova import log as logging
from nova import manager
from nova import utils
from nova.network import manager
from nova.network.quantum import quantum_connection
from nova.network.quantum import fake

LOG = logging.getLogger("quantum_manager")

FLAGS = flags.FLAGS

flags.DEFINE_string('quantum_ipam_lib',
                    'nova.network.quantum.nova_ipam_lib',
                    "Indicates underlying IP address management library")

class QuantumManager(manager.FlatManager):

    def __init__(self, ipam_lib=None, *args, **kwargs):

        if FLAGS.fake_network:
            self.q_conn = fake.FakeQuantumClientConnection()
        else:
            self.q_conn = quantum_connection.QuantumClientConnection()

        if not ipam_lib:
            ipam_lib = FLAGS.quantum_ipam_lib
        self.ipam = utils.import_object(ipam_lib).get_ipam_lib(self)

        super(QuantumManager, self).__init__(*args, **kwargs)

    def create_networks(self, context, label, cidr, multi_host, num_networks,
                        network_size, cidr_v6, gateway_v6, bridge,
                        bridge_interface, dns1=None, dns2=None, **kwargs):
        if num_networks != 1:
            raise Exception("QuantumManager requires that only one"
                            " network is created per call")
        q_tenant_id = kwargs["project_id"] or \
                            FLAGS.quantum_default_tenant_id
        quantum_net_id = bridge
        if quantum_net_id:
            if not self.q_conn.network_exists(q_tenant_id, quantum_net_id):
                    raise Exception("Unable to find existing quantum " \
                        " network for tenant '%s' with net-id '%s'" % \
                         (q_tenant_id, quantum_net_id))
        else:
            # otherwise, create network from default quantum pool
            quantum_net_id = self.q_conn.create_network(q_tenant_id, label)

        ipam_tenant_id = kwargs.get("project_id", None)
        self.ipam.create_subnet(context, label, ipam_tenant_id, quantum_net_id,
                            cidr, gateway_v6, cidr_v6, dns1, dns2)

    def delete_network(self, context, fixed_range):
            project_id = context.project_id
            quantum_net_id = self.ipam.get_network_id_by_cidr(
                                    context, fixed_range, project_id)
            self.ipam.delete_subnets_by_net_id(context, quantum_net_id,
                                                            project_id)
            try:
                q_tenant_id = project_id or FLAGS.quantum_default_tenant_id
                self.q_conn.delete_network(q_tenant_id, quantum_net_id)
            except Exception, e:
                raise Exception("Unable to delete Quantum Network with "
                    "fixed_range = %s (ports still in use?)." % fixed_range)

    def allocate_for_instance(self, context, **kwargs):
        instance_id = kwargs.pop('instance_id')
        instance_type_id = kwargs['instance_type_id']
        host = kwargs.pop('host')
        project_id = kwargs.pop('project_id')
        LOG.debug(_("network allocations for instance %s"), instance_id)

        # if using the create-server-networks extension, 'requested_networks'
        # will be defined, otherwise, just grab relevant nets from IPAM
        requested_networks = kwargs.get('requested_networks')

        if requested_networks:
            net_proj_pairs = [(net_id, project_id) \
                for (net_id, _i) in requested_networks]
        else:
            net_proj_pairs = self.ipam.get_project_and_global_net_ids(context,
                                                                project_id)

        # Create a port via quantum and attach the vif
        for (net_id, project_id) in net_proj_pairs:
            vif_rec = manager.FlatManager.add_virtual_interface(self,
                                      context, instance_id, None)

            q_tenant_id = project_id or FLAGS.quantum_default_tenant_id
            self.q_conn.create_and_attach_port(q_tenant_id, net_id,
                                               vif_rec['uuid'])
            self.ipam.allocate_fixed_ip(context, project_id, net_id, vif_rec)

        return self.get_instance_nw_info(context, instance_id,
                                            instance_type_id, host)

    def get_instance_nw_info(self, context, instance_id,
                                instance_type_id, host):
        network_info = []
        project_id = context.project_id

        admin_context = context.elevated()
        vifs = db.virtual_interface_get_by_instance(admin_context,
                                                    instance_id)
        for vif in vifs:
            q_tenant_id = project_id
            ipam_tenant_id = project_id
            net_id, port_id = self.q_conn.get_port_by_attachment(q_tenant_id,
                                                                 vif['uuid'])
            if not net_id:
                q_tenant_id = FLAGS.quantum_default_tenant_id
                ipam_tenant_id = None
                net_id, port_id = self.q_conn.get_port_by_attachment(
                                             q_tenant_id, vif['uuid'])
            if not net_id:
                raise Exception(_("No network for for virtual interface %s") %\
                                        vif['uuid'])
            (v4_subnet, v6_subnet) = self.ipam.get_subnets_by_net_id(context,
                                        ipam_tenant_id, net_id)
            v4_ips = self.ipam.get_v4_ips_by_interface(context,
                                        net_id, vif['uuid'],
                                        project_id=ipam_tenant_id)
            v6_ips = self.ipam.get_v6_ips_by_interface(context,
                                        net_id, vif['uuid'],
                                        project_id=ipam_tenant_id)

            quantum_net_id = v4_subnet['network_id'] or v6_subnet['network_id']

            def ip_dict(ip, subnet):
                return {
                    "ip": ip,
                    "netmask": subnet["netmask"],
                    "enabled": "1"}

            network_dict = {
                'cidr': v4_subnet['cidr'],
                'injected': True,
                'multi_host': False}

            info = {
                'gateway': v4_subnet['gateway'],
                'dhcp_server': v4_subnet['gateway'],
                'broadcast': v4_subnet['broadcast'],
                'mac': vif['address'],
                'vif_uuid': vif['uuid'],
                'dns': [],
                'ips': [ip_dict(ip, v4_subnet) for ip in v4_ips]}

            if v6_subnet['cidr']:
                network_dict['cidr_v6'] = v6_subnet['cidr']
                info['ip6s'] = [ip_dict(ip, v6_subnet) for ip in v6_ips]
            # TODO(tr3buchet): handle ip6 routes here as well
            if v6_subnet['gateway']:
                info['gateway6'] = v6_subnet['gateway']

            dns_dict = {}
            for s in [v4_subnet, v6_subnet]:
                for k in ['dns1', 'dns2']:
                    if s[k]:
                        dns_dict[s[k]] = None
            info['dns'] = [d for d in dns_dict.keys()]

            network_info.append((network_dict, info))
        return network_info

    def deallocate_for_instance(self, context, **kwargs):
        instance_id = kwargs.get('instance_id')
        project_id = kwargs.pop('project_id', None)

        admin_context = context.elevated()
        vifs = db.virtual_interface_get_by_instance(admin_context,
                                                    instance_id)
        for vif_ref in vifs:
            interface_id = vif_ref['uuid']
            q_tenant_id = project_id
            ipam_tenant_id = project_id
            (net_id, port_id) = self.q_conn.get_port_by_attachment(q_tenant_id,
                                            interface_id)
            if not net_id:
                q_tenant_id = FLAGS.quantum_default_tenant_id
                ipam_tenant_id = None
                (net_id, port_id) = self.q_conn.get_port_by_attachment(
                                        q_tenant_id, interface_id)
            if not net_id:
                LOG.error("Unable to find port with attachment: %s" % \
                                                        (interface_id))
                continue
            self.q_conn.detach_and_delete_port(q_tenant_id,
                                                   net_id, port_id)

            self.ipam.deallocate_ips_by_vif(context, ipam_tenant_id,
                                                    net_id, vif_ref)

        self.net_manager.db.virtual_interface_delete_by_instance(admin_context,
                                                            instance_id)
        self._do_trigger_security_group_members_refresh_for_instance(
                                                                   instance_id)

    # validates that this tenant has quantum networks with the associated
    # UUIDs.  This is called by the 'os-create-server-ext' API extension
    # code so that we can return an API error code to the caller if they
    # request an invalid network.
    def validate_networks(self, context, networks):
        if networks is None:
            return

        project_id = context.project_id
        for (net_id, _i) in networks:
            self.ipam.verify_subnet_exists(context, project_id, net_id)
            if not self.q_conn.network_exists(project_id, net_id):
                raise exception.NetworkNotFound(network_id=net_id)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import

import json
import logging
import socket
from copy import deepcopy
from urllib import urlencode
from datetime import timedelta

import requests
from tornado import gen, queues
from tornado.ioloop import IOLoop
from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPError
from concurrent.futures import TimeoutError

from blackgate.core import component

logger = logging.getLogger(__name__)

class Command(object):

    def __init__(self, request, proxy):
        self.proxy = proxy
        self.request = request
        self.response = {}

    @gen.coroutine
    def async(self):
        client = AsyncHTTPClient()
        self.before_request()
        request = HTTPRequest(
            url='%s?%s' % (self.request['url'], urlencode(self.request['params'])),
            method=self.request['method'].upper(),
            headers=self.request['headers'],
            body=self.request['data'] if self.request['method'].upper() != 'GET' else None,
            connect_timeout=self.options.get('connect_timeout_seconds'),
            request_timeout=self.options.get('timeout_seconds'),
        )
        try:
            resp = yield client.fetch(request)
            self.response = dict(
                status_code=resp.code,
                reason=resp.reason,
                headers=dict(resp.headers),
                content=resp.body,
            )
        except HTTPError as error:
            self.response = dict(
                status_code=error.code,
                reason=error.response.reason,
                headers=dict(error.response.headers),
                content=resp.response.body,
            )
        self.after_response()
        raise gen.Return(self.response)

    def fallback(self, reason=''):
        return dict(
            status_code=502,
            reason='Bad Gateway',
            headers={
                'Content-Type': 'application/json',
            },
            content=json.dumps({
                'code': 502,
                'message': str(reason) or 'gateway is using fallback mechanism',
                'errors': []
            })
        )

    @property
    def options(self):
        return dict(
            group_key=self.proxy['name'],
            command_key='proxy',
            timeout_seconds=self.proxy.get('timeout_seconds') or 30,
            connect_timeout_seconds=self.proxy.get('connect_timeout_seconds') or 3,
            timeout_enabled=bool(self.proxy.get('timeout_enabled')),
       )

    @property
    def circuit_beaker(self):
        return component.get_circuit_beaker(
            self.options.get('command_key'),
            self.options.get('group_key'),
            component.circuit_beaker_impl,
            **component.circuit_beaker_options
        )

    @gen.coroutine
    def queue(self):
        # TODO: implement cache machenism.
        circuit_beaker = self.circuit_beaker
        group_key = self.options.get('group_key')
        executor = component.pools.get_executor(group_key)

        if not circuit_beaker.allow_request():
            result = self.fallback('gateway reject due to circuit beaker open.')
            logger.error('type: circuit_beaker_reject')
            raise gen.Return(result)

        try:
            result = yield executor.submit(self.async)
            circuit_beaker.mark_success() # FIXME: this should be async, possibly network error

        except socket.error:
            result = self.fallback('gateway failed to connect upstream.')
            circuit_beaker.mark_timeout()
            logger.error('type: socket_error')

        except queues.QueueFull:
            result = self.fallback('gateway reject due to too many connections.')
            circuit_beaker.mark_reject()
            logger.error('type: pool_full')

        except gen.TimeoutError:
            result = self.fallback('gateway timeout on fetching upstream.')
            circuit_beaker.mark_timeout()
            logger.error('type: execution_timeout')

        # FIXME: catch more exceptions: submit failure, wrong timeout type, mark_success run failed.

        except Exception as exception: # FIXME: only network error should fallback, others crash it??
            # XXX: use some error aggreation system like sentry to report errors???
            result = self.fallback()
            circuit_beaker.mark_failure()
            import traceback
            traceback.print_exc()

        raise gen.Return(result)

    def before_request(self):
        return

    def after_response(self):
        return
<commit_msg>remove useless imports.<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import

import json
import logging
import socket
from urllib import urlencode

from tornado import gen, queues
from tornado.ioloop import IOLoop
from tornado.httpclient import AsyncHTTPClient, HTTPRequest, HTTPError

from blackgate.core import component

logger = logging.getLogger(__name__)

class Command(object):

    def __init__(self, request, proxy):
        self.proxy = proxy
        self.request = request
        self.response = {}

    @gen.coroutine
    def async(self):
        client = AsyncHTTPClient()
        self.before_request()
        request = HTTPRequest(
            url='%s?%s' % (self.request['url'], urlencode(self.request['params'])),
            method=self.request['method'].upper(),
            headers=self.request['headers'],
            body=self.request['data'] if self.request['method'].upper() != 'GET' else None,
            connect_timeout=self.options.get('connect_timeout_seconds'),
            request_timeout=self.options.get('timeout_seconds'),
        )
        try:
            resp = yield client.fetch(request)
            self.response = dict(
                status_code=resp.code,
                reason=resp.reason,
                headers=dict(resp.headers),
                content=resp.body,
            )
        except HTTPError as error:
            self.response = dict(
                status_code=error.code,
                reason=error.response.reason,
                headers=dict(error.response.headers),
                content=resp.response.body,
            )
        self.after_response()
        raise gen.Return(self.response)

    def fallback(self, reason=''):
        return dict(
            status_code=502,
            reason='Bad Gateway',
            headers={
                'Content-Type': 'application/json',
            },
            content=json.dumps({
                'code': 502,
                'message': str(reason) or 'gateway is using fallback mechanism',
                'errors': []
            })
        )

    @property
    def options(self):
        return dict(
            group_key=self.proxy['name'],
            command_key='proxy',
            timeout_seconds=self.proxy.get('timeout_seconds') or 30,
            connect_timeout_seconds=self.proxy.get('connect_timeout_seconds') or 3,
            timeout_enabled=bool(self.proxy.get('timeout_enabled')),
       )

    @property
    def circuit_beaker(self):
        return component.get_circuit_beaker(
            self.options.get('command_key'),
            self.options.get('group_key'),
            component.circuit_beaker_impl,
            **component.circuit_beaker_options
        )

    @gen.coroutine
    def queue(self):
        # TODO: implement cache machenism.
        circuit_beaker = self.circuit_beaker
        group_key = self.options.get('group_key')
        executor = component.pools.get_executor(group_key)

        if not circuit_beaker.allow_request():
            result = self.fallback('gateway reject due to circuit beaker open.')
            logger.error('type: circuit_beaker_reject')
            raise gen.Return(result)

        try:
            result = yield executor.submit(self.async)
            circuit_beaker.mark_success() # FIXME: this should be async, possibly network error

        except socket.error:
            result = self.fallback('gateway failed to connect upstream.')
            circuit_beaker.mark_timeout()
            logger.error('type: socket_error')

        except queues.QueueFull:
            result = self.fallback('gateway reject due to too many connections.')
            circuit_beaker.mark_reject()
            logger.error('type: pool_full')

        except gen.TimeoutError:
            result = self.fallback('gateway timeout on fetching upstream.')
            circuit_beaker.mark_timeout()
            logger.error('type: execution_timeout')

        # FIXME: catch more exceptions: submit failure, wrong timeout type, mark_success run failed.

        except Exception as exception: # FIXME: only network error should fallback, others crash it??
            # XXX: use some error aggreation system like sentry to report errors???
            result = self.fallback()
            circuit_beaker.mark_failure()
            import traceback
            traceback.print_exc()

        raise gen.Return(result)

    def before_request(self):
        return

    def after_response(self):
        return
<|endoftext|>