<commit_before># Copyright 2012 OpenStack Foundation
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from nova.api.openstack import wsgi
from nova import availability_zones
from nova.compute import api as compute
import nova.conf
from nova.policies import availability_zone as az_policies
from nova import servicegroup

CONF = nova.conf.CONF
ATTRIBUTE_NAME = "availability_zone"

class AvailabilityZoneController(wsgi.Controller):
    """The Availability Zone API controller for the OpenStack API."""

    def __init__(self):
        super(AvailabilityZoneController, self).__init__()
        self.servicegroup_api = servicegroup.API()
        self.host_api = compute.HostAPI()

    def _get_filtered_availability_zones(self, zones, is_available):
        result = []
        for zone in zones:
            # Hide internal_service_availability_zone
            if zone == CONF.internal_service_availability_zone:
                continue
            result.append({'zoneName': zone,
                           'zoneState': {'available': is_available},
                           "hosts": None})
        return result

    def _describe_availability_zones(self, context, **kwargs):
        ctxt = context.elevated()
        available_zones, not_available_zones = (
            availability_zones.get_availability_zones(
                ctxt, self.host_api))

        filtered_available_zones = \
            self._get_filtered_availability_zones(available_zones, True)
        filtered_not_available_zones = \
            self._get_filtered_availability_zones(not_available_zones, False)
        return {'availabilityZoneInfo': filtered_available_zones +
                                        filtered_not_available_zones}

    def _describe_availability_zones_verbose(self, context, **kwargs):
        ctxt = context.elevated()

        services = self.host_api.service_get_all(
            context, set_zones=True, all_cells=True)
        available_zones, not_available_zones = (
            availability_zones.get_availability_zones(
                ctxt, self.host_api, services=services))

        zone_hosts = {}
        host_services = {}
        api_services = ('nova-osapi_compute', 'nova-metadata')
        for service in filter(lambda x: not x.disabled, services):
            if service.binary in api_services:
                # Skip API services in the listing since they are not
                # maintained in the same way as other services
                continue
            zone_hosts.setdefault(service['availability_zone'], set())
            zone_hosts[service['availability_zone']].add(service['host'])

            host_services.setdefault(service['availability_zone'] +
                    service['host'], [])
            host_services[service['availability_zone'] + service['host']].\
                    append(service)

        result = []
        for zone in available_zones:
            hosts = {}
            for host in zone_hosts.get(zone, []):
                hosts[host] = {}
                for service in host_services[zone + host]:
                    alive = self.servicegroup_api.service_is_up(service)
                    hosts[host][service['binary']] = {
                        'available': alive,
                        'active': service['disabled'] is not True,
                        'updated_at': service['updated_at']
                    }
            result.append({'zoneName': zone,
                           'zoneState': {'available': True},
                           "hosts": hosts})

        for zone in not_available_zones:
            result.append({'zoneName': zone,
                           'zoneState': {'available': False},
                           "hosts": None})
        return {'availabilityZoneInfo': result}

    @wsgi.expected_errors(())
    def index(self, req):
        """Returns a summary list of availability zone."""
        context = req.environ['nova.context']
        context.can(az_policies.POLICY_ROOT % 'list')

        return self._describe_availability_zones(context)

    @wsgi.expected_errors(())
    def detail(self, req):
        """Returns a detailed list of availability zone."""
        context = req.environ['nova.context']
        context.can(az_policies.POLICY_ROOT % 'detail')

        return self._describe_availability_zones_verbose(context)
<commit_msg>Pass the actual target in os-availability-zone policy<commit_after># Copyright 2012 OpenStack Foundation
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from nova.api.openstack import wsgi
from nova import availability_zones
from nova.compute import api as compute
import nova.conf
from nova.policies import availability_zone as az_policies
from nova import servicegroup

CONF = nova.conf.CONF
ATTRIBUTE_NAME = "availability_zone"

class AvailabilityZoneController(wsgi.Controller):
    """The Availability Zone API controller for the OpenStack API."""

    def __init__(self):
        super(AvailabilityZoneController, self).__init__()
        self.servicegroup_api = servicegroup.API()
        self.host_api = compute.HostAPI()

    def _get_filtered_availability_zones(self, zones, is_available):
        result = []
        for zone in zones:
            # Hide internal_service_availability_zone
            if zone == CONF.internal_service_availability_zone:
                continue
            result.append({'zoneName': zone,
                           'zoneState': {'available': is_available},
                           "hosts": None})
        return result

    def _describe_availability_zones(self, context, **kwargs):
        ctxt = context.elevated()
        available_zones, not_available_zones = (
            availability_zones.get_availability_zones(
                ctxt, self.host_api))

        filtered_available_zones = \
            self._get_filtered_availability_zones(available_zones, True)
        filtered_not_available_zones = \
            self._get_filtered_availability_zones(not_available_zones, False)
        return {'availabilityZoneInfo': filtered_available_zones +
                                        filtered_not_available_zones}

    def _describe_availability_zones_verbose(self, context, **kwargs):
        ctxt = context.elevated()

        services = self.host_api.service_get_all(
            context, set_zones=True, all_cells=True)
        available_zones, not_available_zones = (
            availability_zones.get_availability_zones(
                ctxt, self.host_api, services=services))

        zone_hosts = {}
        host_services = {}
        api_services = ('nova-osapi_compute', 'nova-metadata')
        for service in filter(lambda x: not x.disabled, services):
            if service.binary in api_services:
                # Skip API services in the listing since they are not
                # maintained in the same way as other services
                continue
            zone_hosts.setdefault(service['availability_zone'], set())
            zone_hosts[service['availability_zone']].add(service['host'])

            host_services.setdefault(service['availability_zone'] +
                    service['host'], [])
            host_services[service['availability_zone'] + service['host']].\
                    append(service)

        result = []
        for zone in available_zones:
            hosts = {}
            for host in zone_hosts.get(zone, []):
                hosts[host] = {}
                for service in host_services[zone + host]:
                    alive = self.servicegroup_api.service_is_up(service)
                    hosts[host][service['binary']] = {
                        'available': alive,
                        'active': service['disabled'] is not True,
                        'updated_at': service['updated_at']
                    }
            result.append({'zoneName': zone,
                           'zoneState': {'available': True},
                           "hosts": hosts})

        for zone in not_available_zones:
            result.append({'zoneName': zone,
                           'zoneState': {'available': False},
                           "hosts": None})
        return {'availabilityZoneInfo': result}

    @wsgi.expected_errors(())
    def index(self, req):
        """Returns a summary list of availability zone."""
        context = req.environ['nova.context']
        context.can(az_policies.POLICY_ROOT % 'list', target={})

        return self._describe_availability_zones(context)

    @wsgi.expected_errors(())
    def detail(self, req):
        """Returns a detailed list of availability zone."""
        context = req.environ['nova.context']
        context.can(az_policies.POLICY_ROOT % 'detail', target={})

        return self._describe_availability_zones_verbose(context)
<|endoftext|>


<commit_before>from django.db import models
from django.db.models import Q
from django.utils.encoding import python_2_unicode_compatible
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext as _
from django.template import Template, Context

from django_hstore import hstore
from parler.models import TranslatableModel, TranslatedFields

from .network import Network

@python_2_unicode_compatible
class Realm(models.Model):
    name = models.CharField(max_length=255)
    slug = models.SlugField(max_length=255)

    created_on = models.DateTimeField()

    def __str__(self):
        return self.name

    def get_types(self):
        return {
            'node': {
                t.slug: t.get_data() for t in self.entitytype_set.all()
            },
            'edge': {
                t.slug: t.get_data() for t in self.relationshiptype_set.all()
            }
        }

class EntityType(TranslatableModel):
    realms = models.ManyToManyField(Realm)
    translations = TranslatedFields(
        name=models.CharField(max_length=255),
        slug=models.SlugField(max_length=255),
        template=models.TextField(blank=True)
    )
    settings = hstore.DictionaryField(blank=True)

    def __str__(self):
        return self.safe_translation_getter('name', language_code='en')

    def get_data(self):
        return {
            'name': self.name,
            'slug': self.slug,
            'settings': self.settings
        }

@python_2_unicode_compatible
class Entity(models.Model):
    realm = models.ForeignKey(Realm)
    type = models.ForeignKey(EntityType, null=True, on_delete=models.SET_NULL)

    name = models.CharField(max_length=255)
    slug = models.SlugField(max_length=255)

    data = hstore.DictionaryField(blank=True)

    objects = hstore.HStoreManager()

    class Meta:
        unique_together = ('realm', 'slug')
        verbose_name_plural = _('Entities')

    def __str__(self):
        return u'{name} ({type})'.format(name=self.name, type=self.type)

    def get_absolute_url(self):
        return reverse('yolodex:entity_detail', kwargs={
            'type': self.type.slug,
            'slug': self.slug
        }, current_app=self.realm.slug)

    def get_network(self, level=1, include_self=True):
        assert level < 4
        entities = set()
        distance = 0
        self.distance = 0
        if not include_self:
            entities.add(self)
        relations = set()
        current_level = level
        entity_lookups = [self.pk]
        while current_level > 0:
            distance += 1
            rels = list(Relationship.objects.filter(
                Q(source_id__in=entity_lookups) | Q(target_id__in=entity_lookups)
                ).select_related('source', 'target'))
            new_entities = set([r.source for r in rels])
            new_entities |= set([r.target for r in rels])
            for e in new_entities:
                e.distance = distance
            relations |= set(rels)
            entity_lookups = [o.pk for o in new_entities]
            entities |= new_entities
            current_level -= 1
        return Network(entities, relations)

class RelationshipType(TranslatableModel):
    realms = models.ManyToManyField(Realm)
    translations = TranslatedFields(
        name=models.CharField(max_length=255),
        slug=models.SlugField(max_length=255),
        verb=models.CharField(max_length=255, blank=True),
        reverse_verb=models.CharField(max_length=255, blank=True)
    )

    settings = hstore.DictionaryField(blank=True)

    def __str__(self):
        return self.safe_translation_getter('name', language_code='en')

    def get_data(self):
        return {
            'name': self.name,
            'slug': self.slug,
            'settings': self.settings
        }

    def _verbify(self, template, edge):
        t = Template(template)
        return t.render(Context({'self': edge.data}))

    def verbify(self, edge):
        return self._verbify(self.verb, edge)

    def reverse_verbify(self, subject):
        return self._verbify(self.reverse_verb, subject)

    def render_with_subject(self, edge, subject=None):
        if (subject is None or subject == edge.source) or not self.reverse_verb:
            return u'{source} {verb} {target}'.format(
                source=edge.source.name,
                verb=self.verbify(edge),
                target=edge.target.name
            )
        return u'{target} {reverse_verb} {source}'.format(
            source=edge.source.name,
            reverse_verb=self.reverse_verbify(edge),
            target=edge.target.name
        )

@python_2_unicode_compatible
class Relationship(models.Model):
    realm = models.ForeignKey(Realm)

    source = models.ForeignKey(Entity,
                               related_name='source_relationships')
    target = models.ForeignKey(Entity,
                               related_name='target_relationships')
    directed = models.BooleanField(default=True)
    type = models.ForeignKey(RelationshipType, null=True, on_delete=models.SET_NULL)

    data = hstore.DictionaryField(blank=True)

    objects = hstore.HStoreManager()

    def __str__(self):
        return u'%s %s %s' % (
            self.source,
            '->' if self.directed else '<->',
            self.target
        )

    def render_with_subject(self, subject=None):
        return self.type.render_with_subject(self, subject)

    def render_without_subject(self, subject=None):
        if subject is None or subject == self.source:
            return u'{target} {reverse_verb}'.format(
                reverse_verb=self.type.reverse_verbify(self),
                target=self.target.name
            )
        return u'{source} {verb}'.format(
            verb=self.type.verbify(self),
            source=self.source.name
        )
<commit_msg>Make network queries slightly better<commit_after>from django.db import models
from django.db.models import Q
from django.utils.encoding import python_2_unicode_compatible
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext as _
from django.template import Template, Context

from django_hstore import hstore
from parler.models import TranslatableModel, TranslatedFields

from .network import Network

@python_2_unicode_compatible
class Realm(models.Model):
    name = models.CharField(max_length=255)
    slug = models.SlugField(max_length=255)

    created_on = models.DateTimeField()

    def __str__(self):
        return self.name

    def get_types(self):
        return {
            'node': {
                t.slug: t.get_data() for t in self.entitytype_set.all()
            },
            'edge': {
                t.slug: t.get_data() for t in self.relationshiptype_set.all()
            }
        }

class EntityType(TranslatableModel):
    realms = models.ManyToManyField(Realm)
    translations = TranslatedFields(
        name=models.CharField(max_length=255),
        slug=models.SlugField(max_length=255),
        template=models.TextField(blank=True)
    )
    settings = hstore.DictionaryField(blank=True)

    def __str__(self):
        return self.safe_translation_getter('name', language_code='en')

    def get_data(self):
        return {
            'name': self.name,
            'slug': self.slug,
            'settings': self.settings
        }

@python_2_unicode_compatible
class Entity(models.Model):
    realm = models.ForeignKey(Realm)
    type = models.ForeignKey(EntityType, null=True, on_delete=models.SET_NULL)

    name = models.CharField(max_length=255)
    slug = models.SlugField(max_length=255)

    data = hstore.DictionaryField(blank=True)

    objects = hstore.HStoreManager()

    class Meta:
        unique_together = ('realm', 'slug')
        verbose_name_plural = _('Entities')

    def __str__(self):
        return u'{name} ({type})'.format(name=self.name, type=self.type)

    def get_absolute_url(self):
        return reverse('yolodex:entity_detail', kwargs={
            'type': self.type.slug,
            'slug': self.slug
        }, current_app=self.realm.slug)

    def get_network(self, level=1, include_self=True):
        assert level < 4
        entities = set()
        distance = 0
        self.distance = 0
        if not include_self:
            entities.add(self)
        relations = set()
        current_level = level
        entity_lookups = [self.pk]
        new_entities = set(entity_lookups)
        while current_level > 0:
            distance += 1
            rels = list(Relationship.objects.filter(
                Q(source_id__in=new_entities) | Q(target_id__in=new_entities)
                ).distinct('id').select_related('source', 'target'))
            new_entities = set([r.source for r in rels])
            new_entities |= set([r.target for r in rels])
            for e in new_entities:
                e.distance = distance
            relations |= set(rels)
            entity_lookups = [o.pk for o in new_entities]
            entities |= new_entities
            current_level -= 1
        return Network(entities, relations)

class RelationshipType(TranslatableModel):
    realms = models.ManyToManyField(Realm)
    translations = TranslatedFields(
        name=models.CharField(max_length=255),
        slug=models.SlugField(max_length=255),
        verb=models.CharField(max_length=255, blank=True),
        reverse_verb=models.CharField(max_length=255, blank=True)
    )

    settings = hstore.DictionaryField(blank=True)

    def __str__(self):
        return self.safe_translation_getter('name', language_code='en')

    def get_data(self):
        return {
            'name': self.name,
            'slug': self.slug,
            'settings': self.settings
        }

    def _verbify(self, template, edge):
        t = Template(template)
        return t.render(Context({'self': edge.data}))

    def verbify(self, edge):
        return self._verbify(self.verb, edge)

    def reverse_verbify(self, subject):
        return self._verbify(self.reverse_verb, subject)

    def render_with_subject(self, edge, subject=None):
        if (subject is None or subject == edge.source) or not self.reverse_verb:
            return u'{source} {verb} {target}'.format(
                source=edge.source.name,
                verb=self.verbify(edge),
                target=edge.target.name
            )
        return u'{target} {reverse_verb} {source}'.format(
            source=edge.source.name,
            reverse_verb=self.reverse_verbify(edge),
            target=edge.target.name
        )

@python_2_unicode_compatible
class Relationship(models.Model):
    realm = models.ForeignKey(Realm)

    source = models.ForeignKey(Entity,
                               related_name='source_relationships')
    target = models.ForeignKey(Entity,
                               related_name='target_relationships')
    directed = models.BooleanField(default=True)
    type = models.ForeignKey(RelationshipType, null=True, on_delete=models.SET_NULL)

    data = hstore.DictionaryField(blank=True)

    objects = hstore.HStoreManager()

    def __str__(self):
        return u'%s %s %s' % (
            self.source,
            '->' if self.directed else '<->',
            self.target
        )

    def render_with_subject(self, subject=None):
        return self.type.render_with_subject(self, subject)

    def render_without_subject(self, subject=None):
        if subject is None or subject == self.source:
            return u'{target} {reverse_verb}'.format(
                reverse_verb=self.type.reverse_verbify(self),
                target=self.target.name
            )
        return u'{source} {verb}'.format(
            verb=self.type.verbify(self),
            source=self.source.name
        )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Tests of session structure."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

from pytest import raises

from ..session import BaseSession, Session
from ..view_model import (WaveformViewModel,
                          FeatureViewModel,
                          )
from ....utils.testing import show_test
from ....utils.tempdir import TemporaryDirectory
from ....utils.logging import set_level
from ....io.mock.artificial import MockModel
from ....io.mock.kwik import create_mock_kwik

#------------------------------------------------------------------------------
# Generic tests
#------------------------------------------------------------------------------

def setup():
    set_level('debug')

def test_session_connect():
    """Test @connect decorator and event system."""
    session = BaseSession()

    # connect names should be on_something().
    with raises(ValueError):
        @session.connect
        def invalid():
            pass

    _track = []

    @session.connect
    def on_my_event():
        _track.append('my event')

    assert _track == []

    session.emit('invalid')
    assert _track == []

    session.emit('my_event')
    assert _track == ['my event']

    # Although the callback doesn't accept a 'data' keyword argument, this does
    # not raise an error because the event system will only pass the argument
    # if it is part of the callback arg spec.
    session.emit('my_event', data='hello')

def test_session_connect_multiple():
    """Test @connect decorator and event system."""
    session = BaseSession()

    _track = []

    @session.connect
    def on_my_event():
        _track.append('my event')

    @session.connect
    def on_my_event():
        _track.append('my event again')

    session.emit('my_event')
    assert _track == ['my event', 'my event again']

def test_session_unconnect():
    """Test unconnect."""
    session = BaseSession()

    _track = []

    @session.connect
    def on_my_event():
        _track.append('my event')

    session.emit('my_event')
    assert _track == ['my event']

    # Unregister and test that the on_my_event() callback is no longer called.
    session.unconnect(on_my_event)
    session.emit('my_event')
    assert _track == ['my event']

def test_session_connect_alternative():
    """Test the alternative @connect() syntax."""
    session = BaseSession()

    _track = []

    assert _track == []

    @session.connect()
    def on_my_event():
        _track.append('my event')

    session.emit('my_event')
    assert _track == ['my event']

def test_action():
    session = BaseSession()
    _track = []

    @session.action(title='My action')
    def my_action():
        _track.append('action')

    session.my_action()
    assert _track == ['action']

    assert session.actions == [{'func': my_action, 'title': 'My action'}]
    session.execute_action(session.actions[0])
    assert _track == ['action', 'action']

def test_action_event():
    session = BaseSession()
    _track = []

    @session.connect
    def on_hello(out, kwarg=''):
        _track.append(out + kwarg)

    # We forgot the 'title=', but this still works.
    @session.action('My action')
    def my_action_hello(data):
        _track.append(data)
        session.emit('hello', data + ' world', kwarg='!')

    # Need one argument.
    with raises(TypeError):
        session.my_action_hello()

    # This triggers the 'hello' event which adds 'hello world' to _track.
    session.my_action_hello('hello')
    assert _track == ['hello', 'hello world!']

#------------------------------------------------------------------------------
# Kwik tests
#------------------------------------------------------------------------------

def _start_manual_clustering(filename=None, model=None, tempdir=None):
    session = Session(store_path=tempdir)
    session.open(filename=filename, model=model)

    return session

def test_session_mock():
    with TemporaryDirectory() as tempdir:
        session = _start_manual_clustering(model=MockModel(),
                                           tempdir=tempdir)

        def _show_waveforms():
            view = session._show_view(WaveformViewModel,
                                      scale_factor=1.,
                                      show=False,
                                      )
            show_test(view)
            return view

        view = _show_waveforms()
        session.select([0])
        view_bis = _show_waveforms()

        session.merge([3, 4])

        view.close()
        view_bis.close()

        session = _start_manual_clustering(model=MockModel(),
                                           tempdir=tempdir)
        session.select([1, 2])
        view = _show_waveforms()

        view.close()

def test_session_kwik():

    n_clusters = 5
    n_spikes = 50
    n_channels = 28
    n_fets = 2
    n_samples_traces = 3000

    with TemporaryDirectory() as tempdir:

        # Create the test HDF5 file in the temporary directory.
        filename = create_mock_kwik(tempdir,
                                    n_clusters=n_clusters,
                                    n_spikes=n_spikes,
                                    n_channels=n_channels,
                                    n_features_per_channel=n_fets,
                                    n_samples_traces=n_samples_traces)

        session = _start_manual_clustering(filename=filename,
                                           tempdir=tempdir)

        def _show_waveforms():
            view = session._show_view(WaveformViewModel,
                                      scale_factor=1.,
                                      show=False,
                                      )
            show_test(view)
            return view

        def _show_features():
            view = session._show_view(FeatureViewModel,
                                      scale_factor=1.,
                                      show=False,
                                      )
            show_test(view)
            return view

        session.select([0])

        # Check the stored items.
        for cluster in range(n_clusters):
            n_spikes = len(session.clustering.spikes_per_cluster[cluster])
            n_unmasked_channels = session.store.n_unmasked_channels(cluster)

            assert session.store.features(cluster).shape == (n_spikes,
                                                             n_channels *
                                                             n_fets)
            assert session.store.masks(cluster).shape == (n_spikes, n_channels)
            assert session.store.mean_masks(cluster).shape == (n_channels,)
            assert n_unmasked_channels <= n_channels
            assert session.store.mean_probe_position(cluster).shape == (2,)
            assert session.store.main_channels(cluster).shape == \
                (n_unmasked_channels,)

        session.merge([3, 4])
        view = _show_waveforms()
        view = _show_features()

        # This won't work but shouldn't raise an error.
        session.select([1000])

        # TODO: more tests
        session.undo()
        session.redo()

        view.close()

def test_session_stats():

    n_clusters = 5
    n_spikes = 50
    n_channels = 28
    n_fets = 2
    n_samples_traces = 3000

    with TemporaryDirectory() as tempdir:

        # Create the test HDF5 file in the temporary directory.
        filename = create_mock_kwik(tempdir,
                                    n_clusters=n_clusters,
                                    n_spikes=n_spikes,
                                    n_channels=n_channels,
                                    n_features_per_channel=n_fets,
                                    n_samples_traces=n_samples_traces)

        session = _start_manual_clustering(filename,
                                           tempdir=tempdir)
        assert session

        # TODO

        # masks = session.stats.cluster_masks(3)
        # assert masks.shape == (n_channels,)

        # session.merge([3, 4])

        # masks = session.stats.cluster_masks(3)
        # assert masks.shape == (n_channels,)

        # masks = session.stats.cluster_masks(n_clusters)
        # assert masks.shape == (n_channels,)
<commit_msg>Added simple session store test.<commit_after># -*- coding: utf-8 -*-

"""Tests of session structure."""

#------------------------------------------------------------------------------
# Imports
#------------------------------------------------------------------------------

from pytest import raises

from ..session import BaseSession, Session
from ..view_model import (WaveformViewModel,
                          FeatureViewModel,
                          )
from ....utils.testing import show_test
from ....utils.tempdir import TemporaryDirectory
from ....utils.logging import set_level
from ....io.mock.artificial import MockModel
from ....io.mock.kwik import create_mock_kwik

#------------------------------------------------------------------------------
# Generic tests
#------------------------------------------------------------------------------

def setup():
    set_level('debug')

def test_session_connect():
    """Test @connect decorator and event system."""
    session = BaseSession()

    # connect names should be on_something().
    with raises(ValueError):
        @session.connect
        def invalid():
            pass

    _track = []

    @session.connect
    def on_my_event():
        _track.append('my event')

    assert _track == []

    session.emit('invalid')
    assert _track == []

    session.emit('my_event')
    assert _track == ['my event']

    # Although the callback doesn't accept a 'data' keyword argument, this does
    # not raise an error because the event system will only pass the argument
    # if it is part of the callback arg spec.
    session.emit('my_event', data='hello')

def test_session_connect_multiple():
    """Test @connect decorator and event system."""
    session = BaseSession()

    _track = []

    @session.connect
    def on_my_event():
        _track.append('my event')

    @session.connect
    def on_my_event():
        _track.append('my event again')

    session.emit('my_event')
    assert _track == ['my event', 'my event again']

def test_session_unconnect():
    """Test unconnect."""
    session = BaseSession()

    _track = []

    @session.connect
    def on_my_event():
        _track.append('my event')

    session.emit('my_event')
    assert _track == ['my event']

    # Unregister and test that the on_my_event() callback is no longer called.
    session.unconnect(on_my_event)
    session.emit('my_event')
    assert _track == ['my event']

def test_session_connect_alternative():
    """Test the alternative @connect() syntax."""
    session = BaseSession()

    _track = []

    assert _track == []

    @session.connect()
    def on_my_event():
        _track.append('my event')

    session.emit('my_event')
    assert _track == ['my event']

def test_action():
    session = BaseSession()
    _track = []

    @session.action(title='My action')
    def my_action():
        _track.append('action')

    session.my_action()
    assert _track == ['action']

    assert session.actions == [{'func': my_action, 'title': 'My action'}]
    session.execute_action(session.actions[0])
    assert _track == ['action', 'action']

def test_action_event():
    session = BaseSession()
    _track = []

    @session.connect
    def on_hello(out, kwarg=''):
        _track.append(out + kwarg)

    # We forgot the 'title=', but this still works.
    @session.action('My action')
    def my_action_hello(data):
        _track.append(data)
        session.emit('hello', data + ' world', kwarg='!')

    # Need one argument.
    with raises(TypeError):
        session.my_action_hello()

    # This triggers the 'hello' event which adds 'hello world' to _track.
    session.my_action_hello('hello')
    assert _track == ['hello', 'hello world!']

#------------------------------------------------------------------------------
# Kwik tests
#------------------------------------------------------------------------------

def _start_manual_clustering(filename=None, model=None, tempdir=None):
    session = Session(store_path=tempdir)
    session.open(filename=filename, model=model)

    return session

def test_session_store():
    with TemporaryDirectory() as tempdir:
        session = _start_manual_clustering(model=MockModel(n_spikes=10),
                                           tempdir=tempdir)

def test_session_mock():
    with TemporaryDirectory() as tempdir:
        session = _start_manual_clustering(model=MockModel(),
                                           tempdir=tempdir)

        def _show_waveforms():
            view = session._show_view(WaveformViewModel,
                                      scale_factor=1.,
                                      show=False,
                                      )
            show_test(view)
            return view

        view = _show_waveforms()
        session.select([0])
        view_bis = _show_waveforms()

        session.merge([3, 4])

        view.close()
        view_bis.close()

        session = _start_manual_clustering(model=MockModel(),
                                           tempdir=tempdir)
        session.select([1, 2])
        view = _show_waveforms()

        view.close()

def test_session_kwik():

    n_clusters = 5
    n_spikes = 50
    n_channels = 28
    n_fets = 2
    n_samples_traces = 3000

    with TemporaryDirectory() as tempdir:

        # Create the test HDF5 file in the temporary directory.
        filename = create_mock_kwik(tempdir,
                                    n_clusters=n_clusters,
                                    n_spikes=n_spikes,
                                    n_channels=n_channels,
                                    n_features_per_channel=n_fets,
                                    n_samples_traces=n_samples_traces)

        session = _start_manual_clustering(filename=filename,
                                           tempdir=tempdir)

        def _show_waveforms():
            view = session._show_view(WaveformViewModel,
                                      scale_factor=1.,
                                      show=False,
                                      )
            show_test(view)
            return view

        def _show_features():
            view = session._show_view(FeatureViewModel,
                                      scale_factor=1.,
                                      show=False,
                                      )
            show_test(view)
            return view

        session.select([0])

        # Check the stored items.
        for cluster in range(n_clusters):
            n_spikes = len(session.clustering.spikes_per_cluster[cluster])
            n_unmasked_channels = session.store.n_unmasked_channels(cluster)

            assert session.store.features(cluster).shape == (n_spikes,
                                                             n_channels *
                                                             n_fets)
            assert session.store.masks(cluster).shape == (n_spikes, n_channels)
            assert session.store.mean_masks(cluster).shape == (n_channels,)
            assert n_unmasked_channels <= n_channels
            assert session.store.mean_probe_position(cluster).shape == (2,)
            assert session.store.main_channels(cluster).shape == \
                (n_unmasked_channels,)

        session.merge([3, 4])
        view = _show_waveforms()
        view = _show_features()

        # This won't work but shouldn't raise an error.
        session.select([1000])

        # TODO: more tests
        session.undo()
        session.redo()

        view.close()

def test_session_stats():

    n_clusters = 5
    n_spikes = 50
    n_channels = 28
    n_fets = 2
    n_samples_traces = 3000

    with TemporaryDirectory() as tempdir:

        # Create the test HDF5 file in the temporary directory.
        filename = create_mock_kwik(tempdir,
                                    n_clusters=n_clusters,
                                    n_spikes=n_spikes,
                                    n_channels=n_channels,
                                    n_features_per_channel=n_fets,
                                    n_samples_traces=n_samples_traces)

        session = _start_manual_clustering(filename,
                                           tempdir=tempdir)
        assert session

        # TODO

        # masks = session.stats.cluster_masks(3)
        # assert masks.shape == (n_channels,)

        # session.merge([3, 4])

        # masks = session.stats.cluster_masks(3)
        # assert masks.shape == (n_channels,)

        # masks = session.stats.cluster_masks(n_clusters)
        # assert masks.shape == (n_channels,)
<|endoftext|>


<commit_before># ---------------------------------------------------------------------------
# DrcogSync.py
# ---------------------------------------------------------------------------
# Synchronize datasets from the DRCOG Regional Data Catalog to OpenColorado
#----------------------------------------------------------------------------

# Imports
import os
import sys
import urllib2
import ckanclient
import logging
from bs4 import BeautifulSoup

# Global variables
base_url = "http://gis.drcog.org"
data_catalog_prefix = "/datacatalog"
subjects_url_prefix = "/datacatalog/subjects/"
dataset_url_prefix = "/datacatalog/content/"
ckan_client = None

ckan_host = "http://test.opencolorado.org/api/2"
ckan_key = sys.argv[1]
ckan_group = "drcog"
ckan_title_prefix = "DRCOG: "
ckan_name_prefix = "drcog-"
ckan_license = "cc-by"

def main():

    # Get a list of subjects from the DRCOG catalog
    subjects = get_subjects()

    # Get all datasets by subject (datasets may be in more than one subject so there could be duplicates here)
    datasets = []
    for subject in subjects:
        datasets = datasets + get_dataset_urls_by_subject(subject)
        break # Just do the first subject for now

    # Remove duplicates
    datasets = list(set(datasets))

    # Begin syncing each dataset to CKAN
    for dataset in datasets:
        print dataset
        dataset_entity = get_dataset_entity(dataset)

        publish_to_ckan(dataset_entity)

        #break #Just do the first dataset for now

def get_subjects():
    """Gets a list of subjects from the DRCOG data catalog

    Returns:
        List[string]
    """
    global base_url, subjects_url_prefix

    subjects = []
    subjects_url = base_url + "/datacatalog/content/welcome-regional-data-catalog?quicktabs_tabbed_menu_homepage=1"

    soup = get_soup_from_url(subjects_url)

    for link in soup.find_all('a'):
        href = link.get('href')
        if href.startswith(subjects_url_prefix):
            subject = href.replace(subjects_url_prefix,"")
            subjects.append(subject)

    return subjects

def get_dataset_urls_by_subject(subject, page_url=None):
    global base_url, subjects_url_prefix, dataset_url_prefix

    datasets = []

    datasets_url = base_url + subjects_url_prefix + subject

    if page_url != None:
        datasets_url = base_url + page_url

    print datasets_url

    soup = get_soup_from_url(datasets_url)

    #print(soup.prettify())

    # Get all of the dataset links on the current page
    for div in soup.findAll("div", { "class" : "node" }):
        if div.h2 != None and div.h2.a != None:
            href = div.h2.a.get('href')
            if href.startswith(dataset_url_prefix):
                dataset = href.replace(dataset_url_prefix,"")
                datasets.append(dataset)
                #print("-- " + dataset)

    # If there is a next link, recursively get the next page of datasets
    pager_next = soup.find("li", { "class" : "pager-next" })

    if pager_next != None:
        next_href = pager_next.a.get('href')
        datasets = datasets + get_dataset_urls_by_subject(subject,next_href)

    return datasets

def get_dataset_entity(dataset):
    global base_url, dataset_url_prefix, ckan_title_prefix, ckan_name_prefix, ckan_license, data_catalog_prefix

    dataset_url = base_url + dataset_url_prefix + dataset

    dataset_entity = {}

    print dataset_url

    soup = get_soup_from_url(dataset_url)

    # Scrape the content from the dataset page
    dataset_entity['name'] = ckan_name_prefix + dataset
    dataset_entity['title'] = ckan_title_prefix + soup.find("h1",{ "id" : "page-title" }).getText()
    dataset_entity['license_id'] = ckan_license
    dataset_entity['url'] = base_url + data_catalog_prefix

    resources = []

    # Get the tags
    tags = []
    terms_element = soup.find("div", { "class" : "terms"})
    if (terms_element != None) :
        for li in terms_element.findAll("li"):
            tag = li.getText()
            tag = tag.lower().replace(' ','-')
            tag = tag.lower().replace('(','')
            tag = tag.lower().replace(')','')
            tags.append(tag)
    dataset_entity['tags'] = tags

    print(tags)

    # Get descriptive information
    for field_item in soup.findAll("div", { "class" : "field-item" }):

        field_item_label = field_item.div.getText().strip().lower()
        field_item_link = field_item.a
        field_item.div.extract()
        field_item_text = field_item.getText().strip()

        #print field_item_label
        #print field_item_link
        #print field_item_text

        # Get dataset attributes
        if (field_item_label.startswith("description:")):
            dataset_entity['notes'] = field_item_text
        elif (field_item_label.startswith("source:")):
            dataset_entity['author'] = field_item_text
        elif (field_item_label.startswith("contact name:")):
            dataset_entity['maintainer'] = field_item_text
        elif (field_item_label.startswith("contact email:")):
            dataset_entity['maintainer_email'] = field_item_text

        # Get resources
        if (field_item_link != None):
            resource_url = field_item_link.get('href')

            resource = {}

            # If paths are relative make them absolute
            if (resource_url.startswith("http")):
                resource["url"] = resource_url
            else:
                resource["url"] = base_url + resource_url

            resource["name"] = dataset_entity['title']

            if (field_item_label.startswith("kml")):
                resource["name"] = dataset_entity['title'] + " - KML"
                resource["mimetype"] = "application/vnd.google-earth.kmz"
            elif (field_item_label.startswith("wms")):
                resource["name"] = dataset_entity['title'] + " - WMS"
                resource["mimetype"] = "application/wms"
            elif (field_item_label.startswith("georss")):
                resource["name"] = dataset_entity['title'] + " - GeoRSS"
                resource["mimetype"] = "application/rss"
            elif (field_item_label.startswith("shapefile")):
                resource["name"] = dataset_entity['title'] + " - SHP"
                resource["mimetype"] = "application/zip"

            resources.append(resource)

    # Add the resources to the dataset
    dataset_entity["resources"] = resources

    return dataset_entity

def publish_to_ckan(dataset_entity):
    """Updates the dataset in the CKAN repository or creates a new dataset

    Returns:
        None
    """
    global ckan_client, ckan_host, ckan_key

    # Initialize the CKAN client
    ckan_client = ckanclient.CkanClient(base_location=ckan_host,api_key=ckan_key)

    # Create the name of the dataset on the CKAN instance
    dataset_id = dataset_entity["name"]

    # Get the dataset from CKAN
    dataset_entity_remote = get_remote_dataset(dataset_id)

    # Check to see if the dataset exists on CKAN or not
    if dataset_entity_remote is None:
        # Create a new dataset
        create_dataset(dataset_entity)
    else:
        # Update an existing dataset
        update_dataset(dataset_entity_remote, dataset_entity)

def create_dataset(dataset_entity):
    """Creates a new dataset and registers it to CKAN

    Parameters:
        dataset_entity - A CKAN dataset entity

    Returns:
        None
    """
    global ckan_group

    try:
        group_entity = ckan_client.group_entity_get(ckan_group)
        if group_entity is not None:
            print('Adding dataset to group: ' + ckan_group)
            dataset_entity['groups'] = [group_entity['id']]
    except ckanclient.CkanApiNotFoundError:
        logger.warn('Group: ' + args.ckan_group_name + ' not found on OpenColorado')
        dataset_entity['groups'] = []

    ckan_client.package_register_post(dataset_entity)

def update_dataset(dataset_entity_remote, dataset_entity):
    """Updates a dataset

    Parameters:
        dataset_entity_remote - The target dataset
        dataset_entity - The updated dataset

    Returns:
        None
    """
    dataset_entity_remote['url'] = dataset_entity["url"]
    dataset_entity_remote['license_id'] = dataset_entity["license_id"]
    dataset_entity_remote['name'] = dataset_entity["name"]
    dataset_entity_remote['title'] = dataset_entity["title"]
    dataset_entity_remote['notes'] = dataset_entity["notes"]
    dataset_entity_remote['author'] = dataset_entity["author"]
    dataset_entity_remote['maintainer'] = dataset_entity["maintainer"]
    dataset_entity_remote['maintainer_email'] = dataset_entity["maintainer_email"]
    dataset_entity_remote['tags'] = dataset_entity["tags"]

    # Process resources
    if not 'resources' in dataset_entity_remote:
       dataset_entity_remote['resources'] = []

    for resource in dataset_entity['resources']:
        mimetype = resource['mimetype']

        # Check if a resource is already present with the same mimetype
        # If so, update it, otherwise add as a new resource
        found = False
        for resource_remote in dataset_entity_remote['resources']:
            if resource_remote['mimetype'] == resource["mimetype"]:
                found = True
                break

        if (found) :
            print("Match found with remote resource, updating:" + resource["mimetype"])
            resource_remote["url"] = resource["url"]
            resource_remote["name"] = resource["name"]
            resource_remote["mimetype"] = resource["mimetype"]
        else:
            print("No match found with remote resource, adding:" + resource["mimetype"])
            print(resource)
            dataset_entity_remote['resources'].append(resource)

    ckan_client.package_entity_put(dataset_entity_remote)

def get_remote_dataset(dataset_id):
    """Gets the dataset from CKAN repository

    Parameters:
        dataset_id - A string representing the unique dataset name

    Returns:
        An object structured the same as the JSON dataset output from
        the CKAN REST API. For more information on the structure look at the
        web service JSON output, or reference:
        http://docs.ckan.org/en/latest/api-v2.html#model-api
    """
    dataset_entity = None

    try:
        # Get the dataset
        dataset_entity = ckan_client.package_entity_get(dataset_id)
        print("Dataset " + dataset_id + " found on OpenColorado")

    except ckanclient.CkanApiNotFoundError:
        print("Dataset " + dataset_id + " not found on OpenColorado")

    return dataset_entity

def get_soup_from_url(url):
    html = "".join(urllib2.urlopen(url).readlines())
    soup = BeautifulSoup(html)
    return soup

#Execute main function
if __name__ == '__main__':
    main()<commit_msg>Added support for PDF links that are part of normal link section<commit_after># ---------------------------------------------------------------------------
# DrcogSync.py
# ---------------------------------------------------------------------------
# Synchronize datasets from the DRCOG Regional Data Catalog to OpenColorado
#----------------------------------------------------------------------------

# Imports
import os
import sys
import urllib2
import ckanclient
import logging
from bs4 import BeautifulSoup

# Global variables
base_url = "http://gis.drcog.org"
data_catalog_prefix = "/datacatalog"
subjects_url_prefix = "/datacatalog/subjects/"
dataset_url_prefix = "/datacatalog/content/"
ckan_client = None

ckan_host = "http://test.opencolorado.org/api/2"
ckan_key = sys.argv[1]
ckan_group = "drcog"
ckan_title_prefix = "DRCOG: "
ckan_name_prefix = "drcog-"
ckan_license = "cc-by"

def main():

    # Get a list of subjects from the DRCOG catalog
    subjects = get_subjects()

    datasets = []

    # Get all datasets by subject (datasets may be in more than one subject so there could be duplicates here)
    for subject in subjects:
        datasets = datasets + get_dataset_urls_by_subject(subject)
        #break # Just do the first subject for now

    # Remove duplicates
    datasets = list(set(datasets))

    # Begin syncing each dataset to CKAN
    for dataset in datasets:
        print dataset
        dataset_entity = get_dataset_entity(dataset)

        publish_to_ckan(dataset_entity)

        #break #Just do the first dataset for now

def get_subjects():
    """Gets a list of subjects from the DRCOG data catalog

    Returns:
        List[string]
    """
    global base_url, subjects_url_prefix

    subjects = []
    subjects_url = base_url + "/datacatalog/content/welcome-regional-data-catalog?quicktabs_tabbed_menu_homepage=1"

    soup = get_soup_from_url(subjects_url)

    for link in soup.find_all('a'):
        href = link.get('href')
        if href.startswith(subjects_url_prefix):
            subject = href.replace(subjects_url_prefix,"")
            subjects.append(subject)

    return subjects

def get_dataset_urls_by_subject(subject, page_url=None):
    global base_url, subjects_url_prefix, dataset_url_prefix

    datasets = []

    datasets_url = base_url + subjects_url_prefix + subject

    if page_url != None:
        datasets_url = base_url + page_url

    print datasets_url

    soup = get_soup_from_url(datasets_url)

    #print(soup.prettify())

    # Get all of the dataset links on the current page
    for div in soup.findAll("div", { "class" : "node" }):
        if div.h2 != None and div.h2.a != None:
            href = div.h2.a.get('href')
            if href.startswith(dataset_url_prefix):
                dataset = href.replace(dataset_url_prefix,"")
                datasets.append(dataset)
                #print("-- " + dataset)

    # If there is a next link, recursively get the next page of datasets
    pager_next = soup.find("li", { "class" : "pager-next" })

    if pager_next != None:
        next_href = pager_next.a.get('href')
        datasets = datasets + get_dataset_urls_by_subject(subject,next_href)

    return datasets

def get_dataset_entity(dataset):
    global base_url, dataset_url_prefix, ckan_title_prefix, ckan_name_prefix, ckan_license, data_catalog_prefix

    dataset_url = base_url + dataset_url_prefix + dataset

    dataset_entity = {}

    print dataset_url

    soup = get_soup_from_url(dataset_url)

    # Scrape the content from the dataset page
    dataset_entity['name'] = ckan_name_prefix + dataset
    dataset_entity['title'] = ckan_title_prefix + soup.find("h1",{ "id" : "page-title" }).getText()
    dataset_entity['license_id'] = ckan_license
    dataset_entity['url'] = dataset_url

    resources = []

    # Get the tags
    tags = []
    terms_element = soup.find("div", { "class" : "terms"})
    if (terms_element != None) :
        for li in terms_element.findAll("li"):
            tag = li.getText()
            tag = tag.lower().replace(' ','-')
            tag = tag.lower().replace('(','')
            tag = tag.lower().replace(')','')
            tags.append(tag)
    dataset_entity['tags'] = tags

    print(tags)

    # Get descriptive information
    for field_item in soup.findAll("div", { "class" : "field-item" }):

        # Get the field label (if it exists)
        field_item_label = ""
        field_item_div = field_item.div
        if (field_item_div != None):
            field_item_label = field_item_div.getText().strip().lower()
            if (len(field_item_label) > 0 and not field_item_label.endswith(".pdf")):
                print(field_item_label + " - extracting")
                field_item.div.extract() # Remove the label tag so we can get the remaining text by itself

        field_item_link = field_item.a

        field_item_text = field_item.getText().strip()

        #print field_item_label
        #print field_item_link
        #print field_item_text

        # Get dataset attributes
        if (field_item_label.startswith("description:")):
            dataset_entity['notes'] = field_item_text
        elif (field_item_label.startswith("source:")):
            dataset_entity['author'] = field_item_text
        elif (field_item_label.startswith("contact name:")):
            dataset_entity['maintainer'] = field_item_text
        elif (field_item_label.startswith("contact email:")):
            dataset_entity['maintainer_email'] = field_item_text

        # Get resources
        if (field_item_link != None):
            resource_url = field_item_link.get('href')

            resource = {}

            # If paths are relative make them absolute
            if (resource_url.startswith("http")):
                resource["url"] = resource_url
            else:
                resource["url"] = base_url + resource_url

            resource["name"] = dataset_entity['title']

            if (field_item_label.startswith("kml")):
                resource["name"] = dataset_entity['title'] + " - KML"
                resource["mimetype"] = "application/vnd.google-earth.kmz"
            elif (field_item_label.startswith("wms")):
                resource["name"] = dataset_entity['title'] + " - WMS"
                resource["mimetype"] = "application/wms"
            elif (field_item_label.startswith("georss")):
                resource["name"] = dataset_entity['title'] + " - GeoRSS"
                resource["mimetype"] = "application/rss"
            elif (field_item_label.startswith("shapefile")):
                resource["name"] = dataset_entity['title'] + " - SHP"
                resource["mimetype"] = "application/zip"

            if ("mimetype" in resource):
                resources.append(resource)

    # Search for other filefields (PDF)
    filefield_file = soup.find("div", { "class" : "filefield-file" })
    if (filefield_file != None):
        filefield_file_label = filefield_file.a.getText().strip()
        filefield_file_link = filefield_file.a.get('href')
        filefield_file_mimetype = filefield_file.a.get('type')

        print(filefield_file_link)
        if (filefield_file_mimetype.startswith("application/pdf")) :
            resource = {}
            if (filefield_file_link.startswith("http")):
                resource["url"] = filefield_file_link
            else:
                resource["url"] = base_url + filefield_file_link

            resource["name"] = filefield_file_label
            resource["mimetype"] = "application/pdf" #Hack, need to get this from URL?
            resources.append(resource)

    # Add the resources to the dataset
    dataset_entity["resources"] = resources

    return dataset_entity

def publish_to_ckan(dataset_entity):
    """Updates the dataset in the CKAN repository or creates a new dataset

    Returns:
        None
    """
    global ckan_client, ckan_host, ckan_key

    # Initialize the CKAN client
    ckan_client = ckanclient.CkanClient(base_location=ckan_host,api_key=ckan_key)

    # Create the name of the dataset on the CKAN instance
    dataset_id = dataset_entity["name"]

    # Get the dataset from CKAN
    dataset_entity_remote = get_remote_dataset(dataset_id)

    # Check to see if the dataset exists on CKAN or not
    if dataset_entity_remote is None:
        # Create a new dataset
        create_dataset(dataset_entity)
    else:
        # Update an existing dataset
        update_dataset(dataset_entity_remote, dataset_entity)

def create_dataset(dataset_entity):
    """Creates a new dataset and registers it to CKAN

    Parameters:
        dataset_entity - A CKAN dataset entity

    Returns:
        None
    """
    global ckan_group

    try:
        group_entity = ckan_client.group_entity_get(ckan_group)
        if group_entity is not None:
            print('Adding dataset to group: ' + ckan_group)
            dataset_entity['groups'] = [group_entity['id']]
    except ckanclient.CkanApiNotFoundError:
        logger.warn('Group: ' + args.ckan_group_name + ' not found on OpenColorado')
        dataset_entity['groups'] = []

    ckan_client.package_register_post(dataset_entity)

def update_dataset(dataset_entity_remote, dataset_entity):
    """Updates a dataset

    Parameters:
        dataset_entity_remote - The target dataset
        dataset_entity - The updated dataset

    Returns:
        None
    """
    dataset_entity_remote['url'] = dataset_entity["url"]
    dataset_entity_remote['license_id'] = dataset_entity["license_id"]
    dataset_entity_remote['name'] = dataset_entity["name"]
    dataset_entity_remote['title'] = dataset_entity["title"]
    dataset_entity_remote['notes'] = dataset_entity["notes"]
    dataset_entity_remote['author'] = dataset_entity["author"]
    dataset_entity_remote['maintainer'] = dataset_entity["maintainer"]
    dataset_entity_remote['maintainer_email'] = dataset_entity["maintainer_email"]
    dataset_entity_remote['tags'] = dataset_entity["tags"]

    # Process resources
    if not 'resources' in dataset_entity_remote:
       dataset_entity_remote['resources'] = []

    for resource in dataset_entity['resources']:

        mimetype = ""
        if 'mimetype' in resource:
            mimetype = resource['mimetype']

        # Check if a resource is already present with the same mimetype
        # If so, update it, otherwise add as a new resource
        found = False
        for resource_remote in dataset_entity_remote['resources']:
            if 'mimetype' in resource_remote and resource_remote['mimetype'] == mimetype:
                found = True
                break

        if (found) :
            print("found")
            resource_remote["url"] = resource["url"]
            resource_remote["name"] = resource["name"]
            resource_remote["mimetype"] = resource["mimetype"]
        else:
            print("not found")
            dataset_entity_remote['resources'].append(resource)

    ckan_client.package_entity_put(dataset_entity_remote)

def get_remote_dataset(dataset_id):
    """Gets the dataset from CKAN repository

    Parameters:
        dataset_id - A string representing the unique dataset name

    Returns:
        An object structured the same as the JSON dataset output from
        the CKAN REST API. For more information on the structure look at the
        web service JSON output, or reference:
        http://docs.ckan.org/en/latest/api-v2.html#model-api
    """
    dataset_entity = None

    try:
        # Get the dataset
        dataset_entity = ckan_client.package_entity_get(dataset_id)
        print("Dataset " + dataset_id + " found on OpenColorado")

    except ckanclient.CkanApiNotFoundError:
        print("Dataset " + dataset_id + " not found on OpenColorado")

    return dataset_entity

def get_soup_from_url(url):
    html = "".join(urllib2.urlopen(url).readlines())
    soup = BeautifulSoup(html)
    return soup

#Execute main function
if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>from __future__ import absolute_import

from typing import Any

from django.views.debug import SafeExceptionReporterFilter
from django.http import HttpRequest, build_request_repr

class ZulipExceptionReporterFilter(SafeExceptionReporterFilter):
    def get_post_parameters(self, request):
        # type: (HttpRequest) -> Dict[str, Any]
        filtered_post = SafeExceptionReporterFilter.get_post_parameters(self, request).copy()
        filtered_vars = ['content', 'secret', 'password', 'key', 'api-key', 'subject', 'stream',
                         'subscriptions', 'to', 'csrfmiddlewaretoken', 'api_key']

        for var in filtered_vars:
            if var in filtered_post:
                filtered_post[var] = '**********'
        return filtered_post

    def get_request_repr(self, request):
        # type: (HttpRequest) -> str
        if request is None:
            return repr(None)
        else:
            return build_request_repr(request,
                                      POST_override=self.get_post_parameters(request),
                                      COOKIES_override="**********",
                                      META_override="**********")
<commit_msg>errors: Remove build_request_repr logic.<commit_after>from __future__ import absolute_import

from typing import Any

from django.views.debug import SafeExceptionReporterFilter
from django.http import HttpRequest

class ZulipExceptionReporterFilter(SafeExceptionReporterFilter):
    def get_post_parameters(self, request):
        # type: (HttpRequest) -> Dict[str, Any]
        filtered_post = SafeExceptionReporterFilter.get_post_parameters(self, request).copy()
        filtered_vars = ['content', 'secret', 'password', 'key', 'api-key', 'subject', 'stream',
                         'subscriptions', 'to', 'csrfmiddlewaretoken', 'api_key']

        for var in filtered_vars:
            if var in filtered_post:
                filtered_post[var] = '**********'
        return filtered_post
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2011 OpenStack LLC.
# Copyright 2011 Grid Dynamics
# Copyright 2011 Eldar Nugaev, Kirill Shileev, Ilya Alekseyev
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License
from webob import exc

from nova import exception
from nova import network
from nova import rpc
from nova.api.openstack import faults
from nova.api.openstack import extensions

LOG = logging.getLogger('nova.api.openstack.contrib.floating_ips')

def _translate_floating_ip_view(floating_ip):
    result = {'id': floating_ip['id'],
              'ip': floating_ip['address']}
    try:
        result['fixed_ip'] = floating_ip['fixed_ip']['address']
    except (TypeError, KeyError):
        result['fixed_ip'] = None
    if 'instance' in floating_ip:
        result['instance_id'] = floating_ip['instance']['id']
    else:
        result['instance_id'] = None
    return {'floating_ip': result}

def _translate_floating_ips_view(floating_ips):
    return {'floating_ips': [_translate_floating_ip_view(floating_ip)
                             for floating_ip in floating_ips]}

class FloatingIPController(object):
    """The Floating IPs API controller for the OpenStack API."""

    _serialization_metadata = {
        'application/xml': {
            "attributes": {
                "floating_ip": [
                    "id",
                    "ip",
                    "instance_id",
                    "fixed_ip",
                    ]}}}

    def __init__(self):
        self.network_api = network.API()
        super(FloatingIPController, self).__init__()

    def show(self, req, id):
        """Return data about the given floating ip."""
        context = req.environ['nova.context']

        try:
            floating_ip = self.network_api.get_floating_ip(context, id)
        except exception.NotFound:
            return faults.Fault(exc.HTTPNotFound())

        return _translate_floating_ip_view(floating_ip)

    def index(self, req):
        context = req.environ['nova.context']

        floating_ips = self.network_api.list_floating_ips(context)

        return _translate_floating_ips_view(floating_ips)

    def create(self, req):
        context = req.environ['nova.context']

        try:
            address = self.network_api.allocate_floating_ip(context)
            ip = self.network_api.get_floating_ip_by_ip(context, address)
        except rpc.RemoteError as ex:
            # NOTE(tr3buchet) - why does this block exist?
            if ex.exc_type == 'NoMoreFloatingIps':
                raise exception.NoMoreFloatingIps()
            else:
                raise

        return {'allocated': {
            "id": ip['id'],
            "floating_ip": ip['address']}}

    def delete(self, req, id):
        context = req.environ['nova.context']
        ip = self.network_api.get_floating_ip(context, id)

        if 'fixed_ip' in ip:
            try:
                self.disassociate(req, id, '')
            except Exception as e:
                LOG.exception(_("Error disassociating fixed_ip %s"), e)

        self.network_api.release_floating_ip(context, address=ip)

        return {'released': {
            "id": ip['id'],
            "floating_ip": ip['address']}}

    def associate(self, req, id, body):
        """ /floating_ips/{id}/associate  fixed ip in body """
        context = req.environ['nova.context']
        floating_ip = self._get_ip_by_id(context, id)

        fixed_ip = body['associate_address']['fixed_ip']

        try:
            self.network_api.associate_floating_ip(context,
                                                   floating_ip, fixed_ip)
        except rpc.RemoteError:
            raise

        return {'associated':
                {
                "floating_ip_id": id,
                "floating_ip": floating_ip,
                "fixed_ip": fixed_ip}}

    def disassociate(self, req, id):
        """ POST /floating_ips/{id}/disassociate """
        context = req.environ['nova.context']
        floating_ip = self.network_api.get_floating_ip(context, id)
        address = floating_ip['address']
        fixed_ip = floating_ip['fixed_ip']['address']

        try:
            self.network_api.disassociate_floating_ip(context, address)
        except rpc.RemoteError:
            raise

        return {'disassociated': {'floating_ip': address,
                                  'fixed_ip': fixed_ip}}

    def _get_ip_by_id(self, context, value):
        """Checks that value is id and then returns its address."""
        return self.network_api.get_floating_ip(context, value)['address']

class Floating_ips(extensions.ExtensionDescriptor):
    def get_name(self):
        return "Floating_ips"

    def get_alias(self):
        return "os-floating-ips"

    def get_description(self):
        return "Floating IPs support"

    def get_namespace(self):
        return "http://docs.openstack.org/ext/floating_ips/api/v1.1"

    def get_updated(self):
        return "2011-06-16T00:00:00+00:00"

    def get_resources(self):
        resources = []

        res = extensions.ResourceExtension('os-floating-ips',
                         FloatingIPController(),
                         member_actions={
                            'associate': 'POST',
                            'disassociate': 'POST'})
        resources.append(res)

        return resources
<commit_msg>adding forgotten import for logging<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright 2011 OpenStack LLC.
# Copyright 2011 Grid Dynamics
# Copyright 2011 Eldar Nugaev, Kirill Shileev, Ilya Alekseyev
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License
from webob import exc

from nova import exception
from nova import log as logging
from nova import network
from nova import rpc
from nova.api.openstack import faults
from nova.api.openstack import extensions

LOG = logging.getLogger('nova.api.openstack.contrib.floating_ips')

def _translate_floating_ip_view(floating_ip):
    result = {'id': floating_ip['id'],
              'ip': floating_ip['address']}
    try:
        result['fixed_ip'] = floating_ip['fixed_ip']['address']
    except (TypeError, KeyError):
        result['fixed_ip'] = None
    if 'instance' in floating_ip:
        result['instance_id'] = floating_ip['instance']['id']
    else:
        result['instance_id'] = None
    return {'floating_ip': result}

def _translate_floating_ips_view(floating_ips):
    return {'floating_ips': [_translate_floating_ip_view(floating_ip)
                             for floating_ip in floating_ips]}

class FloatingIPController(object):
    """The Floating IPs API controller for the OpenStack API."""

    _serialization_metadata = {
        'application/xml': {
            "attributes": {
                "floating_ip": [
                    "id",
                    "ip",
                    "instance_id",
                    "fixed_ip",
                    ]}}}

    def __init__(self):
        self.network_api = network.API()
        super(FloatingIPController, self).__init__()

    def show(self, req, id):
        """Return data about the given floating ip."""
        context = req.environ['nova.context']

        try:
            floating_ip = self.network_api.get_floating_ip(context, id)
        except exception.NotFound:
            return faults.Fault(exc.HTTPNotFound())

        return _translate_floating_ip_view(floating_ip)

    def index(self, req):
        context = req.environ['nova.context']

        floating_ips = self.network_api.list_floating_ips(context)

        return _translate_floating_ips_view(floating_ips)

    def create(self, req):
        context = req.environ['nova.context']

        try:
            address = self.network_api.allocate_floating_ip(context)
            ip = self.network_api.get_floating_ip_by_ip(context, address)
        except rpc.RemoteError as ex:
            # NOTE(tr3buchet) - why does this block exist?
            if ex.exc_type == 'NoMoreFloatingIps':
                raise exception.NoMoreFloatingIps()
            else:
                raise

        return {'allocated': {
            "id": ip['id'],
            "floating_ip": ip['address']}}

    def delete(self, req, id):
        context = req.environ['nova.context']
        ip = self.network_api.get_floating_ip(context, id)

        if 'fixed_ip' in ip:
            try:
                self.disassociate(req, id, '')
            except Exception as e:
                LOG.exception(_("Error disassociating fixed_ip %s"), e)

        self.network_api.release_floating_ip(context, address=ip)

        return {'released': {
            "id": ip['id'],
            "floating_ip": ip['address']}}

    def associate(self, req, id, body):
        """ /floating_ips/{id}/associate  fixed ip in body """
        context = req.environ['nova.context']
        floating_ip = self._get_ip_by_id(context, id)

        fixed_ip = body['associate_address']['fixed_ip']

        try:
            self.network_api.associate_floating_ip(context,
                                                   floating_ip, fixed_ip)
        except rpc.RemoteError:
            raise

        return {'associated':
                {
                "floating_ip_id": id,
                "floating_ip": floating_ip,
                "fixed_ip": fixed_ip}}

    def disassociate(self, req, id):
        """ POST /floating_ips/{id}/disassociate """
        context = req.environ['nova.context']
        floating_ip = self.network_api.get_floating_ip(context, id)
        address = floating_ip['address']
        fixed_ip = floating_ip['fixed_ip']['address']

        try:
            self.network_api.disassociate_floating_ip(context, address)
        except rpc.RemoteError:
            raise

        return {'disassociated': {'floating_ip': address,
                                  'fixed_ip': fixed_ip}}

    def _get_ip_by_id(self, context, value):
        """Checks that value is id and then returns its address."""
        return self.network_api.get_floating_ip(context, value)['address']

class Floating_ips(extensions.ExtensionDescriptor):
    def get_name(self):
        return "Floating_ips"

    def get_alias(self):
        return "os-floating-ips"

    def get_description(self):
        return "Floating IPs support"

    def get_namespace(self):
        return "http://docs.openstack.org/ext/floating_ips/api/v1.1"

    def get_updated(self):
        return "2011-06-16T00:00:00+00:00"

    def get_resources(self):
        resources = []

        res = extensions.ResourceExtension('os-floating-ips',
                         FloatingIPController(),
                         member_actions={
                            'associate': 'POST',
                            'disassociate': 'POST'})
        resources.append(res)

        return resources
<|endoftext|>


<commit_before>import unittest
from contextlib import contextmanager
import sys
import pandas as pd
from io import StringIO
from healthcareai.simple_mode import SimpleDevelopSupervisedModel
from healthcareai.tests.helpers import fixture

class TestSimpleDevelopSupervisedModel(unittest.TestCase):
    def setUp(self):
        self.dataframe = pd.read_csv(fixture('DiabetesClincialSampleData.csv'), na_values=['None'])

        # Drop columns that won't help machine learning
        self.dataframe.drop(['PatientID', 'InTestWindowFLG'], axis=1, inplace=True)

    def test_knn(self):
        hcai = SimpleDevelopSupervisedModel(dataframe=self.dataframe,
                                            predicted_column='ThirtyDayReadmitFLG',
                                            model_type='classification',
                                            impute=True,
                                            grain_column='PatientEncounterID',
                                            verbose=False)

        # Hacky way to capture print output since simple prints output instead of returning it.
        with captured_output() as (out, err):
            hcai.knn()
            output = out.getvalue().strip()

            expected_output_regex = r"Training knn\n{'roc_auc_score': 0.5[0-9]*, 'accuracy': 0.8[0-9]*}"

            self.assertRegexpMatches(output, expected_output_regex)

    def test_random_forest_classification(self):
        hcai = SimpleDevelopSupervisedModel(dataframe=self.dataframe,
                                            predicted_column='ThirtyDayReadmitFLG',
                                            model_type='classification',
                                            impute=True,
                                            grain_column='PatientEncounterID',
                                            verbose=False)

        # Hacky way to capture print output since simple prints output instead of returning it.
        with captured_output() as (out, err):
            hcai.random_forest_classification()
            output = out.getvalue().strip()

            expected_output_regex = r"Training random_forest_classification\n{'roc_auc_score': 0.7[0-9]*, 'accuracy': 0.8[0-9]*}"

            self.assertRegexpMatches(output, expected_output_regex)

    def test_linear_regression(self):
        hcai = SimpleDevelopSupervisedModel(self.dataframe,
                                            'SystolicBPNBR',
                                            'regression',
                                            impute=True,
                                            grain_column='PatientEncounterID')

        # Hacky way to capture print output since simple prints output instead of returning it.
        with captured_output() as (out, err):
            hcai.linear_regression()
            output = out.getvalue().strip()

            expected_output_regex = r"Training linear_regression\n.*\n*{'mean_squared_error': 638\.[0-9]*, 'mean_absolute_error': 20\.[0-9]*}"

            self.assertRegexpMatches(output, expected_output_regex)

@contextmanager
def captured_output():
    new_out, new_err = StringIO(), StringIO()
    old_out, old_err = sys.stdout, sys.stderr
    try:
        sys.stdout, sys.stderr = new_out, new_err
        yield sys.stdout, sys.stderr
    finally:
        sys.stdout, sys.stderr = old_out, old_err

if __name__ == '__main__':
    unittest.main()
<commit_msg>* adjusted regexes in test_simple_develop to not respect order of score metrics (since dictionaries are unordered)<commit_after>import unittest
from contextlib import contextmanager
import sys
import pandas as pd
from io import StringIO
from healthcareai.simple_mode import SimpleDevelopSupervisedModel
from healthcareai.tests.helpers import fixture

class TestSimpleDevelopSupervisedModel(unittest.TestCase):
    def setUp(self):
        self.dataframe = pd.read_csv(fixture('DiabetesClincialSampleData.csv'), na_values=['None'])

        # Drop columns that won't help machine learning
        self.dataframe.drop(['PatientID', 'InTestWindowFLG'], axis=1, inplace=True)

    def test_knn(self):
        hcai = SimpleDevelopSupervisedModel(dataframe=self.dataframe,
                                            predicted_column='ThirtyDayReadmitFLG',
                                            model_type='classification',
                                            impute=True,
                                            grain_column='PatientEncounterID',
                                            verbose=False)

        # Hacky way to capture print output since simple prints output instead of returning it.
        with captured_output() as (out, err):
            hcai.knn()
            output = out.getvalue().strip()

            expected_output_regex = r"Training knn\n({?'roc_auc_score': 0.5[0-9]*.*'accuracy': 0.8[0-9]*|{?'accuracy': 0.8[0-9]*.*'roc_auc_score': 0.5[0-9]*)"

            self.assertRegexpMatches(output, expected_output_regex)

    def test_random_forest_classification(self):
        hcai = SimpleDevelopSupervisedModel(dataframe=self.dataframe,
                                            predicted_column='ThirtyDayReadmitFLG',
                                            model_type='classification',
                                            impute=True,
                                            grain_column='PatientEncounterID',
                                            verbose=False)

        # Hacky way to capture print output since simple prints output instead of returning it.
        with captured_output() as (out, err):
            hcai.random_forest_classification()
            output = out.getvalue().strip()

            expected_output_regex = r"Training random_forest_classification\n({?'roc_auc_score': 0.7[0-9]*, 'accuracy': 0.8[0-9]*|{?'accuracy': 0.8[0-9]*, 'roc_auc_score': 0.7[0-9]*)"

            self.assertRegexpMatches(output, expected_output_regex)

    def test_linear_regression(self):
        hcai = SimpleDevelopSupervisedModel(self.dataframe,
                                            'SystolicBPNBR',
                                            'regression',
                                            impute=True,
                                            grain_column='PatientEncounterID')

        # Hacky way to capture print output since simple prints output instead of returning it.
        with captured_output() as (out, err):
            hcai.linear_regression()
            output = out.getvalue().strip()

            expected_output_regex = r"Training linear_regression\n(.*\n)?({?'mean_squared_error': 638\.[0-9]*, 'mean_absolute_error': 20\.[0-9]*|{?'mean_absolute_error': 20\.[0-9]*, 'mean_squared_error': 638\.[0-9]*)"

            self.assertRegexpMatches(output, expected_output_regex)

@contextmanager
def captured_output():
    new_out, new_err = StringIO(), StringIO()
    old_out, old_err = sys.stdout, sys.stderr
    try:
        sys.stdout, sys.stderr = new_out, new_err
        yield sys.stdout, sys.stderr
    finally:
        sys.stdout, sys.stderr = old_out, old_err

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from eho.server.service import api
from flask import request

from eho.server.utils.api import Rest, render, abort_and_log, request_data

import traceback

rest = Rest('v01', __name__)

@rest.route('/node-templates', methods=['GET', 'POST'])
def templates():
    if request.method == 'GET':
        try:
            return render(
                templates=[nt.dict for nt in api.get_node_templates()])
        except Exception, e:
            abort_and_log(500, "Exception while listing NodeTemplates: %s" % e)
    elif request.method == 'POST':
        data = request_data()
        try:
            return render(api.create_node_template(data).dict)
        except Exception, e:
            abort_and_log(500, "Exception while adding NodeTemplate: %s" % e)

@rest.route('/node-templates/<template_id>', methods=['GET', 'PUT', 'DELETE'])
def template(template_id):
    if request.method == 'GET':
        nt = None
        try:
            nt = api.get_node_template(id=template_id)
        except Exception, e:
            abort_and_log(500, "Exception while getting NodeTemplate by id "
                               "'%s': %s" % (template_id, e))
        if nt is None:
            abort_and_log(404, "NodeTemplate with id '%s' not found"
                               % template_id)

        return render(nt.dict)
    elif request.method == 'PUT':
        raise NotImplementedError("Template update op isn't implemented yet")
    elif request.method == 'DELETE':
        api.terminate_node_template(id=template_id)
        return render(status=204)

@rest.route('/clusters', methods=['GET', 'POST'])
def clusters():
    if request.method == 'GET':
        try:
            return render(clusters=[c.dict for c in api.get_clusters()])
        except Exception, e:
            abort_and_log(500, 'Exception while listing Clusters: %s' % e)
    elif request.method == 'POST':
        data = request_data()
        try:
            return render(api.create_cluster(data).dict)
        except Exception, e:
            traceback.print_exc(e)
            abort_and_log(500, "Exception while adding new Cluster: %s" % e)

@rest.route('/clusters/<cluster_id>', methods=['GET', 'PUT', 'DELETE'])
def cluster(cluster_id):
    if request.method == 'GET':
        c = None
        try:
            c = api.get_cluster(id=cluster_id)
        except Exception, e:
            abort_and_log(500, 'Exception while getting Cluster with id '
                               '\'%s\': %s' % (cluster_id, e))

        if c is None:
            abort_and_log(404, 'Cluster with id \'%s\' not found' % cluster_id)

        return render(c.dict)
    elif request.method == 'PUT':
        raise NotImplementedError("Cluster update op isn't implemented yet")
    elif request.method == 'DELETE':
        api.terminate_cluster(id=cluster_id)
        return render(status=204)
<commit_msg>traceback removed<commit_after>from eho.server.service import api
from flask import request

from eho.server.utils.api import Rest, render, abort_and_log, request_data

rest = Rest('v01', __name__)

@rest.route('/node-templates', methods=['GET', 'POST'])
def templates():
    if request.method == 'GET':
        try:
            return render(
                templates=[nt.dict for nt in api.get_node_templates()])
        except Exception, e:
            abort_and_log(500, "Exception while listing NodeTemplates: %s" % e)
    elif request.method == 'POST':
        data = request_data()
        try:
            return render(api.create_node_template(data).dict)
        except Exception, e:
            abort_and_log(500, "Exception while adding NodeTemplate: %s" % e)

@rest.route('/node-templates/<template_id>', methods=['GET', 'PUT', 'DELETE'])
def template(template_id):
    if request.method == 'GET':
        nt = None
        try:
            nt = api.get_node_template(id=template_id)
        except Exception, e:
            abort_and_log(500, "Exception while getting NodeTemplate by id "
                               "'%s': %s" % (template_id, e))
        if nt is None:
            abort_and_log(404, "NodeTemplate with id '%s' not found"
                               % template_id)

        return render(nt.dict)
    elif request.method == 'PUT':
        raise NotImplementedError("Template update op isn't implemented yet")
    elif request.method == 'DELETE':
        api.terminate_node_template(id=template_id)
        return render(status=204)

@rest.route('/clusters', methods=['GET', 'POST'])
def clusters():
    if request.method == 'GET':
        try:
            return render(clusters=[c.dict for c in api.get_clusters()])
        except Exception, e:
            abort_and_log(500, 'Exception while listing Clusters: %s' % e)
    elif request.method == 'POST':
        data = request_data()
        try:
            return render(api.create_cluster(data).dict)
        except Exception, e:
            abort_and_log(500, "Exception while adding new Cluster: %s" % e)

@rest.route('/clusters/<cluster_id>', methods=['GET', 'PUT', 'DELETE'])
def cluster(cluster_id):
    if request.method == 'GET':
        c = None
        try:
            c = api.get_cluster(id=cluster_id)
        except Exception, e:
            abort_and_log(500, 'Exception while getting Cluster with id '
                               '\'%s\': %s' % (cluster_id, e))

        if c is None:
            abort_and_log(404, 'Cluster with id \'%s\' not found' % cluster_id)

        return render(c.dict)
    elif request.method == 'PUT':
        raise NotImplementedError("Cluster update op isn't implemented yet")
    elif request.method == 'DELETE':
        api.terminate_cluster(id=cluster_id)
        return render(status=204)
<|endoftext|>