<commit_before># Copyright 2013 IBM Corp.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import testtools

from tempest.api.compute import base
from tempest.common import compute
from tempest.common.utils.linux import remote_client
from tempest.common import waiters
from tempest import config
from tempest import test

CONF = config.CONF

class AttachVolumeTestJSON(base.BaseV2ComputeTest):
    max_microversion = '2.19'

    def __init__(self, *args, **kwargs):
        super(AttachVolumeTestJSON, self).__init__(*args, **kwargs)
        self.attachment = None

    @classmethod
    def skip_checks(cls):
        super(AttachVolumeTestJSON, cls).skip_checks()
        if not CONF.service_available.cinder:
            skip_msg = ("%s skipped as Cinder is not available" % cls.__name__)
            raise cls.skipException(skip_msg)

    @classmethod
    def setup_credentials(cls):
        cls.prepare_instance_network()
        super(AttachVolumeTestJSON, cls).setup_credentials()

    @classmethod
    def resource_setup(cls):
        cls.set_validation_resources()

        super(AttachVolumeTestJSON, cls).resource_setup()
        cls.device = CONF.compute.volume_device_name

    def _detach(self, server_id, volume_id):
        if self.attachment:
            self.servers_client.detach_volume(server_id, volume_id)
            waiters.wait_for_volume_status(self.volumes_client,
                                           volume_id, 'available')

    def _create_server(self):
        # Start a server and wait for it to become ready
        server = self.create_test_server(
            validatable=True,
            wait_until='ACTIVE',
            adminPass=self.image_ssh_password)

        # Record addresses so that we can ssh later
        server['addresses'] = self.servers_client.list_addresses(
            server['id'])['addresses']
        return server

    def _create_and_attach_volume(self, server):
        # Create a volume and wait for it to become ready
        volume = self.create_volume()
        self.addCleanup(self.delete_volume, volume['id'])

        # Attach the volume to the server
        self.attachment = self.servers_client.attach_volume(
            server['id'],
            volumeId=volume['id'],
            device='/dev/%s' % self.device)['volumeAttachment']
        waiters.wait_for_volume_status(self.volumes_client,
                                       volume['id'], 'in-use')

        self.addCleanup(self._detach, server['id'], volume['id'])
        return volume

    @test.idempotent_id('52e9045a-e90d-4c0d-9087-79d657faffff')
    def test_attach_detach_volume(self):
        # Stop and Start a server with an attached volume, ensuring that
        # the volume remains attached.
        server = self._create_server()
        volume = self._create_and_attach_volume(server)

        self.servers_client.stop_server(server['id'])
        waiters.wait_for_server_status(self.servers_client, server['id'],
                                       'SHUTOFF')

        self.servers_client.start_server(server['id'])
        waiters.wait_for_server_status(self.servers_client, server['id'],
                                       'ACTIVE')

        if CONF.validation.run_validation:
            linux_client = remote_client.RemoteClient(
                self.get_server_ip(server),
                self.image_ssh_user,
                self.image_ssh_password,
                self.validation_resources['keypair']['private_key'],
                server=server,
                servers_client=self.servers_client)

            partitions = linux_client.get_partitions()
            device_name_to_match = ' ' + self.device + '\n'
            self.assertIn(device_name_to_match, partitions)

        self._detach(server['id'], volume['id'])
        self.attachment = None
        self.servers_client.stop_server(server['id'])
        waiters.wait_for_server_status(self.servers_client, server['id'],
                                       'SHUTOFF')

        self.servers_client.start_server(server['id'])
        waiters.wait_for_server_status(self.servers_client, server['id'],
                                       'ACTIVE')

        if CONF.validation.run_validation:
            linux_client = remote_client.RemoteClient(
                self.get_server_ip(server),
                self.image_ssh_user,
                self.image_ssh_password,
                self.validation_resources['keypair']['private_key'],
                server=server,
                servers_client=self.servers_client)

            partitions = linux_client.get_partitions()
            self.assertNotIn(device_name_to_match, partitions)

    @test.idempotent_id('7fa563fe-f0f7-43eb-9e22-a1ece036b513')
    def test_list_get_volume_attachments(self):
        # Create Server, Volume and attach that Volume to Server
        server = self._create_server()
        volume = self._create_and_attach_volume(server)

        # List Volume attachment of the server
        body = self.servers_client.list_volume_attachments(
            server['id'])['volumeAttachments']
        self.assertEqual(1, len(body))
        self.assertIn(self.attachment, body)

        # Get Volume attachment of the server
        body = self.servers_client.show_volume_attachment(
            server['id'],
            self.attachment['id'])['volumeAttachment']
        self.assertEqual(server['id'], body['serverId'])
        self.assertEqual(volume['id'], body['volumeId'])
        self.assertEqual(self.attachment['id'], body['id'])

class AttachVolumeShelveTestJSON(AttachVolumeTestJSON):
    """Testing volume with shelved instance.

    This test checks the attaching and detaching volumes from
    a shelved or shelved offload instance.
    """

    min_microversion = '2.20'
    max_microversion = 'latest'

    def _count_volumes(self, server):
        # Count number of volumes on an instance
        volumes = 0
        if CONF.validation.run_validation:
            linux_client = remote_client.RemoteClient(
                self.get_server_ip(server),
                self.image_ssh_user,
                self.image_ssh_password,
                self.validation_resources['keypair']['private_key'],
                server=server,
                servers_client=self.servers_client)

            command = 'grep -c -E [vs]d.$ /proc/partitions'
            volumes = int(linux_client.exec_command(command).strip())
        return volumes

    def _shelve_server(self, server):
        # NOTE(andreaf) If we are going to shelve a server, we should
        # check first whether the server is ssh-able. Otherwise we
        # won't be able to distinguish failures introduced by shelve
        # from pre-existing ones. Also it's good to wait for cloud-init
        # to be done and sshd server to be running before shelving to
        # avoid breaking the VM
        if CONF.validation.run_validation:
            linux_client = remote_client.RemoteClient(
                self.get_server_ip(server),
                self.image_ssh_user,
                self.image_ssh_password,
                self.validation_resources['keypair']['private_key'],
                server=server,
                servers_client=self.servers_client)
            linux_client.validate_authentication()

        # If validation went ok, or it was skipped, shelve the server
        compute.shelve_server(self.servers_client, server['id'])

    def _unshelve_server_and_check_volumes(self, server, number_of_volumes):
        # Unshelve the instance and check that there are expected volumes
        self.servers_client.unshelve_server(server['id'])
        waiters.wait_for_server_status(self.servers_client,
                                       server['id'],
                                       'ACTIVE')
        if CONF.validation.run_validation:
            counted_volumes = self._count_volumes(server)
            self.assertEqual(number_of_volumes, counted_volumes)

    @test.idempotent_id('13a940b6-3474-4c3c-b03f-29b89112bfee')
    @testtools.skipUnless(CONF.compute_feature_enabled.shelve,
                          'Shelve is not available.')
    def test_attach_volume_shelved_or_offload_server(self):
        # Create server, count number of volumes on it, shelve
        # server and attach pre-created volume to shelved server
        server = self._create_server()
        num_vol = self._count_volumes(server)
        self._shelve_server(server)
        self._create_and_attach_volume(server)

        # Unshelve the instance and check that attached volume exists
        self._unshelve_server_and_check_volumes(server, num_vol + 1)

        # Get Volume attachment of the server
        volume_attachment = self.servers_client.show_volume_attachment(
            server['id'],
            self.attachment['id'])['volumeAttachment']
        self.assertEqual(server['id'], volume_attachment['serverId'])
        self.assertEqual(self.attachment['id'], volume_attachment['id'])
        # Check the mountpoint is not None after unshelve server even in
        # case of shelved_offloaded.
        self.assertIsNotNone(volume_attachment['device'])

    @test.idempotent_id('b54e86dd-a070-49c4-9c07-59ae6dae15aa')
    @testtools.skipUnless(CONF.compute_feature_enabled.shelve,
                          'Shelve is not available.')
    def test_detach_volume_shelved_or_offload_server(self):
        # Create server, count number of volumes on it, shelve
        # server and attach pre-created volume to shelved server
        server = self._create_server()
        num_vol = self._count_volumes(server)
        self._shelve_server(server)
        volume = self._create_and_attach_volume(server)

        # Detach the volume
        self._detach(server['id'], volume['id'])
        self.attachment = None

        # Unshelve the instance and check that we have the expected number of
        # volume(s)
        self._unshelve_server_and_check_volumes(server, num_vol)
<commit_msg>Make the parameter 'device' optional<commit_after># Copyright 2013 IBM Corp.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import testtools

from tempest.api.compute import base
from tempest.common import compute
from tempest.common.utils.linux import remote_client
from tempest.common import waiters
from tempest import config
from tempest import test

CONF = config.CONF

class AttachVolumeTestJSON(base.BaseV2ComputeTest):
    max_microversion = '2.19'

    def __init__(self, *args, **kwargs):
        super(AttachVolumeTestJSON, self).__init__(*args, **kwargs)
        self.attachment = None

    @classmethod
    def skip_checks(cls):
        super(AttachVolumeTestJSON, cls).skip_checks()
        if not CONF.service_available.cinder:
            skip_msg = ("%s skipped as Cinder is not available" % cls.__name__)
            raise cls.skipException(skip_msg)

    @classmethod
    def setup_credentials(cls):
        cls.prepare_instance_network()
        super(AttachVolumeTestJSON, cls).setup_credentials()

    @classmethod
    def resource_setup(cls):
        cls.set_validation_resources()

        super(AttachVolumeTestJSON, cls).resource_setup()
        cls.device = CONF.compute.volume_device_name

    def _detach(self, server_id, volume_id):
        if self.attachment:
            self.servers_client.detach_volume(server_id, volume_id)
            waiters.wait_for_volume_status(self.volumes_client,
                                           volume_id, 'available')

    def _create_server(self):
        # Start a server and wait for it to become ready
        server = self.create_test_server(
            validatable=True,
            wait_until='ACTIVE',
            adminPass=self.image_ssh_password)

        # Record addresses so that we can ssh later
        server['addresses'] = self.servers_client.list_addresses(
            server['id'])['addresses']
        return server

    def _create_and_attach_volume(self, server, device=None):
        # Create a volume and wait for it to become ready
        volume = self.create_volume()
        self.addCleanup(self.delete_volume, volume['id'])

        # Attach the volume to the server
        kwargs = {'volumeId': volume['id']}
        if device:
            kwargs.update({'device': '/dev/%s' % device})
        self.attachment = self.servers_client.attach_volume(
            server['id'], **kwargs)['volumeAttachment']
        waiters.wait_for_volume_status(self.volumes_client,
                                       volume['id'], 'in-use')

        self.addCleanup(self._detach, server['id'], volume['id'])
        return volume

    @test.idempotent_id('52e9045a-e90d-4c0d-9087-79d657faffff')
    def test_attach_detach_volume(self):
        # Stop and Start a server with an attached volume, ensuring that
        # the volume remains attached.
        server = self._create_server()
        volume = self._create_and_attach_volume(server, device=self.device)

        self.servers_client.stop_server(server['id'])
        waiters.wait_for_server_status(self.servers_client, server['id'],
                                       'SHUTOFF')

        self.servers_client.start_server(server['id'])
        waiters.wait_for_server_status(self.servers_client, server['id'],
                                       'ACTIVE')

        if CONF.validation.run_validation:
            linux_client = remote_client.RemoteClient(
                self.get_server_ip(server),
                self.image_ssh_user,
                self.image_ssh_password,
                self.validation_resources['keypair']['private_key'],
                server=server,
                servers_client=self.servers_client)

            partitions = linux_client.get_partitions()
            device_name_to_match = ' ' + self.device + '\n'
            self.assertIn(device_name_to_match, partitions)

        self._detach(server['id'], volume['id'])
        self.attachment = None
        self.servers_client.stop_server(server['id'])
        waiters.wait_for_server_status(self.servers_client, server['id'],
                                       'SHUTOFF')

        self.servers_client.start_server(server['id'])
        waiters.wait_for_server_status(self.servers_client, server['id'],
                                       'ACTIVE')

        if CONF.validation.run_validation:
            linux_client = remote_client.RemoteClient(
                self.get_server_ip(server),
                self.image_ssh_user,
                self.image_ssh_password,
                self.validation_resources['keypair']['private_key'],
                server=server,
                servers_client=self.servers_client)

            partitions = linux_client.get_partitions()
            self.assertNotIn(device_name_to_match, partitions)

    @test.idempotent_id('7fa563fe-f0f7-43eb-9e22-a1ece036b513')
    def test_list_get_volume_attachments(self):
        # Create Server, Volume and attach that Volume to Server
        server = self._create_server()
        volume = self._create_and_attach_volume(server, device=self.device)

        # List Volume attachment of the server
        body = self.servers_client.list_volume_attachments(
            server['id'])['volumeAttachments']
        self.assertEqual(1, len(body))
        self.assertIn(self.attachment, body)

        # Get Volume attachment of the server
        body = self.servers_client.show_volume_attachment(
            server['id'],
            self.attachment['id'])['volumeAttachment']
        self.assertEqual(server['id'], body['serverId'])
        self.assertEqual(volume['id'], body['volumeId'])
        self.assertEqual(self.attachment['id'], body['id'])

class AttachVolumeShelveTestJSON(AttachVolumeTestJSON):
    """Testing volume with shelved instance.

    This test checks the attaching and detaching volumes from
    a shelved or shelved offload instance.
    """

    min_microversion = '2.20'
    max_microversion = 'latest'

    def _count_volumes(self, server):
        # Count number of volumes on an instance
        volumes = 0
        if CONF.validation.run_validation:
            linux_client = remote_client.RemoteClient(
                self.get_server_ip(server),
                self.image_ssh_user,
                self.image_ssh_password,
                self.validation_resources['keypair']['private_key'],
                server=server,
                servers_client=self.servers_client)

            command = 'grep -c -E [vs]d.$ /proc/partitions'
            volumes = int(linux_client.exec_command(command).strip())
        return volumes

    def _shelve_server(self, server):
        # NOTE(andreaf) If we are going to shelve a server, we should
        # check first whether the server is ssh-able. Otherwise we
        # won't be able to distinguish failures introduced by shelve
        # from pre-existing ones. Also it's good to wait for cloud-init
        # to be done and sshd server to be running before shelving to
        # avoid breaking the VM
        if CONF.validation.run_validation:
            linux_client = remote_client.RemoteClient(
                self.get_server_ip(server),
                self.image_ssh_user,
                self.image_ssh_password,
                self.validation_resources['keypair']['private_key'],
                server=server,
                servers_client=self.servers_client)
            linux_client.validate_authentication()

        # If validation went ok, or it was skipped, shelve the server
        compute.shelve_server(self.servers_client, server['id'])

    def _unshelve_server_and_check_volumes(self, server, number_of_volumes):
        # Unshelve the instance and check that there are expected volumes
        self.servers_client.unshelve_server(server['id'])
        waiters.wait_for_server_status(self.servers_client,
                                       server['id'],
                                       'ACTIVE')
        if CONF.validation.run_validation:
            counted_volumes = self._count_volumes(server)
            self.assertEqual(number_of_volumes, counted_volumes)

    @test.idempotent_id('13a940b6-3474-4c3c-b03f-29b89112bfee')
    @testtools.skipUnless(CONF.compute_feature_enabled.shelve,
                          'Shelve is not available.')
    def test_attach_volume_shelved_or_offload_server(self):
        # Create server, count number of volumes on it, shelve
        # server and attach pre-created volume to shelved server
        server = self._create_server()
        num_vol = self._count_volumes(server)
        self._shelve_server(server)
        self._create_and_attach_volume(server, device=self.device)

        # Unshelve the instance and check that attached volume exists
        self._unshelve_server_and_check_volumes(server, num_vol + 1)

        # Get Volume attachment of the server
        volume_attachment = self.servers_client.show_volume_attachment(
            server['id'],
            self.attachment['id'])['volumeAttachment']
        self.assertEqual(server['id'], volume_attachment['serverId'])
        self.assertEqual(self.attachment['id'], volume_attachment['id'])
        # Check the mountpoint is not None after unshelve server even in
        # case of shelved_offloaded.
        self.assertIsNotNone(volume_attachment['device'])

    @test.idempotent_id('b54e86dd-a070-49c4-9c07-59ae6dae15aa')
    @testtools.skipUnless(CONF.compute_feature_enabled.shelve,
                          'Shelve is not available.')
    def test_detach_volume_shelved_or_offload_server(self):
        # Create server, count number of volumes on it, shelve
        # server and attach pre-created volume to shelved server
        server = self._create_server()
        num_vol = self._count_volumes(server)
        self._shelve_server(server)
        volume = self._create_and_attach_volume(server, device=self.device)

        # Detach the volume
        self._detach(server['id'], volume['id'])
        self.attachment = None

        # Unshelve the instance and check that we have the expected number of
        # volume(s)
        self._unshelve_server_and_check_volumes(server, num_vol)
<|endoftext|>


<commit_before>from django.db import models
from django.utils.translation import ugettext as _
from event_kiosk.events.models import Event
from datetime import datetime

class Presentation(models.Model):
    name = models.CharField(_('name'), max_length=255, help_text=_('Reference name for this presentation. Only visible in the admin.'))
    transitionTime = models.PositiveIntegerField(_('transition time'), help_text=_('Number of seconds to display each slide.'))
    pauseTimeOnTouch = models.PositiveIntegerField(_('pause time on touch'), help_text=_('If someone interacts with the kiosk, stop the slideshow for this number of seconds.'))

    headerImage = models.ImageField(_('header image'), upload_to='headers/', null=True, blank=True)

    displayMenu = models.BooleanField(_('display menu'), default=True, help_text=_('Whether or not the menu should appear on this presentation.'))
    displayIndicators = models.BooleanField(_('display indicators'), default=True, help_text=_('Whether or not to display the little dots at the bottom that show slideshow progress.'))

    def __str__(self):
        return self.name

class Slide(models.Model):
    IMAGE = 'image'
    EVENT = 'event'
    EVENT_LIST = 'eventList'
    WEATHER = 'weather'

    SLIDE_TYPE_CHOICES = (
        ('', ''),
        (IMAGE, _('Image')),
        (EVENT, _('Event')),
        (EVENT_LIST, _('Event list')),
        (WEATHER, _('Weather')),
    )

    MONTHLY = 'MONTHLY'
    DAILY = 'DAILY'

    EVENT_LIST_SLIDE_FORMAT_CHOICES = (
        (MONTHLY, _('Monthly')),
        (DAILY, _('Daily'))
    )

    presentation = models.ForeignKey(to=Presentation, related_name="slides", null=False, blank=False)
    type = models.CharField(_('type'), max_length=32, choices=SLIDE_TYPE_CHOICES, default=None, blank=False, null=False)

    # Image slide background image
    image = models.ImageField(_('image'), upload_to='slides/%Y/%m/%d/', null=True, blank=True)

    # Event slide event
    event = models.ForeignKey(to=Event, related_name="slides", null=True, blank=True)

    # Event list slide display type
    displayType = models.CharField(_('event list display type'), max_length=32,
                                          choices=EVENT_LIST_SLIDE_FORMAT_CHOICES, default=MONTHLY,
                                          blank=False, null=False)

    position = models.PositiveIntegerField(default=0, blank=False, null=False)

    # Weather slide information
    location = models.CharField(_('location'), help_text=_('A comma-delimited country name and country code (ISO 3166)'), max_length=255, null=True, blank=True)
    lat = models.DecimalField(_('latitude'), null=True, blank=True, max_digits=10, decimal_places=7)
    lon = models.DecimalField(_('longitude'), null=True, blank=True, max_digits=10, decimal_places=7)

    def to_json(self):
        if (self.type == 'image'):
            return {
                'type': 'image',
                'title': '',
                'img': self.image.url
            }

        elif (self.type == 'event'):
            return {
                'type': 'event',
                'title': self.event.shortTitle if self.event.shortTitle != '' else self.event.title,
                'event': self.event.to_json()
            }

        elif (self.type == 'weather'):
            return {
                'type': 'weather',
                'location': self.location,
                'lat': self.lat,
                'lon': self.lon,
                'title': ''
            }

        elif (self.type == 'eventList'):
            events = Event.objects.filter(date__gte=datetime.today())
            events_json = []
            for event in events:
                events_json.append(event.to_json())

            return {
                'type': 'eventList',
                'title': _('Upcoming events'),
                'displayType': self.displayType,
                'events': events_json
            }

    def __str__(self):
        return 'Slide'

    class Meta(object):
        ordering = ('position', )
<commit_msg>Added an ordering on events<commit_after>from django.db import models
from django.utils.translation import ugettext as _
from event_kiosk.events.models import Event
from datetime import datetime

class Presentation(models.Model):
    name = models.CharField(_('name'), max_length=255, help_text=_('Reference name for this presentation. Only visible in the admin.'))
    transitionTime = models.PositiveIntegerField(_('transition time'), help_text=_('Number of seconds to display each slide.'))
    pauseTimeOnTouch = models.PositiveIntegerField(_('pause time on touch'), help_text=_('If someone interacts with the kiosk, stop the slideshow for this number of seconds.'))

    headerImage = models.ImageField(_('header image'), upload_to='headers/', null=True, blank=True)

    displayMenu = models.BooleanField(_('display menu'), default=True, help_text=_('Whether or not the menu should appear on this presentation.'))
    displayIndicators = models.BooleanField(_('display indicators'), default=True, help_text=_('Whether or not to display the little dots at the bottom that show slideshow progress.'))

    def __str__(self):
        return self.name

class Slide(models.Model):
    IMAGE = 'image'
    EVENT = 'event'
    EVENT_LIST = 'eventList'
    WEATHER = 'weather'

    SLIDE_TYPE_CHOICES = (
        ('', ''),
        (IMAGE, _('Image')),
        (EVENT, _('Event')),
        (EVENT_LIST, _('Event list')),
        (WEATHER, _('Weather')),
    )

    MONTHLY = 'MONTHLY'
    DAILY = 'DAILY'

    EVENT_LIST_SLIDE_FORMAT_CHOICES = (
        (MONTHLY, _('Monthly')),
        (DAILY, _('Daily'))
    )

    presentation = models.ForeignKey(to=Presentation, related_name="slides", null=False, blank=False)
    type = models.CharField(_('type'), max_length=32, choices=SLIDE_TYPE_CHOICES, default=None, blank=False, null=False)

    # Image slide background image
    image = models.ImageField(_('image'), upload_to='slides/%Y/%m/%d/', null=True, blank=True)

    # Event slide event
    event = models.ForeignKey(to=Event, related_name="slides", null=True, blank=True)

    # Event list slide display type
    displayType = models.CharField(_('event list display type'), max_length=32,
                                          choices=EVENT_LIST_SLIDE_FORMAT_CHOICES, default=MONTHLY,
                                          blank=False, null=False)

    position = models.PositiveIntegerField(default=0, blank=False, null=False)

    # Weather slide information
    location = models.CharField(_('location'), help_text=_('A comma-delimited country name and country code (ISO 3166)'), max_length=255, null=True, blank=True)
    lat = models.DecimalField(_('latitude'), null=True, blank=True, max_digits=10, decimal_places=7)
    lon = models.DecimalField(_('longitude'), null=True, blank=True, max_digits=10, decimal_places=7)

    def to_json(self):
        if (self.type == 'image'):
            return {
                'type': 'image',
                'title': '',
                'img': self.image.url
            }

        elif (self.type == 'event'):
            return {
                'type': 'event',
                'title': self.event.shortTitle if self.event.shortTitle != '' else self.event.title,
                'event': self.event.to_json()
            }

        elif (self.type == 'weather'):
            return {
                'type': 'weather',
                'location': self.location,
                'lat': self.lat,
                'lon': self.lon,
                'title': ''
            }

        elif (self.type == 'eventList'):
            events = Event.objects.filter(
                date__gte=datetime.today()
            ).order_by('date', 'startTime', 'shortTitle', 'title')
            events_json = []
            for event in events:
                events_json.append(event.to_json())

            return {
                'type': 'eventList',
                'title': _('Upcoming events'),
                'displayType': self.displayType,
                'events': events_json
            }

    def __str__(self):
        return 'Slide'

    class Meta(object):
        ordering = ('position', )
<|endoftext|>


<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _
from frappe.model.document import Document

class WebsiteTheme(Document):
	def validate(self):
		self.validate_if_customizable()
		self.validate_colors()

	def on_update(self):
		if (not self.custom
			and frappe.local.conf.get('developer_mode')
			and not (frappe.flags.in_import or frappe.flags.in_test)):

			self.export_doc()

		self.clear_cache_if_current_theme()

	def is_standard_and_not_valid_user(self):
		return (not self.custom
			and not frappe.local.conf.get('developer_mode')
			and not (frappe.flags.in_import or frappe.flags.in_test))

	def on_trash(self):
		if self.is_standard_and_not_valid_user():
			frappe.throw(_("You are not allowed to delete a standard Website Theme"),
				frappe.PermissionError)

	def validate_if_customizable(self):
		if self.is_standard_and_not_valid_user():
			frappe.throw(_("Please Duplicate this Website Theme to customize."))

	def validate_colors(self):
		if (self.top_bar_color or self.top_bar_text_color) and \
			self.top_bar_color==self.top_bar_text_color:
				frappe.throw(_("Top Bar Color and Text Color are the same. They should be have good contrast to be readable."))

	def export_doc(self):
		"""Export to standard folder `[module]/website_theme/[name]/[name].json`."""
		from frappe.modules.export_file import export_to_files
		export_to_files(record_list=[['Website Theme', self.name]])

	def clear_cache_if_current_theme(self):
		website_settings = frappe.get_doc("Website Settings", "Website Settings")
		if getattr(website_settings, "website_theme", None) == self.name:
			website_settings.clear_cache()

	def use_theme(self):
		use_theme(self.name)

@frappe.whitelist()
def use_theme(theme):
	website_settings = frappe.get_doc("Website Settings", "Website Settings")
	website_settings.website_theme = theme
	website_settings.ignore_validate = True
	website_settings.save()

def add_website_theme(context):
	bootstrap = frappe.get_hooks("bootstrap")[0]
	website_theme = get_active_theme()
	context.theme = website_theme or frappe._dict()
	if website_theme:
		if website_theme.bootstrap:
			bootstrap = website_theme.bootstrap

		context.no_sidebar = website_theme.no_sidebar

	context.web_include_css = [bootstrap] + context.web_include_css

def get_active_theme():
	website_theme = frappe.db.get_value("Website Settings", "Website Settings", "website_theme")
	if website_theme:
		return frappe.get_doc("Website Theme", website_theme)

<commit_msg>[hotfix] website theme<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _
from frappe.model.document import Document

class WebsiteTheme(Document):
	def validate(self):
		self.validate_if_customizable()
		self.validate_colors()

	def on_update(self):
		if (not self.custom
			and frappe.local.conf.get('developer_mode')
			and not (frappe.flags.in_import or frappe.flags.in_test)):

			self.export_doc()

		self.clear_cache_if_current_theme()

	def is_standard_and_not_valid_user(self):
		return (not self.custom
			and not frappe.local.conf.get('developer_mode')
			and not (frappe.flags.in_import or frappe.flags.in_test))

	def on_trash(self):
		if self.is_standard_and_not_valid_user():
			frappe.throw(_("You are not allowed to delete a standard Website Theme"),
				frappe.PermissionError)

	def validate_if_customizable(self):
		if self.is_standard_and_not_valid_user():
			frappe.throw(_("Please Duplicate this Website Theme to customize."))

	def validate_colors(self):
		if (self.top_bar_color or self.top_bar_text_color) and \
			self.top_bar_color==self.top_bar_text_color:
				frappe.throw(_("Top Bar Color and Text Color are the same. They should be have good contrast to be readable."))

	def export_doc(self):
		"""Export to standard folder `[module]/website_theme/[name]/[name].json`."""
		from frappe.modules.export_file import export_to_files
		export_to_files(record_list=[['Website Theme', self.name]])

	def clear_cache_if_current_theme(self):
		website_settings = frappe.get_doc("Website Settings", "Website Settings")
		if getattr(website_settings, "website_theme", None) == self.name:
			website_settings.clear_cache()

	def use_theme(self):
		use_theme(self.name)

@frappe.whitelist()
def use_theme(theme):
	website_settings = frappe.get_doc("Website Settings", "Website Settings")
	website_settings.website_theme = theme
	website_settings.ignore_validate = True
	website_settings.save()

def add_website_theme(context):
	bootstrap = frappe.get_hooks("bootstrap")[0]
	website_theme = get_active_theme()
	context.theme = website_theme and website_theme.as_dict() or frappe._dict()
	if website_theme:
		if website_theme.bootstrap:
			bootstrap = website_theme.bootstrap

		context.no_sidebar = website_theme.no_sidebar

	context.web_include_css = [bootstrap] + context.web_include_css

def get_active_theme():
	website_theme = frappe.db.get_value("Website Settings", "Website Settings", "website_theme")
	if website_theme:
		return frappe.get_doc("Website Theme", website_theme)

<|endoftext|>


<commit_before>import json

from django.conf import settings
from django.http import Http404, HttpResponseBadRequest, JsonResponse
from django.utils.decorators import method_decorator
from django.views import View
from django.views.decorators.csrf import csrf_exempt

from corehq.apps.domain.auth import formplayer_auth
from corehq.apps.domain.models import Domain
from corehq.apps.hqadmin.utils import get_django_user_from_session, get_session
from corehq.apps.users.models import CouchUser

@method_decorator(csrf_exempt, name='dispatch')
@method_decorator(formplayer_auth, name='dispatch')
class SessionDetailsView(View):
    """
    Internal API to allow formplayer to get the Django user ID
    from the session key.

    Authentication is done by HMAC signing of the request body:

        secret = settings.FORMPLAYER_INTERNAL_AUTH_KEY
        data = '{"session_id": "123"}'
        digest = base64.b64encode(hmac.new(secret, data, hashlib.sha256).digest())
        requests.post(url, data=data, headers={'X-MAC-DIGEST': digest})
    """
    urlname = 'session_details'
    http_method_names = ['post']

    def post(self, request, *args, **kwargs):
        try:
            data = json.loads(request.body.decode('utf-8'))
        except ValueError:
            return HttpResponseBadRequest()

        if not data or not isinstance(data, dict):
            return HttpResponseBadRequest()

        session_id = data.get('sessionId', None)
        if not session_id:
            return HttpResponseBadRequest()

        session = get_session(session_id)
        user = get_django_user_from_session(session)
        if user:
            couch_user = CouchUser.get_by_username(user.username)
            if not couch_user:
                raise Http404
        else:
            raise Http404

        # reset the session's expiry if there's some formplayer activity
        secure_session = session.get('secure_session')
        if secure_session:
            timeout = settings.SECURE_TIMEOUT
            domain = data.get('domain')
            if domain:
                domain_obj = Domain.get_by_name(domain)
                if domain_obj:
                    timeout = domain_obj.secure_timeout or secure_session_timeout
        else:
            timeout = settings.INACTIVITY_TIMEOUT

        session.set_expiry(timeout * 60)
        session.save()

        return JsonResponse({
            'username': user.username,
            'djangoUserId': user.pk,
            'superUser': user.is_superuser,
            'authToken': None,
            'domains': couch_user.domains,
            'anonymous': False
        })
<commit_msg>Fixed one more outdated reference<commit_after>import json

from django.conf import settings
from django.http import Http404, HttpResponseBadRequest, JsonResponse
from django.utils.decorators import method_decorator
from django.views import View
from django.views.decorators.csrf import csrf_exempt

from corehq.apps.domain.auth import formplayer_auth
from corehq.apps.domain.models import Domain
from corehq.apps.hqadmin.utils import get_django_user_from_session, get_session
from corehq.apps.users.models import CouchUser

@method_decorator(csrf_exempt, name='dispatch')
@method_decorator(formplayer_auth, name='dispatch')
class SessionDetailsView(View):
    """
    Internal API to allow formplayer to get the Django user ID
    from the session key.

    Authentication is done by HMAC signing of the request body:

        secret = settings.FORMPLAYER_INTERNAL_AUTH_KEY
        data = '{"session_id": "123"}'
        digest = base64.b64encode(hmac.new(secret, data, hashlib.sha256).digest())
        requests.post(url, data=data, headers={'X-MAC-DIGEST': digest})
    """
    urlname = 'session_details'
    http_method_names = ['post']

    def post(self, request, *args, **kwargs):
        try:
            data = json.loads(request.body.decode('utf-8'))
        except ValueError:
            return HttpResponseBadRequest()

        if not data or not isinstance(data, dict):
            return HttpResponseBadRequest()

        session_id = data.get('sessionId', None)
        if not session_id:
            return HttpResponseBadRequest()

        session = get_session(session_id)
        user = get_django_user_from_session(session)
        if user:
            couch_user = CouchUser.get_by_username(user.username)
            if not couch_user:
                raise Http404
        else:
            raise Http404

        # reset the session's expiry if there's some formplayer activity
        secure_session = session.get('secure_session')
        if secure_session:
            timeout = settings.SECURE_TIMEOUT
            domain = data.get('domain')
            if domain:
                domain_obj = Domain.get_by_name(domain)
                if domain_obj:
                    timeout = domain_obj.secure_timeout or timeout
        else:
            timeout = settings.INACTIVITY_TIMEOUT

        session.set_expiry(timeout * 60)
        session.save()

        return JsonResponse({
            'username': user.username,
            'djangoUserId': user.pk,
            'superUser': user.is_superuser,
            'authToken': None,
            'domains': couch_user.domains,
            'anonymous': False
        })
<|endoftext|>


<commit_before># Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Barrnap(Package):
    """Barrnap predicts the location of ribosomal RNA genes in genomes."""

    homepage = "https://github.com/tseemann/barrnap"
    url      = "https://github.com/tseemann/barrnap/archive/0.8.tar.gz"

    version('0.8', 'd02ccb800d60fa824bae946dd4fa2358')

    depends_on('hmmer@3.1b:', type='run')

    def install(self, spec, prefix):
        install_tree('bin', prefix.bin)
        install_tree('db', prefix.db)
<commit_msg>barrnap package: Add versions 0.6 and 0.7 (#9820)<commit_after># Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Barrnap(Package):
    """Barrnap predicts the location of ribosomal RNA genes in genomes."""

    homepage = "https://github.com/tseemann/barrnap"
    url      = "https://github.com/tseemann/barrnap/archive/0.8.tar.gz"

    version('0.8', 'd02ccb800d60fa824bae946dd4fa2358')
    version('0.7', 'ef2173e250f06cca7569c03404c9d4ab6a908ef7643e28901fbe9a732d20c09b')
    version('0.6', '272642a41634623bda34dccdce487ab791925fa769e3e575d53014956a1f9dce')

    depends_on('hmmer@3.1b:', type='run')

    def install(self, spec, prefix):
        install_tree('bin', prefix.bin)
        install_tree('db', prefix.db)
<|endoftext|>


<commit_before>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest.api.volume import base
from tempest import config
from tempest.lib import decorators

CONF = config.CONF

class VolumesSnapshotListTestJSON(base.BaseVolumeTest):

    @classmethod
    def skip_checks(cls):
        super(VolumesSnapshotListTestJSON, cls).skip_checks()
        if not CONF.volume_feature_enabled.snapshot:
            raise cls.skipException("Cinder volume snapshots are disabled")

    @classmethod
    def resource_setup(cls):
        super(VolumesSnapshotListTestJSON, cls).resource_setup()
        cls.snapshot_id_list = []
        volume_origin = cls.create_volume()

        # Create snapshots with params
        for _ in range(3):
            snapshot = cls.create_snapshot(volume_origin['id'])
            cls.snapshot_id_list.append(snapshot['id'])
        cls.snapshot = snapshot

    def _list_by_param_values_and_assert(self, with_detail=False, **params):
        """list or list_details with given params and validates result."""

        fetched_snap_list = self.snapshots_client.list_snapshots(
            detail=with_detail, **params)['snapshots']

        # Validating params of fetched snapshots
        for snap in fetched_snap_list:
            for key in params:
                msg = "Failed to list snapshots %s by %s" % \
                      ('details' if with_detail else '', key)
                self.assertEqual(params[key], snap[key], msg)

    def _list_snapshots_by_param_limit(self, limit, expected_elements):
        """list snapshots by limit param"""
        # Get snapshots list using limit parameter
        fetched_snap_list = self.snapshots_client.list_snapshots(
            limit=limit)['snapshots']
        # Validating filtered snapshots length equals to expected_elements
        self.assertEqual(expected_elements, len(fetched_snap_list))

    @decorators.idempotent_id('59f41f43-aebf-48a9-ab5d-d76340fab32b')
    def test_snapshots_list_with_params(self):
        """list snapshots with params."""
        # Verify list snapshots by display_name filter
        params = {'name': self.snapshot['name']}
        self._list_by_param_values_and_assert(**params)

        # Verify list snapshots by status filter
        params = {'status': 'available'}
        self._list_by_param_values_and_assert(**params)

        # Verify list snapshots by status and display name filter
        params = {'status': 'available',
                  'name': self.snapshot['name']}
        self._list_by_param_values_and_assert(**params)

    @decorators.idempotent_id('220a1022-1fcd-4a74-a7bd-6b859156cda2')
    def test_snapshots_list_details_with_params(self):
        """list snapshot details with params."""
        # Verify list snapshot details by display_name filter
        params = {'name': self.snapshot['name']}
        self._list_by_param_values_and_assert(with_detail=True, **params)
        # Verify list snapshot details by status filter
        params = {'status': 'available'}
        self._list_by_param_values_and_assert(with_detail=True, **params)
        # Verify list snapshot details by status and display name filter
        params = {'status': 'available',
                  'name': self.snapshot['name']}
        self._list_by_param_values_and_assert(with_detail=True, **params)

    @decorators.idempotent_id('db4d8e0a-7a2e-41cc-a712-961f6844e896')
    def test_snapshot_list_param_limit(self):
        # List returns limited elements
        self._list_snapshots_by_param_limit(limit=1, expected_elements=1)

    @decorators.idempotent_id('a1427f61-420e-48a5-b6e3-0b394fa95400')
    def test_snapshot_list_param_limit_equals_infinite(self):
        # List returns all elements when request limit exceeded
        # snapshots number
        snap_list = self.snapshots_client.list_snapshots()['snapshots']
        self._list_snapshots_by_param_limit(limit=100000,
                                            expected_elements=len(snap_list))

    @decorators.idempotent_id('e3b44b7f-ae87-45b5-8a8c-66110eb24d0a')
    def test_snapshot_list_param_limit_equals_zero(self):
        # List returns zero elements
        self._list_snapshots_by_param_limit(limit=0, expected_elements=0)

    def _list_snapshots_param_sort(self, sort_key, sort_dir):
        """list snapshots by sort param"""
        snap_list = self.snapshots_client.list_snapshots(
            sort_key=sort_key, sort_dir=sort_dir)['snapshots']
        self.assertNotEmpty(snap_list)
        if sort_key is 'display_name':
            sort_key = 'name'
        # Note: On Cinder API, 'display_name' works as a sort key
        # on a request, a volume name appears as 'name' on the response.
        # So Tempest needs to change the key name here for this inconsistent
        # API behavior.
        sorted_list = [snapshot[sort_key] for snapshot in snap_list]
        msg = 'The list of snapshots was not sorted correctly.'
        self.assertEqual(sorted(sorted_list, reverse=(sort_dir == 'desc')),
                         sorted_list, msg)

    @decorators.idempotent_id('c5513ada-64c1-4d28-83b9-af3307ec1388')
    def test_snapshot_list_param_sort_id_asc(self):
        self._list_snapshots_param_sort(sort_key='id', sort_dir='asc')

    @decorators.idempotent_id('8a7fe058-0b41-402a-8afd-2dbc5a4a718b')
    def test_snapshot_list_param_sort_id_desc(self):
        self._list_snapshots_param_sort(sort_key='id', sort_dir='desc')

    @decorators.idempotent_id('4052c3a0-2415-440a-a8cc-305a875331b0')
    def test_snapshot_list_param_sort_created_at_asc(self):
        self._list_snapshots_param_sort(sort_key='created_at', sort_dir='asc')

    @decorators.idempotent_id('dcbbe24a-f3c0-4ec8-9274-55d48db8d1cf')
    def test_snapshot_list_param_sort_created_at_desc(self):
        self._list_snapshots_param_sort(sort_key='created_at', sort_dir='desc')

    @decorators.idempotent_id('d58b5fed-0c37-42d3-8c5d-39014ac13c00')
    def test_snapshot_list_param_sort_name_asc(self):
        self._list_snapshots_param_sort(sort_key='display_name',
                                        sort_dir='asc')

    @decorators.idempotent_id('96ba6f4d-1f18-47e1-b4bc-76edc6c21250')
    def test_snapshot_list_param_sort_name_desc(self):
        self._list_snapshots_param_sort(sort_key='display_name',
                                        sort_dir='desc')

    @decorators.idempotent_id('05489dde-44bc-4961-a1f5-3ce7ee7824f7')
    def test_snapshot_list_param_marker(self):
        # The list of snapshots should end before the provided marker
        params = {'marker': self.snapshot_id_list[1]}
        snap_list = self.snapshots_client.list_snapshots(**params)['snapshots']
        fetched_list_id = [snap['id'] for snap in snap_list]
        # Verify the list of snapshots ends before the provided
        # marker(second snapshot), therefore only the first snapshot
        # should displayed.
        self.assertEqual(self.snapshot_id_list[:1], fetched_list_id)
<commit_msg>Fix test_snapshot_list_param_marker when using predefined user<commit_after>#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from tempest.api.volume import base
from tempest import config
from tempest.lib import decorators

CONF = config.CONF

class VolumesSnapshotListTestJSON(base.BaseVolumeTest):

    @classmethod
    def skip_checks(cls):
        super(VolumesSnapshotListTestJSON, cls).skip_checks()
        if not CONF.volume_feature_enabled.snapshot:
            raise cls.skipException("Cinder volume snapshots are disabled")

    @classmethod
    def resource_setup(cls):
        super(VolumesSnapshotListTestJSON, cls).resource_setup()
        volume_origin = cls.create_volume()

        # Create snapshots with params
        for _ in range(3):
            snapshot = cls.create_snapshot(volume_origin['id'])
        cls.snapshot = snapshot

    def _list_by_param_values_and_assert(self, with_detail=False, **params):
        """list or list_details with given params and validates result."""

        fetched_snap_list = self.snapshots_client.list_snapshots(
            detail=with_detail, **params)['snapshots']

        # Validating params of fetched snapshots
        for snap in fetched_snap_list:
            for key in params:
                msg = "Failed to list snapshots %s by %s" % \
                      ('details' if with_detail else '', key)
                self.assertEqual(params[key], snap[key], msg)

    def _list_snapshots_by_param_limit(self, limit, expected_elements):
        """list snapshots by limit param"""
        # Get snapshots list using limit parameter
        fetched_snap_list = self.snapshots_client.list_snapshots(
            limit=limit)['snapshots']
        # Validating filtered snapshots length equals to expected_elements
        self.assertEqual(expected_elements, len(fetched_snap_list))

    @decorators.idempotent_id('59f41f43-aebf-48a9-ab5d-d76340fab32b')
    def test_snapshots_list_with_params(self):
        """list snapshots with params."""
        # Verify list snapshots by display_name filter
        params = {'name': self.snapshot['name']}
        self._list_by_param_values_and_assert(**params)

        # Verify list snapshots by status filter
        params = {'status': 'available'}
        self._list_by_param_values_and_assert(**params)

        # Verify list snapshots by status and display name filter
        params = {'status': 'available',
                  'name': self.snapshot['name']}
        self._list_by_param_values_and_assert(**params)

    @decorators.idempotent_id('220a1022-1fcd-4a74-a7bd-6b859156cda2')
    def test_snapshots_list_details_with_params(self):
        """list snapshot details with params."""
        # Verify list snapshot details by display_name filter
        params = {'name': self.snapshot['name']}
        self._list_by_param_values_and_assert(with_detail=True, **params)
        # Verify list snapshot details by status filter
        params = {'status': 'available'}
        self._list_by_param_values_and_assert(with_detail=True, **params)
        # Verify list snapshot details by status and display name filter
        params = {'status': 'available',
                  'name': self.snapshot['name']}
        self._list_by_param_values_and_assert(with_detail=True, **params)

    @decorators.idempotent_id('db4d8e0a-7a2e-41cc-a712-961f6844e896')
    def test_snapshot_list_param_limit(self):
        # List returns limited elements
        self._list_snapshots_by_param_limit(limit=1, expected_elements=1)

    @decorators.idempotent_id('a1427f61-420e-48a5-b6e3-0b394fa95400')
    def test_snapshot_list_param_limit_equals_infinite(self):
        # List returns all elements when request limit exceeded
        # snapshots number
        snap_list = self.snapshots_client.list_snapshots()['snapshots']
        self._list_snapshots_by_param_limit(limit=100000,
                                            expected_elements=len(snap_list))

    @decorators.idempotent_id('e3b44b7f-ae87-45b5-8a8c-66110eb24d0a')
    def test_snapshot_list_param_limit_equals_zero(self):
        # List returns zero elements
        self._list_snapshots_by_param_limit(limit=0, expected_elements=0)

    def _list_snapshots_param_sort(self, sort_key, sort_dir):
        """list snapshots by sort param"""
        snap_list = self.snapshots_client.list_snapshots(
            sort_key=sort_key, sort_dir=sort_dir)['snapshots']
        self.assertNotEmpty(snap_list)
        if sort_key is 'display_name':
            sort_key = 'name'
        # Note: On Cinder API, 'display_name' works as a sort key
        # on a request, a volume name appears as 'name' on the response.
        # So Tempest needs to change the key name here for this inconsistent
        # API behavior.
        sorted_list = [snapshot[sort_key] for snapshot in snap_list]
        msg = 'The list of snapshots was not sorted correctly.'
        self.assertEqual(sorted(sorted_list, reverse=(sort_dir == 'desc')),
                         sorted_list, msg)

    @decorators.idempotent_id('c5513ada-64c1-4d28-83b9-af3307ec1388')
    def test_snapshot_list_param_sort_id_asc(self):
        self._list_snapshots_param_sort(sort_key='id', sort_dir='asc')

    @decorators.idempotent_id('8a7fe058-0b41-402a-8afd-2dbc5a4a718b')
    def test_snapshot_list_param_sort_id_desc(self):
        self._list_snapshots_param_sort(sort_key='id', sort_dir='desc')

    @decorators.idempotent_id('4052c3a0-2415-440a-a8cc-305a875331b0')
    def test_snapshot_list_param_sort_created_at_asc(self):
        self._list_snapshots_param_sort(sort_key='created_at', sort_dir='asc')

    @decorators.idempotent_id('dcbbe24a-f3c0-4ec8-9274-55d48db8d1cf')
    def test_snapshot_list_param_sort_created_at_desc(self):
        self._list_snapshots_param_sort(sort_key='created_at', sort_dir='desc')

    @decorators.idempotent_id('d58b5fed-0c37-42d3-8c5d-39014ac13c00')
    def test_snapshot_list_param_sort_name_asc(self):
        self._list_snapshots_param_sort(sort_key='display_name',
                                        sort_dir='asc')

    @decorators.idempotent_id('96ba6f4d-1f18-47e1-b4bc-76edc6c21250')
    def test_snapshot_list_param_sort_name_desc(self):
        self._list_snapshots_param_sort(sort_key='display_name',
                                        sort_dir='desc')

    @decorators.idempotent_id('05489dde-44bc-4961-a1f5-3ce7ee7824f7')
    def test_snapshot_list_param_marker(self):
        # The list of snapshots should end before the provided marker
        snap_list = self.snapshots_client.list_snapshots()['snapshots']
        # list_snapshots will take the reverse order as they are created.
        snapshot_id_list = [snap['id'] for snap in snap_list][::-1]

        params = {'marker': snapshot_id_list[1]}
        snap_list = self.snapshots_client.list_snapshots(**params)['snapshots']
        fetched_list_id = [snap['id'] for snap in snap_list]
        # Verify the list of snapshots ends before the provided
        # marker(second snapshot), therefore only the first snapshot
        # should displayed.
        self.assertEqual(snapshot_id_list[:1], fetched_list_id)
<|endoftext|>


<commit_before>from typing import List

import numpy as np
import logging
import pandas as pd
from sqlalchemy import create_engine
import uuid
import shutil
from bayesianpy.jni import bayesServer, bayesServerAnalysis, bayesServerDiscovery, jp
import os
from bayesianpy.decorators import listify
import dask.dataframe as dd
from typing import Iterable
import bayesianpy.dask as dk
from collections import defaultdict

class DataFrameReader:
    def __init__(self, df):
        self._df = df
        self._row = None
        self._columns = df.columns.tolist()
        self.reset()
        self.row_index = 0
        self._writer = DataFrameWriter()

    def __getattr__(self, key):
        return self.__getitem__(key)

    def read(self):
        self._row = next(self._iterator, None)
        self.row_index += 1
        return self._row is not None

    def reset(self):
        self._iterator = self._df.itertuples()

    def get_index(self):
        return self._row[0]

    def to_dict(self):
        return self.row()

    def tolist(self, cols):
        return [self.__getitem__(c) for c in cols]

    def row(self):
        return {c : self.__getitem__(c) for c in self._columns}

    def writer(self):
        if self._row is not None:
            return self._writer.with_index(self.get_index())
        else:
            return self._writer

    def __getitem__(self, key):
        # the df index of the row is at index 0
        try:
            ix = self._columns.index(key) + 1
            return self._row[ix]
        except BaseException as e:
            return None

    def __next__(self):
        if self.read():
            return self
        else:
            raise StopIteration

    def __iter__(self):
        return self

class DataFrameWriter:

    def __init__(self):
        self._columns = defaultdict(list)
        self._row_indices = set()

    def with_index(self, index):
        if index not in self._row_indices:
            self._row_indices.add(index)

        self._current_row_index = index
        return self

    def set_value(self, column, value):
        self._columns[column].append(value)

    def as_dataframe(self):
        self._columns['ix'] = list(self._row_indices)
        df = pd.DataFrame(self._columns)
        return df.set_index('ix')

    def flush(self, df: pd.DataFrame):
        df1 = self.as_dataframe()
        return df.join(df1)

class AutoType:
    def __init__(self, df, discrete=[], continuous=[], continuous_to_discrete_limit = 20):
        self._df = df
        self._continuous_to_discrete_limit = continuous_to_discrete_limit
        self._continuous = continuous
        self._discrete = discrete

    @listify
    def get_continuous_variables(self):
        cols = self._df.columns.tolist()
        for col in cols:
            if not DataFrame.is_float(self._df[col].dtype) and not DataFrame.is_int(self._df[col].dtype):
                continue
            if col in self._discrete:
                continue
            elif col in self._continuous:
                yield col
            elif len(self._df[col].unique()) > self._continuous_to_discrete_limit:
                yield col

    @listify
    def get_discrete_variables(self):
        continuous = set(self.get_continuous_variables())
        for col in self._df.columns.tolist():
            if col in self._continuous:
                continue

            if DataFrame.is_timestamp(self._df[col].dtype):
                continue

            elif col in self._discrete:
                yield col
            elif col not in continuous:
                yield col

class DataFrame:

    def __init__(self, df):
        self._df = df

    @staticmethod
    def is_timestamp(dtype):
        for ts in ['timestamp64', 'timedelta64', 'datetime64']:
            if ts in str(dtype):
                return True

        return False

    @staticmethod
    def is_numeric(dtype):
        return DataFrame.is_float(dtype) or DataFrame.is_int(dtype)

    @staticmethod
    def is_float(dtype):
        return str(dtype) in {"float32", "float64"}

    @staticmethod
    def is_int(dtype):
        return str(dtype) in {"int32", "int64"}

    @staticmethod
    def is_bool(dtype):
        return str(dtype) == "bool"

    @staticmethod
    def is_string(dtype):
        return str(dtype) == "object" or str(dtype) == "O"

    @staticmethod
    def could_be_int(col):
        if DataFrame.is_int(col.dtype):
            return True

        if DataFrame.is_float(col.dtype):
            for val in col.dropna().unique():
                if int(val) != val:
                    return False

            return True

        return False

    @staticmethod
    def coerce_to_numeric(df: pd.DataFrame, logger: logging.Logger, cutoff=0.10, ignore=[]) -> pd.DataFrame:
        for col in df.columns:
            if DataFrame.is_timestamp(df[col].dtype):
                continue

            if col in ignore:
                continue

            values = df[col].dropna().unique()
            ratio = 0
            pre_length = len(values)

            if pre_length > 0:
                new_values = pd.to_numeric(df[col].dropna().unique(), errors='coerce')
                post_length = len(new_values[~np.isnan(new_values)])
                ratio = (pre_length - post_length) / pre_length

            if ratio <= cutoff:
                logger.debug("Converting column {} to numeric (ratio: {})".format(col, ratio))
                df[col] = pd.to_numeric(df[col], errors='coerce')
            else:
                logger.debug("Not converting column {} (ratio: {})".format(col, ratio))

        return df

    @staticmethod
    def coerce_to_boolean(df: pd.DataFrame, ignore=[]):
        for col in df.columns:
            if col in ignore:
                continue

            if df[col].min() == df[col].max():
                continue

            if len(df[col].dropna().unique()) != 2:
                continue

            values = df[col].dropna().unique()
            series = df[col].astype(bool)

            if len(df[df[col] == values[0]]) != len(series[series == True]) and \
                            len(df[df[col] == values[1]]) != len(series[series == True]):
                continue

            df[col] = series

        return df

    @staticmethod
    def cast2(dtype, value):
        if DataFrame.is_int(dtype):
            return int(value)
        if DataFrame.is_bool(dtype):
            return value == True
        if DataFrame.is_float(dtype):
            return float(value)

        return value

    @staticmethod
    def cast(df, col_name, value):
        if DataFrame.is_int(df[col_name].dtype):
            return int(value)
        if DataFrame.is_bool(df[col_name].dtype):
            return value == True
        if DataFrame.is_float(df[col_name].dtype):
            return float(value)

        return value

    @staticmethod
    def replace_0_with_normal_dist(df: pd.DataFrame, columns):
        for col in columns:
            if col in df.columns:
                df.loc[df[col] == 0, col] = df[col].apply(lambda x: np.random.normal(0, 3))

class DaskDataFrame:

    def __init__(self, df: dd.DataFrame):
        self._df = df
        self.empty = all(p.empty for p in self._get_df_partitions())

    def __len__(self):
        return len(self._df)

    def __getattribute__(self, item):
        try:
            return super().__getattribute__(item)
        except AttributeError:
            return self._df.__getattribute__(item)

    def __getitem__(self, key):
        # the df index of the row is at index 0
        return self._df.__getitem__(key)

    def _get_df_partitions(self) -> Iterable[pd.DataFrame]:
        for partitition in range(0, self._df.npartitions):
            yield self._df.get_partition(partitition).compute()

    def to_sql(self, table, engine, index_label='ix', index=True):
        for partition in range(0, self._df.npartitions):
            df = self._df.get_partition(partition)
            df.compute().to_sql(table, engine, if_exists='append', index_label=index_label, index=True)

class Filter:
    @staticmethod
    def remove_static_variables(df: pd.DataFrame, cutoff=1, logger:logging.Logger=None):
        column_names = df.apply(lambda x: len(x.unique()) > cutoff)

        if logger is not None:
            logger.info("Removing {}".format(", ".join(df.columns[column_names == True])))

        return df[column_names[column_names == True].index.tolist()].copy()

    @staticmethod
    def remove_variable_variables(df: pd.DataFrame):
        column_names = df.apply(lambda x: len(x.unique()) != len(df))
        return df[column_names[column_names == True].index.tolist()]

    @staticmethod
    def remove_discrete_variables_with_too_many_states(df: pd.DataFrame, num_states = 30):
        column_names = df.select_dtypes(include=['object']).apply(lambda x: len(x.unique()) >= num_states)
        cols = list(set(df.columns.tolist()) - set(column_names[column_names == True].index.tolist()))
        return df[cols]

    @staticmethod
    def apply(df: pd.DataFrame):
        return Filter.remove_discrete_variables_with_too_many_states(Filter.remove_variable_variables(Filter.remove_static_variables(df)))

def create_histogram(series):
    hdo = bayesServerAnalysis().HistogramDensityOptions()
    values = []
    for value in series.iteritems():
        v = bayesServerDiscovery().WeightedValue()
        if np.isnan(value[1]) or np.isinf(value[1]):
            continue
        v.setValue(jp.java.lang.Double(value[1]))
        v.setWeight(1.0)
        values.append(v)

    return bayesServerAnalysis().HistogramDensity.learn(jp.java.util.Arrays.asList(values), hdo)

class DataSet:
    def __init__(self, df: pd.DataFrame, db_folder: str, logger: logging.Logger,
                 identifier: str=None, weight_column: str=None):
        if identifier is None:
            self.uuid = str(uuid.uuid4()).replace("-","")
        else:
            self.uuid = identifier

        self._db_dir = db_folder
        self._create_folder()
        filename = "sqlite:///{}.db".format(os.path.join(self._db_dir, "db", self.uuid))
        self._engine = create_engine(filename)
        self.table = "table_" + self.uuid
        self._logger = logger
        self.data = df
        self._weight_column = weight_column

    def subset(self, indices:List[int]) -> 'DataSet':
        return DataSet(self.data.loc[indices], self._db_dir, self._logger, identifier=self.uuid)

    def get_reader_options(self):
        return bayesServer().data.ReaderOptions("ix") if self._weight_column is None \
            else bayesServer().data.ReaderOptions("ix", self._weight_column)

    def get_dataframe(self) -> pd.DataFrame:
        return self.data

    def get_connection(self):
        return "jdbc:sqlite:{}.db".format(os.path.join(self._db_dir, "db", self.uuid))

    def _create_folder(self):
        if not os.path.exists(os.path.join(self._db_dir, "db")):
            os.makedirs(os.path.join(self._db_dir, "db"))

    def write(self):
        self._logger.info("Writing rows to storage")
        dk.to_sql(self.data, self.table, self._engine)
        self._logger.info("Finished writing rows to storage")

    def create_data_reader_command(self, indexes=[]):
        """
        Get the data reader
        :param indexes: training/ testing indexes
        :return: a a DatabaseDataReaderCommand
        """

        if len(indexes) > 0:
            query = "select * from {} where ix in ({}) order by ix asc".format(self.table, ",".join(str(i) for i in indexes))
        else:
            query = "select * from {} order by ix asc".format(self.table)

        data_reader_command = bayesServer().data.DatabaseDataReaderCommand(
            self.get_connection(),
            query)

        return data_reader_command

    def cleanup(self):
        self._logger.debug("Cleaning up: deleting db folder")
        try:
            shutil.rmtree(os.path.join(self._db_dir, "db"))
        except:
            self._logger.error("Could not delete the db folder {} for some reason.".format(self._db_dir))

    def __enter__(self):
        self.write()
        return self

    def __exit__(self, type, value, traceback):
        self.cleanup()

def as_probability(series, output_column='cdf'):
    hist = create_histogram(series)
    df = pd.DataFrame(series)
    df[output_column] = series.apply(lambda x: hist.cdf(x))
    return df
<commit_msg>Fixing logging<commit_after>from typing import List

import numpy as np
import logging
import pandas as pd
from sqlalchemy import create_engine
import uuid
import shutil
from bayesianpy.jni import bayesServer, bayesServerAnalysis, bayesServerDiscovery, jp
import os
from bayesianpy.decorators import listify
import dask.dataframe as dd
from typing import Iterable
import bayesianpy.dask as dk
from collections import defaultdict

class DataFrameReader:
    def __init__(self, df):
        self._df = df
        self._row = None
        self._columns = df.columns.tolist()
        self.reset()
        self.row_index = 0
        self._writer = DataFrameWriter()

    def __getattr__(self, key):
        return self.__getitem__(key)

    def read(self):
        self._row = next(self._iterator, None)
        self.row_index += 1
        return self._row is not None

    def reset(self):
        self._iterator = self._df.itertuples()

    def get_index(self):
        return self._row[0]

    def to_dict(self):
        return self.row()

    def tolist(self, cols):
        return [self.__getitem__(c) for c in cols]

    def row(self):
        return {c : self.__getitem__(c) for c in self._columns}

    def writer(self):
        if self._row is not None:
            return self._writer.with_index(self.get_index())
        else:
            return self._writer

    def __getitem__(self, key):
        # the df index of the row is at index 0
        try:
            ix = self._columns.index(key) + 1
            return self._row[ix]
        except BaseException as e:
            return None

    def __next__(self):
        if self.read():
            return self
        else:
            raise StopIteration

    def __iter__(self):
        return self

class DataFrameWriter:

    def __init__(self):
        self._columns = defaultdict(list)
        self._row_indices = set()

    def with_index(self, index):
        if index not in self._row_indices:
            self._row_indices.add(index)

        self._current_row_index = index
        return self

    def set_value(self, column, value):
        self._columns[column].append(value)

    def as_dataframe(self):
        self._columns['ix'] = list(self._row_indices)
        df = pd.DataFrame(self._columns)
        return df.set_index('ix')

    def flush(self, df: pd.DataFrame):
        df1 = self.as_dataframe()
        return df.join(df1)

class AutoType:
    def __init__(self, df, discrete=[], continuous=[], continuous_to_discrete_limit = 20):
        self._df = df
        self._continuous_to_discrete_limit = continuous_to_discrete_limit
        self._continuous = continuous
        self._discrete = discrete

    @listify
    def get_continuous_variables(self):
        cols = self._df.columns.tolist()
        for col in cols:
            if not DataFrame.is_float(self._df[col].dtype) and not DataFrame.is_int(self._df[col].dtype):
                continue
            if col in self._discrete:
                continue
            elif col in self._continuous:
                yield col
            elif len(self._df[col].unique()) > self._continuous_to_discrete_limit:
                yield col

    @listify
    def get_discrete_variables(self):
        continuous = set(self.get_continuous_variables())
        for col in self._df.columns.tolist():
            if col in self._continuous:
                continue

            if DataFrame.is_timestamp(self._df[col].dtype):
                continue

            elif col in self._discrete:
                yield col
            elif col not in continuous:
                yield col

class DataFrame:

    def __init__(self, df):
        self._df = df

    @staticmethod
    def is_timestamp(dtype):
        for ts in ['timestamp64', 'timedelta64', 'datetime64']:
            if ts in str(dtype):
                return True

        return False

    @staticmethod
    def is_numeric(dtype):
        return DataFrame.is_float(dtype) or DataFrame.is_int(dtype)

    @staticmethod
    def is_float(dtype):
        return str(dtype) in {"float32", "float64"}

    @staticmethod
    def is_int(dtype):
        return str(dtype) in {"int32", "int64"}

    @staticmethod
    def is_bool(dtype):
        return str(dtype) == "bool"

    @staticmethod
    def is_string(dtype):
        return str(dtype) == "object" or str(dtype) == "O"

    @staticmethod
    def could_be_int(col):
        if DataFrame.is_int(col.dtype):
            return True

        if DataFrame.is_float(col.dtype):
            for val in col.dropna().unique():
                if int(val) != val:
                    return False

            return True

        return False

    @staticmethod
    def coerce_to_numeric(df: pd.DataFrame, logger: logging.Logger, cutoff=0.10, ignore=[]) -> pd.DataFrame:
        for col in df.columns:
            if DataFrame.is_timestamp(df[col].dtype):
                continue

            if col in ignore:
                continue

            values = df[col].dropna().unique()
            ratio = 0
            pre_length = len(values)

            if pre_length > 0:
                new_values = pd.to_numeric(df[col].dropna().unique(), errors='coerce')
                post_length = len(new_values[~np.isnan(new_values)])
                ratio = (pre_length - post_length) / pre_length

            if ratio <= cutoff:
                logger.debug("Converting column {} to numeric (ratio: {})".format(col, ratio))
                df[col] = pd.to_numeric(df[col], errors='coerce')
            else:
                logger.debug("Not converting column {} (ratio: {})".format(col, ratio))

        return df

    @staticmethod
    def coerce_to_boolean(df: pd.DataFrame, ignore=[]):
        for col in df.columns:
            if col in ignore:
                continue

            if df[col].min() == df[col].max():
                continue

            if len(df[col].dropna().unique()) != 2:
                continue

            values = df[col].dropna().unique()
            series = df[col].astype(bool)

            if len(df[df[col] == values[0]]) != len(series[series == True]) and \
                            len(df[df[col] == values[1]]) != len(series[series == True]):
                continue

            df[col] = series

        return df

    @staticmethod
    def cast2(dtype, value):
        if DataFrame.is_int(dtype):
            return int(value)
        if DataFrame.is_bool(dtype):
            return value == True
        if DataFrame.is_float(dtype):
            return float(value)

        return value

    @staticmethod
    def cast(df, col_name, value):
        if DataFrame.is_int(df[col_name].dtype):
            return int(value)
        if DataFrame.is_bool(df[col_name].dtype):
            return value == True
        if DataFrame.is_float(df[col_name].dtype):
            return float(value)

        return value

    @staticmethod
    def replace_0_with_normal_dist(df: pd.DataFrame, columns):
        for col in columns:
            if col in df.columns:
                df.loc[df[col] == 0, col] = df[col].apply(lambda x: np.random.normal(0, 3))

class DaskDataFrame:

    def __init__(self, df: dd.DataFrame):
        self._df = df
        self.empty = all(p.empty for p in self._get_df_partitions())

    def __len__(self):
        return len(self._df)

    def __getattribute__(self, item):
        try:
            return super().__getattribute__(item)
        except AttributeError:
            return self._df.__getattribute__(item)

    def __getitem__(self, key):
        # the df index of the row is at index 0
        return self._df.__getitem__(key)

    def _get_df_partitions(self) -> Iterable[pd.DataFrame]:
        for partitition in range(0, self._df.npartitions):
            yield self._df.get_partition(partitition).compute()

    def to_sql(self, table, engine, index_label='ix', index=True):
        for partition in range(0, self._df.npartitions):
            df = self._df.get_partition(partition)
            df.compute().to_sql(table, engine, if_exists='append', index_label=index_label, index=True)

class Filter:
    @staticmethod
    def remove_static_variables(df: pd.DataFrame, cutoff=1, logger:logging.Logger=None):
        column_names = df.apply(lambda x: len(x.unique()) > cutoff)

        if logger is not None:
            logger.info("Removing {}".format(", ".join(df.columns[column_names == False])))

        return df[column_names[column_names == True].index.tolist()].copy()

    @staticmethod
    def remove_variable_variables(df: pd.DataFrame):
        column_names = df.apply(lambda x: len(x.unique()) != len(df))
        return df[column_names[column_names == True].index.tolist()]

    @staticmethod
    def remove_discrete_variables_with_too_many_states(df: pd.DataFrame, num_states = 30):
        column_names = df.select_dtypes(include=['object']).apply(lambda x: len(x.unique()) >= num_states)
        cols = list(set(df.columns.tolist()) - set(column_names[column_names == True].index.tolist()))
        return df[cols]

    @staticmethod
    def apply(df: pd.DataFrame):
        return Filter.remove_discrete_variables_with_too_many_states(Filter.remove_variable_variables(Filter.remove_static_variables(df)))

def create_histogram(series):
    hdo = bayesServerAnalysis().HistogramDensityOptions()
    values = []
    for value in series.iteritems():
        v = bayesServerDiscovery().WeightedValue()
        if np.isnan(value[1]) or np.isinf(value[1]):
            continue
        v.setValue(jp.java.lang.Double(value[1]))
        v.setWeight(1.0)
        values.append(v)

    return bayesServerAnalysis().HistogramDensity.learn(jp.java.util.Arrays.asList(values), hdo)

class DataSet:
    def __init__(self, df: pd.DataFrame, db_folder: str, logger: logging.Logger,
                 identifier: str=None, weight_column: str=None):
        if identifier is None:
            self.uuid = str(uuid.uuid4()).replace("-","")
        else:
            self.uuid = identifier

        self._db_dir = db_folder
        self._create_folder()
        filename = "sqlite:///{}.db".format(os.path.join(self._db_dir, "db", self.uuid))
        self._engine = create_engine(filename)
        self.table = "table_" + self.uuid
        self._logger = logger
        self.data = df
        self._weight_column = weight_column

    def subset(self, indices:List[int]) -> 'DataSet':
        return DataSet(self.data.loc[indices], self._db_dir, self._logger, identifier=self.uuid)

    def get_reader_options(self):
        return bayesServer().data.ReaderOptions("ix") if self._weight_column is None \
            else bayesServer().data.ReaderOptions("ix", self._weight_column)

    def get_dataframe(self) -> pd.DataFrame:
        return self.data

    def get_connection(self):
        return "jdbc:sqlite:{}.db".format(os.path.join(self._db_dir, "db", self.uuid))

    def _create_folder(self):
        if not os.path.exists(os.path.join(self._db_dir, "db")):
            os.makedirs(os.path.join(self._db_dir, "db"))

    def write(self):
        self._logger.info("Writing rows to storage")
        dk.to_sql(self.data, self.table, self._engine)
        self._logger.info("Finished writing rows to storage")

    def create_data_reader_command(self, indexes=[]):
        """
        Get the data reader
        :param indexes: training/ testing indexes
        :return: a a DatabaseDataReaderCommand
        """

        if len(indexes) > 0:
            query = "select * from {} where ix in ({}) order by ix asc".format(self.table, ",".join(str(i) for i in indexes))
        else:
            query = "select * from {} order by ix asc".format(self.table)

        data_reader_command = bayesServer().data.DatabaseDataReaderCommand(
            self.get_connection(),
            query)

        return data_reader_command

    def cleanup(self):
        self._logger.debug("Cleaning up: deleting db folder")
        try:
            shutil.rmtree(os.path.join(self._db_dir, "db"))
        except:
            self._logger.error("Could not delete the db folder {} for some reason.".format(self._db_dir))

    def __enter__(self):
        self.write()
        return self

    def __exit__(self, type, value, traceback):
        self.cleanup()

def as_probability(series, output_column='cdf'):
    hist = create_histogram(series)
    df = pd.DataFrame(series)
    df[output_column] = series.apply(lambda x: hist.cdf(x))
    return df
<|endoftext|>