<commit_before># Licensed to the Software Freedom Conservancy (SFC) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The SFC licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# module for getting the lan ip address of the computer
import os
import socket

if os.name != "nt":
    import fcntl
    import struct

    def get_interface_ip(ifname):
        def _bytes(value, encoding):
            try:
                return bytes(value, encoding)  # Python 3
            except TypeError:
                return value  # Python 2

        sckt = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        return socket.inet_ntoa(fcntl.ioctl(
            sckt.fileno(),
            0x8915,  # SIOCGIFADDR
            struct.pack('256s', _bytes(ifname[:15], 'utf-8'))
        )[20:24])

def get_lan_ip():
    try:
        ip = socket.gethostbyname(socket.gethostname())
    except Exception:
        return '0.0.0.0'
    if ip.startswith("127.") and os.name != "nt":
        interfaces = ["eth0", "eth1", "eth2", "en0", "en1", "en2", "en3",
                      "en4", "wlan0", "wlan1", "wifi0", "ath0", "ath1", "ppp0"]
        for ifname in interfaces:
            try:
                ip = get_interface_ip(ifname)
                break
            except IOError:
                pass
    return ip
<commit_msg>[py] Returning fixed IP in GitHub actions (#9550)<commit_after># Licensed to the Software Freedom Conservancy (SFC) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The SFC licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# module for getting the lan ip address of the computer
import os
import socket

if os.name != "nt":
    import fcntl
    import struct

    def get_interface_ip(ifname):
        def _bytes(value, encoding):
            try:
                return bytes(value, encoding)  # Python 3
            except TypeError:
                return value  # Python 2

        sckt = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        return socket.inet_ntoa(fcntl.ioctl(
            sckt.fileno(),
            0x8915,  # SIOCGIFADDR
            struct.pack('256s', _bytes(ifname[:15], 'utf-8'))
        )[20:24])

def get_lan_ip():
    if os.environ.get('CI') == 'true':
        return '0.0.0.0'

    try:
        ip = socket.gethostbyname(socket.gethostname())
    except Exception:
        return '0.0.0.0'
    if ip.startswith("127.") and os.name != "nt":
        interfaces = ["eth0", "eth1", "eth2", "en0", "en1", "en2", "en3",
                      "en4", "wlan0", "wlan1", "wifi0", "ath0", "ath1", "ppp0"]
        for ifname in interfaces:
            try:
                ip = get_interface_ip(ifname)
                break
            except IOError:
                pass
    return ip
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import datetime
import errno
import json
import os
import pytz
import re

def camelcase(value):
    """
    Convert a module name or string with underscores and periods to camel case.

    :param value: the string to convert
    :type value: str
    :returns: the value converted to camel case.
    """
    return ''.join(x.capitalize() if x else '_' for x in
                   re.split("[._]+", value))

def mkdir(path, mode=0o777, recurse=True, safe=True):
    """
    Create a new directory or ensure a directory already exists.

    :param path: The directory to create.
    :type path: str
    :param mode: The mode (permission mask) prior to applying umask.
    :type mode: int
    :param recurse: Whether intermediate missing dirs should be created.
    :type recurse: bool
    :param safe: Set to True to suppress the error if the dir already exists.
    :type safe: bool
    """
    method = os.makedirs if recurse else os.mkdir

    try:
        method(path, mode)
    except OSError as exc:
        if safe and exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

class JsonEncoder(json.JSONEncoder):
    """
    This extends the standard json.JSONEncoder to allow for more types to be
    sensibly serialized. This is used in Girder's REST layer to serialize
    route return values when JSON is requested.
    """
    def default(self, obj):
        if isinstance(obj, set):
            return tuple(obj)
        elif isinstance(obj, datetime.datetime):
            return obj.replace(tzinfo=pytz.UTC).isoformat()
        return str(obj)
<commit_msg>Rename safe -> existOk<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import datetime
import errno
import json
import os
import pytz
import re

def camelcase(value):
    """
    Convert a module name or string with underscores and periods to camel case.

    :param value: the string to convert
    :type value: str
    :returns: the value converted to camel case.
    """
    return ''.join(x.capitalize() if x else '_' for x in
                   re.split("[._]+", value))

def mkdir(path, mode=0o777, recurse=True, existOk=True):
    """
    Create a new directory or ensure a directory already exists.

    :param path: The directory to create.
    :type path: str
    :param mode: The mode (permission mask) prior to applying umask.
    :type mode: int
    :param recurse: Whether intermediate missing dirs should be created.
    :type recurse: bool
    :param existOk: Set to True to suppress the error if the dir exists.
    :type existOk: bool
    """
    method = os.makedirs if recurse else os.mkdir

    try:
        method(path, mode)
    except OSError as exc:
        if existOk and exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

class JsonEncoder(json.JSONEncoder):
    """
    This extends the standard json.JSONEncoder to allow for more types to be
    sensibly serialized. This is used in Girder's REST layer to serialize
    route return values when JSON is requested.
    """
    def default(self, obj):
        if isinstance(obj, set):
            return tuple(obj)
        elif isinstance(obj, datetime.datetime):
            return obj.replace(tzinfo=pytz.UTC).isoformat()
        return str(obj)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
from django.test import TestCase, Client
from django.contrib.staticfiles.testing import StaticLiveServerTestCase
from django.contrib.auth.models import User
from django.core import mail
from django.test.utils import override_settings
import factory
from faker import Faker
from splinter import Browser

fake = Faker()

class UserFactory(factory.django.DjangoModelFactory):
    class Meta:
        model = User

    username = fake.username()
    email = fake.email()

# # # # # # # # # # # # # #
# Client Tests for Views  #
# # # # # # # # # # # # # #

class PlayClientTests(TestCase):

    # Test 1
    # Check that /play/ page loads the correct template
    def test_play_template(self):
        response = Client().get('/play/')
        self.assertTemplateUsed(response, 'typingtest2.html')

    # Test 2
    # Check that /play/multi/ page loads the correct template
    def test_multi_template(self):
        response = Client().get('/play/multi/')
        self.assertTemplateUsed(response, 'typingtest3.html')

    # Test 3
    # Check that /play/content/ page loads the correct content
    def test_content_api(self):
        response = Client().post(
            '/play/content/',
            {
                'user': 'tpeek',
                'repo': 'Copy-n-Haste',
                'path': 'README.md'
            }
        )
        self.assertEqual(response.content[:14], '# Copy-n-Haste')
<commit_msg>Add 4 webtests for playing game<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
from django.test import TestCase, Client
from django.contrib.staticfiles.testing import StaticLiveServerTestCase
from django.contrib.auth.models import User
from django.test.utils import override_settings
import factory
from faker import Faker
from splinter import Browser

fake = Faker()

class UserFactory(factory.django.DjangoModelFactory):
    class Meta:
        model = User

    username = fake.username()
    email = fake.email()

# # # # # # # # # # # # # #
# Client Tests for Views  #
# # # # # # # # # # # # # #

class PlayClientTests(TestCase):

    # Test 1
    # Check that /play/ page loads the correct template
    def test_play_template(self):
        response = Client().get('/play/')
        self.assertTemplateUsed(response, 'typingtest2.html')

    # Test 2
    # Check that /play/multi/ page loads the correct template
    def test_multi_template(self):
        response = Client().get('/play/multi/')
        self.assertTemplateUsed(response, 'typingtest3.html')

    # Test 3
    # Check that /play/content/ page loads the correct content
    def test_content_api(self):
        response = Client().post(
            '/play/content/',
            {
                'user': 'tpeek',
                'repo': 'Copy-n-Haste',
                'path': 'README.md'
            }
        )
        self.assertEqual(response.content[:14], '# Copy-n-Haste')

# # # # # # # # # # # # # # #
# Web Tests for Play Pages  #
# # # # # # # # # # # # # # #

@override_settings(DEBUG=True)
class PlayPagesWebTests(StaticLiveServerTestCase):

    def setUp(self):
        self.user1 = UserFactory.build()
        self.user1.set_password('abc')
        self.user1.save()
        self.browser = Browser()

    def tearDown(self):
        self.browser.quit()

    def login_helper(self, username, password):
        self.browser.visit('%s%s' % (self.live_server_url, '/accounts/login/'))

        self.browser.fill('username', username)
        self.browser.fill('password', password)
        self.browser.find_by_value('Log in').first.click()

    # Test 4
    # Check for anonymous playing single player game
    def test_anon_single_player(self):
        pass

    # Test 5
    # Check for anonymous playing multiplayer game
    def test_anon_multiplayer(self):
        pass

    # Test 6
    # Check for user playing single player game
    def test_user_single_player(self):
        pass

    # Test 7
    # Check for user playing multiplayer game
    def test_user_multiplayer(self):
        pass
<|endoftext|>


<commit_before># Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
"""
Tests For Scheduler Host Filter Drivers.
"""

import json

from nova import exception
from nova import flags
from nova import test
from nova.scheduler import host_filter

FLAGS = flags.FLAGS

class FakeZoneManager:
    pass

class HostFilterTestCase(test.TestCase):
    """Test case for host filter drivers."""

    def _host_caps(self, multiplier):
        # Returns host capabilities in the following way:
        # host1 = memory:free 10 (100max)
        #         disk:available 100 (1000max)
        # hostN = memory:free 10 + 10N
        #         disk:available 100 + 100N
        # in other words: hostN has more resources than host0
        # which means ... don't go above 10 hosts.
        return {'host_name-description': 'XenServer %s' % multiplier,
                'host_hostname': 'xs-%s' % multiplier,
                'host_memory_total': 100,
                'host_memory_overhead': 10,
                'host_memory_free': 10 + multiplier * 10,
                'host_memory_free-computed': 10 + multiplier * 10,
                'host_other-config': {},
                'host_ip_address': '192.168.1.%d' % (100 + multiplier),
                'host_cpu_info': {},
                'disk_available': 100 + multiplier * 100,
                'disk_total': 1000,
                'disk_used': 0,
                'host_uuid': 'xxx-%d' % multiplier,
                'host_name-label': 'xs-%s' % multiplier}

    def setUp(self):
        self.old_flag = FLAGS.default_host_filter_driver
        FLAGS.default_host_filter_driver = \
                            'nova.scheduler.host_filter.AllHostsFilter'
        self.instance_type = dict(name='tiny',
                memory_mb=50,
                vcpus=10,
                local_gb=500,
                flavorid=1,
                swap=500,
                rxtx_quota=30000,
                rxtx_cap=200)

        self.zone_manager = FakeZoneManager()
        states = {}
        for x in xrange(10):
            states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)}
        self.zone_manager.service_states = states

    def tearDown(self):
        FLAGS.default_host_filter_driver = self.old_flag

    def test_choose_driver(self):
        # Test default driver ...
        driver = host_filter.choose_driver()
        self.assertEquals(driver._full_name(),
                        'nova.scheduler.host_filter.AllHostsFilter')
        # Test valid driver ...
        driver = host_filter.choose_driver(
                        'nova.scheduler.host_filter.FlavorFilter')
        self.assertEquals(driver._full_name(),
                        'nova.scheduler.host_filter.FlavorFilter')
        # Test invalid driver ...
        try:
            host_filter.choose_driver('does not exist')
            self.fail("Should not find driver")
        except exception.SchedulerHostFilterDriverNotFound:
            pass

    def test_all_host_driver(self):
        driver = host_filter.AllHostsFilter()
        cooked = driver.instance_type_to_filter(self.instance_type)
        hosts = driver.filter_hosts(self.zone_manager, cooked)
        self.assertEquals(10, len(hosts))
        for host, capabilities in hosts:
            self.assertTrue(host.startswith('host'))

    def test_flavor_driver(self):
        driver = host_filter.FlavorFilter()
        # filter all hosts that can support 50 ram and 500 disk
        name, cooked = driver.instance_type_to_filter(self.instance_type)
        self.assertEquals('nova.scheduler.host_filter.FlavorFilter', name)
        hosts = driver.filter_hosts(self.zone_manager, cooked)
        self.assertEquals(6, len(hosts))
        just_hosts = [host for host, caps in hosts]
        just_hosts.sort()
        self.assertEquals('host05', just_hosts[0])
        self.assertEquals('host10', just_hosts[5])

    def test_json_driver(self):
        driver = host_filter.JsonFilter()
        # filter all hosts that can support 50 ram and 500 disk
        name, cooked = driver.instance_type_to_filter(self.instance_type)
        self.assertEquals('nova.scheduler.host_filter.JsonFilter', name)
        hosts = driver.filter_hosts(self.zone_manager, cooked)
        self.assertEquals(6, len(hosts))
        just_hosts = [host for host, caps in hosts]
        just_hosts.sort()
        self.assertEquals('host05', just_hosts[0])
        self.assertEquals('host10', just_hosts[5])

        # Try some custom queries

        raw = ['or',
                   ['and',
                       ['<', '$compute.host_memory_free', 30],
                       ['<', '$compute.disk_available', 300]],
                   ['and',
                       ['>', '$compute.host_memory_free', 70],
                       ['>', '$compute.disk_available', 700]]]
        cooked = json.dumps(raw)
        hosts = driver.filter_hosts(self.zone_manager, cooked)

        self.assertEquals(5, len(hosts))
        just_hosts = [host for host, caps in hosts]
        just_hosts.sort()
        for index, host in zip([1, 2, 8, 9, 10], just_hosts):
            self.assertEquals('host%02d' % index, host)

        raw = ['not',
                  ['=', '$compute.host_memory_free', 30],]
        cooked = json.dumps(raw)
        hosts = driver.filter_hosts(self.zone_manager, cooked)

        self.assertEquals(9, len(hosts))
        just_hosts = [host for host, caps in hosts]
        just_hosts.sort()
        for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts):
            self.assertEquals('host%02d' % index, host)

        raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100]
        cooked = json.dumps(raw)
        hosts = driver.filter_hosts(self.zone_manager, cooked)

        self.assertEquals(5, len(hosts))
        just_hosts = [host for host, caps in hosts]
        just_hosts.sort()
        for index, host in zip([2, 4, 6, 8, 10], just_hosts):
            self.assertEquals('host%02d' % index, host)

        # Try some bogus input ...
        raw = ['unknown command', ]
        cooked = json.dumps(raw)
        try:
            driver.filter_hosts(self.zone_manager, cooked)
            self.fail("Should give KeyError")
        except KeyError, e:
            pass

        self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps([])))
        self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps({})))
        self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps(
                ['not', True, False, True, False])))

        try:
            driver.filter_hosts(self.zone_manager, json.dumps(
                'not', True, False, True, False))
            self.fail("Should give KeyError")
        except KeyError, e:
            pass

        self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps(
                ['=', '$foo', 100])))
        self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps(
                ['=', '$.....', 100])))
        self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps(
            ['>', ['and', ['or', ['not', ['<', ['>=',
                ['<=', ['in', ]]]]]]]])))

        self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps(
                ['=', {}, ['>', '$missing....foo']])))
<commit_msg>pep8 fixed<commit_after># Copyright 2011 OpenStack LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.
"""
Tests For Scheduler Host Filter Drivers.
"""

import json

from nova import exception
from nova import flags
from nova import test
from nova.scheduler import host_filter

FLAGS = flags.FLAGS

class FakeZoneManager:
    pass

class HostFilterTestCase(test.TestCase):
    """Test case for host filter drivers."""

    def _host_caps(self, multiplier):
        # Returns host capabilities in the following way:
        # host1 = memory:free 10 (100max)
        #         disk:available 100 (1000max)
        # hostN = memory:free 10 + 10N
        #         disk:available 100 + 100N
        # in other words: hostN has more resources than host0
        # which means ... don't go above 10 hosts.
        return {'host_name-description': 'XenServer %s' % multiplier,
                'host_hostname': 'xs-%s' % multiplier,
                'host_memory_total': 100,
                'host_memory_overhead': 10,
                'host_memory_free': 10 + multiplier * 10,
                'host_memory_free-computed': 10 + multiplier * 10,
                'host_other-config': {},
                'host_ip_address': '192.168.1.%d' % (100 + multiplier),
                'host_cpu_info': {},
                'disk_available': 100 + multiplier * 100,
                'disk_total': 1000,
                'disk_used': 0,
                'host_uuid': 'xxx-%d' % multiplier,
                'host_name-label': 'xs-%s' % multiplier}

    def setUp(self):
        self.old_flag = FLAGS.default_host_filter_driver
        FLAGS.default_host_filter_driver = \
                            'nova.scheduler.host_filter.AllHostsFilter'
        self.instance_type = dict(name='tiny',
                memory_mb=50,
                vcpus=10,
                local_gb=500,
                flavorid=1,
                swap=500,
                rxtx_quota=30000,
                rxtx_cap=200)

        self.zone_manager = FakeZoneManager()
        states = {}
        for x in xrange(10):
            states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)}
        self.zone_manager.service_states = states

    def tearDown(self):
        FLAGS.default_host_filter_driver = self.old_flag

    def test_choose_driver(self):
        # Test default driver ...
        driver = host_filter.choose_driver()
        self.assertEquals(driver._full_name(),
                        'nova.scheduler.host_filter.AllHostsFilter')
        # Test valid driver ...
        driver = host_filter.choose_driver(
                        'nova.scheduler.host_filter.FlavorFilter')
        self.assertEquals(driver._full_name(),
                        'nova.scheduler.host_filter.FlavorFilter')
        # Test invalid driver ...
        try:
            host_filter.choose_driver('does not exist')
            self.fail("Should not find driver")
        except exception.SchedulerHostFilterDriverNotFound:
            pass

    def test_all_host_driver(self):
        driver = host_filter.AllHostsFilter()
        cooked = driver.instance_type_to_filter(self.instance_type)
        hosts = driver.filter_hosts(self.zone_manager, cooked)
        self.assertEquals(10, len(hosts))
        for host, capabilities in hosts:
            self.assertTrue(host.startswith('host'))

    def test_flavor_driver(self):
        driver = host_filter.FlavorFilter()
        # filter all hosts that can support 50 ram and 500 disk
        name, cooked = driver.instance_type_to_filter(self.instance_type)
        self.assertEquals('nova.scheduler.host_filter.FlavorFilter', name)
        hosts = driver.filter_hosts(self.zone_manager, cooked)
        self.assertEquals(6, len(hosts))
        just_hosts = [host for host, caps in hosts]
        just_hosts.sort()
        self.assertEquals('host05', just_hosts[0])
        self.assertEquals('host10', just_hosts[5])

    def test_json_driver(self):
        driver = host_filter.JsonFilter()
        # filter all hosts that can support 50 ram and 500 disk
        name, cooked = driver.instance_type_to_filter(self.instance_type)
        self.assertEquals('nova.scheduler.host_filter.JsonFilter', name)
        hosts = driver.filter_hosts(self.zone_manager, cooked)
        self.assertEquals(6, len(hosts))
        just_hosts = [host for host, caps in hosts]
        just_hosts.sort()
        self.assertEquals('host05', just_hosts[0])
        self.assertEquals('host10', just_hosts[5])

        # Try some custom queries

        raw = ['or',
                   ['and',
                       ['<', '$compute.host_memory_free', 30],
                       ['<', '$compute.disk_available', 300]],
                   ['and',
                       ['>', '$compute.host_memory_free', 70],
                       ['>', '$compute.disk_available', 700]]]
        cooked = json.dumps(raw)
        hosts = driver.filter_hosts(self.zone_manager, cooked)

        self.assertEquals(5, len(hosts))
        just_hosts = [host for host, caps in hosts]
        just_hosts.sort()
        for index, host in zip([1, 2, 8, 9, 10], just_hosts):
            self.assertEquals('host%02d' % index, host)

        raw = ['not',
                  ['=', '$compute.host_memory_free', 30], ]
        cooked = json.dumps(raw)
        hosts = driver.filter_hosts(self.zone_manager, cooked)

        self.assertEquals(9, len(hosts))
        just_hosts = [host for host, caps in hosts]
        just_hosts.sort()
        for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts):
            self.assertEquals('host%02d' % index, host)

        raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100]
        cooked = json.dumps(raw)
        hosts = driver.filter_hosts(self.zone_manager, cooked)

        self.assertEquals(5, len(hosts))
        just_hosts = [host for host, caps in hosts]
        just_hosts.sort()
        for index, host in zip([2, 4, 6, 8, 10], just_hosts):
            self.assertEquals('host%02d' % index, host)

        # Try some bogus input ...
        raw = ['unknown command', ]
        cooked = json.dumps(raw)
        try:
            driver.filter_hosts(self.zone_manager, cooked)
            self.fail("Should give KeyError")
        except KeyError, e:
            pass

        self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps([])))
        self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps({})))
        self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps(
                ['not', True, False, True, False])))

        try:
            driver.filter_hosts(self.zone_manager, json.dumps(
                'not', True, False, True, False))
            self.fail("Should give KeyError")
        except KeyError, e:
            pass

        self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps(
                ['=', '$foo', 100])))
        self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps(
                ['=', '$.....', 100])))
        self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps(
            ['>', ['and', ['or', ['not', ['<', ['>=',
                ['<=', ['in', ]]]]]]]])))

        self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps(
                ['=', {}, ['>', '$missing....foo']])))
<|endoftext|>


<commit_before># Copyright (c) 2010, Roger Light <roger@atchoo.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# 3. Neither the name of mosquitto nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from ctypes import *
from ctypes.util import find_library

# Log destinations
MOSQ_LOG_NONE=0x00
MOSQ_LOG_STDOUT=0x04
MOSQ_LOG_STDERR=0x08

# Log types
MOSQ_LOG_INFO=0x01
MOSQ_LOG_NOTICE=0x02
MOSQ_LOG_WARNING=0x04
MOSQ_LOG_ERR=0x08
MOSQ_LOG_DEBUG=0x10

class Mosquitto:
	"""MQTT version 3 client class"""

	def __init__(self, id, obj=None):
		#==================================================
		# Library loading
		#==================================================
		self._libmosq = cdll.LoadLibrary(find_library("mosquitto"))
		self._mosquitto_new = self._libmosq.mosquitto_new
		self._mosquitto_new.argtypes = [c_char_p, c_void_p]
		self._mosquitto_new.restype = c_void_p

		self._mosquitto_destroy = self._libmosq.mosquitto_destroy
		self._mosquitto_destroy.argtypes = [c_void_p]
		self._mosquitto_destroy.restype = None

		self._mosquitto_connect = self._libmosq.mosquitto_connect
		self._mosquitto_connect.argtypes = [c_void_p, c_char_p, c_int, c_int, c_bool]
		self._mosquitto_connect.restype = c_int

		self._mosquitto_disconnect = self._libmosq.mosquitto_disconnect
		self._mosquitto_disconnect.argtypes = [c_void_p]
		self._mosquitto_disconnect.restype = c_int

		self._mosquitto_publish = self._libmosq.mosquitto_publish
		self._mosquitto_publish.argtypes = [c_void_p, POINTER(c_uint16), c_char_p, c_uint32, POINTER(c_uint8), c_int, c_bool]
		self._mosquitto_publish.restype = c_int

		self._mosquitto_subscribe = self._libmosq.mosquitto_subscribe
		self._mosquitto_subscribe.argtypes = [c_void_p, POINTER(c_uint16), c_char_p, c_int]
		self._mosquitto_subscribe.restype = c_int

		self._mosquitto_unsubscribe = self._libmosq.mosquitto_unsubscribe
		self._mosquitto_unsubscribe.argtypes = [c_void_p, POINTER(c_uint16), c_char_p]
		self._mosquitto_unsubscribe.restype = c_int

		self._mosquitto_loop = self._libmosq.mosquitto_loop
		self._mosquitto_loop.argtypes = [c_void_p, c_int]
		self._mosquitto_loop.restype = c_int

		self._mosquitto_will_set = self._libmosq.mosquitto_will_set
		self._mosquitto_will_set.argtypes = [c_void_p, c_bool, c_char_p, c_uint32, POINTER(c_uint8), c_int, c_bool]
		self._mosquitto_will_set.restype = c_int

		self._mosquitto_log_init = self._libmosq.mosquitto_log_init
		self._mosquitto_log_init.argtypes = [c_void_p, c_int, c_int]
		self._mosquitto_log_init.restype = c_int

		self._mosquitto_connect_callback_set = self._libmosq.mosquitto_connect_callback_set
		#self._mosquitto_connect_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_connect_callback_set.restype = None

		self._mosquitto_disconnect_callback_set = self._libmosq.mosquitto_disconnect_callback_set
		#self._mosquitto_disconnect_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_disconnect_callback_set.restype = None

		self._mosquitto_publish_callback_set = self._libmosq.mosquitto_publish_callback_set
		#self._mosquitto_publish_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_publish_callback_set.restype = None

		self._mosquitto_message_callback_set = self._libmosq.mosquitto_message_callback_set
		#self._mosquitto_message_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_message_callback_set.restype = None

		self._mosquitto_subscribe_callback_set = self._libmosq.mosquitto_subscribe_callback_set
		#self._mosquitto_subscribe_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_subscribe_callback_set.restype = None

		self._mosquitto_unsubscribe_callback_set = self._libmosq.mosquitto_unsubscribe_callback_set
		#self._mosquitto_unsubscribe_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_unsubscribe_callback_set.restype = None

		self._MOSQ_CONNECT_FUNC = CFUNCTYPE(None, c_void_p, c_int)
		self._MOSQ_DISCONNECT_FUNC = CFUNCTYPE(None, c_void_p)
		self._MOSQ_PUBLISH_FUNC = CFUNCTYPE(None, c_void_p, c_uint16)
		self._MOSQ_MESSAGE_FUNC = CFUNCTYPE(None, c_void_p, POINTER(c_MosquittoMessage))
		self._MOSQ_SUBSCRIBE_FUNC = CFUNCTYPE(None, c_void_p, c_uint16, c_int, POINTER(c_uint8))
		self._MOSQ_UNSUBSCRIBE_FUNC = CFUNCTYPE(None, c_void_p, c_uint16)
		#==================================================
		# End library loading
		#==================================================

		self._mosq = self._mosquitto_new(id, obj)

		#==================================================
		# Configure callbacks
		#==================================================
		self._internal_on_connect_cast = self._MOSQ_CONNECT_FUNC(self._internal_on_connect)
		self._mosquitto_connect_callback_set(self._mosq, self._internal_on_connect_cast)
		self._on_connect = None

		self._internal_on_disconnect_cast = self._MOSQ_DISCONNECT_FUNC(self._internal_on_disconnect)
		self._mosquitto_disconnect_callback_set(self._mosq, self._internal_on_disconnect_cast)
		self._on_disconnect = None

		self._internal_on_message_cast = self._MOSQ_MESSAGE_FUNC(self._internal_on_message)
		self._mosquitto_message_callback_set(self._mosq, self._internal_on_message_cast)
		self.on_message = None

		self._internal_on_publish_cast = self._MOSQ_PUBLISH_FUNC(self._internal_on_publish)
		self._mosquitto_publish_callback_set(self._mosq, self._internal_on_publish_cast)
		self.on_publish = None

		self._internal_on_subscribe_cast = self._MOSQ_SUBSCRIBE_FUNC(self._internal_on_subscribe)
		self._mosquitto_subscribe_callback_set(self._mosq, self._internal_on_subscribe_cast)
		self.on_subscribe = None

		self._internal_on_unsubscribe_cast = self._MOSQ_UNSUBSCRIBE_FUNC(self._internal_on_unsubscribe)
		self._mosquitto_unsubscribe_callback_set(self._mosq, self._internal_on_unsubscribe_cast)
		self.on_unsubscribe = None
		#==================================================
		# End configure callbacks
		#==================================================

	def __del__(self):
		if self._mosq:
			self._mosquitto_destroy(self._mosq)

	def connect(self, hostname="127.0.0.1", port=1883, keepalive=60, clean_session=True):
		return self._mosquitto_connect(self._mosq, hostname, port, keepalive, clean_session)

	def disconnect(self):
		return self._mosquitto_disconnect(self._mosq)

	def log_init(self, priorities, destinations):
		return self._mosquitto_log_init(self._mosq, priorities, destinations)

	def loop(self, timeout=0):
		return self._mosquitto_loop(self._mosq, timeout)

	def subscribe(self, sub, qos):
		return self._mosquitto_subscribe(self._mosq, None, sub, qos)

	def unsubscribe(self, sub):
		return self._mosquitto_unsubscribe(self._mosq, None, sub)

	def publish(self, topic, payload, qos=0, retain=False):
		return self._mosquitto_publish(self._mosq, None, topic, len(payload), cast(payload, POINTER(c_uint8)), qos, retain)

	def will_set(self, will, topic, payloadlen, payload, qos=0, retain=False):
		return self._mosquitto_will_set(self._mosq, will, topic, payloadlen, payload, qos, retain)

	def _internal_on_connect(self, obj, rc):
		if self.on_connect:
			self.on_connect(rc)

	def _internal_on_disconnect(self):
		if self.on_disconnect:
			self.on_disconnect()

	def _internal_on_message(self, obj, message):
		if self.on_message:
			topic = message.contents.topic
			payload = message.contents.payload
			qos = message.contents.qos
			retain = message.contents.retain
			msg = MosquittoMessage(topic, payload, qos, retain)
			self.on_message(msg)

	def _internal_on_publish(self, obj, mid):
		if self.on_publish:
			self.on_publish(mid)

	def _internal_on_subscribe(self, obj, mid, qos_count, granted_qos):
		if self.on_subscribe:
			qos_list = []
			for i in range(qos_count):
				qos_list.append(granted_qos[i])
			self.on_subscribe(mid, qos_list)

	def _internal_on_unsubscribe(self, obj, mid):
		if self.on_unsubscribe:
			self.on_unsubscribe(mid)

class c_MosquittoMessage(Structure):
	_fields_ = [("mid", c_uint16),
				("topic", c_char_p),
				("payload", c_char_p),
				("payloadlen", c_uint32),
				("qos", c_int),
				("retain", c_bool)]

class MosquittoMessage:
	"""MQTT message class"""
	def __init__(self, topic, payload, qos, retain):
		self.topic = topic
		self.payload = payload
		self.qos = qos
		self.retain = retain

<commit_msg>Default loop timeout value should be -1, not 0.<commit_after># Copyright (c) 2010, Roger Light <roger@atchoo.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#   this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in the
#   documentation and/or other materials provided with the distribution.
# 3. Neither the name of mosquitto nor the names of its
#   contributors may be used to endorse or promote products derived from
#   this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

from ctypes import *
from ctypes.util import find_library

# Log destinations
MOSQ_LOG_NONE=0x00
MOSQ_LOG_STDOUT=0x04
MOSQ_LOG_STDERR=0x08

# Log types
MOSQ_LOG_INFO=0x01
MOSQ_LOG_NOTICE=0x02
MOSQ_LOG_WARNING=0x04
MOSQ_LOG_ERR=0x08
MOSQ_LOG_DEBUG=0x10

class Mosquitto:
	"""MQTT version 3 client class"""

	def __init__(self, id, obj=None):
		#==================================================
		# Library loading
		#==================================================
		self._libmosq = cdll.LoadLibrary(find_library("mosquitto"))
		self._mosquitto_new = self._libmosq.mosquitto_new
		self._mosquitto_new.argtypes = [c_char_p, c_void_p]
		self._mosquitto_new.restype = c_void_p

		self._mosquitto_destroy = self._libmosq.mosquitto_destroy
		self._mosquitto_destroy.argtypes = [c_void_p]
		self._mosquitto_destroy.restype = None

		self._mosquitto_connect = self._libmosq.mosquitto_connect
		self._mosquitto_connect.argtypes = [c_void_p, c_char_p, c_int, c_int, c_bool]
		self._mosquitto_connect.restype = c_int

		self._mosquitto_disconnect = self._libmosq.mosquitto_disconnect
		self._mosquitto_disconnect.argtypes = [c_void_p]
		self._mosquitto_disconnect.restype = c_int

		self._mosquitto_publish = self._libmosq.mosquitto_publish
		self._mosquitto_publish.argtypes = [c_void_p, POINTER(c_uint16), c_char_p, c_uint32, POINTER(c_uint8), c_int, c_bool]
		self._mosquitto_publish.restype = c_int

		self._mosquitto_subscribe = self._libmosq.mosquitto_subscribe
		self._mosquitto_subscribe.argtypes = [c_void_p, POINTER(c_uint16), c_char_p, c_int]
		self._mosquitto_subscribe.restype = c_int

		self._mosquitto_unsubscribe = self._libmosq.mosquitto_unsubscribe
		self._mosquitto_unsubscribe.argtypes = [c_void_p, POINTER(c_uint16), c_char_p]
		self._mosquitto_unsubscribe.restype = c_int

		self._mosquitto_loop = self._libmosq.mosquitto_loop
		self._mosquitto_loop.argtypes = [c_void_p, c_int]
		self._mosquitto_loop.restype = c_int

		self._mosquitto_will_set = self._libmosq.mosquitto_will_set
		self._mosquitto_will_set.argtypes = [c_void_p, c_bool, c_char_p, c_uint32, POINTER(c_uint8), c_int, c_bool]
		self._mosquitto_will_set.restype = c_int

		self._mosquitto_log_init = self._libmosq.mosquitto_log_init
		self._mosquitto_log_init.argtypes = [c_void_p, c_int, c_int]
		self._mosquitto_log_init.restype = c_int

		self._mosquitto_connect_callback_set = self._libmosq.mosquitto_connect_callback_set
		#self._mosquitto_connect_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_connect_callback_set.restype = None

		self._mosquitto_disconnect_callback_set = self._libmosq.mosquitto_disconnect_callback_set
		#self._mosquitto_disconnect_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_disconnect_callback_set.restype = None

		self._mosquitto_publish_callback_set = self._libmosq.mosquitto_publish_callback_set
		#self._mosquitto_publish_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_publish_callback_set.restype = None

		self._mosquitto_message_callback_set = self._libmosq.mosquitto_message_callback_set
		#self._mosquitto_message_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_message_callback_set.restype = None

		self._mosquitto_subscribe_callback_set = self._libmosq.mosquitto_subscribe_callback_set
		#self._mosquitto_subscribe_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_subscribe_callback_set.restype = None

		self._mosquitto_unsubscribe_callback_set = self._libmosq.mosquitto_unsubscribe_callback_set
		#self._mosquitto_unsubscribe_callback_set.argtypes = [c_void_p, c_void_p]
		self._mosquitto_unsubscribe_callback_set.restype = None

		self._MOSQ_CONNECT_FUNC = CFUNCTYPE(None, c_void_p, c_int)
		self._MOSQ_DISCONNECT_FUNC = CFUNCTYPE(None, c_void_p)
		self._MOSQ_PUBLISH_FUNC = CFUNCTYPE(None, c_void_p, c_uint16)
		self._MOSQ_MESSAGE_FUNC = CFUNCTYPE(None, c_void_p, POINTER(c_MosquittoMessage))
		self._MOSQ_SUBSCRIBE_FUNC = CFUNCTYPE(None, c_void_p, c_uint16, c_int, POINTER(c_uint8))
		self._MOSQ_UNSUBSCRIBE_FUNC = CFUNCTYPE(None, c_void_p, c_uint16)
		#==================================================
		# End library loading
		#==================================================

		self._mosq = self._mosquitto_new(id, obj)

		#==================================================
		# Configure callbacks
		#==================================================
		self._internal_on_connect_cast = self._MOSQ_CONNECT_FUNC(self._internal_on_connect)
		self._mosquitto_connect_callback_set(self._mosq, self._internal_on_connect_cast)
		self._on_connect = None

		self._internal_on_disconnect_cast = self._MOSQ_DISCONNECT_FUNC(self._internal_on_disconnect)
		self._mosquitto_disconnect_callback_set(self._mosq, self._internal_on_disconnect_cast)
		self._on_disconnect = None

		self._internal_on_message_cast = self._MOSQ_MESSAGE_FUNC(self._internal_on_message)
		self._mosquitto_message_callback_set(self._mosq, self._internal_on_message_cast)
		self.on_message = None

		self._internal_on_publish_cast = self._MOSQ_PUBLISH_FUNC(self._internal_on_publish)
		self._mosquitto_publish_callback_set(self._mosq, self._internal_on_publish_cast)
		self.on_publish = None

		self._internal_on_subscribe_cast = self._MOSQ_SUBSCRIBE_FUNC(self._internal_on_subscribe)
		self._mosquitto_subscribe_callback_set(self._mosq, self._internal_on_subscribe_cast)
		self.on_subscribe = None

		self._internal_on_unsubscribe_cast = self._MOSQ_UNSUBSCRIBE_FUNC(self._internal_on_unsubscribe)
		self._mosquitto_unsubscribe_callback_set(self._mosq, self._internal_on_unsubscribe_cast)
		self.on_unsubscribe = None
		#==================================================
		# End configure callbacks
		#==================================================

	def __del__(self):
		if self._mosq:
			self._mosquitto_destroy(self._mosq)

	def connect(self, hostname="127.0.0.1", port=1883, keepalive=60, clean_session=True):
		return self._mosquitto_connect(self._mosq, hostname, port, keepalive, clean_session)

	def disconnect(self):
		return self._mosquitto_disconnect(self._mosq)

	def log_init(self, priorities, destinations):
		return self._mosquitto_log_init(self._mosq, priorities, destinations)

	def loop(self, timeout=-1):
		return self._mosquitto_loop(self._mosq, timeout)

	def subscribe(self, sub, qos):
		return self._mosquitto_subscribe(self._mosq, None, sub, qos)

	def unsubscribe(self, sub):
		return self._mosquitto_unsubscribe(self._mosq, None, sub)

	def publish(self, topic, payload, qos=0, retain=False):
		return self._mosquitto_publish(self._mosq, None, topic, len(payload), cast(payload, POINTER(c_uint8)), qos, retain)

	def will_set(self, will, topic, payloadlen, payload, qos=0, retain=False):
		return self._mosquitto_will_set(self._mosq, will, topic, payloadlen, payload, qos, retain)

	def _internal_on_connect(self, obj, rc):
		if self.on_connect:
			self.on_connect(rc)

	def _internal_on_disconnect(self):
		if self.on_disconnect:
			self.on_disconnect()

	def _internal_on_message(self, obj, message):
		if self.on_message:
			topic = message.contents.topic
			payload = message.contents.payload
			qos = message.contents.qos
			retain = message.contents.retain
			msg = MosquittoMessage(topic, payload, qos, retain)
			self.on_message(msg)

	def _internal_on_publish(self, obj, mid):
		if self.on_publish:
			self.on_publish(mid)

	def _internal_on_subscribe(self, obj, mid, qos_count, granted_qos):
		if self.on_subscribe:
			qos_list = []
			for i in range(qos_count):
				qos_list.append(granted_qos[i])
			self.on_subscribe(mid, qos_list)

	def _internal_on_unsubscribe(self, obj, mid):
		if self.on_unsubscribe:
			self.on_unsubscribe(mid)

class c_MosquittoMessage(Structure):
	_fields_ = [("mid", c_uint16),
				("topic", c_char_p),
				("payload", c_char_p),
				("payloadlen", c_uint32),
				("qos", c_int),
				("retain", c_bool)]

class MosquittoMessage:
	"""MQTT message class"""
	def __init__(self, topic, payload, qos, retain):
		self.topic = topic
		self.payload = payload
		self.qos = qos
		self.retain = retain

<|endoftext|>


<commit_before>"""Functions to parse Golem report files."""
import json
import os
import base64

def get_date_time_from_timestamp(timestamp):
    """Get the date time from a timestamp.

    The timestamp must have the following format:
    'year.month.day.hour.minutes'
    Example:
    '2017.12.20.10.31' -> '2017/12/20 10:31'
    """
    sp = timestamp.split('.')
    date_time_string = '{0}/{1}/{2} {3}:{4}'.format(sp[0], sp[1], sp[2], sp[3], sp[4])
    return date_time_string

def get_last_executions(root_path, projects=None, suite=None, limit=5):
    """Get the last n executions.

    Get the executions of one suite or all the suites,
    one project or all the projects.

    Returns a list of executions (timestamps strings)
    ordered in descendant order by execution time and
    limited by `limit`
    """
    last_execution_data = {}
    path = os.path.join(root_path, 'projects')
    # if no projects provided, search every project
    if not projects:
        projects = os.walk(path).__next__()[1]
    for project in projects:
        last_execution_data[project] = {}
        report_path = os.path.join(path, project, 'reports')
        # use one suite or all the suites
        if suite:
            executed_suites = [suite]
        else:
            executed_suites = os.walk(report_path).__next__()[1]
            executed_suites = [x for x in executed_suites if x != 'single_tests']
        for exec_suite in executed_suites:
            last_execution_data[project][exec_suite] = []
            suite_path = os.path.join(report_path, exec_suite)
            suite_executions = os.walk(suite_path).__next__()[1]
            last_executions = sorted(suite_executions)
            limit = int(limit)
            last_executions = last_executions[-limit:]
            for execution in last_executions:
                last_execution_data[project][exec_suite].append(execution)
    return last_execution_data

def _parse_execution_data(execution_directory=None, workspace=None, project=None,
                          suite=None, execution=None):
    execution_data = {
        'tests': [],
        'total_tests': 0,
        'totals_by_result': {}
    }
    if not execution_directory:
        execution_directory = os.path.join(workspace, 'projects', project,
                                           'reports', suite, execution)
    test_cases = []
    if os.path.isdir(execution_directory):
        test_cases = next(os.walk(execution_directory))[1]
    for test_case in test_cases:
        # each test case may have n >= 1 test sets
        # each test set is considered a different test
        test_case_path = os.path.join(execution_directory, test_case)
        test_sets = os.walk(test_case_path).__next__()[1]

        for test_set in test_sets:
            new_test_case = {
                'test_set': '',
                'module': '',
                'sub_modules': '',
                'name': '',
                'full_name': '',
                'result': '',
                'test_elapsed_time': '',
                'start_date_time': '',
                'browser': '',
                'data': '',
                'environment': '',
                'set_name': ''
            }
            execution_data['total_tests'] += 1
            new_test_case['test_set'] = test_set
            module = ''
            sub_modules = []
            test_case_split = test_case.split('.')
            if len(test_case_split) > 1:
                module = test_case_split[0]
                if len(test_case_split) > 2:
                    sub_modules = test_case_split[1:-1]
            new_test_case['module'] = module
            test_case_name = test_case_split[-1]
            new_test_case['sub_modules'] = sub_modules
            new_test_case['name'] = test_case_name
            new_test_case['full_name'] = test_case

            test_set_path = os.path.join(test_case_path, test_set)

            # if the suite is being executed, the report dir might exist,
            # but the report.json may still not be generated
            # check if the test has finished (report.json file exists)
            report_json_path = os.path.join(test_set_path, 'report.json')
            if not os.path.exists(report_json_path):
                # test has not finished, add empty test case to the list
                new_test_case['result'] = 'pending'
                execution_data['tests'].append(new_test_case)
                execution_data['totals_by_result']['pending'] = execution_data['totals_by_result'].get('pending', 0) + 1
            else:
                # test has finished
                with open(os.path.join(test_set_path, 'report.json'), 'r') as json_file:
                    report_data = json.load(json_file)
                new_test_case['result'] = report_data['result']
                status = report_data['result']
                execution_data['totals_by_result'][status] = execution_data['totals_by_result'].get(status, 0) + 1
                new_test_case['test_elapsed_time'] = report_data['test_elapsed_time']
                start_date_time = get_date_time_from_timestamp(report_data['test_timestamp'])
                new_test_case['start_date_time'] = start_date_time
                new_test_case['browser'] = report_data['browser']
                new_test_case['data'] = report_data['test_data']
                new_test_case['environment'] = report_data['environment']
                # TODO, previous versions won't have set_name
                # remove the if when retro-compatibility is not required
                if 'set_name' in report_data:
                    new_test_case['set_name'] = report_data['set_name']
                execution_data['tests'].append(new_test_case)
    return execution_data

def get_execution_data(execution_directory=None, workspace=None,
                       project=None, suite=None, execution=None):
    """Retrieve the data of all the tests of a suite execution.

    From the execution_report.json if it exists, otherwise it parses
    the tests one by one.

    The `execution_report.json` should be generated when the suite
    execution ends.
    """
    has_finished = False
    if execution_directory:
        report_path = os.path.join(execution_directory, 'execution_report.json')
    else:
        report_path = os.path.join(workspace, 'projects', project,
                                   'reports', suite, execution,
                                   'execution_report.json')
    if os.path.exists(report_path):
        # get execution_report.json data
        with open(report_path) as json_file:
            data = json.load(json_file)
            has_finished = True
    else:
        # execution_report.json does not exist
        data = _parse_execution_data(execution_directory, workspace,
                                     project, suite, execution)
    data['has_finished'] = has_finished
    return data

def get_test_case_data(root_path, project, test, suite=None, execution=None,
                       test_set=None, is_single=False):
    """Retrieves all the data of a single test case execution."""
    # TODO execution and test_set are not optional
    test_case_data = {
        'module': '',
        'sub_modules': '',
        'name': '',
        'full_name': '',
        'description': '',
        'result': '',
        'test_elapsed_time': '',
        'start_date_time': '',
        'errors': [],
        'browser': '',
        'environment': '',
        'steps': [],
        'debug_log': [],
        'info_log': [],
    }
    if is_single:
        test_case_dir = os.path.join(root_path, 'projects', project, 'reports',
                                     'single_tests', test, execution, test_set)
    else:
        test_case_dir = os.path.join(root_path, 'projects', project, 'reports',
                                     suite, execution, test, test_set)
    report_json_path = os.path.join(test_case_dir, 'report.json')
    if os.path.exists(report_json_path):
        with open(report_json_path, 'r') as json_file:
            report_data = json.load(json_file)
            module = ''
            sub_modules = []
            test_case_split = test.split('.')
            if len(test_case_split) > 1:
                module = test_case_split[0]
                if len(test_case_split) > 2:
                    sub_modules = test_case_split[1:-1]
            test_case_data['module'] = module
            test_case_name = test_case_split[-1]
            test_case_data['sub_modules'] = sub_modules
            test_case_data['name'] = test_case_name
            test_case_data['full_name'] = test
            test_case_data['description'] = report_data['description']
            test_case_data['result'] = report_data['result']
            test_case_data['test_elapsed_time'] = report_data['test_elapsed_time']
            start_date_time = get_date_time_from_timestamp(report_data['test_timestamp'])
            test_case_data['start_date_time'] = start_date_time
            test_case_data['errors'] = report_data['errors']
            test_case_data['browser'] = report_data['browser']
            test_case_data['environment'] = report_data['environment']
            test_case_data['steps'] = report_data['steps']
            test_case_data['test_set'] = test_set
            test_case_data['execution'] = execution
            test_case_data['data'] = report_data['test_data']
            if 'set_name' in report_data:
                test_case_data['set_name'] = report_data['set_name']

    debug_log_path = os.path.join(test_case_dir, 'execution_debug.log')
    info_log_path = os.path.join(test_case_dir, 'execution_info.log')
    if os.path.isfile(debug_log_path):
        with open(debug_log_path) as log_file:
            log = log_file.readlines()
            test_case_data['debug_log'] = log
    if os.path.isfile(info_log_path):
        with open(info_log_path) as log_file:
            log = log_file.readlines()
            test_case_data['info_log'] = log
    return test_case_data

def is_execution_finished(path, sets):
    """Is a suite execution finished.

    It is considered finished when all the tests contain
    a `report.json` file
    """
    if sets:
        is_finished = True
        for data_set in sets:
            report_path = os.path.join(path, data_set, 'report.json')
            if not os.path.exists(report_path):
                is_finished = False
    else:
        is_finished = False
    return is_finished

def generate_execution_report(execution_directory, elapsed_time):
    """Generate a json report of the entire suite execution so
    it is not required to parse the entire execution test by test
    each time it is requested by the reports module."""
    data = _parse_execution_data(execution_directory=execution_directory)
    data['net_elapsed_time'] = elapsed_time
    report_path = os.path.join(execution_directory, 'execution_report.json')
    with open(report_path, 'w', encoding='utf-8') as json_file:
        json.dump(data, json_file, indent=4)

def return_html_test_steps(test_case_data, file_name):
    report_html = """
    """
    for step in test_case_data['steps']:
        if step['screenshot'] is not None:
            image_filename = os.path.join(file_name, step['screenshot'] + '.png')
            if (os.path.isfile(image_filename)):
                b64 = base64.encodestring(open(image_filename, "rb").read())

                image_img = '' % (
                'test', 'test', b64.decode("utf8"))
                report_html += """
{}
{}
""".format(step['message'], image_img)
        else:
            report_html += """
{}
""".format(step['message'])
    return report_html<commit_msg>added generate_junit_execution_report to report_parser<commit_after>"""Functions to parse Golem report files."""
import json
import os
import base64

def get_date_time_from_timestamp(timestamp):
    """Get the date time from a timestamp.

    The timestamp must have the following format:
    'year.month.day.hour.minutes'
    Example:
    '2017.12.20.10.31' -> '2017/12/20 10:31'
    """
    sp = timestamp.split('.')
    date_time_string = '{0}/{1}/{2} {3}:{4}'.format(sp[0], sp[1], sp[2], sp[3], sp[4])
    return date_time_string

def get_last_executions(root_path, projects=None, suite=None, limit=5):
    """Get the last n executions.

    Get the executions of one suite or all the suites,
    one project or all the projects.

    Returns a list of executions (timestamps strings)
    ordered in descendant order by execution time and
    limited by `limit`
    """
    last_execution_data = {}
    path = os.path.join(root_path, 'projects')
    # if no projects provided, search every project
    if not projects:
        projects = os.walk(path).__next__()[1]
    for project in projects:
        last_execution_data[project] = {}
        report_path = os.path.join(path, project, 'reports')
        # use one suite or all the suites
        if suite:
            executed_suites = [suite]
        else:
            executed_suites = os.walk(report_path).__next__()[1]
            executed_suites = [x for x in executed_suites if x != 'single_tests']
        for exec_suite in executed_suites:
            last_execution_data[project][exec_suite] = []
            suite_path = os.path.join(report_path, exec_suite)
            suite_executions = os.walk(suite_path).__next__()[1]
            last_executions = sorted(suite_executions)
            limit = int(limit)
            last_executions = last_executions[-limit:]
            for execution in last_executions:
                last_execution_data[project][exec_suite].append(execution)
    return last_execution_data

def _parse_execution_data(execution_directory=None, workspace=None, project=None,
                          suite=None, execution=None):
    execution_data = {
        'tests': [],
        'total_tests': 0,
        'totals_by_result': {}
    }
    if not execution_directory:
        execution_directory = os.path.join(workspace, 'projects', project,
                                           'reports', suite, execution)
    test_cases = []
    if os.path.isdir(execution_directory):
        test_cases = next(os.walk(execution_directory))[1]
    for test_case in test_cases:
        # each test case may have n >= 1 test sets
        # each test set is considered a different test
        test_case_path = os.path.join(execution_directory, test_case)
        test_sets = os.walk(test_case_path).__next__()[1]

        for test_set in test_sets:
            new_test_case = {
                'test_set': '',
                'module': '',
                'sub_modules': '',
                'name': '',
                'full_name': '',
                'result': '',
                'test_elapsed_time': '',
                'start_date_time': '',
                'browser': '',
                'data': '',
                'environment': '',
                'set_name': ''
            }
            execution_data['total_tests'] += 1
            new_test_case['test_set'] = test_set
            module = ''
            sub_modules = []
            test_case_split = test_case.split('.')
            if len(test_case_split) > 1:
                module = test_case_split[0]
                if len(test_case_split) > 2:
                    sub_modules = test_case_split[1:-1]
            new_test_case['module'] = module
            test_case_name = test_case_split[-1]
            new_test_case['sub_modules'] = sub_modules
            new_test_case['name'] = test_case_name
            new_test_case['full_name'] = test_case

            test_set_path = os.path.join(test_case_path, test_set)

            # if the suite is being executed, the report dir might exist,
            # but the report.json may still not be generated
            # check if the test has finished (report.json file exists)
            report_json_path = os.path.join(test_set_path, 'report.json')
            if not os.path.exists(report_json_path):
                # test has not finished, add empty test case to the list
                new_test_case['result'] = 'pending'
                execution_data['tests'].append(new_test_case)
                execution_data['totals_by_result']['pending'] = execution_data['totals_by_result'].get('pending', 0) + 1
            else:
                # test has finished
                with open(os.path.join(test_set_path, 'report.json'), 'r') as json_file:
                    report_data = json.load(json_file)
                new_test_case['result'] = report_data['result']
                status = report_data['result']
                execution_data['totals_by_result'][status] = execution_data['totals_by_result'].get(status, 0) + 1
                new_test_case['test_elapsed_time'] = report_data['test_elapsed_time']
                start_date_time = get_date_time_from_timestamp(report_data['test_timestamp'])
                new_test_case['start_date_time'] = start_date_time
                new_test_case['browser'] = report_data['browser']
                new_test_case['data'] = report_data['test_data']
                new_test_case['environment'] = report_data['environment']
                # TODO, previous versions won't have set_name
                # remove the if when retro-compatibility is not required
                if 'set_name' in report_data:
                    new_test_case['set_name'] = report_data['set_name']
                execution_data['tests'].append(new_test_case)
    return execution_data

def get_execution_data(execution_directory=None, workspace=None,
                       project=None, suite=None, execution=None):
    """Retrieve the data of all the tests of a suite execution.

    From the execution_report.json if it exists, otherwise it parses
    the tests one by one.

    The `execution_report.json` should be generated when the suite
    execution ends.
    """
    has_finished = False
    if execution_directory:
        report_path = os.path.join(execution_directory, 'execution_report.json')
    else:
        report_path = os.path.join(workspace, 'projects', project,
                                   'reports', suite, execution,
                                   'execution_report.json')
    if os.path.exists(report_path):
        # get execution_report.json data
        with open(report_path) as json_file:
            data = json.load(json_file)
            has_finished = True
    else:
        # execution_report.json does not exist
        data = _parse_execution_data(execution_directory, workspace,
                                     project, suite, execution)
    data['has_finished'] = has_finished
    return data

def get_test_case_data(root_path, project, test, suite=None, execution=None,
                       test_set=None, is_single=False):
    """Retrieves all the data of a single test case execution."""
    # TODO execution and test_set are not optional
    test_case_data = {
        'module': '',
        'sub_modules': '',
        'name': '',
        'full_name': '',
        'description': '',
        'result': '',
        'test_elapsed_time': '',
        'start_date_time': '',
        'errors': [],
        'browser': '',
        'environment': '',
        'steps': [],
        'debug_log': [],
        'info_log': [],
    }
    if is_single:
        test_case_dir = os.path.join(root_path, 'projects', project, 'reports',
                                     'single_tests', test, execution, test_set)
    else:
        test_case_dir = os.path.join(root_path, 'projects', project, 'reports',
                                     suite, execution, test, test_set)
    report_json_path = os.path.join(test_case_dir, 'report.json')
    if os.path.exists(report_json_path):
        with open(report_json_path, 'r') as json_file:
            report_data = json.load(json_file)
            module = ''
            sub_modules = []
            test_case_split = test.split('.')
            if len(test_case_split) > 1:
                module = test_case_split[0]
                if len(test_case_split) > 2:
                    sub_modules = test_case_split[1:-1]
            test_case_data['module'] = module
            test_case_name = test_case_split[-1]
            test_case_data['sub_modules'] = sub_modules
            test_case_data['name'] = test_case_name
            test_case_data['full_name'] = test
            test_case_data['description'] = report_data['description']
            test_case_data['result'] = report_data['result']
            test_case_data['test_elapsed_time'] = report_data['test_elapsed_time']
            start_date_time = get_date_time_from_timestamp(report_data['test_timestamp'])
            test_case_data['start_date_time'] = start_date_time
            test_case_data['errors'] = report_data['errors']
            test_case_data['browser'] = report_data['browser']
            test_case_data['environment'] = report_data['environment']
            test_case_data['steps'] = report_data['steps']
            test_case_data['test_set'] = test_set
            test_case_data['execution'] = execution
            test_case_data['data'] = report_data['test_data']
            if 'set_name' in report_data:
                test_case_data['set_name'] = report_data['set_name']

    debug_log_path = os.path.join(test_case_dir, 'execution_debug.log')
    info_log_path = os.path.join(test_case_dir, 'execution_info.log')
    if os.path.isfile(debug_log_path):
        with open(debug_log_path) as log_file:
            log = log_file.readlines()
            test_case_data['debug_log'] = log
    if os.path.isfile(info_log_path):
        with open(info_log_path) as log_file:
            log = log_file.readlines()
            test_case_data['info_log'] = log
    return test_case_data

def is_execution_finished(path, sets):
    """Is a suite execution finished.

    It is considered finished when all the tests contain
    a `report.json` file
    """
    if sets:
        is_finished = True
        for data_set in sets:
            report_path = os.path.join(path, data_set, 'report.json')
            if not os.path.exists(report_path):
                is_finished = False
    else:
        is_finished = False
    return is_finished

def generate_execution_report(execution_directory, elapsed_time):
    """Generate a json report of the entire suite execution so
    it is not required to parse the entire execution test by test
    each time it is requested by the reports module."""
    data = _parse_execution_data(execution_directory=execution_directory)
    data['net_elapsed_time'] = elapsed_time
    report_path = os.path.join(execution_directory, 'execution_report.json')
    with open(report_path, 'w', encoding='utf-8') as json_file:
        json.dump(data, json_file, indent=4)

def generate_junit_execution_report(suite_name, execution_directory, elapsed_time):
    """Generate a junit xml report of the entire suite execution for use with CI tools."""

    xml = """<?xml version="1.0" encoding="UTF-8"?><testsuites><testsuite name="{0}" tests="{1}" failures="{2}" errors="{3}" time="{4}">{5}</testsuite></testsuites>"""

    testcase = """<testcase name="{0}" classname="{1}" time="{2}">{3}</testcase>"""

    data = _parse_execution_data(execution_directory=execution_directory)
    data['net_elapsed_time'] = elapsed_time

    testcase_data = ""

    for test in data['tests']:
        test_class_name = ''
        if test['module'] != '':
            test_class_name += test['module'] + '.'

        if test['module'] != '':
            test_class_name += test['sub_modules'] + '.'

        test_class_name += test['name']
        name = test['full_name']
        test_elapsed_time = test['test_elapsed_time']

        if test['result'] == 'code error':
            result_text = '<error message="code error" type=""></error>'
        elif test['result'] == 'failure':
            result_text = '<failure message="test failed" type=""></failure>'
        elif test['result'] == 'success':
            result_text = 'success'
        else:
            result_text = ''

        testcase_data += testcase.format(name, test_class_name, test_elapsed_time, result_text)

    if 'failure' not in data['totals_by_result']:
        data['totals_by_result']['failure'] = 0

    if 'code error' not in data['totals_by_result']:
        data['totals_by_result']['code error'] = 0

    xml_report = xml.format(
        suite_name,
        data['total_tests'],
        data['totals_by_result']['failure'],
        data['totals_by_result']['code error'],
        data['net_elapsed_time'],
        testcase_data)

    report_path = os.path.join(execution_directory, 'execution_report.xml')
    with open(report_path, 'w', encoding='utf-8') as xml_file:
        xml_file.write(xml_report)

def return_html_test_steps(test_case_data, file_name):
    report_html = """
    """
    for step in test_case_data['steps']:
        if step['screenshot'] is not None:
            image_filename = os.path.join(file_name, step['screenshot'] + '.png')
            if (os.path.isfile(image_filename)):
                b64 = base64.encodestring(open(image_filename, "rb").read())

                image_img = '' % (
                'test', 'test', b64.decode("utf8"))
                report_html += """
{}
{}
""".format(step['message'], image_img)
        else:
            report_html += """
{}
""".format(step['message'])
    return report_html
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# MIT License

'''
A Bookmark combines a set of Tags and a single Object ('class Filename' in the inclided example).
'''

import datetime
from sqlalchemy import Table
from sqlalchemy import Column
from sqlalchemy import ForeignKey
from sqlalchemy import UniqueConstraint
from sqlalchemy import Integer
from sqlalchemy.orm import relationship
from sqlalchemy.orm import backref
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.ext.associationproxy import association_proxy
from .get_one_or_create import get_one_or_create
from .BaseMixin import BASE
from .Timestamp import Timestamp

tagbookmarks_table = \
    Table('tagbookmarks', BASE.metadata,
          Column('bookmark_id',
                 Integer,
                 ForeignKey("bookmark.id"),
                 primary_key=True),
          Column('tag_id',
                 Integer,
                 ForeignKey("tag.id"),
                 primary_key=True),
          UniqueConstraint('bookmark_id', 'tag_id'))

@classmethod
def construct(cls, session, tag, **kwargs):
    #tag = Tag.construct(session=session, tag=tag) # could demand to get a tag obj...
    result = get_one_or_create(session, cls, **kwargs)
    result.tag_rel.add(tag)
    result.timestamp = Timestamp()
    return result

def repr(self):
    return str(getattr(self, self.target_name)) + ' ' + str(self.tags)

class BookmarkClassConstructor():
    def __new__(cls, mapper_to_bookmark):
        future_class_attr = {}
        future_class_attr['id'] = Column(Integer, primary_key=True)
        future_class_attr['tag_rel'] = relationship("Tag",
                                                    secondary=lambda: tagbookmarks_table,
                                                    collection_class=set,
                                                    backref=backref('bookmarks'))
        future_class_attr['tags'] = association_proxy('tag_rel', 'tag')
        target_class_name = mapper_to_bookmark.__name__
        target_name = target_class_name.lower().split('.')[-1] # 'filename' usually

        future_class_attr[target_name+'_id'] = Column(Integer, ForeignKey(target_name+'.id'), unique=False, nullable=False)
        future_class_attr[target_name] = relationship(target_class_name, backref='bookmarks')
        future_class_attr['target_class_name'] = target_class_name
        future_class_attr['target_name'] = target_name

        future_class_attr['timestamp_id'] = \
            Column(Integer, ForeignKey('timestamp.id'), unique=False, nullable=False)
        future_class_attr['timestamp'] = relationship('Timestamp', backref='bookmarks')

        future_class_attr['construct'] = construct
        future_class_attr['__repr__'] = repr
        return type('Bookmark', (BASE,), future_class_attr)
<commit_msg>auto-commit<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# MIT License

'''
A Bookmark combines a set of Tags and a single Object ('class Filename' in the inclided example).
'''

import datetime
from sqlalchemy import Table
from sqlalchemy import Column
from sqlalchemy import ForeignKey
from sqlalchemy import UniqueConstraint
from sqlalchemy import Integer
from sqlalchemy.orm import relationship
from sqlalchemy.orm import backref
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.ext.associationproxy import association_proxy
from .get_one_or_create import get_one_or_create
from .BaseMixin import BASE
from .Timestamp import Timestamp

tagbookmarks_table = \
    Table('tagbookmarks', BASE.metadata,
          Column('bookmark_id',
                 Integer,
                 ForeignKey("bookmark.id"),
                 primary_key=True),
          Column('tag_id',
                 Integer,
                 ForeignKey("tag.id"),
                 primary_key=True),
          UniqueConstraint('bookmark_id', 'tag_id'))

@classmethod
def construct(cls, session, tag, **kwargs):
    #tag = Tag.construct(session=session, tag=tag) # could demand to get a tag obj...
    result = get_one_or_create(session, cls, **kwargs)
    result.tag_rel.add(tag)
    #result.timestamp = Timestamp()
    return result

def repr(self):
    return str(getattr(self, self.target_name)) + ' ' + str(self.tags)

class BookmarkClassConstructor():
    def __new__(cls, mapper_to_bookmark):
        future_class_attr = {}
        future_class_attr['id'] = Column(Integer, primary_key=True)
        future_class_attr['tag_rel'] = relationship("Tag",
                                                    secondary=lambda: tagbookmarks_table,
                                                    collection_class=set,
                                                    backref=backref('bookmarks'))
        future_class_attr['tags'] = association_proxy('tag_rel', 'tag')
        target_class_name = mapper_to_bookmark.__name__
        target_name = target_class_name.lower().split('.')[-1] # 'filename' usually

        future_class_attr[target_name+'_id'] = Column(Integer, ForeignKey(target_name+'.id'), unique=False, nullable=False)
        future_class_attr[target_name] = relationship(target_class_name, backref='bookmarks')
        future_class_attr['target_class_name'] = target_class_name
        future_class_attr['target_name'] = target_name

        future_class_attr['timestamp_id'] = \
            Column(Integer, ForeignKey('timestamp.id'), unique=False, nullable=False)
        future_class_attr['timestamp'] = relationship('Timestamp', backref='bookmarks')

        future_class_attr['construct'] = construct
        future_class_attr['__repr__'] = repr
        return type('Bookmark', (BASE,), future_class_attr)

<|endoftext|>


<commit_before>from datetime import datetime
from sqlalchemy.orm import reconstructor, relationship, backref
from sqlalchemy.schema import Column, ForeignKey
from sqlalchemy.types import Integer, Unicode, Boolean, DateTime
from sqlalchemy import BigInteger
from sqlalchemy.sql.expression import false, or_
from sqlalchemy.ext.associationproxy import association_proxy

from openspending.core import db

from openspending.model.source import Source
from openspending.model.dataorg import DataOrg
from openspending.model.metadataorg import MetadataOrg
#from openspending.model import Source, Account, DataOrg
from openspending.model.common import (MutableDict, JSONType,
                                       DatasetFacetMixin)

from slugify import slugify

class Dataset(db.Model):

    """ The dataset is the core entity of any access to data. All
    requests to the actual data store are routed through it, as well
    as data loading and model generation.

    """
    __tablename__ = 'dataset'
    __searchable__ = ['label', 'description']

    id = Column(Integer, primary_key=True)
    name = Column(Unicode(255), unique=True)
    label = Column(Unicode(2000))
    description = Column(Unicode())

    created_at = Column(DateTime, default=datetime.utcnow)
    updated_at = Column(DateTime, default=datetime.utcnow,
                        onupdate=datetime.utcnow)

    datalastupdated = Column(DateTime, default=datetime.utcnow)

    source_id = Column(Integer, ForeignKey('source.id'))
    source = relationship(Source, backref=backref("dataset", uselist=False))

    mapping = Column(MutableDict.as_mutable(JSONType), default=dict)

    ORoperations = Column( MutableDict.as_mutable(JSONType), default=dict)

    prefuncs = Column(MutableDict.as_mutable(JSONType), default=dict)

    dataType = Column(Unicode(2000))

    published = Column(Boolean, default=False)

    loaded = Column(Boolean, default=False)

    tested = Column(Boolean, default=False)

    dataorg_id = Column(Integer, ForeignKey('dataorg.id'))
    dataorg = relationship(DataOrg,
                           backref=backref('datasets', lazy='dynamic'))

    metadataorg_id = Column(Integer, ForeignKey('metadataorg.id'))
    metadataorg = relationship(MetadataOrg,
                           backref=backref('datasets', lazy='dynamic'))

    years = Column(Unicode(1000))

    stats = Column(Unicode(50))

    update_freq = Column(Unicode(255))
    orgurl = Column(Unicode(500))
    units = Column(Unicode(255))

    webservice = Column(Unicode(500))
    agency = Column(Unicode(255))
    organization = Column(Unicode(255))
    notes = Column(Unicode(4000))
    update_cycle = Column(Unicode(255))
    #TODO
    #tag stuff

    def __init__(self, data = None):
        if data == None:
            return
        self.label = data.get('label')
        if (data.get('name', None)):
            self.name = slugify(str(data.get('name')), max_length=30, separator="_")
        else:
            self.name = slugify(str(data.get('label')), max_length=30, separator="_")

        #check if name is already taken
        if Dataset.by_name(self.name):
            for x in range(10):
                newname = self.name + "_" + str(x)
                if not Dataset.by_name(newname):
                    self.name = newname
                    break

        self.description = data.get('description')
        self.ORoperations = data.get('ORoperations', {})
        self.mapping = data.get('mapping', {})
        self.prefuncs = data.get('prefuncs', {})
        self.created_at = datetime.utcnow()
        self.dataType = data.get('dataType')
        if type(data.get('dataorg')) == int:
            self.dataorg = DataOrg.by_id(data.get('dataorg'))
        else:
            try:
                self.dataorg = data.get('dataorg')
            except Exception, e:
                print "failed to load the dataorg for dataset"
                print e

    #def get_count_query(self):
    #    return self.session.query(func.count(dataType))

    def to_json_dump(self):
        """ Returns a JSON representation of an SQLAlchemy-backed object.
        """

        json = {}
        json['fields'] = {}
        json['pk'] = getattr(self, 'id')
        json['model'] = "Dataset"

        fields = ['name', 'label', 'description',
                 'source_id', 'mapping', 'ORoperations', 'prefuncs', 'dataType','published','loaded', 'tested','dataorg_id']

        for field in fields:
            json['fields'][field] = getattr(self, field)

        return json

    @classmethod
    def import_json_dump(cls, theobj):
        fields = ['name', 'label', 'description',
                 'source_id', 'mapping', 'ORoperations', 'prefuncs', 'dataType','published','loaded', 'tested','dataorg_id']
        classobj = cls()
        for field in fields:
            setattr(classobj, field, theobj['fields'][field])

        db.session.add(classobj)
        db.session.commit()

        return classobj.id

    @property
    def has_data(self):
        if self.source_id:
            return True
        else:
            return False

    @property
    def tags_str(self):
        output = []
        for t in self.tags:
            output.append(str(t))
        return ",".join(output)

    def touch(self):
        """ Update the dataset timestamp. This is used for cache
        invalidation. """
        self.updated_at = datetime.utcnow()
        db.session.add(self)

    def __repr__(self):
        return "%s (%s)" % (self.label, self.id, )

    def update(self, data):
        #not to update name
        self.label = data.get('label')
        if (data.get('name', None)):
            self.name = slugify(str(data.get('name')), max_length=30, separator="_")
        else:
            self.name = slugify(str(data.get('label')), max_length=30, separator="_")

        self.description = data.get('description')
        self.dataType = data.get('dataType')
        self.dataorg = DataOrg.by_id(data.get('dataorg'))
        self.stats = data.get('stats')

    def as_dict(self):
        the_years=[]
        load_status = "Need Source"
        if self.source:
            load_status = self.source.load_status
        dataset_dict=None
        if self.dataorg:
            dataset_dict=self.dataorg.label
        if self.years:
            the_years=self.years.split(",")
            the_years=map(int,the_years)
            the_years.sort()
        return {
            'label': self.label,
            'name': self.name,
            'description': self.description,
            'dataType': self.dataType,
            'dataorg': dataset_dict,
            'has_data': self.has_data,
            'source': self.source_id,
            'status': load_status,
            'years':the_years,
            'stats':self.stats
        }

    def detailed_dict(self):
        the_years=[]
        dataorg_dict=None
        if self.dataorg:
            dataset_dict=self.dataorg.label
        if self.years:
            the_years=self.years.split(",")
            the_years=map(int,the_years)
            the_years.sort()
        return {
            'label': self.label,
            'name': self.name,
            'description': self.description,
            'dataType': self.dataType,
            'dataorg': dataorg_dict,
            'created_at': self.created_at,
            'years':the_years
        }

    @classmethod
    def all_by_account(cls, account, order=True):
        """ Query available datasets based on dataset visibility. """
        from openspending.model.account import Account
        #limit to certain published/loaded/tested
        criteria = []
        if isinstance(account, Account) and account.is_authenticated():
            criteria += ["1=1" if account.admin else "1=2",
                         cls.managers.any(Account.id == account.id)]
        q = db.session.query(cls).filter(or_(*criteria))
        if order:
            q = q.order_by(cls.label.asc())
        return q

    @classmethod
    def get_all_admin(cls, order=True):
        """ Query available datasets based on dataset visibility. """
        q = db.session.query(cls)
        if order:
            q = q.order_by(cls.label.asc())
        return q

    @classmethod
    def all(cls, order=True):
        """ Query available datasets based on dataset visibility. """
        q = db.session.query(cls)
        if order:
            q = q.order_by(cls.label.asc())
        return q

    @classmethod
    def by_name(cls, name):
        return db.session.query(cls).filter_by(name=name).first()

    @classmethod
    def by_label(cls, label):
        return db.session.query(cls).filter_by(label=label).first()

    @classmethod
    def by_id(cls, id):
        return db.session.query(cls).filter_by(id=id).first()

<commit_msg>add source url fixes #235<commit_after>from datetime import datetime
from sqlalchemy.orm import reconstructor, relationship, backref
from sqlalchemy.schema import Column, ForeignKey
from sqlalchemy.types import Integer, Unicode, Boolean, DateTime
from sqlalchemy import BigInteger
from sqlalchemy.sql.expression import false, or_
from sqlalchemy.ext.associationproxy import association_proxy

from openspending.core import db

from openspending.model.source import Source
from openspending.model.dataorg import DataOrg
from openspending.model.metadataorg import MetadataOrg
#from openspending.model import Source, Account, DataOrg
from openspending.model.common import (MutableDict, JSONType,
                                       DatasetFacetMixin)

from slugify import slugify

class Dataset(db.Model):

    """ The dataset is the core entity of any access to data. All
    requests to the actual data store are routed through it, as well
    as data loading and model generation.

    """
    __tablename__ = 'dataset'
    __searchable__ = ['label', 'description']

    id = Column(Integer, primary_key=True)
    name = Column(Unicode(255), unique=True)
    label = Column(Unicode(2000))
    description = Column(Unicode())

    created_at = Column(DateTime, default=datetime.utcnow)
    updated_at = Column(DateTime, default=datetime.utcnow,
                        onupdate=datetime.utcnow)

    datalastupdated = Column(DateTime, default=datetime.utcnow)

    source_id = Column(Integer, ForeignKey('source.id'))
    source = relationship(Source, backref=backref("dataset", uselist=False))

    mapping = Column(MutableDict.as_mutable(JSONType), default=dict)

    ORoperations = Column( MutableDict.as_mutable(JSONType), default=dict)

    prefuncs = Column(MutableDict.as_mutable(JSONType), default=dict)

    dataType = Column(Unicode(2000))

    published = Column(Boolean, default=False)

    loaded = Column(Boolean, default=False)

    tested = Column(Boolean, default=False)

    dataorg_id = Column(Integer, ForeignKey('dataorg.id'))
    dataorg = relationship(DataOrg,
                           backref=backref('datasets', lazy='dynamic'))

    metadataorg_id = Column(Integer, ForeignKey('metadataorg.id'))
    metadataorg = relationship(MetadataOrg,
                           backref=backref('datasets', lazy='dynamic'))

    years = Column(Unicode(1000))

    stats = Column(Unicode(50))

    update_freq = Column(Unicode(255))
    orgurl = Column(Unicode(500))
    units = Column(Unicode(255))

    webservice = Column(Unicode(500))
    agency = Column(Unicode(255))
    organization = Column(Unicode(255))
    notes = Column(Unicode(4000))
    update_cycle = Column(Unicode(255))
    #TODO
    #tag stuff

    def __init__(self, data = None):
        if data == None:
            return
        self.label = data.get('label')
        if (data.get('name', None)):
            self.name = slugify(str(data.get('name')), max_length=30, separator="_")
        else:
            self.name = slugify(str(data.get('label')), max_length=30, separator="_")

        #check if name is already taken
        if Dataset.by_name(self.name):
            for x in range(10):
                newname = self.name + "_" + str(x)
                if not Dataset.by_name(newname):
                    self.name = newname
                    break

        self.description = data.get('description')
        self.ORoperations = data.get('ORoperations', {})
        self.mapping = data.get('mapping', {})
        self.prefuncs = data.get('prefuncs', {})
        self.created_at = datetime.utcnow()
        self.dataType = data.get('dataType')
        if type(data.get('dataorg')) == int:
            self.dataorg = DataOrg.by_id(data.get('dataorg'))
        else:
            try:
                self.dataorg = data.get('dataorg')
            except Exception, e:
                print "failed to load the dataorg for dataset"
                print e

    #def get_count_query(self):
    #    return self.session.query(func.count(dataType))

    def to_json_dump(self):
        """ Returns a JSON representation of an SQLAlchemy-backed object.
        """

        json = {}
        json['fields'] = {}
        json['pk'] = getattr(self, 'id')
        json['model'] = "Dataset"

        fields = ['name', 'label', 'description',
                 'source_id', 'mapping', 'ORoperations', 'prefuncs', 'dataType','published','loaded', 'tested','dataorg_id']

        for field in fields:
            json['fields'][field] = getattr(self, field)

        return json

    @classmethod
    def import_json_dump(cls, theobj):
        fields = ['name', 'label', 'description',
                 'source_id', 'mapping', 'ORoperations', 'prefuncs', 'dataType','published','loaded', 'tested','dataorg_id']
        classobj = cls()
        for field in fields:
            setattr(classobj, field, theobj['fields'][field])

        db.session.add(classobj)
        db.session.commit()

        return classobj.id

    @property
    def has_data(self):
        if self.source_id:
            return True
        else:
            return False

    @property
    def tags_str(self):
        output = []
        for t in self.tags:
            output.append(str(t))
        return ",".join(output)

    def touch(self):
        """ Update the dataset timestamp. This is used for cache
        invalidation. """
        self.updated_at = datetime.utcnow()
        db.session.add(self)

    def __repr__(self):
        return "%s (%s)" % (self.label, self.id, )

    def update(self, data):
        #not to update name
        self.label = data.get('label')
        if (data.get('name', None)):
            self.name = slugify(str(data.get('name')), max_length=30, separator="_")
        else:
            self.name = slugify(str(data.get('label')), max_length=30, separator="_")

        self.description = data.get('description')
        self.dataType = data.get('dataType')
        self.dataorg = DataOrg.by_id(data.get('dataorg'))
        self.stats = data.get('stats')

    def as_dict(self):
        the_years=[]
        load_status = "Need Source"
        source_url=""
        if self.source:
            load_status = self.source.load_status
            source_url = self.source.url
        dataset_dict=None
        if self.dataorg:
            dataset_dict=self.dataorg.label
        if self.years:
            the_years=self.years.split(",")
            the_years=map(int,the_years)
            the_years.sort()
        return {
            'label': self.label,
            'name': self.name,
            'description': self.description,
            'dataType': self.dataType,
            'dataorg': dataset_dict,
            'has_data': self.has_data,
            'source': self.source_id,
            'status': load_status,
            'years':the_years,
            'stats':self.stats,
            'url':source_url
        }

    def detailed_dict(self):
        the_years=[]
        dataorg_dict=None
        if self.dataorg:
            dataset_dict=self.dataorg.label
        if self.years:
            the_years=self.years.split(",")
            the_years=map(int,the_years)
            the_years.sort()
        return {
            'label': self.label,
            'name': self.name,
            'description': self.description,
            'dataType': self.dataType,
            'dataorg': dataorg_dict,
            'created_at': self.created_at,
            'years':the_years
        }

    @classmethod
    def all_by_account(cls, account, order=True):
        """ Query available datasets based on dataset visibility. """
        from openspending.model.account import Account
        #limit to certain published/loaded/tested
        criteria = []
        if isinstance(account, Account) and account.is_authenticated():
            criteria += ["1=1" if account.admin else "1=2",
                         cls.managers.any(Account.id == account.id)]
        q = db.session.query(cls).filter(or_(*criteria))
        if order:
            q = q.order_by(cls.label.asc())
        return q

    @classmethod
    def get_all_admin(cls, order=True):
        """ Query available datasets based on dataset visibility. """
        q = db.session.query(cls)
        if order:
            q = q.order_by(cls.label.asc())
        return q

    @classmethod
    def all(cls, order=True):
        """ Query available datasets based on dataset visibility. """
        q = db.session.query(cls)
        if order:
            q = q.order_by(cls.label.asc())
        return q

    @classmethod
    def by_name(cls, name):
        return db.session.query(cls).filter_by(name=name).first()

    @classmethod
    def by_label(cls, label):
        return db.session.query(cls).filter_by(label=label).first()

    @classmethod
    def by_id(cls, id):
        return db.session.query(cls).filter_by(id=id).first()

<|endoftext|>