<commit_before>#!/usr/bin/env python3

import argparse
import logging
import sys
import traceback

from raven.handlers.logging import SentryHandler
from threading import Thread
from time import sleep
from uuid import UUID

from ..api import ChangesApi
from ..container import Container
from ..log_reporter import LogReporter

DESCRIPTION = "LXC Wrapper for running Changes jobs"

DEFAULT_RELEASE = 'precise'

class CommandError(Exception):
    pass

class WrappedOutput(object):
    def __init__(self, stream, reporter):
        self.stream = stream
        self.reporter = reporter

    def write(self, chunk):
        self.stream.write(chunk)
        self.reporter.write(chunk)

    def flush(self):
        self.stream.flush()
        self.reporter.flush()

class WrapperCommand(object):
    def __init__(self, argv=None):
        self.argv = argv
        self.stdout = sys.stdout
        self.stderr = sys.stderr

    def get_arg_parser(self):
        parser = argparse.ArgumentParser(description=DESCRIPTION)
        parser.add_argument('--snapshot', '-s', type=UUID,
                            help="Snapshot ID of the container")
        parser.add_argument('--release', '-r',
                            help="Ubuntu release (default: {})".format(DEFAULT_RELEASE))
        parser.add_argument('--keep', action='store_true', default=False,
                            help="Don't destroy the container after running cmd/build")
        parser.add_argument('--no-validate', action='store_false', default=True, dest='validate',
                            help="Don't validate downloaded images")
        parser.add_argument('--save-snapshot', action='store_true', default=False,
                            help="Create an image from this container")
        parser.add_argument('--clean', action='store_true', default=False,
                            help="Use a fresh container from Ubuntu minimal install")
        parser.add_argument('--flush-cache', action='store_true', default=False,
                            help="Rebuild Ubuntu minimal install cache")
        parser.add_argument('--api-url',
                            help="API URL to Changes (i.e. https://changes.example.com/api/0/)")
        parser.add_argument('--jobstep-id',
                            help="Jobstep ID for Changes")
        parser.add_argument('--pre-launch',
                            help="Command to run before container is launched")
        parser.add_argument('--post-launch',
                            help="Command to run after container is launched")
        parser.add_argument('--user', '-u', default='ubuntu',
                            help="User to run command (or script) as")
        parser.add_argument('--script',
                            help="Script to execute as command")
        parser.add_argument('--s3-bucket',
                            help="S3 Bucket to store/fetch images from")
        parser.add_argument('--log-level', default='WARN')
        parser.add_argument('cmd', nargs=argparse.REMAINDER,
                            help="Command to run inside the container")
        return parser

    def configure_logging(self, level):
        logging.basicConfig(level=level)

        root = logging.getLogger()
        root.addHandler(SentryHandler())

    def patch_system_logging(self, reporter):
        sys.stdout = WrappedOutput(sys.stdout, reporter)
        sys.stderr = WrappedOutput(sys.stderr, reporter)

    def run(self):
        parser = self.get_arg_parser()
        args = parser.parse_args(self.argv)

        try:
            args.cmd.remove('--')
        except ValueError:
            pass

        self.configure_logging(args.log_level)

        if args.jobstep_id:
            return self.run_remote(args)
        return self.run_local(args)

    def run_local(self, args):
        """
        Run a local-only build (i.e. for testing).
        """
        snapshot = str(args.snapshot) if args.snapshot else None
        release = args.release or DEFAULT_RELEASE

        self.run_build_script(
            snapshot=snapshot,
            release=release,
            validate=args.validate,
            s3_bucket=args.s3_bucket,
            pre_launch=args.pre_launch,
            post_launch=args.post_launch,
            clean=args.clean,
            flush_cache=args.flush_cache,
            save_snapshot=args.save_snapshot,
            user=args.user,
            cmd=args.cmd,
            script=args.script,
            keep=args.keep,
        )

    def run_remote(self, args):
        """
        Run a build script from upstream (Changes), pulling any required
        information from the remote server, as well as pushing up status
        changes and log information.
        """
        if not args.api_url:
            raise CommandError('jobstep_id passed, but missing api_url')

        # we wrap the actual run routine to make it easier to catch
        # top level exceptions and report them via the log
        def inner_run(api, jobstep_id):
            try:
                # fetch build information to set defaults for things like snapshot
                # TODO(dcramer): make this support a small amount of downtime
                # TODO(dcramer): make this verify the snapshot
                resp = api.get_jobstep(jobstep_id)
                if resp['status']['id'] == 'finished':
                    raise Exception('JobStep already marked as finished, aborting.')

                release = resp['data'].get('release') or DEFAULT_RELEASE

                # If we're expected a snapshot output we need to override
                # any snapshot parameters, and also ensure we're creating a clean
                # image
                if resp['expectedSnapshot']:
                    snapshot = str(UUID(resp['expectedSnapshot']['id']))
                    save_snapshot = True
                    clean = True

                else:
                    if resp['snapshot']:
                        snapshot = str(UUID(resp['snapshot']['id']))
                    else:
                        snapshot = None
                    save_snapshot = False
                    clean = False

                api.update_jobstep(jobstep_id, {"status": "in_progress"})

                cmd = [
                    'changes-client',
                    '--server', args.api_url,
                    '--jobstep_id', jobstep_id,
                    '--run_as_child',
                ]

                self.run_build_script(
                    snapshot=snapshot,
                    release=release,
                    validate=args.validate,
                    s3_bucket=args.s3_bucket,
                    pre_launch=args.pre_launch,
                    post_launch=args.post_launch,
                    clean=clean,
                    flush_cache=args.flush_cache,
                    save_snapshot=save_snapshot,
                    user=args.user,
                    cmd=cmd,
                    keep=args.keep,
                )

            except Exception:
                reporter.write(traceback.format_exc())

                api.update_jobstep(jobstep_id, {"status": "finished", "result": "failed"})
                if args.save_snapshot:
                    api.update_snapshot_image(snapshot, {"status": "failed"})

                raise

            else:
                api.update_jobstep(jobstep_id, {"status": "finished"})
                if args.save_snapshot:
                    api.update_snapshot_image(snapshot, {"status": "active"})

        api = ChangesApi(args.api_url)
        jobstep_id = args.jobstep_id

        reporter = LogReporter(api, jobstep_id)
        reporter_thread = Thread(target=reporter.process)
        reporter_thread.start()
        self.patch_system_logging(reporter)

        run_thread = Thread(target=inner_run, args=[api, jobstep_id])
        run_thread.daemon = True
        run_thread.start()
        while run_thread.is_alive():
            try:
                run_thread.join(10)
            except Exception:
                reporter.write(traceback.format_exc())
                break
            sleep(1)

        if run_thread.is_alive():
            reporter.write('==> Signal received from upstream, terminating.\n')
            # give it a second chance in case there was a race between the heartbeat
            # and the builder
            run_thread.join(5)

        reporter.close()

        reporter_thread.join(60)

    def run_build_script(self, snapshot, release, validate, s3_bucket, pre_launch,
                         post_launch, clean, flush_cache, save_snapshot,
                         user, cmd=None, script=None, keep=False):
        """
        Run the given build script inside of the LXC container.
        """
        assert clean or not (save_snapshot and snapshot), \
            "You cannot create a snapshot from an existing snapshot"

        assert not (cmd and script), \
            'Only one of cmd or script can be specified'

        assert cmd or script, \
            'Missing build command'

        container = Container(
            snapshot=snapshot,
            release=release,
            validate=validate,
            s3_bucket=s3_bucket,
        )

        try:
            container.launch(pre_launch, post_launch, clean, flush_cache)

            # TODO(dcramer): we should assert only one type of command arg is set
            if cmd:
                container.run(cmd, user=user)
            elif script:
                container.run_script(script, user=user)

            if save_snapshot:
                snapshot = container.create_image()
                print("==> Snapshot saved: {}".format(snapshot))
                if s3_bucket:
                    container.upload_image(snapshot=snapshot)
        except Exception as e:
            logging.exception(e)
            raise e
        finally:
            if not keep:
                container.destroy()
            else:
                print("==> Container kept at {}".format(container.rootfs))
                print("==> SSH available via:")
                print("==>   $ sudo lxc-attach --name={}".format(container.name))

def main():
    command = WrapperCommand()
    command.run()

if __name__ == '__main__':
    main()
<commit_msg>Missing name arg<commit_after>#!/usr/bin/env python3

import argparse
import logging
import sys
import traceback

from raven.handlers.logging import SentryHandler
from threading import Thread
from time import sleep
from uuid import UUID, uuid4

from ..api import ChangesApi
from ..container import Container
from ..log_reporter import LogReporter

DESCRIPTION = "LXC Wrapper for running Changes jobs"

DEFAULT_RELEASE = 'precise'

class CommandError(Exception):
    pass

class WrappedOutput(object):
    def __init__(self, stream, reporter):
        self.stream = stream
        self.reporter = reporter

    def write(self, chunk):
        self.stream.write(chunk)
        self.reporter.write(chunk)

    def flush(self):
        self.stream.flush()
        self.reporter.flush()

class WrapperCommand(object):
    def __init__(self, argv=None):
        self.argv = argv
        self.stdout = sys.stdout
        self.stderr = sys.stderr

    def get_arg_parser(self):
        parser = argparse.ArgumentParser(description=DESCRIPTION)
        parser.add_argument('--snapshot', '-s', type=UUID,
                            help="Snapshot ID of the container")
        parser.add_argument('--release', '-r',
                            help="Ubuntu release (default: {})".format(DEFAULT_RELEASE))
        parser.add_argument('--keep', action='store_true', default=False,
                            help="Don't destroy the container after running cmd/build")
        parser.add_argument('--no-validate', action='store_false', default=True, dest='validate',
                            help="Don't validate downloaded images")
        parser.add_argument('--save-snapshot', action='store_true', default=False,
                            help="Create an image from this container")
        parser.add_argument('--clean', action='store_true', default=False,
                            help="Use a fresh container from Ubuntu minimal install")
        parser.add_argument('--flush-cache', action='store_true', default=False,
                            help="Rebuild Ubuntu minimal install cache")
        parser.add_argument('--api-url',
                            help="API URL to Changes (i.e. https://changes.example.com/api/0/)")
        parser.add_argument('--jobstep-id',
                            help="Jobstep ID for Changes")
        parser.add_argument('--pre-launch',
                            help="Command to run before container is launched")
        parser.add_argument('--post-launch',
                            help="Command to run after container is launched")
        parser.add_argument('--user', '-u', default='ubuntu',
                            help="User to run command (or script) as")
        parser.add_argument('--script',
                            help="Script to execute as command")
        parser.add_argument('--s3-bucket',
                            help="S3 Bucket to store/fetch images from")
        parser.add_argument('--log-level', default='WARN')
        parser.add_argument('cmd', nargs=argparse.REMAINDER,
                            help="Command to run inside the container")
        return parser

    def configure_logging(self, level):
        logging.basicConfig(level=level)

        root = logging.getLogger()
        root.addHandler(SentryHandler())

    def patch_system_logging(self, reporter):
        sys.stdout = WrappedOutput(sys.stdout, reporter)
        sys.stderr = WrappedOutput(sys.stderr, reporter)

    def run(self):
        parser = self.get_arg_parser()
        args = parser.parse_args(self.argv)

        try:
            args.cmd.remove('--')
        except ValueError:
            pass

        self.configure_logging(args.log_level)

        if args.jobstep_id:
            return self.run_remote(args)
        return self.run_local(args)

    def run_local(self, args):
        """
        Run a local-only build (i.e. for testing).
        """
        snapshot = str(args.snapshot) if args.snapshot else None
        release = args.release or DEFAULT_RELEASE

        self.run_build_script(
            snapshot=snapshot,
            release=release,
            validate=args.validate,
            s3_bucket=args.s3_bucket,
            pre_launch=args.pre_launch,
            post_launch=args.post_launch,
            clean=args.clean,
            flush_cache=args.flush_cache,
            save_snapshot=args.save_snapshot,
            user=args.user,
            cmd=args.cmd,
            script=args.script,
            keep=args.keep,
        )

    def run_remote(self, args):
        """
        Run a build script from upstream (Changes), pulling any required
        information from the remote server, as well as pushing up status
        changes and log information.
        """
        if not args.api_url:
            raise CommandError('jobstep_id passed, but missing api_url')

        # we wrap the actual run routine to make it easier to catch
        # top level exceptions and report them via the log
        def inner_run(api, jobstep_id):
            try:
                # fetch build information to set defaults for things like snapshot
                # TODO(dcramer): make this support a small amount of downtime
                # TODO(dcramer): make this verify the snapshot
                resp = api.get_jobstep(jobstep_id)
                if resp['status']['id'] == 'finished':
                    raise Exception('JobStep already marked as finished, aborting.')

                release = resp['data'].get('release') or DEFAULT_RELEASE

                # If we're expected a snapshot output we need to override
                # any snapshot parameters, and also ensure we're creating a clean
                # image
                if resp['expectedSnapshot']:
                    snapshot = str(UUID(resp['expectedSnapshot']['id']))
                    save_snapshot = True
                    clean = True

                else:
                    if resp['snapshot']:
                        snapshot = str(UUID(resp['snapshot']['id']))
                    else:
                        snapshot = None
                    save_snapshot = False
                    clean = False

                api.update_jobstep(jobstep_id, {"status": "in_progress"})

                cmd = [
                    'changes-client',
                    '--server', args.api_url,
                    '--jobstep_id', jobstep_id,
                    '--run_as_child',
                ]

                self.run_build_script(
                    snapshot=snapshot,
                    release=release,
                    validate=args.validate,
                    s3_bucket=args.s3_bucket,
                    pre_launch=args.pre_launch,
                    post_launch=args.post_launch,
                    clean=clean,
                    flush_cache=args.flush_cache,
                    save_snapshot=save_snapshot,
                    user=args.user,
                    cmd=cmd,
                    keep=args.keep,
                )

            except Exception:
                reporter.write(traceback.format_exc())

                api.update_jobstep(jobstep_id, {"status": "finished", "result": "failed"})
                if args.save_snapshot:
                    api.update_snapshot_image(snapshot, {"status": "failed"})

                raise

            else:
                api.update_jobstep(jobstep_id, {"status": "finished"})
                if args.save_snapshot:
                    api.update_snapshot_image(snapshot, {"status": "active"})

        api = ChangesApi(args.api_url)
        jobstep_id = args.jobstep_id

        reporter = LogReporter(api, jobstep_id)
        reporter_thread = Thread(target=reporter.process)
        reporter_thread.start()
        self.patch_system_logging(reporter)

        run_thread = Thread(target=inner_run, args=[api, jobstep_id])
        run_thread.daemon = True
        run_thread.start()
        while run_thread.is_alive():
            try:
                run_thread.join(10)
            except Exception:
                reporter.write(traceback.format_exc())
                break
            sleep(1)

        if run_thread.is_alive():
            reporter.write('==> Signal received from upstream, terminating.\n')
            # give it a second chance in case there was a race between the heartbeat
            # and the builder
            run_thread.join(5)

        reporter.close()

        reporter_thread.join(60)

    def run_build_script(self, snapshot, release, validate, s3_bucket, pre_launch,
                         post_launch, clean, flush_cache, save_snapshot,
                         user, cmd=None, script=None, keep=False):
        """
        Run the given build script inside of the LXC container.
        """
        assert clean or not (save_snapshot and snapshot), \
            "You cannot create a snapshot from an existing snapshot"

        assert not (cmd and script), \
            'Only one of cmd or script can be specified'

        assert cmd or script, \
            'Missing build command'

        container = Container(
            name=str(uuid4()),
            snapshot=snapshot,
            release=release,
            validate=validate,
            s3_bucket=s3_bucket,
        )

        try:
            container.launch(pre_launch, post_launch, clean, flush_cache)

            # TODO(dcramer): we should assert only one type of command arg is set
            if cmd:
                container.run(cmd, user=user)
            elif script:
                container.run_script(script, user=user)

            if save_snapshot:
                snapshot = container.create_image()
                print("==> Snapshot saved: {}".format(snapshot))
                if s3_bucket:
                    container.upload_image(snapshot=snapshot)
        except Exception as e:
            logging.exception(e)
            raise e
        finally:
            if not keep:
                container.destroy()
            else:
                print("==> Container kept at {}".format(container.rootfs))
                print("==> SSH available via:")
                print("==>   $ sudo lxc-attach --name={}".format(container.name))

def main():
    command = WrapperCommand()
    command.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import logging
import uuid
from datetime import datetime
import gevent
import six
from gevent.queue import Queue, Empty
from pyramid.view import view_config
from pyramid.httpexceptions import HTTPUnauthorized
from pyramid.security import forget
import channelstream
from channelstream.user import User
from channelstream.connection import Connection
from channelstream.channel import Channel
from ..ext_json import json

log = logging.getLogger(__name__)

def pass_message(msg, stats):
    if msg.get('timestamp'):
        # if present lets use timestamp provided in the message
        if '.' in msg['timestamp']:
            timestmp = datetime.strptime(msg['timestamp'],
                                         '%Y-%m-%dT%H:%M:%S.%f')
        else:
            timestmp = datetime.strptime(msg['timestamp'],
                                         '%Y-%m-%dT%H:%M:%S')
    else:
        timestmp = datetime.utcnow()
    message = {'uuid': str(uuid.uuid4()).replace('-', ''),
               'user': msg.get('user'),
               'message': msg['message'],
               'type': 'message',
               'timestamp': timestmp}
    pm_users = msg.get('pm_users', [])
    total_sent = 0
    stats['total_unique_messages'] += 1
    exclude_users = msg.get('exclude_users') or []
    if msg.get('channel'):
        channel_inst = channelstream.CHANNELS.get(msg['channel'])
        if channel_inst:
            total_sent += channel_inst.add_message(message,
                                                   pm_users=pm_users,
                                                   exclude_users=exclude_users)
    elif pm_users:
        # if pm then iterate over all users and notify about new message!
        for username in pm_users:
            user_inst = channelstream.USERS.get(username)
            if user_inst:
                total_sent += user_inst.add_message(message)
    stats['total_messages'] += total_sent

def get_connection_channels(connection):
    found_channels = []
    for channel in six.itervalues(channelstream.CHANNELS):
        user_conns = channel.connections.get(connection.username) or []
        if connection in user_conns:
            found_channels.append(channel.name)
    return found_channels

class ServerViews(object):
    def __init__(self, request):
        self.request = request

    def _get_channel_info(self, req_channels=None, include_history=True,
                          include_connections=False, include_users=False,
                          exclude_channels=None):
        """
        Gets channel information for req_channels or all channels
        if req_channels is not present
        :param: include_history (bool) will include message history
                for the channel
        :param: include_connections (bool) will include connection list
                for the channel
        :param: include_users (bool) will include user list for the channel
        :param: exclude_channels (bool) will exclude specific channels
                from info list (handy to exclude global broadcast)
        """

        if not exclude_channels:
            exclude_channels = []
        start_time = datetime.utcnow()

        json_data = {"channels": {},
                     "unique_users": len(channelstream.USERS),
                     "users": []}

        users_to_list = set()

        # select everything for empty list
        if not req_channels:
            channel_instances = six.itervalues(channelstream.CHANNELS)
        else:
            channel_instances = [channelstream.CHANNELS[c]
                                 for c in req_channels]

        for channel_inst in channel_instances:
            if channel_inst.name in exclude_channels:
                continue

            channel_info = channel_inst.get_info(
                include_history=include_history,
                include_connections=include_connections,
                include_users=include_users
            )
            json_data["channels"][channel_inst.name] = channel_info
            users_to_list.update([x['user'] for x in channel_info['users']])

        for username in users_to_list:
            json_data['users'].append(
                {'user': username,
                 'state': channelstream.USERS[username].state})
        log.info('info time: %s' % (datetime.utcnow() - start_time))
        return json_data

    def get_common_info(self, channels, info_config):
        include_history = info_config.get('include_history', True)
        include_users = info_config.get('include_users', True)
        exclude_channels = info_config.get('exclude_channels', [])
        include_connections = info_config.get('include_connections', False)
        channels_info = self._get_channel_info(
            channels, include_history=include_history,
            include_connections=include_connections,
            include_users=include_users, exclude_channels=exclude_channels)
        return channels_info

    @view_config(route_name='action', match_param='action=connect',
                 renderer='json', permission='access')
    def connect(self):
        """
        return the id of connected users - will be secured with password string
        for webapp to internally call the server - we combine conn string
        with user id, and we tell which channels the user is allowed to
        subscribe to
        """
        json_body = self.request.json_body
        username = json_body.get('username')
        fresh_user_state = json_body.get('fresh_user_state', {})
        update_user_state = json_body.get('user_state', {})
        channel_configs = json_body.get('channel_configs', {})
        state_public_keys = json_body.get('state_public_keys', None)
        conn_id = json_body.get('conn_id')
        subscribe_to_channels = json_body.get('channels')
        if username is None:
            self.request.response.status = 400
            return {'error': "No username specified"}
        if not subscribe_to_channels:
            self.request.response.status = 400
            return {'error': "No channels specified"}

        # everything is ok so lets add new connection to
        # channel and connection list
        with channelstream.lock:
            if username not in channelstream.USERS:
                user = User(username)
                user.state_from_dict(fresh_user_state)
                channelstream.USERS[username] = user
            else:
                user = channelstream.USERS[username]
            if state_public_keys is not None:
                user.state_public_keys = state_public_keys

            user.state_from_dict(update_user_state)
            connection = Connection(username, conn_id)
            if connection.id not in channelstream.CONNECTIONS:
                channelstream.CONNECTIONS[connection.id] = connection
            user.add_connection(connection)
            for channel_name in subscribe_to_channels:
                # user gets assigned to a channel
                if channel_name not in channelstream.CHANNELS:
                    channel = Channel(channel_name,
                                      channel_configs=channel_configs)
                    channelstream.CHANNELS[channel_name] = channel
                channelstream.CHANNELS[channel_name].add_connection(connection)
            log.info('connecting %s with uuid %s' % (username, connection.id))

        # get info config for channel information
        info_config = json_body.get('info') or {}
        channels_info = self.get_common_info(subscribe_to_channels,
                                             info_config)
        return {'conn_id': connection.id, 'state': user.state,
                'channels': subscribe_to_channels,
                'channels_info': channels_info}

    @view_config(route_name='action', match_param='action=subscribe',
                 renderer='json', permission='access')
    def subscribe(self, *args):
        """ call this to subscribe specific connection to new channels """
        json_body = self.request.json_body
        conn_id = json_body.get('conn_id', self.request.GET.get('conn_id'))
        connection = channelstream.CONNECTIONS.get(conn_id)
        subscribe_to_channels = json_body.get('channels')
        channel_configs = json_body.get('channel_configs', {})
        if not connection:
            self.request.response.status = 403
            return {'error': "Unknown connection"}
        if not subscribe_to_channels:
            self.request.response.status = 400
            return {'error': "No channels specified"}
        # everything is ok so lets add new connection to channel
        # and connection list
        # lets lock it just in case
        # find the right user
        user = channelstream.USERS.get(connection.username)
        with channelstream.lock:
            if user:
                for channel_name in subscribe_to_channels:
                    if channel_name not in channelstream.CHANNELS:
                        channel = Channel(channel_name,
                                          channel_configs=channel_configs)
                        channelstream.CHANNELS[channel_name] = channel
                    channelstream.CHANNELS[channel_name].add_connection(
                        connection)

        # get info config for channel information
        current_channels = get_connection_channels(connection)
        info_config = json_body.get('info') or {}
        channels_info = self.get_common_info(current_channels, info_config)
        return {"channels": current_channels,
                "channels_info": channels_info}

    @view_config(route_name='action', match_param='action=unsubscribe',
                 renderer='json', permission='access')
    def unsubscribe(self, *args):
        """ call this to unsubscribe specific connection from channels """
        json_body = self.request.json_body
        conn_id = json_body.get('conn_id', self.request.GET.get('conn_id'))
        connection = channelstream.CONNECTIONS.get(conn_id)
        unsubscribe_channels = json_body.get('channels')
        if not connection:
            self.request.response.status = 403
            return {'error': "Unknown connection"}
        if not unsubscribe_channels:
            self.request.response.status = 400
            return {'error': "No channels specified"}
        # everything is ok so lets add new connection to channel
        # and connection list
        # lets lock it just in case
        # find the right user
        user = channelstream.USERS.get(connection.username)
        with channelstream.lock:
            if user:
                for channel_name in unsubscribe_channels:
                    channelstream.CHANNELS[channel_name].remove_connection(
                        connection)

        # get info config for channel information
        current_channels = get_connection_channels(connection)
        info_config = json_body.get('info') or {}
        channels_info = self.get_common_info(current_channels, info_config)

        return {"channels": current_channels,
                "channels_info": channels_info}

    @view_config(route_name='action', match_param='action=listen',
                 request_method="OPTIONS", renderer='string')
    def handle_CORS(self):
        self.request.handle_cors()
        return ''

    @view_config(route_name='action', match_param='action=listen',
                 renderer='string')
    def listen(self):
        self.request.handle_cors()
        config = self.request.registry.settings
        self.conn_id = self.request.params.get('conn_id')
        connection = channelstream.CONNECTIONS.get(self.conn_id)
        if not connection:
            raise HTTPUnauthorized()
        # mark the conn active
        connection.last_active = datetime.utcnow()
        # attach a queue to connection
        connection.queue = Queue()

        def yield_response():
            # for chrome issues
            # yield ' ' * 1024
            # wait for this to wake up
            messages = []
            # block for first message - wake up after a while
            try:
                messages.extend(connection.queue.get(
                    timeout=config['wake_connections_after']))
            except Empty:
                pass
            # get more messages if enqueued takes up total 0.25s
            while True:
                try:
                    messages.extend(connection.queue.get(timeout=0.25))
                except Empty:
                    break
            cb = self.request.params.get('callback')
            if cb:
                yield cb + '(' + json.dumps(messages) + ')'
            else:
                yield json.dumps(messages)

        self.request.response.app_iter = yield_response()
        return self.request.response

    @view_config(route_name='action', match_param='action=user_state',
                 renderer='json', permission='access')
    def user_state(self):
        """ set the status of specific user """
        json_body = self.request.json_body
        username = json_body.get('user')
        user_state = json_body.get('user_state')
        state_public_keys = json_body.get('state_public_keys', None)
        if not username:
            self.request.response.status = 400
            return {'error': "No username specified"}

        user_inst = channelstream.USERS.get(username)
        if user_inst:
            user_inst.state_from_dict(user_state)
            if state_public_keys is not None:
                user_inst.state_public_keys = state_public_keys
            # mark active
            user_inst.last_active = datetime.utcnow()
        return user_inst.state

    @view_config(route_name='action', match_param='action=message',
                 renderer='json', permission='access')
    def message(self):
        msg_list = self.request.json_body
        for msg in msg_list:
            if not msg.get('channel') and not msg.get('pm_users', []):
                continue
            gevent.spawn(pass_message, msg, channelstream.stats)

    @view_config(route_name='action', match_param='action=disconnect',
                 renderer='json', permission='access')
    def disconnect(self):
        json_body = self.request.json_body
        conn_id = json_body.get('conn_id', self.request.GET.get('conn_id'))
        conn = channelstream.CONNECTIONS.get(conn_id)
        if conn is not None:
            conn.mark_for_gc()

    @view_config(route_name='action', match_param='action=channel_config',
                 renderer='json', permission='access')
    def channel_config(self):
        """ call this to reconfigure channels """
        json_body = self.request.json_body
        if not json_body:
            self.request.response.status = 400
            return {'error': "No channels specified"}

        with channelstream.lock:
            for channel_name, config in json_body.items():
                if not channelstream.CHANNELS.get(channel_name):
                    channel = Channel(channel_name,
                                      channel_configs=json_body)
                    channelstream.CHANNELS[channel_name] = channel
                else:
                    channel = channelstream.CHANNELS[channel_name]
                    channel.reconfigure_from_dict(json_body)
        channels_info = self._get_channel_info(json_body.keys(),
                                               include_history=False,
                                               include_users=False)
        return channels_info

    @view_config(
        context='channelstream.wsgi_views.wsgi_security:RequestBasicChallenge')
    def admin_challenge(self):
        response = HTTPUnauthorized()
        response.headers.update(forget(self.request))
        return response

    @view_config(route_name='admin',
                 renderer='templates/admin.jinja2', permission='access')
    @view_config(route_name='admin_json',
                 renderer='json', permission='access')
    def admin(self):
        uptime = str(datetime.utcnow() - channelstream.stats['started_on'])
        remembered_user_count = len(
            [user for user in six.iteritems(channelstream.USERS)])
        unique_user_count = len(
            [user for user in six.itervalues(channelstream.USERS)
             if user.connections])
        total_connections = sum([len(user.connections)
             for user in six.itervalues(channelstream.USERS)])
        total_sys_connections = len(channelstream.CONNECTIONS.values())
        return {
            "remembered_user_count": remembered_user_count,
            "unique_user_count": unique_user_count,
            "total_connections": total_connections,
            "total_sys_connections": total_sys_connections,
            "total_channels": len(channelstream.CHANNELS.keys()),
            "total_messages": channelstream.stats['total_messages'],
            "total_unique_messages": channelstream.stats[
                'total_unique_messages'],
            "channels": channelstream.CHANNELS,
            "users": channelstream.USERS, "uptime": uptime
        }

    @view_config(route_name='action', match_param='action=info',
                 renderer='json', permission='access')
    def info(self):
        if not self.request.body:
            req_channels = channelstream.CHANNELS.keys()
            info_config = {
                'include_history': True,
                'include_users': True,
                'exclude_channels': [],
                'include_connections': True
            }
        else:
            json_body = self.request.json_body
            # get info config for channel information
            info_config = json_body.get('info') or {}
            req_channels = info_config.get('channels', None)
            info_config['include_connections'] = info_config.get(
                'include_connections', True)
        channels_info = self.get_common_info(req_channels, info_config)
        return channels_info
<commit_msg>wsgi app: fix counts on info dictionary<commit_after>import logging
import uuid
from datetime import datetime
import gevent
import six
from gevent.queue import Queue, Empty
from pyramid.view import view_config
from pyramid.httpexceptions import HTTPUnauthorized
from pyramid.security import forget
import channelstream
from channelstream.user import User
from channelstream.connection import Connection
from channelstream.channel import Channel
from ..ext_json import json

log = logging.getLogger(__name__)

def pass_message(msg, stats):
    if msg.get('timestamp'):
        # if present lets use timestamp provided in the message
        if '.' in msg['timestamp']:
            timestmp = datetime.strptime(msg['timestamp'],
                                         '%Y-%m-%dT%H:%M:%S.%f')
        else:
            timestmp = datetime.strptime(msg['timestamp'],
                                         '%Y-%m-%dT%H:%M:%S')
    else:
        timestmp = datetime.utcnow()
    message = {'uuid': str(uuid.uuid4()).replace('-', ''),
               'user': msg.get('user'),
               'message': msg['message'],
               'type': 'message',
               'timestamp': timestmp}
    pm_users = msg.get('pm_users', [])
    total_sent = 0
    stats['total_unique_messages'] += 1
    exclude_users = msg.get('exclude_users') or []
    if msg.get('channel'):
        channel_inst = channelstream.CHANNELS.get(msg['channel'])
        if channel_inst:
            total_sent += channel_inst.add_message(message,
                                                   pm_users=pm_users,
                                                   exclude_users=exclude_users)
    elif pm_users:
        # if pm then iterate over all users and notify about new message!
        for username in pm_users:
            user_inst = channelstream.USERS.get(username)
            if user_inst:
                total_sent += user_inst.add_message(message)
    stats['total_messages'] += total_sent

def get_connection_channels(connection):
    found_channels = []
    for channel in six.itervalues(channelstream.CHANNELS):
        user_conns = channel.connections.get(connection.username) or []
        if connection in user_conns:
            found_channels.append(channel.name)
    return found_channels

class ServerViews(object):
    def __init__(self, request):
        self.request = request

    def _get_channel_info(self, req_channels=None, include_history=True,
                          include_connections=False, include_users=False,
                          exclude_channels=None):
        """
        Gets channel information for req_channels or all channels
        if req_channels is not present
        :param: include_history (bool) will include message history
                for the channel
        :param: include_connections (bool) will include connection list
                for the channel
        :param: include_users (bool) will include user list for the channel
        :param: exclude_channels (bool) will exclude specific channels
                from info list (handy to exclude global broadcast)
        """

        if not exclude_channels:
            exclude_channels = []
        start_time = datetime.utcnow()

        remembered_user_count = len(
            [user for user in six.iteritems(channelstream.USERS)])
        unique_user_count = len(
            [user for user in six.itervalues(channelstream.USERS)
             if user.connections])

        json_data = {"channels": {},
                     "unique_users": unique_user_count,
                     "remembered_user_count": remembered_user_count,
                     "users": []}

        users_to_list = set()

        # select everything for empty list
        if not req_channels:
            channel_instances = six.itervalues(channelstream.CHANNELS)
        else:
            channel_instances = [channelstream.CHANNELS[c]
                                 for c in req_channels]

        for channel_inst in channel_instances:
            if channel_inst.name in exclude_channels:
                continue

            channel_info = channel_inst.get_info(
                include_history=include_history,
                include_connections=include_connections,
                include_users=include_users
            )
            json_data["channels"][channel_inst.name] = channel_info
            users_to_list.update([x['user'] for x in channel_info['users']])

        for username in users_to_list:
            json_data['users'].append(
                {'user': username,
                 'state': channelstream.USERS[username].state})
        log.info('info time: %s' % (datetime.utcnow() - start_time))
        return json_data

    def get_common_info(self, channels, info_config):
        include_history = info_config.get('include_history', True)
        include_users = info_config.get('include_users', True)
        exclude_channels = info_config.get('exclude_channels', [])
        include_connections = info_config.get('include_connections', False)
        channels_info = self._get_channel_info(
            channels, include_history=include_history,
            include_connections=include_connections,
            include_users=include_users, exclude_channels=exclude_channels)
        return channels_info

    @view_config(route_name='action', match_param='action=connect',
                 renderer='json', permission='access')
    def connect(self):
        """
        return the id of connected users - will be secured with password string
        for webapp to internally call the server - we combine conn string
        with user id, and we tell which channels the user is allowed to
        subscribe to
        """
        json_body = self.request.json_body
        username = json_body.get('username')
        fresh_user_state = json_body.get('fresh_user_state', {})
        update_user_state = json_body.get('user_state', {})
        channel_configs = json_body.get('channel_configs', {})
        state_public_keys = json_body.get('state_public_keys', None)
        conn_id = json_body.get('conn_id')
        subscribe_to_channels = json_body.get('channels')
        if username is None:
            self.request.response.status = 400
            return {'error': "No username specified"}
        if not subscribe_to_channels:
            self.request.response.status = 400
            return {'error': "No channels specified"}

        # everything is ok so lets add new connection to
        # channel and connection list
        with channelstream.lock:
            if username not in channelstream.USERS:
                user = User(username)
                user.state_from_dict(fresh_user_state)
                channelstream.USERS[username] = user
            else:
                user = channelstream.USERS[username]
            if state_public_keys is not None:
                user.state_public_keys = state_public_keys

            user.state_from_dict(update_user_state)
            connection = Connection(username, conn_id)
            if connection.id not in channelstream.CONNECTIONS:
                channelstream.CONNECTIONS[connection.id] = connection
            user.add_connection(connection)
            for channel_name in subscribe_to_channels:
                # user gets assigned to a channel
                if channel_name not in channelstream.CHANNELS:
                    channel = Channel(channel_name,
                                      channel_configs=channel_configs)
                    channelstream.CHANNELS[channel_name] = channel
                channelstream.CHANNELS[channel_name].add_connection(connection)
            log.info('connecting %s with uuid %s' % (username, connection.id))

        # get info config for channel information
        info_config = json_body.get('info') or {}
        channels_info = self.get_common_info(subscribe_to_channels,
                                             info_config)
        return {'conn_id': connection.id, 'state': user.state,
                'channels': subscribe_to_channels,
                'channels_info': channels_info}

    @view_config(route_name='action', match_param='action=subscribe',
                 renderer='json', permission='access')
    def subscribe(self, *args):
        """ call this to subscribe specific connection to new channels """
        json_body = self.request.json_body
        conn_id = json_body.get('conn_id', self.request.GET.get('conn_id'))
        connection = channelstream.CONNECTIONS.get(conn_id)
        subscribe_to_channels = json_body.get('channels')
        channel_configs = json_body.get('channel_configs', {})
        if not connection:
            self.request.response.status = 403
            return {'error': "Unknown connection"}
        if not subscribe_to_channels:
            self.request.response.status = 400
            return {'error': "No channels specified"}
        # everything is ok so lets add new connection to channel
        # and connection list
        # lets lock it just in case
        # find the right user
        user = channelstream.USERS.get(connection.username)
        with channelstream.lock:
            if user:
                for channel_name in subscribe_to_channels:
                    if channel_name not in channelstream.CHANNELS:
                        channel = Channel(channel_name,
                                          channel_configs=channel_configs)
                        channelstream.CHANNELS[channel_name] = channel
                    channelstream.CHANNELS[channel_name].add_connection(
                        connection)

        # get info config for channel information
        current_channels = get_connection_channels(connection)
        info_config = json_body.get('info') or {}
        channels_info = self.get_common_info(current_channels, info_config)
        return {"channels": current_channels,
                "channels_info": channels_info}

    @view_config(route_name='action', match_param='action=unsubscribe',
                 renderer='json', permission='access')
    def unsubscribe(self, *args):
        """ call this to unsubscribe specific connection from channels """
        json_body = self.request.json_body
        conn_id = json_body.get('conn_id', self.request.GET.get('conn_id'))
        connection = channelstream.CONNECTIONS.get(conn_id)
        unsubscribe_channels = json_body.get('channels')
        if not connection:
            self.request.response.status = 403
            return {'error': "Unknown connection"}
        if not unsubscribe_channels:
            self.request.response.status = 400
            return {'error': "No channels specified"}
        # everything is ok so lets add new connection to channel
        # and connection list
        # lets lock it just in case
        # find the right user
        user = channelstream.USERS.get(connection.username)
        with channelstream.lock:
            if user:
                for channel_name in unsubscribe_channels:
                    channelstream.CHANNELS[channel_name].remove_connection(
                        connection)

        # get info config for channel information
        current_channels = get_connection_channels(connection)
        info_config = json_body.get('info') or {}
        channels_info = self.get_common_info(current_channels, info_config)

        return {"channels": current_channels,
                "channels_info": channels_info}

    @view_config(route_name='action', match_param='action=listen',
                 request_method="OPTIONS", renderer='string')
    def handle_CORS(self):
        self.request.handle_cors()
        return ''

    @view_config(route_name='action', match_param='action=listen',
                 renderer='string')
    def listen(self):
        self.request.handle_cors()
        config = self.request.registry.settings
        self.conn_id = self.request.params.get('conn_id')
        connection = channelstream.CONNECTIONS.get(self.conn_id)
        if not connection:
            raise HTTPUnauthorized()
        # mark the conn active
        connection.last_active = datetime.utcnow()
        # attach a queue to connection
        connection.queue = Queue()

        def yield_response():
            # for chrome issues
            # yield ' ' * 1024
            # wait for this to wake up
            messages = []
            # block for first message - wake up after a while
            try:
                messages.extend(connection.queue.get(
                    timeout=config['wake_connections_after']))
            except Empty:
                pass
            # get more messages if enqueued takes up total 0.25s
            while True:
                try:
                    messages.extend(connection.queue.get(timeout=0.25))
                except Empty:
                    break
            cb = self.request.params.get('callback')
            if cb:
                yield cb + '(' + json.dumps(messages) + ')'
            else:
                yield json.dumps(messages)

        self.request.response.app_iter = yield_response()
        return self.request.response

    @view_config(route_name='action', match_param='action=user_state',
                 renderer='json', permission='access')
    def user_state(self):
        """ set the status of specific user """
        json_body = self.request.json_body
        username = json_body.get('user')
        user_state = json_body.get('user_state')
        state_public_keys = json_body.get('state_public_keys', None)
        if not username:
            self.request.response.status = 400
            return {'error': "No username specified"}

        user_inst = channelstream.USERS.get(username)
        if user_inst:
            user_inst.state_from_dict(user_state)
            if state_public_keys is not None:
                user_inst.state_public_keys = state_public_keys
            # mark active
            user_inst.last_active = datetime.utcnow()
        return user_inst.state

    @view_config(route_name='action', match_param='action=message',
                 renderer='json', permission='access')
    def message(self):
        msg_list = self.request.json_body
        for msg in msg_list:
            if not msg.get('channel') and not msg.get('pm_users', []):
                continue
            gevent.spawn(pass_message, msg, channelstream.stats)

    @view_config(route_name='action', match_param='action=disconnect',
                 renderer='json', permission='access')
    def disconnect(self):
        json_body = self.request.json_body
        conn_id = json_body.get('conn_id', self.request.GET.get('conn_id'))
        conn = channelstream.CONNECTIONS.get(conn_id)
        if conn is not None:
            conn.mark_for_gc()

    @view_config(route_name='action', match_param='action=channel_config',
                 renderer='json', permission='access')
    def channel_config(self):
        """ call this to reconfigure channels """
        json_body = self.request.json_body
        if not json_body:
            self.request.response.status = 400
            return {'error': "No channels specified"}

        with channelstream.lock:
            for channel_name, config in json_body.items():
                if not channelstream.CHANNELS.get(channel_name):
                    channel = Channel(channel_name,
                                      channel_configs=json_body)
                    channelstream.CHANNELS[channel_name] = channel
                else:
                    channel = channelstream.CHANNELS[channel_name]
                    channel.reconfigure_from_dict(json_body)
        channels_info = self._get_channel_info(json_body.keys(),
                                               include_history=False,
                                               include_users=False)
        return channels_info

    @view_config(
        context='channelstream.wsgi_views.wsgi_security:RequestBasicChallenge')
    def admin_challenge(self):
        response = HTTPUnauthorized()
        response.headers.update(forget(self.request))
        return response

    @view_config(route_name='admin',
                 renderer='templates/admin.jinja2', permission='access')
    @view_config(route_name='admin_json',
                 renderer='json', permission='access')
    def admin(self):
        uptime = str(datetime.utcnow() - channelstream.stats['started_on'])
        remembered_user_count = len(
            [user for user in six.iteritems(channelstream.USERS)])
        unique_user_count = len(
            [user for user in six.itervalues(channelstream.USERS)
             if user.connections])
        total_connections = sum([len(user.connections)
             for user in six.itervalues(channelstream.USERS)])
        total_sys_connections = len(channelstream.CONNECTIONS.values())
        return {
            "remembered_user_count": remembered_user_count,
            "unique_user_count": unique_user_count,
            "total_connections": total_connections,
            "total_sys_connections": total_sys_connections,
            "total_channels": len(channelstream.CHANNELS.keys()),
            "total_messages": channelstream.stats['total_messages'],
            "total_unique_messages": channelstream.stats[
                'total_unique_messages'],
            "channels": channelstream.CHANNELS,
            "users": channelstream.USERS, "uptime": uptime
        }

    @view_config(route_name='action', match_param='action=info',
                 renderer='json', permission='access')
    def info(self):
        if not self.request.body:
            req_channels = channelstream.CHANNELS.keys()
            info_config = {
                'include_history': True,
                'include_users': True,
                'exclude_channels': [],
                'include_connections': True
            }
        else:
            json_body = self.request.json_body
            # get info config for channel information
            info_config = json_body.get('info') or {}
            req_channels = info_config.get('channels', None)
            info_config['include_connections'] = info_config.get(
                'include_connections', True)
        channels_info = self.get_common_info(req_channels, info_config)
        return channels_info
<|endoftext|>


<commit_before>import logging
import datetime

from sqlalchemy import Table
from sqlalchemy import Column
from sqlalchemy import types
from sqlalchemy import Index
from sqlalchemy.engine.reflection import Inspector

from ckan import model
from ckan.model.meta import metadata, mapper, Session
from ckan.model.types import make_uuid
from ckan.model.domain_object import DomainObject

log = logging.getLogger(__name__)

__all__ = [
    'OntologyObject', 'ontology_table',
    'NodeObject', 'node_table',
    'DatasetOntologyRelation', 'dataset_ontology_relation_table'
]
#Ontology = 'ontology_table'
ontology_table = None
node_table = None
dataset_ontology_relation_table = None

def setup():
    if ontology_table is None:
        define_ontology_tables()
        log.debug('Ontology tables defined in memory')

    if not model.package_table.exists():
        log.debug('Ontology table creation deferred')
        return

    if not ontology_table.exists():

        # Create each table individually rather than
        # using metadata.create_all()
        ontology_table.create()
        node_table.create()
        dataset_ontology_relation_table.create()
        log.debug('Ontology tables created')
    else:
        log.debug('Ontology tables already exist')

class OntologyError(Exception):
    pass

class OntologyDomainObject(DomainObject):
    """
        Convenience methods for searching objects
    """
    key_attr = 'id'

    @classmethod
    def get(cls, key, default=None, attr=None):
        """
            Finds a single entity in the register.
        """

        if attr == None:
            attr = cls.key_attr
        kwds = {attr: key}
        o = cls.filter(**kwds).first()
        if o:
            return o
        else:
            return default

    @classmethod
    def filter(cls, **kwds):
        query = Session.query(cls).autoflush(False)
        return query.filter_by(**kwds)

    @classmethod
    def get_all(cls, key=None, attr=None):
        query = Session.query(cls).autoflush(False)
        return query.all()

class OntologyObject(OntologyDomainObject):
    """
        A Harvest Object is created every time an element is fetched from a
        harvest source. Its contents can be processed and imported to ckan
        packages, RDF graphs, etc.
    """

    def as_dict(self):
        return {'id': self.id, 'name': self.name, 'description': self.description,
                'ontology':self.ontology, 'json_ontology':self.json_ontology, 'active': self.active}

    def as_simple_dict(self):
        return {'id': self.id, 'name': self.name, 'description': self.description, 'active': self.active}

    def __repr__(self):
        return '<OntologyObject id=%s name=%s description=%s json_ontology=%s active=%r>' % \
               (self.id, self.name, self.description, self.json_ontology, self.active)

    def __str__(self):
        return self.__repr__().encode('ascii', 'ignore')

class NodeObject(OntologyDomainObject):
    def __repr__(self):
        return '<DatasetOntologyRelation id=%s URI=%s ontology_id=%s name=%r>' % \
               (self.id, self.URI, self.ontology_id, self.name)

    def __str__(self):
        return self.__repr__().encode('ascii', 'ignore')

class DatasetOntologyRelation(OntologyDomainObject):
    '''Keep the relations between a dataset and an ontology.'''

    def __repr__(self):
        return '<DatasetOntologyRelation id=%s dataset_id=%s ontology_id=%s node_id=%r>' % \
               (self.id, self.dataset_id, self.ontology_id, self.node_id)

    def __str__(self):
        return self.__repr__().encode('ascii', 'ignore')

def define_ontology_tables():
    global ontology_table
    global node_table
    global dataset_ontology_relation_table

    ontology_table = Table('ontology', metadata,
        Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
        Column('name', types.UnicodeText, default=u''),
        Column('description', types.UnicodeText, default=u''),
        Column('ontology', types.UnicodeText, default=u''),
        Column('json_ontology', types.UnicodeText, default=u''),
        Column('created', types.DateTime, default=datetime.datetime.utcnow),
        Column('type', types.UnicodeText, nullable=False),
        Column('active', types.Boolean, default=True),
        Column('user_id', types.UnicodeText, default=u'')
    )

    dataset_ontology_relation_table = Table('dataset_ontology_relation', metadata,
        Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
        Column('dataset_id', types.UnicodeText, primary_key=True, default=u''),
        Column('ontology_id', types.UnicodeText, primary_key=True, default=u''),
        Column('node_id', types.UnicodeText, primary_key=True, default=u'')
    )

    node_table = Table('node', metadata,
       Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
       Column('URI', types.UnicodeText, primary_key=True, default=u''),
       Column('name', types.UnicodeText, primary_key=True, default=u''),
       Column('ontology_id', types.UnicodeText, primary_key=True, default=u'')
    )

    mapper(
        OntologyObject,
        ontology_table
    )

    mapper(
        NodeObject,
        node_table
    )

    mapper(
        DatasetOntologyRelation,
        dataset_ontology_relation_table,
    )
<commit_msg>hotfix<commit_after>import logging
import datetime

from sqlalchemy import Table
from sqlalchemy import Column
from sqlalchemy import types
from sqlalchemy import Index
from sqlalchemy.engine.reflection import Inspector

from ckan import model
from ckan.model.meta import metadata, mapper, Session
from ckan.model.types import make_uuid
from ckan.model.domain_object import DomainObject

log = logging.getLogger(__name__)

__all__ = [
    'OntologyObject', 'ontology_table',
    'NodeObject', 'node_table',
    'DatasetOntologyRelation', 'dataset_ontology_relation_table'
]
#Ontology = 'ontology_table'
ontology_table = None
node_table = None
dataset_ontology_relation_table = None

def setup():
    if ontology_table is None:
        define_ontology_tables()
        log.debug('Ontology tables defined in memory')

    if not model.package_table.exists():
        log.debug('Ontology table creation deferred')
        return

    if not ontology_table.exists():

        # Create each table individually rather than
        # using metadata.create_all()
        ontology_table.create()
        node_table.create()
        dataset_ontology_relation_table.create()
        log.debug('Ontology tables created')
    else:
        log.debug('Ontology tables already exist')

class OntologyError(Exception):
    pass

class OntologyDomainObject(DomainObject):
    """
        Convenience methods for searching objects
    """
    key_attr = 'id'

    @classmethod
    def get(cls, key, default=None, attr=None):
        """
            Finds a single entity in the register.
        """

        if attr == None:
            attr = cls.key_attr
        kwds = {attr: key}
        o = cls.filter(**kwds).first()
        if o:
            return o
        else:
            return default

    @classmethod
    def filter(cls, **kwds):
        query = Session.query(cls).autoflush(False)
        return query.filter_by(**kwds)

    @classmethod
    def get_all(cls, key=None, attr=None):
        query = Session.query(cls).autoflush(False)
        return query.all()

class OntologyObject(OntologyDomainObject):
    """
        A Harvest Object is created every time an element is fetched from a
        harvest source. Its contents can be processed and imported to ckan
        packages, RDF graphs, etc.
    """

    def as_dict(self):
        return {'id': self.id, 'name': self.name, 'description': self.description,
                'json_ontology':self.json_ontology, 'active': self.active}

    def as_simple_dict(self):
        return {'id': self.id, 'name': self.name, 'description': self.description, 'active': self.active}

    def __repr__(self):
        return '<OntologyObject id=%s name=%s description=%s json_ontology=%s active=%r>' % \
               (self.id, self.name, self.description, self.json_ontology, self.active)

    def __str__(self):
        return self.__repr__().encode('ascii', 'ignore')

class NodeObject(OntologyDomainObject):
    def __repr__(self):
        return '<DatasetOntologyRelation id=%s URI=%s ontology_id=%s name=%r>' % \
               (self.id, self.URI, self.ontology_id, self.name)

    def __str__(self):
        return self.__repr__().encode('ascii', 'ignore')

class DatasetOntologyRelation(OntologyDomainObject):
    '''Keep the relations between a dataset and an ontology.'''

    def __repr__(self):
        return '<DatasetOntologyRelation id=%s dataset_id=%s ontology_id=%s node_id=%r>' % \
               (self.id, self.dataset_id, self.ontology_id, self.node_id)

    def __str__(self):
        return self.__repr__().encode('ascii', 'ignore')

def define_ontology_tables():
    global ontology_table
    global node_table
    global dataset_ontology_relation_table

    ontology_table = Table('ontology', metadata,
        Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
        Column('name', types.UnicodeText, default=u''),
        Column('description', types.UnicodeText, default=u''),
        Column('json_ontology', types.JSON, default=u''),
        Column('created', types.DateTime, default=datetime.datetime.utcnow),
        Column('type', types.UnicodeText, nullable=False),
        Column('active', types.Boolean, default=True),
        Column('user_id', types.UnicodeText, default=u'')
    )

    dataset_ontology_relation_table = Table('dataset_ontology_relation', metadata,
        Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
        Column('dataset_id', types.UnicodeText, primary_key=True, default=u''),
        Column('ontology_id', types.UnicodeText, primary_key=True, default=u''),
        Column('node_id', types.UnicodeText, primary_key=True, default=u'')
    )

    node_table = Table('node', metadata,
       Column('id', types.UnicodeText, primary_key=True, default=make_uuid),
       Column('URI', types.UnicodeText, primary_key=True, default=u''),
       Column('name', types.UnicodeText, primary_key=True, default=u''),
       Column('ontology_id', types.UnicodeText, primary_key=True, default=u'')
    )

    mapper(
        OntologyObject,
        ontology_table
    )

    mapper(
        NodeObject,
        node_table
    )

    mapper(
        DatasetOntologyRelation,
        dataset_ontology_relation_table,
    )
<|endoftext|>


<commit_before>import ckan.plugins.toolkit as toolkit
from xmltodict import unparse
from pylons import config
import xml.dom.minidom as minidom
import collections

import json

@toolkit.side_effect_free
def package_export(context, data_dict):
    '''Return the given CKAN converted to a format.

    :param id: the ID of the dataset
    :type id: string
    :format id: string

    :param format: the format name
    :type format: string
    :format format: string

    :returns: the package metadata
    :rtype: string
    '''
    try:
        package_id = data_dict['id']
    except KeyError:
        raise toolkit.ValidationError({'id': 'missing id'})

    dataset_dict = toolkit.get_action('package_show')(context,
                                                      {'id': package_id})
    file_format = data_dict.get("format", "")

    converted_package = "No converter available for format " + file_format

    if (file_format.lower()=='datacite'):
         converted_package = _datacite_converter(dataset_dict)

    return converted_package

def _datacite_converter(dataset_dict):

    datacite_dict = collections.OrderedDict()
    datacite_dict['resource']=collections.OrderedDict()
    datacite_dict['resource']["@xsi:schemaLocation"] = "http://datacite.org/schema/kernel-3 http://schema.datacite.org/meta/kernel-3/metadata.xsd"
    datacite_dict['resource']["@xmlns"]="http://datacite.org/schema/kernel-3"
    datacite_dict['resource']["@xmlns:xsi"]="http://www.w3.org/2001/XMLSchema-instance"

    # Identifier
    datacite_dict['resource']['identifier'] = {"#text":dataset_dict.get('doi', ''), '@identifierType':"DOI"}

    # Creators
    try:
        pkg_authors = json.loads(dataset_dict.get("author", "[]"))
    except:
        pkg_authors = []

    dc_creators_list = []
    for author in pkg_authors:
        dc_creator ={}
        if author.get('name', False):
            dc_creator['creatorName'] = author.get('name', '')
        if author.get('affiliation', False):
            dc_creator['affiliation'] = author.get('affiliation', '')
        if author.get('identifier', False):
            dc_creator['nameIdentifier'] = {"#text":author.get('identifier', '')}
        if author.get('identifier_scheme', False):
            dc_creator['nameIdentifier']['@nameIdentifierScheme'] = author.get('identifier_scheme', '').upper()
        if dc_creator:
            dc_creators_list += [ dc_creator ]
    if dc_creators_list:
        datacite_dict['resource']['creators'] = {'creator': dc_creators_list }

    # Titles
    dc_titles = []
    if dataset_dict.get("title", ""):
        dc_titles += [ {"#text": dataset_dict.get("title", "")} ]

    title_type_dict = { "alternative_title":"AlternativeTitle", "subtitle":"Subtitle", "translated_title":"TranslatedTitle", "other":"Other" }
    try:
        pkg_subtitles = json.loads(dataset_dict.get("subtitle", "[]"))
    except:
        pkg_subtitles = []
    for subtitle in pkg_subtitles:
        dc_title = {}
        if subtitle.get('subtitle', False):
            dc_title['#text'] = subtitle.get('subtitle', '')
        if subtitle.get('type', False):
            dc_title['@titleType'] = title_type_dict.get(subtitle.get('type', ''), "Other")
        if subtitle.get('language', False):
            dc_title['@xml:lang'] = subtitle.get('language', '')
        if dc_title:
             dc_titles += [ dc_title ]
    if dc_titles:
        datacite_dict['resource']['titles'] = {'title': dc_titles }

    # Publisher & publication year
    try:
        pkg_publication = json.loads(dataset_dict.get("publication", "{}"))
    except:
        pkg_publication = {}
    if  pkg_publication.get("publisher",""):
        datacite_dict['resource']['publisher'] = {"#text": pkg_publication.get("publisher","")}
    if  pkg_publication.get("publication_year",""):
        datacite_dict['resource']['publicationYear'] = {"#text": pkg_publication.get("publication_year","")}

    # Subjects
    dc_subjects = []
    for tag in dataset_dict.get("tags", []):
        tag_name = tag.get("display_name", tag.get("name",""))
        dc_subjects += [{ "@xml:lang":"en-us", "#text":tag_name}]
    if dc_subjects:
        datacite_dict['resource']['subjects'] = {'subject':dc_subjects}

    # Contributor (contact person)
    try:
        pkg_contributor = json.loads(dataset_dict.get("maintainer", "{}"))
    except:
        pkg_contributor = {}
    if pkg_contributor:
        dc_contributor ={}
        if pkg_contributor.get('name', False):
            dc_contributor['contributorName'] = author.get('name', '')
        if pkg_contributor.get('affiliation', False):
            dc_contributor['affiliation'] = author.get('affiliation', '')
        if pkg_contributor.get('identifier', False):
            dc_contributor['nameIdentifier'] = {"#text":author.get('identifier', '')}
        if pkg_contributor.get('identifier_scheme', False):
            dc_contributor['nameIdentifier']['@nameIdentifierScheme'] = author.get('identifier_scheme', '').upper()
        if dc_contributor:
            dc_contributor['contributorType'] = 'ContactPerson'
            datacite_dict['resource']['contributors'] = {'contributor': [dc_contributor] }

    # Dates (TODO)
    try:
        pkg_dates = json.loads(dataset_dict.get("date", "[]"))
    except:
        pkg_dates = []
    dates_list = []
    for date in  pkg_dates:
        dc_date = {}
        if date.get('date', ""):
            dc_date['#text'] = date.get('date', "")
            if date.get('end_date', ""):
                dc_date['#text'] += '/' + date.get('end_date', "")
        if date.get('date_type', ""):
            dc_date['@dateType'] = date.get('date_type', "").title()
        if dc_date:
            dates_list += [dc_date]
    if dates_list:
        datacite_dict['resource']['dates']={'date':dates_list}

    # Language
    datacite_dict['resource']['language'] = dataset_dict.get("language", "en")

    # ResourceType
    resource_type_general_dict ={"audiovisual": "Audiovisual", "collection": "Collection", "dataset": "Dataset", "event": "Event", "image": "Image",
                                 "interactive_resource": "InteractiveResource", "model": "Model", "physical_object": "PhysicalObject",
                                 "service": "Service", "software": "Software", "sound": "Sound", "text": "Text", "workflow": "Workflow", "other": "Other"}
    if dataset_dict.get("resource_type", ""):
        resource_type_general = resource_type_general_dict.get(dataset_dict.get("resource_type_general", "other"), "Other")
        datacite_dict['resource']['resourceType'] = {"#text": dataset_dict.get("resource_type",""), '@resourceTypeGeneral':resource_type_general}

    # Alternate Identifier (CKAN URL)
    ckan_package_url = config.get('ckan.site_url',"") + toolkit.url_for(controller='package', action='read', id=dataset_dict.get('name', ' '))
    datacite_dict['resource']['alternateIdentifiers']={'AlternateIdentifier':[{"#text":ckan_package_url, '@alternateIdentifierType':"URL"}]}
    # legacy
    if dataset_dict.get('url', ''):
        datacite_dict['resource']['alternateIdentifiers']['AlternateIdentifier'] += [{"#text": dataset_dict.get('url', ''), '@alternateIdentifierType':"URL"}]

    # Related identifiers (TODO)

    # Sizes
    datacite_sizes = []
    for resource in dataset_dict.get("resources", []):
        if resource.get("size", ""):
            datacite_sizes += [{"#text": resource.get("size", " ") + " bytes"}]
    if datacite_sizes:
         datacite_dict['resource']['sizes'] = {'size': datacite_sizes}

    # Formats
    datacite_formats = []
    for resource in dataset_dict.get("resources", []):
        if resource.get("format", resource.get("mimetype", resource.get("mimetype_inner", ""))):
            format = {"#text": resource.get("format", "")}
            if format not in datacite_formats:
                datacite_formats += [format]
    if datacite_formats:
         datacite_dict['resource']['formats'] = {'format': datacite_formats}

    # Version
    if  dataset_dict.get("version", ""):
        datacite_dict['resource']['version'] = dataset_dict.get("version", "")

    # Rights
    if dataset_dict.get("license_title", ""):
        datacite_dict['resource']['rightsList'] = {"rights":[{"#text": dataset_dict.get("license_title", "")}]}

    # Description
    if dataset_dict.get("notes", ""):
        datacite_dict['resource']['descriptions'] = {'description': {"#text":dataset_dict.get("notes", " "), "@xml:lang":"en-us", "@descriptionType":"Abstract"}}

    # GeoLocation
    datacite_geolocation = {}
    if  dataset_dict.get("spatial_info", ""):
       datacite_geolocation["geoLocationPlace"] = dataset_dict.get("spatial_info", "")

    try:
        pkg_spatial = json.loads(dataset_dict.get("spatial", "{}"))
    except:
        pkg_spatial = {}
    if pkg_spatial:
       coordinates_list = _flatten_list( pkg_spatial.get("coordinates", "[]"))
       coordinates = " ".join(coordinates_list)
       if pkg_spatial.get("type", "").lower() == "polygon" :
            datacite_geolocation["geoLocationBox"] = coordinates
       else:
            datacite_geolocation["geoLocationPoint"] = coordinates

    if datacite_geolocation:
       datacite_dict['resource']['geoLocations'] = {"geoLocation": [datacite_geolocation]}

    # Converto to xml
    converted_package = unparse(datacite_dict)

    return converted_package

def _flatten_list(input_list):
    output_list = []
    for item in input_list:
        if type(item) is not list:
            output_list += [str(item)]
        else:
            output_list += _flatten_list(item)
    return output_list

<commit_msg>fixed issues with datacite xml<commit_after>import ckan.plugins.toolkit as toolkit
from xmltodict import unparse
from pylons import config
import xml.dom.minidom as minidom
import collections

import json

@toolkit.side_effect_free
def package_export(context, data_dict):
    '''Return the given CKAN converted to a format.

    :param id: the ID of the dataset
    :type id: string
    :format id: string

    :param format: the format name
    :type format: string
    :format format: string

    :returns: the package metadata
    :rtype: string
    '''
    try:
        package_id = data_dict['id']
    except KeyError:
        raise toolkit.ValidationError({'id': 'missing id'})

    dataset_dict = toolkit.get_action('package_show')(context,
                                                      {'id': package_id})
    file_format = data_dict.get("format", "")

    converted_package = "No converter available for format " + file_format

    if (file_format.lower()=='datacite'):
         converted_package = _datacite_converter(dataset_dict)

    return converted_package

def _datacite_converter(dataset_dict):

    datacite_dict = collections.OrderedDict()
    datacite_dict['resource']=collections.OrderedDict()
    datacite_dict['resource']["@xsi:schemaLocation"] = "http://datacite.org/schema/kernel-3 http://schema.datacite.org/meta/kernel-3/metadata.xsd"
    datacite_dict['resource']["@xmlns"]="http://datacite.org/schema/kernel-3"
    datacite_dict['resource']["@xmlns:xsi"]="http://www.w3.org/2001/XMLSchema-instance"

    # Identifier
    datacite_dict['resource']['identifier'] = {"#text":dataset_dict.get('doi', ''), '@identifierType':"DOI"}

    # Creators
    try:
        pkg_authors = json.loads(dataset_dict.get("author", "[]"))
    except:
        pkg_authors = []

    dc_creators_list = []
    for author in pkg_authors:
        dc_creator = collections.OrderedDict()
        if author.get('name', False):
            dc_creator['creatorName'] = author.get('name', '')
        if author.get('affiliation', False):
            dc_creator['affiliation'] = author.get('affiliation', '')
        if author.get('identifier', False):
            dc_creator['nameIdentifier'] = {"#text":author.get('identifier', '')}
        if author.get('identifier_scheme', False):
            dc_creator['nameIdentifier']['@nameIdentifierScheme'] = author.get('identifier_scheme', '').upper()
        if dc_creator:
            dc_creators_list += [ dc_creator ]
    if dc_creators_list:
        datacite_dict['resource']['creators'] = {'creator': dc_creators_list }

    # Titles
    dc_titles = []
    if dataset_dict.get("title", ""):
        dc_titles += [ {"#text": dataset_dict.get("title", "")} ]

    title_type_dict = { "alternative_title":"AlternativeTitle", "subtitle":"Subtitle", "translated_title":"TranslatedTitle", "other":"Other" }
    try:
        pkg_subtitles = json.loads(dataset_dict.get("subtitle", "[]"))
    except:
        pkg_subtitles = []
    for subtitle in pkg_subtitles:
        dc_title = collections.OrderedDict()
        if subtitle.get('subtitle', False):
            dc_title['#text'] = subtitle.get('subtitle', '')
        if subtitle.get('type', False):
            dc_title['@titleType'] = title_type_dict.get(subtitle.get('type', ''), "Other")
        if subtitle.get('language', False):
            dc_title['@xml:lang'] = subtitle.get('language', '')
        if dc_title:
             dc_titles += [ dc_title ]
    if dc_titles:
        datacite_dict['resource']['titles'] = {'title': dc_titles }

    # Publisher & publication year
    try:
        pkg_publication = json.loads(dataset_dict.get("publication", "{}"))
    except:
        pkg_publication = {}
    if  pkg_publication.get("publisher",""):
        datacite_dict['resource']['publisher'] = {"#text": pkg_publication.get("publisher","")}
    if  pkg_publication.get("publication_year",""):
        datacite_dict['resource']['publicationYear'] = {"#text": pkg_publication.get("publication_year","")}

    # Subjects
    dc_subjects = []
    for tag in dataset_dict.get("tags", []):
        tag_name = tag.get("display_name", tag.get("name",""))
        dc_subjects += [{ "@xml:lang":"en-us", "#text":tag_name}]
    if dc_subjects:
        datacite_dict['resource']['subjects'] = {'subject':dc_subjects}

    # Contributor (contact person)
    try:
        pkg_contributor = json.loads(dataset_dict.get("maintainer", "{}"))
    except:
        pkg_contributor = {}
    if pkg_contributor:
        dc_contributor = collections.OrderedDict()
        if pkg_contributor.get('name', False):
            dc_contributor['contributorName'] = author.get('name', '')
        if pkg_contributor.get('affiliation', False):
            dc_contributor['affiliation'] = author.get('affiliation', '')
        if pkg_contributor.get('identifier', False):
            dc_contributor['nameIdentifier'] = {"#text":author.get('identifier', '')}
        if pkg_contributor.get('identifier_scheme', False):
            dc_contributor['nameIdentifier']['@nameIdentifierScheme'] = author.get('identifier_scheme', '').upper()
        if dc_contributor:
            dc_contributor['@contributorType'] = 'ContactPerson'
            datacite_dict['resource']['contributors'] = {'contributor': [dc_contributor] }

    # Dates (TODO)
    try:
        pkg_dates = json.loads(dataset_dict.get("date", "[]"))
    except:
        pkg_dates = []
    dates_list = []
    for date in  pkg_dates:
        dc_date =  collections.OrderedDict()
        if date.get('date', ""):
            dc_date['#text'] = date.get('date', "")
            if date.get('end_date', ""):
                dc_date['#text'] += '/' + date.get('end_date', "")
        if date.get('date_type', ""):
            dc_date['@dateType'] = date.get('date_type', "").title()
        if dc_date:
            dates_list += [dc_date]
    if dates_list:
        datacite_dict['resource']['dates']={'date':dates_list}

    # Language
    datacite_dict['resource']['language'] = dataset_dict.get("language", "en")

    # ResourceType
    resource_type_general_dict ={"audiovisual": "Audiovisual", "collection": "Collection", "dataset": "Dataset", "event": "Event", "image": "Image",
                                 "interactive_resource": "InteractiveResource", "model": "Model", "physical_object": "PhysicalObject",
                                 "service": "Service", "software": "Software", "sound": "Sound", "text": "Text", "workflow": "Workflow", "other": "Other"}
    if dataset_dict.get("resource_type", ""):
        resource_type_general = resource_type_general_dict.get(dataset_dict.get("resource_type_general", "other"), "Other")
        datacite_dict['resource']['resourceType'] = {"#text": dataset_dict.get("resource_type",""), '@resourceTypeGeneral':resource_type_general}

    # Alternate Identifier (CKAN URL)
    ckan_package_url = config.get('ckan.site_url',"") + toolkit.url_for(controller='package', action='read', id=dataset_dict.get('name', ' '))
    datacite_dict['resource']['alternateIdentifiers']={'AlternateIdentifier':[{"#text":ckan_package_url, '@alternateIdentifierType':"URL"}]}
    # legacy
    if dataset_dict.get('url', ''):
        datacite_dict['resource']['alternateIdentifiers']['alternateIdentifier'] += [{"#text": dataset_dict.get('url', ''), '@alternateIdentifierType':"URL"}]

    # Related identifiers (TODO)

    # Sizes
    datacite_sizes = []
    for resource in dataset_dict.get("resources", []):
        if resource.get("size", ""):
            datacite_sizes += [{"#text": resource.get("size", " ") + " bytes"}]
    if datacite_sizes:
         datacite_dict['resource']['sizes'] = {'size': datacite_sizes}

    # Formats
    datacite_formats = []
    for resource in dataset_dict.get("resources", []):
        if resource.get("format", resource.get("mimetype", resource.get("mimetype_inner", ""))):
            format = {"#text": resource.get("format", "")}
            if format not in datacite_formats:
                datacite_formats += [format]
    if datacite_formats:
         datacite_dict['resource']['formats'] = {'format': datacite_formats}

    # Version
    if  dataset_dict.get("version", ""):
        datacite_dict['resource']['version'] = dataset_dict.get("version", "")

    # Rights
    if dataset_dict.get("license_title", ""):
        datacite_dict['resource']['rightsList'] = {"rights":[{"#text": dataset_dict.get("license_title", "")}]}

    # Description
    if dataset_dict.get("notes", ""):
        datacite_dict['resource']['descriptions'] = {'description': {"#text":dataset_dict.get("notes", " "), "@xml:lang":"en-us", "@descriptionType":"Abstract"}}

    # GeoLocation
    datacite_geolocation = {}
    if  dataset_dict.get("spatial_info", ""):
       datacite_geolocation["geoLocationPlace"] = dataset_dict.get("spatial_info", "")

    try:
        pkg_spatial = json.loads(dataset_dict.get("spatial", "{}"))
    except:
        pkg_spatial =  collections.OrderedDict()
    if pkg_spatial:
       coordinates_list = _flatten_list( pkg_spatial.get("coordinates", "[]"))
       coordinates = " ".join(coordinates_list)
       if pkg_spatial.get("type", "").lower() == "polygon" :
            datacite_geolocation["geoLocationBox"] = coordinates
       else:
            datacite_geolocation["geoLocationPoint"] = coordinates

    if datacite_geolocation:
       datacite_dict['resource']['geoLocations'] = {"geoLocation": [datacite_geolocation]}

    # Converto to xml
    converted_package = unparse(datacite_dict)

    return converted_package

def _flatten_list(input_list):
    output_list = []
    for item in input_list:
        if type(item) is not list:
            output_list += [str(item)]
        else:
            output_list += _flatten_list(item)
    return output_list

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2017-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import os
import unittest
from openquake.baselib import config

class ConfigPathsTestCase(unittest.TestCase):
    """Make sure that the config path search logic is tested"""

    def test_venv(self):
        venv = os.environ.get('VIRTUAL_ENV')
        self.assertTrue(venv, 'You cannot run the tests, you must use a '
                        'development installation with a virtualenv!')
        self.assertIn(os.path.join(venv, 'openquake.cfg'), config.paths)

    def test_config_file(self):
        cfgfile = os.environ.get('OQ_CONFIG_FILE')
        if cfgfile:
            self.assertEqual(config.paths[0], cfgfile)
<commit_msg>Removed an assertion in ConfigPathsTestCase<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2017-2018 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import os
import logging
import unittest
from openquake.baselib import config

class ConfigPathsTestCase(unittest.TestCase):
    """Make sure that the config path search logic is tested"""

    def test_venv(self):
        venv = os.environ.get('VIRTUAL_ENV')
        if venv:
            self.assertIn(os.path.join(venv, 'openquake.cfg'), config.paths)
        else:
            logging.warn('To run the tests, you should use a '
                         'development installation with a virtualenv')

    def test_config_file(self):
        cfgfile = os.environ.get('OQ_CONFIG_FILE')
        if cfgfile:
            self.assertEqual(config.paths[0], cfgfile)
<|endoftext|>


<commit_before># The Hazard Library
# Copyright (C) 2013-2014, GEM Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
Module exports :class:`AkkarEtAl2013`.
"""
from __future__ import division

import warnings
from openquake.hazardlib.gsim.akkar_2014 import AkkarEtAlRjb2014

class AkkarEtAl2013(AkkarEtAlRjb2014):
    """
    To ensure backwards compatibility with existing seismic hazard models,
    the call AkkarEtAl2013 is retained as legacy. The AkkarEtAl2013 GMPE
    is now implemented as AkkarEtAlRjb2014
    """
    def __init__(self):
        warnings.warn("AkkarEtAl2013 is deprecated - AkkarEtAlRjb2014 instead",
                      DeprecationWarning)
        super(AkkarEtAl2013, self).__init__()
<commit_msg>Updates to new deprecation mechanism<commit_after># The Hazard Library
# Copyright (C) 2013-2014, GEM Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
Module exports :class:`AkkarEtAl2013`.
"""
from __future__ import division

import warnings
from openquake.hazardlib.gsim.akkar_2014 import AkkarEtAlRjb2014

class AkkarEtAl2013(AkkarEtAlRjb2014):
    """
    To ensure backwards compatibility with existing seismic hazard models,
    the call AkkarEtAl2013 is retained as legacy. The AkkarEtAl2013 GMPE
    is now implemented as AkkarEtAlRjb2014
    """
    deprecated = True
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

from django import test
from django.core import mail
from mailer import engine
from mailer import send_mail

class DjangoMailerPresenceTest(test.TestCase):
    def test_mailsent(self):
        send_mail('subject', 'message_body', 'from@test.com', ['to@test.com'])
        engine.send_all()
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].subject, 'subject')
<commit_msg>Add comment ot openstack test<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4
''' Test for django mailer.

This test is pretty much worthless, and should be removed once real testing of
views that send emails is implemented
'''

from django import test
from django.core import mail
from mailer import engine
from mailer import send_mail

class DjangoMailerPresenceTest(test.TestCase):
    def test_mailsent(self):
        send_mail('subject', 'message_body', 'from@test.com', ['to@test.com'])
        engine.send_all()
        self.assertEqual(len(mail.outbox), 1)
        self.assertEqual(mail.outbox[0].subject, 'subject')
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import utils

import can
import can_bridge
import serial_datagrams

def read_frame(fdesc):
    """
    Reads a full CAN datagram from the CAN <-> serial bridge.
    """
    buf = bytes()
    datagram = None

    frame = serial_datagrams.read_datagram(fdesc)
    frame = can_bridge.decode_frame(frame)

    return frame

def send_frame(fdesc, frame):
    bridge_frame = can_bridge.encode_frame_command(frame)
    datagram = serial_datagrams.datagram_encode(bridge_frame)
    fdesc.write(datagram)
    fdesc.flush()

def main():
    DATA = "CVRA"

    parser = utils.ConnectionArgumentParser(description='Tests the CAN loopback over the Brige')
    args = parser.parse_args()

    connection = utils.open_connection(args)

    frame = can.Frame(data=DATA.encode())
    send_frame(connection, frame)
    print("Sent {}".format(DATA))

    answer = read_frame(connection)

    print("Got {}".format(answer.data.decode()))

if __name__ == "__main__":
    main()
<commit_msg>Delete CAN bridge test tool<commit_after><|endoftext|>