<commit_before>from __future__ import absolute_import, unicode_literals

import datetime
import logging
import numbers

import gi
gi.require_version('Gst', '1.0')
from gi.repository import Gst

from mopidy import compat, httpclient
from mopidy.models import Album, Artist, Track

logger = logging.getLogger(__name__)

def calculate_duration(num_samples, sample_rate):
    """Determine duration of samples using GStreamer helper for precise
    math."""
    return Gst.util_uint64_scale(num_samples, Gst.SECOND, sample_rate)

def create_buffer(data, capabilites=None, timestamp=None, duration=None):
    """Create a new GStreamer buffer based on provided data.

    Mainly intended to keep gst imports out of non-audio modules.
    """
    buffer_ = Gst.Buffer(data)
    if capabilites:
        if isinstance(capabilites, compat.string_types):
            capabilites = Gst.caps_from_string(capabilites)
        buffer_.set_caps(capabilites)
    if timestamp:
        buffer_.pts = timestamp
    if duration:
        buffer_.duration = duration
    return buffer_

def millisecond_to_clocktime(value):
    """Convert a millisecond time to internal GStreamer time."""
    return value * Gst.MSECOND

def clocktime_to_millisecond(value):
    """Convert an internal GStreamer time to millisecond time."""
    return value // Gst.MSECOND

def supported_uri_schemes(uri_schemes):
    """Determine which URIs we can actually support from provided whitelist.

    :param uri_schemes: list/set of URIs to check support for.
    :type uri_schemes: list or set or URI schemes as strings.
    :rtype: set of URI schemes we can support via this GStreamer install.
    """
    supported_schemes = set()
    registry = Gst.Registry.get()

    for factory in registry.get_feature_list(Gst.ElementFactory):
        for uri in factory.get_uri_protocols():
            if uri in uri_schemes:
                supported_schemes.add(uri)

    return supported_schemes

def _artists(tags, artist_name, artist_id=None, artist_sortname=None):
    # Name missing, don't set artist
    if not tags.get(artist_name):
        return None
    # One artist name and either id or sortname, include all available fields
    if len(tags[artist_name]) == 1 and \
            (artist_id in tags or artist_sortname in tags):
        attrs = {'name': tags[artist_name][0]}
        if artist_id in tags:
            attrs['musicbrainz_id'] = tags[artist_id][0]
        if artist_sortname in tags:
            attrs['sortname'] = tags[artist_sortname][0]
        return [Artist(**attrs)]

    # Multiple artist, provide artists with name only to avoid ambiguity.
    return [Artist(name=name) for name in tags[artist_name]]

# TODO: split based on "stream" and "track" based conversion? i.e. handle data
# from radios in it's own helper instead?
def convert_tags_to_track(tags):
    """Convert our normalized tags to a track.

    :param  tags: dictionary of tag keys with a list of values
    :type tags: :class:`dict`
    :rtype: :class:`mopidy.models.Track`
    """
    album_kwargs = {}
    track_kwargs = {}

    track_kwargs['composers'] = _artists(tags, Gst.TAG_COMPOSER)
    track_kwargs['performers'] = _artists(tags, Gst.TAG_PERFORMER)
    track_kwargs['artists'] = _artists(tags, Gst.TAG_ARTIST,
                                       'musicbrainz-artistid',
                                       'musicbrainz-sortname')
    album_kwargs['artists'] = _artists(
        tags, Gst.TAG_ALBUM_ARTIST, 'musicbrainz-albumartistid')

    track_kwargs['genre'] = '; '.join(tags.get(Gst.TAG_GENRE, []))
    track_kwargs['name'] = '; '.join(tags.get(Gst.TAG_TITLE, []))
    if not track_kwargs['name']:
        track_kwargs['name'] = '; '.join(tags.get(Gst.TAG_ORGANIZATION, []))

    track_kwargs['comment'] = '; '.join(tags.get('comment', []))
    if not track_kwargs['comment']:
        track_kwargs['comment'] = '; '.join(tags.get(Gst.TAG_LOCATION, []))
    if not track_kwargs['comment']:
        track_kwargs['comment'] = '; '.join(tags.get(Gst.TAG_COPYRIGHT, []))

    track_kwargs['track_no'] = tags.get(Gst.TAG_TRACK_NUMBER, [None])[0]
    track_kwargs['disc_no'] = tags.get(Gst.TAG_ALBUM_VOLUME_NUMBER, [None])[0]
    track_kwargs['bitrate'] = tags.get(Gst.TAG_BITRATE, [None])[0]
    track_kwargs['musicbrainz_id'] = tags.get('musicbrainz-trackid', [None])[0]

    album_kwargs['name'] = tags.get(Gst.TAG_ALBUM, [None])[0]
    album_kwargs['num_tracks'] = tags.get(Gst.TAG_TRACK_COUNT, [None])[0]
    album_kwargs['num_discs'] = tags.get(Gst.TAG_ALBUM_VOLUME_COUNT, [None])[0]
    album_kwargs['musicbrainz_id'] = tags.get('musicbrainz-albumid', [None])[0]

    if tags.get(Gst.TAG_DATE) and tags.get(Gst.TAG_DATE)[0]:
        track_kwargs['date'] = tags[Gst.TAG_DATE][0].isoformat()

    # Clear out any empty values we found
    track_kwargs = {k: v for k, v in track_kwargs.items() if v}
    album_kwargs = {k: v for k, v in album_kwargs.items() if v}

    # Only bother with album if we have a name to show.
    if album_kwargs.get('name'):
        track_kwargs['album'] = Album(**album_kwargs)

    return Track(**track_kwargs)

def setup_proxy(element, config):
    """Configure a GStreamer element with proxy settings.

    :param element: element to setup proxy in.
    :type element: :class:`Gst.GstElement`
    :param config: proxy settings to use.
    :type config: :class:`dict`
    """
    if not hasattr(element.props, 'proxy') or not config.get('hostname'):
        return

    element.set_property('proxy', httpclient.format_proxy(config, auth=False))
    element.set_property('proxy-id', config.get('username'))
    element.set_property('proxy-pw', config.get('password'))

def convert_taglist(taglist):
    """Convert a :class:`Gst.Taglist` to plain Python types.

    Knows how to convert:

    - Dates
    - Buffers
    - Numbers
    - Strings
    - Booleans

    Unknown types will be ignored and debug logged. Tag keys are all strings
    defined as part GStreamer under GstTagList_.

    .. _GstTagList: http://gstreamer.freedesktop.org/data/doc/gstreamer/\
0.10.36/gstreamer/html/gstreamer-GstTagList.html

    :param taglist: A GStreamer taglist to be converted.
    :type taglist: :class:`Gst.Taglist`
    :rtype: dictionary of tag keys with a list of values.
    """
    result = {}

    # Taglists are not really dicts, hence the lack of .items() and
    # explicit use of .keys()
    for key in taglist.keys():
        result.setdefault(key, [])

        values = taglist[key]
        if not isinstance(values, list):
            values = [values]

        for value in values:
            if isinstance(value, Gst.Date):
                try:
                    date = datetime.date(value.year, value.month, value.day)
                    result[key].append(date)
                except ValueError:
                    logger.debug('Ignoring invalid date: %r = %r', key, value)
            elif isinstance(value, Gst.Buffer):
                result[key].append(bytes(value))
            elif isinstance(
                    value, (compat.string_types, bool, numbers.Number)):
                result[key].append(value)
            else:
                logger.debug('Ignoring unknown data: %r = %r', key, value)

    return result
<commit_msg>gst1: Buffers no longer have caps<commit_after>from __future__ import absolute_import, unicode_literals

import datetime
import logging
import numbers

import gi
gi.require_version('Gst', '1.0')
from gi.repository import Gst

from mopidy import httpclient
from mopidy.models import Album, Artist, Track

logger = logging.getLogger(__name__)

def calculate_duration(num_samples, sample_rate):
    """Determine duration of samples using GStreamer helper for precise
    math."""
    return Gst.util_uint64_scale(num_samples, Gst.SECOND, sample_rate)

def create_buffer(data, capabilites=None, timestamp=None, duration=None):
    """Create a new GStreamer buffer based on provided data.

    Mainly intended to keep gst imports out of non-audio modules.

    .. versionchanged:: 1.2
        ``capabilites`` argument is no longer in use
    """
    buffer_ = Gst.Buffer.new_wrapped(data)
    if timestamp:
        buffer_.pts = timestamp
    if duration:
        buffer_.duration = duration
    return buffer_

def millisecond_to_clocktime(value):
    """Convert a millisecond time to internal GStreamer time."""
    return value * Gst.MSECOND

def clocktime_to_millisecond(value):
    """Convert an internal GStreamer time to millisecond time."""
    return value // Gst.MSECOND

def supported_uri_schemes(uri_schemes):
    """Determine which URIs we can actually support from provided whitelist.

    :param uri_schemes: list/set of URIs to check support for.
    :type uri_schemes: list or set or URI schemes as strings.
    :rtype: set of URI schemes we can support via this GStreamer install.
    """
    supported_schemes = set()
    registry = Gst.Registry.get()

    for factory in registry.get_feature_list(Gst.ElementFactory):
        for uri in factory.get_uri_protocols():
            if uri in uri_schemes:
                supported_schemes.add(uri)

    return supported_schemes

def _artists(tags, artist_name, artist_id=None, artist_sortname=None):
    # Name missing, don't set artist
    if not tags.get(artist_name):
        return None
    # One artist name and either id or sortname, include all available fields
    if len(tags[artist_name]) == 1 and \
            (artist_id in tags or artist_sortname in tags):
        attrs = {'name': tags[artist_name][0]}
        if artist_id in tags:
            attrs['musicbrainz_id'] = tags[artist_id][0]
        if artist_sortname in tags:
            attrs['sortname'] = tags[artist_sortname][0]
        return [Artist(**attrs)]

    # Multiple artist, provide artists with name only to avoid ambiguity.
    return [Artist(name=name) for name in tags[artist_name]]

# TODO: split based on "stream" and "track" based conversion? i.e. handle data
# from radios in it's own helper instead?
def convert_tags_to_track(tags):
    """Convert our normalized tags to a track.

    :param  tags: dictionary of tag keys with a list of values
    :type tags: :class:`dict`
    :rtype: :class:`mopidy.models.Track`
    """
    album_kwargs = {}
    track_kwargs = {}

    track_kwargs['composers'] = _artists(tags, Gst.TAG_COMPOSER)
    track_kwargs['performers'] = _artists(tags, Gst.TAG_PERFORMER)
    track_kwargs['artists'] = _artists(tags, Gst.TAG_ARTIST,
                                       'musicbrainz-artistid',
                                       'musicbrainz-sortname')
    album_kwargs['artists'] = _artists(
        tags, Gst.TAG_ALBUM_ARTIST, 'musicbrainz-albumartistid')

    track_kwargs['genre'] = '; '.join(tags.get(Gst.TAG_GENRE, []))
    track_kwargs['name'] = '; '.join(tags.get(Gst.TAG_TITLE, []))
    if not track_kwargs['name']:
        track_kwargs['name'] = '; '.join(tags.get(Gst.TAG_ORGANIZATION, []))

    track_kwargs['comment'] = '; '.join(tags.get('comment', []))
    if not track_kwargs['comment']:
        track_kwargs['comment'] = '; '.join(tags.get(Gst.TAG_LOCATION, []))
    if not track_kwargs['comment']:
        track_kwargs['comment'] = '; '.join(tags.get(Gst.TAG_COPYRIGHT, []))

    track_kwargs['track_no'] = tags.get(Gst.TAG_TRACK_NUMBER, [None])[0]
    track_kwargs['disc_no'] = tags.get(Gst.TAG_ALBUM_VOLUME_NUMBER, [None])[0]
    track_kwargs['bitrate'] = tags.get(Gst.TAG_BITRATE, [None])[0]
    track_kwargs['musicbrainz_id'] = tags.get('musicbrainz-trackid', [None])[0]

    album_kwargs['name'] = tags.get(Gst.TAG_ALBUM, [None])[0]
    album_kwargs['num_tracks'] = tags.get(Gst.TAG_TRACK_COUNT, [None])[0]
    album_kwargs['num_discs'] = tags.get(Gst.TAG_ALBUM_VOLUME_COUNT, [None])[0]
    album_kwargs['musicbrainz_id'] = tags.get('musicbrainz-albumid', [None])[0]

    if tags.get(Gst.TAG_DATE) and tags.get(Gst.TAG_DATE)[0]:
        track_kwargs['date'] = tags[Gst.TAG_DATE][0].isoformat()

    # Clear out any empty values we found
    track_kwargs = {k: v for k, v in track_kwargs.items() if v}
    album_kwargs = {k: v for k, v in album_kwargs.items() if v}

    # Only bother with album if we have a name to show.
    if album_kwargs.get('name'):
        track_kwargs['album'] = Album(**album_kwargs)

    return Track(**track_kwargs)

def setup_proxy(element, config):
    """Configure a GStreamer element with proxy settings.

    :param element: element to setup proxy in.
    :type element: :class:`Gst.GstElement`
    :param config: proxy settings to use.
    :type config: :class:`dict`
    """
    if not hasattr(element.props, 'proxy') or not config.get('hostname'):
        return

    element.set_property('proxy', httpclient.format_proxy(config, auth=False))
    element.set_property('proxy-id', config.get('username'))
    element.set_property('proxy-pw', config.get('password'))

def convert_taglist(taglist):
    """Convert a :class:`Gst.Taglist` to plain Python types.

    Knows how to convert:

    - Dates
    - Buffers
    - Numbers
    - Strings
    - Booleans

    Unknown types will be ignored and debug logged. Tag keys are all strings
    defined as part GStreamer under GstTagList_.

    .. _GstTagList: http://gstreamer.freedesktop.org/data/doc/gstreamer/\
0.10.36/gstreamer/html/gstreamer-GstTagList.html

    :param taglist: A GStreamer taglist to be converted.
    :type taglist: :class:`Gst.Taglist`
    :rtype: dictionary of tag keys with a list of values.
    """
    result = {}

    # Taglists are not really dicts, hence the lack of .items() and
    # explicit use of .keys()
    for key in taglist.keys():
        result.setdefault(key, [])

        values = taglist[key]
        if not isinstance(values, list):
            values = [values]

        for value in values:
            if isinstance(value, Gst.Date):
                try:
                    date = datetime.date(value.year, value.month, value.day)
                    result[key].append(date)
                except ValueError:
                    logger.debug('Ignoring invalid date: %r = %r', key, value)
            elif isinstance(value, Gst.Buffer):
                result[key].append(bytes(value))
            elif isinstance(
                    value, (compat.string_types, bool, numbers.Number)):
                result[key].append(value)
            else:
                logger.debug('Ignoring unknown data: %r = %r', key, value)

    return result
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""
Test related to code in db/alchemy/db_utils.py
"""

import mock
import os
import unittest

from db.alchemy.db_utils import SessionCache

class InitSessionTestCase(unittest.TestCase):
    """Tests the behaviour of alchemy.db_utils.SessionCache.init_session()."""

    def tearDown(self):
        SessionCache().__sessions__.clear()

    def test_init_session_with_empty_user(self):
        """
        _init_session() raises an `AssertionError` if the user name parameter
        is empty or `None`.
        """
        self.assertRaises(AssertionError, SessionCache()._init_session, "", "")
        self.assertRaises(
            AssertionError, SessionCache()._init_session, None, "")

    def test_init_session_with_repeated_initialization(self):
        """
        _init_session() raises an `AssertionError` upon repeated
        initialization.
        """
        sc = SessionCache()
        sc.__sessions__["oq_uiapi_writer"] = object()
        self.assertRaises(
            AssertionError, SessionCache()._init_session, "oq_uiapi_writer", "")

    def test_init_session_with_oq_engine_db_name(self):
        """
        _init_session() observes the `OQ_ENGINE_DB_NAME` environment variable.
        """
        os.environ["OQ_ENGINE_DB_NAME"] = "abc123"
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr1", "")
                self.assertEqual(1, ce_mock.call_count)
                args, kwargs = ce_mock.call_args
                self.assertEqual(
                    ('postgresql+psycopg2://usr1:@localhost/abc123',), args)
                self.assertEqual({}, kwargs)
        del os.environ["OQ_ENGINE_DB_NAME"]

    def test_init_session_without_oq_engine_db_name(self):
        """
        In the absence of the `OQ_ENGINE_DB_NAME` environment variable
        the session will be established for the "geonode" database.
        """
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr2", "")
                self.assertEqual(1, ce_mock.call_count)
                args, kwargs = ce_mock.call_args
                self.assertEqual(
                    ('postgresql+psycopg2://usr2:@localhost/geonode',), args)
                self.assertEqual({}, kwargs)

    def test_init_session_with_oq_engine_db_host(self):
        """
        _init_session() observes the `OQ_ENGINE_DB_HOST` environment variable.
        """
        os.environ["OQ_ENGINE_DB_HOST"] = "bcd234"
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr3", "")
                self.assertEqual(1, ce_mock.call_count)
                args, kwargs = ce_mock.call_args
                self.assertEqual(
                    ('postgresql+psycopg2://usr3:@bcd234/geonode',), args)
                self.assertEqual({}, kwargs)
        del os.environ["OQ_ENGINE_DB_HOST"]

    def test_init_session_without_oq_engine_db_host(self):
        """
        In the absence of the `OQ_ENGINE_DB_HOST` environment variable
        the session will be established for the `localhost`.
        """
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr4", "")
                self.assertEqual(1, ce_mock.call_count)
                args, kwargs = ce_mock.call_args
                self.assertEqual(
                    ('postgresql+psycopg2://usr4:@localhost/geonode',), args)
                self.assertEqual({}, kwargs)

    def test_init_session_with_none_password(self):
        """
        _init_session() will use an empty string for `None` passwords.
        """
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr5", None)
                self.assertEqual(1, ce_mock.call_count)
                args, kwargs = ce_mock.call_args
                self.assertEqual(
                    ('postgresql+psycopg2://usr5:@localhost/geonode',), args)
                self.assertEqual({}, kwargs)

    def test_init_session_with_empty_password(self):
        """
        _init_session() will use an empty password properly.
        """
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr6", "")
                self.assertEqual(1, ce_mock.call_count)
                args, kwargs = ce_mock.call_args
                self.assertEqual(
                    ('postgresql+psycopg2://usr6:@localhost/geonode',), args)
                self.assertEqual({}, kwargs)

    def test_init_session_with_non_empty_password(self):
        """
        _init_session() will use a non-empty password correctly.
        """
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr7", "s3cr3t")
                self.assertEqual(1, ce_mock.call_count)
                args, kwargs = ce_mock.call_args
                self.assertEqual(
                    ('postgresql+psycopg2://usr7:s3cr3t@localhost/geonode',),
                    args)
                self.assertEqual({}, kwargs)

    def test_init_session_updates_internal_dict(self):
        """
        _init_session() will add newly created sessions to the internal
        `__sessions__` dictionary.
        """
        session = object()
        sc = SessionCache()
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = lambda: session
                self.assertTrue(sc.__sessions__.get("usr8") is None)
                sc._init_session("usr8", "t0ps3cr3t")
                self.assertEqual(session, sc.__sessions__.get("usr8"))
<commit_msg>added first unit test for SessionCache.get()<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

"""
Test related to code in db/alchemy/db_utils.py
"""

import mock
import os
import unittest

from db.alchemy.db_utils import SessionCache

class SessionCacheInitSessionTestCase(unittest.TestCase):
    """Tests the behaviour of alchemy.db_utils.SessionCache.init_session()."""

    def tearDown(self):
        SessionCache().__sessions__.clear()

    def test_init_session_with_empty_user(self):
        """
        _init_session() raises an `AssertionError` if the user name parameter
        is empty or `None`.
        """
        self.assertRaises(AssertionError, SessionCache()._init_session, "", "")
        self.assertRaises(
            AssertionError, SessionCache()._init_session, None, "")

    def test_init_session_with_repeated_initialization(self):
        """
        _init_session() raises an `AssertionError` upon repeated
        initialization.
        """
        sc = SessionCache()
        sc.__sessions__["oq_uiapi_writer"] = object()
        self.assertRaises(
            AssertionError, SessionCache()._init_session, "oq_uiapi_writer", "")

    def test_init_session_with_oq_engine_db_name(self):
        """
        _init_session() observes the `OQ_ENGINE_DB_NAME` environment variable.
        """
        os.environ["OQ_ENGINE_DB_NAME"] = "abc123"
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr1", "")
                self.assertEqual(1, ce_mock.call_count)
                (arg,), kwargs = ce_mock.call_args
                self.assertEqual(
                    'postgresql+psycopg2://usr1:@localhost/abc123', arg)
                self.assertEqual({}, kwargs)
        del os.environ["OQ_ENGINE_DB_NAME"]

    def test_init_session_without_oq_engine_db_name(self):
        """
        In the absence of the `OQ_ENGINE_DB_NAME` environment variable
        the session will be established for the "geonode" database.
        """
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr2", "")
                self.assertEqual(1, ce_mock.call_count)
                (arg,), kwargs = ce_mock.call_args
                self.assertEqual(
                    'postgresql+psycopg2://usr2:@localhost/geonode', arg)
                self.assertEqual({}, kwargs)

    def test_init_session_with_oq_engine_db_host(self):
        """
        _init_session() observes the `OQ_ENGINE_DB_HOST` environment variable.
        """
        os.environ["OQ_ENGINE_DB_HOST"] = "bcd234"
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr3", "")
                self.assertEqual(1, ce_mock.call_count)
                (arg,), kwargs = ce_mock.call_args
                self.assertEqual(
                    'postgresql+psycopg2://usr3:@bcd234/geonode', arg)
                self.assertEqual({}, kwargs)
        del os.environ["OQ_ENGINE_DB_HOST"]

    def test_init_session_without_oq_engine_db_host(self):
        """
        In the absence of the `OQ_ENGINE_DB_HOST` environment variable
        the session will be established for the `localhost`.
        """
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr4", "")
                self.assertEqual(1, ce_mock.call_count)
                (arg,), kwargs = ce_mock.call_args
                self.assertEqual(
                    'postgresql+psycopg2://usr4:@localhost/geonode', arg)
                self.assertEqual({}, kwargs)

    def test_init_session_with_none_password(self):
        """
        _init_session() will use an empty string for `None` passwords.
        """
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr5", None)
                self.assertEqual(1, ce_mock.call_count)
                (arg,), kwargs = ce_mock.call_args
                self.assertEqual(
                    'postgresql+psycopg2://usr5:@localhost/geonode', arg)
                self.assertEqual({}, kwargs)

    def test_init_session_with_empty_password(self):
        """
        _init_session() will use an empty password properly.
        """
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr6", "")
                self.assertEqual(1, ce_mock.call_count)
                (arg,), kwargs = ce_mock.call_args
                self.assertEqual(
                    'postgresql+psycopg2://usr6:@localhost/geonode', arg)
                self.assertEqual({}, kwargs)

    def test_init_session_with_non_empty_password(self):
        """
        _init_session() will use a non-empty password correctly.
        """
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = object
                SessionCache()._init_session("usr7", "s3cr3t")
                self.assertEqual(1, ce_mock.call_count)
                (arg,), kwargs = ce_mock.call_args
                self.assertEqual(
                    'postgresql+psycopg2://usr7:s3cr3t@localhost/geonode', arg)
                self.assertEqual({}, kwargs)

    def test_init_session_updates_internal_dict(self):
        """
        _init_session() will add newly created sessions to the internal
        `__sessions__` dictionary.
        """
        session = object()
        sc = SessionCache()
        with mock.patch('sqlalchemy.create_engine') as ce_mock:
            with mock.patch('sqlalchemy.orm.sessionmaker') as sm_mock:
                sm_mock.return_value = lambda: session
                self.assertTrue(sc.__sessions__.get("usr8") is None)
                sc._init_session("usr8", "t0ps3cr3t")
                self.assertEqual(session, sc.__sessions__.get("usr8"))

class SessionCacheGetTestCase(unittest.TestCase):
    """Tests the behaviour of alchemy.db_utils.SessionCache.get()."""

    def tearDown(self):
        SessionCache().__sessions__.clear()

    def test_get_with_no_session_for_user(self):
        """
        get() will call _init_session() if this is the first time a session is
        requested for the given database user.
        """
        sc = SessionCache()
        expected_session = object()

        def fake_init_session(user, password):
            """
            This will merely set the session for the given user to
            `expected_session`.
            """
            sc.__sessions__[user] = expected_session

        original_method = sc._init_session
        mock_method = mock.Mock()
        mock_method.side_effect = fake_init_session
        sc._init_session = mock_method

        self.assertEqual(expected_session, sc.get("usr1", ""))
        self.assertEqual(1, mock_method.call_count)
        (user, passwd), kwargs = mock_method.call_args
        self.assertEqual("usr1", user)
        self.assertEqual("", passwd)
        sc._init_session = original_method
<|endoftext|>


<commit_before># Copyright 2016 Raytheon BBN Technologies
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0

import asyncio, concurrent
import itertools
import h5py
import numpy as np
import os.path

from auspex.parameter import Parameter
from auspex.stream import DataStreamDescriptor
from auspex.log import logger
from auspex.filters.filter import Filter, InputConnector, OutputConnector
from tqdm import tqdm, tqdm_notebook

class WriteToHDF5(Filter):
    """Writes data to file."""

    sink = InputConnector()
    filename = Parameter()

    def __init__(self, filename=None, compress=True, **kwargs):
        super(WriteToHDF5, self).__init__(**kwargs)
        self.compress = compress
        self.filename.value = filename
        self.points_taken = 0
        self.file = None
        self.up_to_date = False

    def final_init(self):
        if not self.filename.value:
            raise Exception("Filename never supplied to writer.")
        self.file = self.new_file()

    def new_filename(self):
        # Increment the filename until we find one we want.
        i = 0
        ext = self.filename.find('.h5')
        if ext > -1:
            filename = self.filename[:ext]
        while os.path.exists("{}-{:04d}.h5".format(filename,i)):
            i += 1
        return "{}-{:04d}.h5".format(filename,i)

    def new_file(self):
        """ Open a new data file to write """
        # Close the current file, if any
        if self.file is not None:
            try:
                self.file.close()
            except Exception as e:
                logger.error("Encounter exception: {}".format(e))
                logger.error("Cannot close file '{}'. File may be damaged.".format(self.file.filename))
        # Get new file name
        self.filename = self.new_filename()
        head = os.path.dirname(self.filename)
        head = os.path.normpath(head)
        dirs = head.split(os.sep)
        # Check if path exists. If not, create new one(s).
        fulldir = ''
        for d in dirs:
            fulldir = os.path.join(fulldir, d)
            if not os.path.exists(fulldir):
                logger.debug("Create new directory: {}.".format(fulldir))
                os.mkdir(fulldir)
        logger.debug("Create new data file: %s." %self.filename)
        return h5py.File(self.filename, 'w', libver='latest')

    async def run(self):
        stream     = self.sink.input_streams[0]
        desc       = stream.descriptor
        axes       = stream.descriptor.axes
        params     = stream.descriptor.params
        axis_names = desc.axis_names()

        params['exp_src'] = stream.descriptor.exp_src
        num_axes   = len(axes)

        # All of the combinations for the present values of the sweep parameters only
        tuples     = stream.descriptor.tuples()

        # Create a 2D dataset with a 1D data column
        dtype = [(a, 'f') for a in axis_names]
        logger.debug("Data type for HDF5: %s", dtype)
        dtype.append((desc.data_name, 'f'))
        if self.compress:
            self.data = self.file.create_dataset('data', (len(tuples),), dtype=dtype,
                                        chunks=True, maxshape=(None,),
                                        compression='gzip')
            # self.file.swmr_mode = True
        else:
            self.data = self.file.create_dataset('data', (len(tuples),), dtype=dtype,
                                        chunks=True, maxshape=(None,))
            # self.file.swmr_mode = True

        # Write params into attrs
        for k,v in params.items():
            if k not in axis_names:
                self.data.attrs[k] = v

        # Create a table for the DataStreamDescriptor
        ref_dtype = h5py.special_dtype(ref=h5py.Reference)
        self.descriptor = self.file.create_dataset("descriptor", (len(axes),), dtype=ref_dtype)

        # Associated axis dimensions with the data and add
        # references to the descriptor.
        for i, a in enumerate(axes):
            if a.unstructured:
                name = "+".join(a.name)
                dtype = [(p.name, 'f') for p in a.parameter]
                self.file.create_dataset(name, (a.num_points(),), dtype=dtype, maxshape=(None,) )
                for j, (col_name, col_unit) in enumerate(zip(a.name, a.unit)):
                    self.file[name][col_name,:] = a.points[:,j]
                    self.file[name].attrs['unit_'+col_name] = col_unit
            else:
                name = a.name
                self.file[name] = a.points
                self.file[name].attrs['unit'] = "None" if a.unit is None else a.unit
            self.data.dims.create_scale(self.file[name], name)
            self.data.dims[0].attach_scale(self.file[name])
            self.descriptor[i] = self.file[name].ref

        # Write the initial coordinate tuples
        for i, a in enumerate(axis_names):
            self.data[a,:] = tuples[:,i]

        # Write pointer
        w_idx = 0

        while True:

            message = await stream.queue.get()
            message_type = message['type']
            message_data = message['data']
            message_comp = message['compression']

            if message_comp == 'zlib':
                message_data = pickle.loads(zlib.decompress(message_data))
            # If we receive a message
            if message['type'] == 'event':
                logger.debug('%s "%s" received event "%s"', self.__class__.__name__, self.name, message_data)
                if message['data'] == 'done':
                    break
            elif message['type'] == 'data':
                if not hasattr(message_data, 'size'):
                    message_data = np.array([message_data])

                logger.debug('%s "%s" received %d points', self.__class__.__name__, self.name, message_data.size)
                logger.debug("Now has %d of %d points.", stream.points_taken, stream.num_points())

                self.up_to_date = (w_idx == self.data.len())

                # Resize if necessary, also get the new set of sweep tuples since the axes must have changed
                if w_idx + message_data.size > self.data.len():
                    # Get new data size
                    num_points = stream.descriptor.num_points()
                    self.data.resize((num_points,))
                    logger.debug("HDF5 stream was resized to %d points", w_idx + message_data.size)

                    # Get and write new coordinate tuples to the main
                    # data set as well as the individual axis tables.
                    tuples = stream.descriptor.tuples()
                    for i, axis_name in enumerate(axis_names):
                        self.data[axis_name, w_idx:num_points] = tuples[w_idx:num_points, i]
                    for i, a in enumerate(axes):
                        if a.unstructured:
                            name = "+".join(a.name)
                            if a.num_points() > self.file[name].len():
                                self.file[name].resize((a.num_points(),))
                                for j, col_name in enumerate(a.name):
                                    self.file[name][col_name,:] = a.points[:,j]

                self.data[desc.data_name, w_idx:w_idx+message_data.size] = message_data

                self.file.flush()
                w_idx += message_data.size
                self.points_taken = w_idx

                logger.debug("HDF5: Write index at %d", w_idx)
                logger.debug("HDF5: %s has written %d points", stream.name, w_idx)

        try:
            self.file.close()
        except:
            # This doesn't seem to happen when we don't used named columns
            logger.debug("Ignoring 'dictionary changed sized during iteration' error.")

class ProgressBar(Filter):
    """ Display progress bar(s) on the terminal/notebook.

    num: number of progress bars to be display, \
    corresponding to the number of axes (counting from outer most)

        For running in Jupyter Notebook:
    Needs to open '_tqdm_notebook.py',\
    search for 'n = int(s[:npos])'\
    then replace it with 'n = float(s[:npos])'
    """
    sink = InputConnector()
    def __init__(self, num=0, notebook=False):
        super(ProgressBar,self).__init__()
        self.num    = num
        self.notebook = notebook
        self.bars   = []
        self.w_id   = 0

    async def run(self):
        self.stream = self.sink.input_streams[0]
        axes = self.stream.descriptor.axes
        num_axes = len(axes)
        totals = [self.stream.descriptor.num_points_through_axis(axis) for axis in range(num_axes)]
        chunk_sizes = [max(1,self.stream.descriptor.num_points_through_axis(axis+1)) for axis in range(num_axes)]
        self.num = min(self.num, num_axes)

        self.bars   = []
        for i in range(self.num):
            if self.notebook:
                self.bars.append(tqdm_notebook(total=totals[i]/chunk_sizes[i]))
            else:
                self.bars.append(tqdm(total=totals[i]/chunk_sizes[i]))
        self.w_id   = 0
        while True:
            if self.stream.done() and self.w_id==self.stream.num_points():
                break

            new_data = np.array(await self.stream.queue.get()).flatten()
            while self.stream.queue.qsize() > 0:
                new_data = np.append(new_data, np.array(self.stream.queue.get_nowait()).flatten())
            self.w_id += new_data.size
            num_data = self.stream.points_taken
            for i in range(self.num):
                if num_data == 0:
                    if self.notebook:
                        self.bars[i].sp(close=True)
                        # Reset the progress bar with a new one
                        self.bars[i] = tqdm_notebook(total=totals[i]/chunk_sizes[i])
                    else:
                        # Reset the progress bar with a new one
                        self.bars[i].close()
                        self.bars[i] = tqdm(total=totals[i]/chunk_sizes[i])
                pos = int(10*num_data / chunk_sizes[i])/10.0 # One decimal is good enough
                if pos > self.bars[i].n:
                    self.bars[i].update(pos - self.bars[i].n)
                num_data = num_data % chunk_sizes[i]
<commit_msg>Parameter value fixes<commit_after># Copyright 2016 Raytheon BBN Technologies
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0

import asyncio, concurrent
import itertools
import h5py
import numpy as np
import os.path

from auspex.parameter import Parameter
from auspex.stream import DataStreamDescriptor
from auspex.log import logger
from auspex.filters.filter import Filter, InputConnector, OutputConnector
from tqdm import tqdm, tqdm_notebook

class WriteToHDF5(Filter):
    """Writes data to file."""

    sink = InputConnector()
    filename = Parameter()

    def __init__(self, filename=None, compress=True, **kwargs):
        super(WriteToHDF5, self).__init__(**kwargs)
        self.compress = compress
        self.filename.value = filename
        self.points_taken = 0
        self.file = None
        self.up_to_date = False

    def final_init(self):
        if not self.filename.value:
            raise Exception("Filename never supplied to writer.")
        self.file = self.new_file()

    def new_filename(self):
        # Increment the filename until we find one we want.
        i = 0
        filename = self.filename.value
        ext = filename.find('.h5')
        if ext > -1:
            filename = filename[:ext]
        while os.path.exists("{}-{:04d}.h5".format(filename,i)):
            i += 1
        return "{}-{:04d}.h5".format(filename,i)

    def new_file(self):
        """ Open a new data file to write """
        # Close the current file, if any
        if self.file is not None:
            try:
                self.file.close()
            except Exception as e:
                logger.error("Encounter exception: {}".format(e))
                logger.error("Cannot close file '{}'. File may be damaged.".format(self.file.filename))
        # Get new file name
        self.filename.value = self.new_filename()
        head = os.path.dirname(self.filename.value)
        head = os.path.normpath(head)
        dirs = head.split(os.sep)
        # Check if path exists. If not, create new one(s).
        fulldir = ''
        for d in dirs:
            fulldir = os.path.join(fulldir, d)
            if not os.path.exists(fulldir):
                logger.debug("Create new directory: {}.".format(fulldir))
                os.mkdir(fulldir)
        logger.debug("Create new data file: %s." % self.filename.value)
        return h5py.File(self.filename.value, 'w', libver='latest')

    async def run(self):
        stream     = self.sink.input_streams[0]
        desc       = stream.descriptor
        axes       = stream.descriptor.axes
        params     = stream.descriptor.params
        axis_names = desc.axis_names()

        params['exp_src'] = stream.descriptor.exp_src
        num_axes   = len(axes)

        # All of the combinations for the present values of the sweep parameters only
        tuples     = stream.descriptor.tuples()

        # Create a 2D dataset with a 1D data column
        dtype = [(a, 'f') for a in axis_names]
        logger.debug("Data type for HDF5: %s", dtype)
        dtype.append((desc.data_name, 'f'))
        if self.compress:
            self.data = self.file.create_dataset('data', (len(tuples),), dtype=dtype,
                                        chunks=True, maxshape=(None,),
                                        compression='gzip')
            # self.file.swmr_mode = True
        else:
            self.data = self.file.create_dataset('data', (len(tuples),), dtype=dtype,
                                        chunks=True, maxshape=(None,))
            # self.file.swmr_mode = True

        # Write params into attrs
        for k,v in params.items():
            if k not in axis_names:
                self.data.attrs[k] = v

        # Create a table for the DataStreamDescriptor
        ref_dtype = h5py.special_dtype(ref=h5py.Reference)
        self.descriptor = self.file.create_dataset("descriptor", (len(axes),), dtype=ref_dtype)

        # Associated axis dimensions with the data and add
        # references to the descriptor.
        for i, a in enumerate(axes):
            if a.unstructured:
                name = "+".join(a.name)
                dtype = [(p.name, 'f') for p in a.parameter]
                self.file.create_dataset(name, (a.num_points(),), dtype=dtype, maxshape=(None,) )
                for j, (col_name, col_unit) in enumerate(zip(a.name, a.unit)):
                    self.file[name][col_name,:] = a.points[:,j]
                    self.file[name].attrs['unit_'+col_name] = col_unit
            else:
                name = a.name
                self.file[name] = a.points
                self.file[name].attrs['unit'] = "None" if a.unit is None else a.unit
            self.data.dims.create_scale(self.file[name], name)
            self.data.dims[0].attach_scale(self.file[name])
            self.descriptor[i] = self.file[name].ref

        # Write the initial coordinate tuples
        for i, a in enumerate(axis_names):
            self.data[a,:] = tuples[:,i]

        # Write pointer
        w_idx = 0

        while True:

            message = await stream.queue.get()
            message_type = message['type']
            message_data = message['data']
            message_comp = message['compression']

            if message_comp == 'zlib':
                message_data = pickle.loads(zlib.decompress(message_data))
            # If we receive a message
            if message['type'] == 'event':
                logger.debug('%s "%s" received event "%s"', self.__class__.__name__, self.name, message_data)
                if message['data'] == 'done':
                    break
            elif message['type'] == 'data':
                if not hasattr(message_data, 'size'):
                    message_data = np.array([message_data])

                logger.debug('%s "%s" received %d points', self.__class__.__name__, self.name, message_data.size)
                logger.debug("Now has %d of %d points.", stream.points_taken, stream.num_points())

                self.up_to_date = (w_idx == self.data.len())

                # Resize if necessary, also get the new set of sweep tuples since the axes must have changed
                if w_idx + message_data.size > self.data.len():
                    # Get new data size
                    num_points = stream.descriptor.num_points()
                    self.data.resize((num_points,))
                    logger.debug("HDF5 stream was resized to %d points", w_idx + message_data.size)

                    # Get and write new coordinate tuples to the main
                    # data set as well as the individual axis tables.
                    tuples = stream.descriptor.tuples()
                    for i, axis_name in enumerate(axis_names):
                        self.data[axis_name, w_idx:num_points] = tuples[w_idx:num_points, i]
                    for i, a in enumerate(axes):
                        if a.unstructured:
                            name = "+".join(a.name)
                            if a.num_points() > self.file[name].len():
                                self.file[name].resize((a.num_points(),))
                                for j, col_name in enumerate(a.name):
                                    self.file[name][col_name,:] = a.points[:,j]

                self.data[desc.data_name, w_idx:w_idx+message_data.size] = message_data

                self.file.flush()
                w_idx += message_data.size
                self.points_taken = w_idx

                logger.debug("HDF5: Write index at %d", w_idx)
                logger.debug("HDF5: %s has written %d points", stream.name, w_idx)

        try:
            self.file.close()
        except:
            # This doesn't seem to happen when we don't used named columns
            logger.debug("Ignoring 'dictionary changed sized during iteration' error.")

class ProgressBar(Filter):
    """ Display progress bar(s) on the terminal/notebook.

    num: number of progress bars to be display, \
    corresponding to the number of axes (counting from outer most)

        For running in Jupyter Notebook:
    Needs to open '_tqdm_notebook.py',\
    search for 'n = int(s[:npos])'\
    then replace it with 'n = float(s[:npos])'
    """
    sink = InputConnector()
    def __init__(self, num=0, notebook=False):
        super(ProgressBar,self).__init__()
        self.num    = num
        self.notebook = notebook
        self.bars   = []
        self.w_id   = 0

    async def run(self):
        self.stream = self.sink.input_streams[0]
        axes = self.stream.descriptor.axes
        num_axes = len(axes)
        totals = [self.stream.descriptor.num_points_through_axis(axis) for axis in range(num_axes)]
        chunk_sizes = [max(1,self.stream.descriptor.num_points_through_axis(axis+1)) for axis in range(num_axes)]
        self.num = min(self.num, num_axes)

        self.bars   = []
        for i in range(self.num):
            if self.notebook:
                self.bars.append(tqdm_notebook(total=totals[i]/chunk_sizes[i]))
            else:
                self.bars.append(tqdm(total=totals[i]/chunk_sizes[i]))
        self.w_id   = 0
        while True:
            if self.stream.done() and self.w_id==self.stream.num_points():
                break

            new_data = np.array(await self.stream.queue.get()).flatten()
            while self.stream.queue.qsize() > 0:
                new_data = np.append(new_data, np.array(self.stream.queue.get_nowait()).flatten())
            self.w_id += new_data.size
            num_data = self.stream.points_taken
            for i in range(self.num):
                if num_data == 0:
                    if self.notebook:
                        self.bars[i].sp(close=True)
                        # Reset the progress bar with a new one
                        self.bars[i] = tqdm_notebook(total=totals[i]/chunk_sizes[i])
                    else:
                        # Reset the progress bar with a new one
                        self.bars[i].close()
                        self.bars[i] = tqdm(total=totals[i]/chunk_sizes[i])
                pos = int(10*num_data / chunk_sizes[i])/10.0 # One decimal is good enough
                if pos > self.bars[i].n:
                    self.bars[i].update(pos - self.bars[i].n)
                num_data = num_data % chunk_sizes[i]
<|endoftext|>


<commit_before>try:
    from urlparse import urlsplit
    from StringIO import StringIO
except ImportError:
    from urllib.parse import urlsplit
    from io import BytesIO as StringIO
import mock
from lxml import html
from ...helpers import LoggedInApplicationTest

@mock.patch('app.main.views.users.data_api_client')
class TestUsersView(LoggedInApplicationTest):

    def test_should_be_a_404_if_user_not_found(self, data_api_client):
        data_api_client.get_user.return_value = None
        response = self.client.get('/admin/users?email_address=some@email.com')
        self.assertEquals(response.status_code, 404)

        document = html.fromstring(response.get_data(as_text=True))

        page_title = document.xpath(
            '//p[@class="banner-message"]//text()')[0].strip()
        self.assertEqual("Sorry, we couldn't find an account with that email address", page_title)

        page_title = document.xpath(
            '//p[@class="summary-item-no-content"]//text()')[0].strip()
        self.assertEqual("No users to show", page_title)

    def test_should_be_a_404_if_no_email_provided(self, data_api_client):
        data_api_client.get_user.return_value = None
        response = self.client.get('/admin/users?email_address=')
        self.assertEquals(response.status_code, 404)

        document = html.fromstring(response.get_data(as_text=True))

        page_title = document.xpath(
            '//p[@class="banner-message"]//text()')[0].strip()
        self.assertEqual("Sorry, we couldn't find an account with that email address", page_title)

        page_title = document.xpath(
            '//p[@class="summary-item-no-content"]//text()')[0].strip()
        self.assertEqual("No users to show", page_title)

    def test_should_be_a_404_if_no_email_param_provided(self, data_api_client):
        data_api_client.get_user.return_value = None
        response = self.client.get('/admin/users')
        self.assertEquals(response.status_code, 404)

        document = html.fromstring(response.get_data(as_text=True))

        page_title = document.xpath(
            '//p[@class="banner-message"]//text()')[0].strip()
        self.assertEqual("Sorry, we couldn't find an account with that email address", page_title)

        page_title = document.xpath(
            '//p[@class="summary-item-no-content"]//text()')[0].strip()
        self.assertEqual("No users to show", page_title)

    def test_should_show_buyer_user(self, data_api_client):
        buyer = self.load_example_listing("user_response")
        buyer.pop('supplier', None)
        buyer['users']['role'] = 'buyer'
        data_api_client.get_user.return_value = buyer
        response = self.client.get('/admin/users?email_address=test.user@sme.com')
        self.assertEquals(response.status_code, 200)

        document = html.fromstring(response.get_data(as_text=True))

        email_address = document.xpath(
            '//header[@class="page-heading page-heading-without-breadcrumb"]//h1/text()')[0].strip()
        self.assertEqual("test.user@sme.com", email_address)

        name = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[0].strip()
        self.assertEqual("Test User", name)

        role = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[1].strip()
        self.assertEqual("buyer", role)

        supplier = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[2].strip()
        self.assertEquals('', supplier)

        last_login = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[3].strip()
        self.assertEquals('10:33:53', last_login)

        last_login_day = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[4].strip()
        self.assertEquals('23 July', last_login_day)

        last_password_changed = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[5].strip()
        self.assertEquals('13:46:01', last_password_changed)

        last_password_changed_day = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[6].strip()
        self.assertEquals('29 June', last_password_changed_day)

        locked = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[7].strip()
        self.assertEquals('No', locked)

        button = document.xpath(
            '//input[@class="button-destructive"]')[0].value
        self.assertEquals('Deactivate', button)

    def test_should_show_supplier_user(self, data_api_client):
        buyer = self.load_example_listing("user_response")
        data_api_client.get_user.return_value = buyer
        response = self.client.get('/admin/users?email_address=test.user@sme.com')
        self.assertEquals(response.status_code, 200)

        document = html.fromstring(response.get_data(as_text=True))

        email_address = document.xpath(
            '//header[@class="page-heading page-heading-without-breadcrumb"]//h1/text()')[0].strip()
        self.assertEqual("test.user@sme.com", email_address)

        role = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[1].strip()
        self.assertEqual("supplier", role)

        supplier = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/a/text()')[0].strip()
        self.assertEquals('SME Corp UK Limited', supplier)

        supplier_link = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/a')[0]
        self.assertEquals('/admin/suppliers/users?supplier_id=1000', supplier_link.attrib['href'])

    def test_should_show_unlock_button(self, data_api_client):
        buyer = self.load_example_listing("user_response")
        buyer['users']['locked'] = True

        data_api_client.get_user.return_value = buyer
        response = self.client.get('/admin/users?email_address=test.user@sme.com')
        self.assertEquals(response.status_code, 200)

        document = html.fromstring(response.get_data(as_text=True))

        unlock_button = document.xpath(
            '//input[@class="button-secondary"]')[0].attrib['value']
        unlock_link = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/form')[0]
        return_link = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/form/input')[1]
        self.assertEquals('/admin/suppliers/users/999/unlock', unlock_link.attrib['action'])
        self.assertEquals('Unlock', unlock_button)
        self.assertEquals('/admin/users?email_address=test.user%40sme.com', return_link.attrib['value'])

    def test_should_show_deactivate_button(self, data_api_client):
        buyer = self.load_example_listing("user_response")

        data_api_client.get_user.return_value = buyer
        response = self.client.get('/admin/users?email_address=test.user@sme.com')
        self.assertEquals(response.status_code, 200)

        document = html.fromstring(response.get_data(as_text=True))

        deactivate_button = document.xpath(
            '//input[@class="button-destructive"]')[0].attrib['value']
        deactivate_link = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/form')[0]
        return_link = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/form/input')[1]
        self.assertEquals('/admin/suppliers/users/999/deactivate', deactivate_link.attrib['action'])
        self.assertEquals('Deactivate', deactivate_button)
        self.assertEquals('/admin/users?email_address=test.user%40sme.com', return_link.attrib['value'])
<commit_msg>Add tests<commit_after>try:
    from urlparse import urlsplit
    from StringIO import StringIO
except ImportError:
    from urllib.parse import urlsplit
    from io import BytesIO as StringIO
import mock
from lxml import html
from ...helpers import LoggedInApplicationTest

@mock.patch('app.main.views.users.data_api_client')
class TestUsersView(LoggedInApplicationTest):

    def test_should_be_a_404_if_user_not_found(self, data_api_client):
        data_api_client.get_user.return_value = None
        response = self.client.get('/admin/users?email_address=some@email.com')
        self.assertEquals(response.status_code, 404)

        document = html.fromstring(response.get_data(as_text=True))

        page_title = document.xpath(
            '//p[@class="banner-message"]//text()')[0].strip()
        self.assertEqual("Sorry, we couldn't find an account with that email address", page_title)

        page_title = document.xpath(
            '//p[@class="summary-item-no-content"]//text()')[0].strip()
        self.assertEqual("No users to show", page_title)

    def test_should_be_a_404_if_no_email_provided(self, data_api_client):
        data_api_client.get_user.return_value = None
        response = self.client.get('/admin/users?email_address=')
        self.assertEquals(response.status_code, 404)

        document = html.fromstring(response.get_data(as_text=True))

        page_title = document.xpath(
            '//p[@class="banner-message"]//text()')[0].strip()
        self.assertEqual("Sorry, we couldn't find an account with that email address", page_title)

        page_title = document.xpath(
            '//p[@class="summary-item-no-content"]//text()')[0].strip()
        self.assertEqual("No users to show", page_title)

    def test_should_be_a_404_if_no_email_param_provided(self, data_api_client):
        data_api_client.get_user.return_value = None
        response = self.client.get('/admin/users')
        self.assertEquals(response.status_code, 404)

        document = html.fromstring(response.get_data(as_text=True))

        page_title = document.xpath(
            '//p[@class="banner-message"]//text()')[0].strip()
        self.assertEqual("Sorry, we couldn't find an account with that email address", page_title)

        page_title = document.xpath(
            '//p[@class="summary-item-no-content"]//text()')[0].strip()
        self.assertEqual("No users to show", page_title)

    def test_should_show_buyer_user(self, data_api_client):
        buyer = self.load_example_listing("user_response")
        buyer.pop('supplier', None)
        buyer['users']['role'] = 'buyer'
        data_api_client.get_user.return_value = buyer
        response = self.client.get('/admin/users?email_address=test.user@sme.com')
        self.assertEquals(response.status_code, 200)

        document = html.fromstring(response.get_data(as_text=True))

        email_address = document.xpath(
            '//header[@class="page-heading page-heading-without-breadcrumb"]//h1/text()')[0].strip()
        self.assertEqual("test.user@sme.com", email_address)

        name = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[0].strip()
        self.assertEqual("Test User", name)

        role = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[1].strip()
        self.assertEqual("buyer", role)

        supplier = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[2].strip()
        self.assertEquals('', supplier)

        last_login = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[3].strip()
        self.assertEquals('10:33:53', last_login)

        last_login_day = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[4].strip()
        self.assertEquals('23 July', last_login_day)

        last_password_changed = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[5].strip()
        self.assertEquals('13:46:01', last_password_changed)

        last_password_changed_day = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[6].strip()
        self.assertEquals('29 June', last_password_changed_day)

        locked = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[7].strip()
        self.assertEquals('No', locked)

        button = document.xpath(
            '//input[@class="button-destructive"]')[0].value
        self.assertEquals('Deactivate', button)

    def test_should_show_supplier_user(self, data_api_client):
        buyer = self.load_example_listing("user_response")
        data_api_client.get_user.return_value = buyer
        response = self.client.get('/admin/users?email_address=test.user@sme.com')
        self.assertEquals(response.status_code, 200)

        document = html.fromstring(response.get_data(as_text=True))

        email_address = document.xpath(
            '//header[@class="page-heading page-heading-without-breadcrumb"]//h1/text()')[0].strip()
        self.assertEqual("test.user@sme.com", email_address)

        role = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/text()')[1].strip()
        self.assertEqual("supplier", role)

        supplier = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/a/text()')[0].strip()
        self.assertEquals('SME Corp UK Limited', supplier)

        supplier_link = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/a')[0]
        self.assertEquals('/admin/suppliers/users?supplier_id=1000', supplier_link.attrib['href'])

    def test_should_show_unlock_button(self, data_api_client):
        buyer = self.load_example_listing("user_response")
        buyer['users']['locked'] = True

        data_api_client.get_user.return_value = buyer
        response = self.client.get('/admin/users?email_address=test.user@sme.com')
        self.assertEquals(response.status_code, 200)

        document = html.fromstring(response.get_data(as_text=True))

        unlock_button = document.xpath(
            '//input[@class="button-secondary"]')[0].attrib['value']
        unlock_link = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/form')[0]
        return_link = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/form/input')[1]
        self.assertEquals('/admin/suppliers/users/999/unlock', unlock_link.attrib['action'])
        self.assertEquals('Unlock', unlock_button)
        self.assertEquals('/admin/users?email_address=test.user%40sme.com', return_link.attrib['value'])

    def test_should_show_deactivate_button(self, data_api_client):
        buyer = self.load_example_listing("user_response")

        data_api_client.get_user.return_value = buyer
        response = self.client.get('/admin/users?email_address=test.user@sme.com')
        self.assertEquals(response.status_code, 200)

        document = html.fromstring(response.get_data(as_text=True))

        deactivate_button = document.xpath(
            '//input[@class="button-destructive"]')[0].attrib['value']
        deactivate_link = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/form')[0]
        return_link = document.xpath(
            '//tr[@class="summary-item-row"]//td/span/form/input')[1]
        self.assertEquals('/admin/suppliers/users/999/deactivate', deactivate_link.attrib['action'])
        self.assertEquals('Deactivate', deactivate_button)
        self.assertEquals('/admin/users?email_address=test.user%40sme.com', return_link.attrib['value'])

@mock.patch('app.main.views.users.data_api_client')
class TestUsersExport(LoggedInApplicationTest):
    _bad_statuses = ['coming', 'expired']

    _valid_framework = {
        'name': 'G-Cloud 7',
        'slug': 'g-cloud-7',
        'status': 'live'
    }

    _invalid_framework = {
        'name': 'G-Cloud 8',
        'slug': 'g-cloud-8',
        'status': 'coming'
    }

    def _return_get_user_export_response(self, data_api_client, frameworks):
        data_api_client.find_frameworks.return_value = {"frameworks": frameworks}
        return self.client.get('/admin/users/export')

    def _assert_things_about_frameworks(self, response, frameworks):

        def _assert_things_about_valid_frameworks(options, frameworks):
            valid_frameworks = [
                framework for framework in frameworks if framework['status'] not in self._bad_statuses]

            assert len(options) == len(valid_frameworks)
            for framework in valid_frameworks:
                assert framework['slug'] in [option.xpath('input')[0].attrib['value'] for option in options]
                assert framework['name'] in ["".join(option.xpath('text()')).strip() for option in options]

        def _assert_things_about_invalid_frameworks(options, frameworks):
            invalid_frameworks = [
                framework for framework in frameworks if framework['status'] in self._bad_statuses]

            for framework in invalid_frameworks:
                assert framework['slug'] not in [option.xpath('input')[0].attrib['value'] for option in options]
                assert framework['name'] not in ["".join(option.xpath('text()')).strip() for option in options]

        document = html.fromstring(response.get_data(as_text=True))

        options = document.xpath(
            '//fieldset[@id="framework_slug"]/label')

        assert response.status_code == 200
        _assert_things_about_valid_frameworks(options, frameworks)
        _assert_things_about_invalid_frameworks(options, frameworks)

    def _return_post_user_export_response(self, data_api_client, frameworks, users, framework_slug=None):
        # collection of users is modified in the route
        data_api_client.export_users.return_value = {"users": users.copy()}
        data_api_client.find_frameworks.return_value = {"frameworks": frameworks}

        if framework_slug is None:
            framework_slug = frameworks[0]['slug']

        return self.client.post(
            '/admin/users/export',
            data={'framework_slug': framework_slug}
        )

    def _assert_things_about_user_export(self, response, users):

        rows = [line.split(",") for line in response.get_data(as_text=True).splitlines()]

        assert len(rows) == len(users) + 1

        if users:
            common_header_values = set(users[0].keys()) & set(rows[0])
            assert len(common_header_values) == len(list(users[0].keys())) == len(rows[0])

            for index, user in enumerate(users):
                common_user_values = set(map(str, user.values())) & set(rows[index+1])
                assert len(common_user_values) == len(list(user.values())) == len(rows[index+1])

    ##########################################################################

    def test_get_form_with_valid_framework(self, data_api_client):
        frameworks = [self._valid_framework]
        response = self._return_get_user_export_response(data_api_client, frameworks)
        self._assert_things_about_frameworks(response, frameworks)

    def test_get_form_with_invalid_framework(self, data_api_client):
        frameworks = [self._invalid_framework]
        response = self._return_get_user_export_response(data_api_client, frameworks)
        assert self.strip_all_whitespace("No valid frameworks were found.") in \
            self.strip_all_whitespace(response.get_data(as_text=True))
        self._assert_things_about_frameworks(response, frameworks)

    def test_get_form_with_valid_and_framework(self, data_api_client):
        frameworks = [self._valid_framework, self._invalid_framework]
        response = self._return_get_user_export_response(data_api_client, frameworks)
        self._assert_things_about_frameworks(response, frameworks)

    def test_post_user_export_with_no_users(self, data_api_client):
        frameworks = [self._valid_framework]
        users = []

        response = self._return_post_user_export_response(data_api_client, frameworks, users)
        self._assert_things_about_user_export(response, users)

    def test_post_user_export_with_one_user(self, data_api_client):
        frameworks = [self._valid_framework]
        users = [{
            "application_result": "fail",
            "application_status": "no_application",
            "declaration_status": "unstarted",
            "framework_agreement": False,
            "supplier_id": 1,
            "user_email": "test.user@sme.com",
            "user_name": "Tess User"
        }]

        response = self._return_post_user_export_response(data_api_client, frameworks, users)
        self._assert_things_about_user_export(response, users)

    def test_post_user_export_with_two_users(self, data_api_client):
        frameworks = [self._valid_framework]
        users = [{
            "application_result": "fail",
            "application_status": "no_application",
            "declaration_status": "unstarted",
            "framework_agreement": False,
            "supplier_id": 1,
            "user_email": "test.user@sme.com",
            "user_name": "Tess User"
        }, {
            "application_result": "pass",
            "application_status": "application",
            "declaration_status": "complete",
            "framework_agreement": True,
            "supplier_id": 2,
            "user_email": "paul@paul.paul",
            "user_name": "Paul"
        }]

        response = self._return_post_user_export_response(data_api_client, frameworks, users)
        self._assert_things_about_user_export(response, users)

    def test_cannot_post_user_export_with_no_framework(self, data_api_client):
        frameworks = [self._valid_framework]
        framework_slug = ''
        users = []
        response = self._return_post_user_export_response(data_api_client, frameworks, users, framework_slug)
        assert self.strip_all_whitespace("A list of users cannot be generated unless you select a valid framework") in \
            self.strip_all_whitespace(response.get_data(as_text=True))

    def test_cannot_post_user_export_with_bad_framework_slug(self, data_api_client):
        frameworks = [self._valid_framework]
        framework_slug = 'all-the-frameworks-in-the-uk'
        users = []
        response = self._return_post_user_export_response(data_api_client, frameworks, users, framework_slug)
        assert self.strip_all_whitespace("A list of users cannot be generated unless you select a valid framework") in \
            self.strip_all_whitespace(response.get_data(as_text=True))
<|endoftext|>


<commit_before>from __future__ import print_function, unicode_literals

import sys
import json
import re
from datetime import datetime
from enum import Enum
from six import StringIO

from azure.cli.extensions import event_dispatcher

class OutputFormatException(Exception):
    pass

def format_json(obj):
    input_dict = obj.__dict__ if hasattr(obj, '__dict__') else obj
    return json.dumps(input_dict, indent=2, sort_keys=True, separators=(',', ': '))

def format_table(obj):
    obj_list = obj if isinstance(obj, list) else [obj]
    to = TableOutput()
    try:
        for item in obj_list:
            for item_key in sorted(item):
                to.cell(item_key, item[item_key])
            to.end_row()
        return to.dump()
    except TypeError:
        return ''

def format_text(obj):
    obj_list = obj if isinstance(obj, list) else [obj]
    to = TextOutput()
    try:
        for item in obj_list:
            for item_key in sorted(item):
                to.add(item_key, item[item_key])
        return to.dump()
    except TypeError:
        return ''

def format_list(obj):
    obj_list = obj if isinstance(obj, list) else [obj]
    lo = ListOutput()
    return lo.dump(obj_list)

class OutputProducer(object): #pylint: disable=too-few-public-methods

    format_dict = {
        'json': format_json,
        'table': format_table,
        'text': format_text,
        'list': format_list
    }

    KEYS_CAMELCASE_PATTERN = re.compile('(?!^)_([a-zA-Z])')

    def __init__(self, formatter=format_list, file=sys.stdout): #pylint: disable=redefined-builtin
        self.formatter = formatter
        self.file = file

    def out(self, obj):
        obj = OutputProducer.todict(obj)
        event_data = {'result': obj}
        event_dispatcher.raise_event(event_dispatcher.TRANSFORM_RESULT, event_data)
        event_dispatcher.raise_event(event_dispatcher.FILTER_RESULT, event_data)
        print(self.formatter(event_data['result']), file=self.file)

    @staticmethod
    def todict(obj): #pylint: disable=too-many-return-statements
        def to_camelcase(s):
            return re.sub(OutputProducer.KEYS_CAMELCASE_PATTERN, lambda x: x.group(1).upper(), s)

        if isinstance(obj, dict):
            return {k: OutputProducer.todict(v) for (k, v) in obj.items()}
        elif isinstance(obj, list):
            return [OutputProducer.todict(a) for a in obj]
        elif isinstance(obj, Enum):
            return obj.value
        elif isinstance(obj, datetime):
            return obj.isoformat()
        elif hasattr(obj, '_asdict'):
            return OutputProducer.todict(obj._asdict())
        elif hasattr(obj, '__dict__'):
            return dict([(to_camelcase(k), OutputProducer.todict(v))
                         for k, v in obj.__dict__.items()
                         if not callable(v) and not k.startswith('_')])
        else:
            return obj

    @staticmethod
    def get_formatter(format_type):
        return OutputProducer.format_dict.get(format_type, format_list)

class ListOutput(object): #pylint: disable=too-few-public-methods

    # Match the capital letters in a camel case string
    FORMAT_KEYS_PATTERN = re.compile('([A-Z][^A-Z]*)')

    def __init__(self):
        self._formatted_keys_cache = {}

    @staticmethod
    def _get_max_key_len(keys):
        return len(max(keys, key=len)) if keys else 0

    @staticmethod
    def _sort_key_func(key, item):
        # We want dictionaries to be last so use ASCII char 126 ~ to
        # prefix dictionary and list key names.
        if isinstance(item[key], dict):
            return '~~'+key
        elif isinstance(item[key], list):
            return '~'+key
        else:
            return key

    def _get_formatted_key(self, key):
        def _format_key(key):
            words = [word for word in re.split(ListOutput.FORMAT_KEYS_PATTERN, key) if word]
            return ' '.join(words).title()

        try:
            return self._formatted_keys_cache[key]
        except KeyError:
            self._formatted_keys_cache[key] = _format_key(key)
            return self._formatted_keys_cache[key]

    @staticmethod
    def _dump_line(io, line, indent):
        io.write('   ' * indent)
        io.write(line)
        io.write('\n')

    def _dump_object(self, io, obj, indent):
        if isinstance(obj, list):
            for array_item in obj:
                self._dump_object(io, array_item, indent)
        elif isinstance(obj, dict):
            obj_fk = {k: self._get_formatted_key(k) for k in obj}
            key_width = ListOutput._get_max_key_len(obj_fk.values())
            for key in sorted(obj, key=lambda x: ListOutput._sort_key_func(x, obj)):
                if isinstance(obj[key], dict) or isinstance(obj[key], list):
                    # complex object
                    line = '%s :' % (self._get_formatted_key(key).ljust(key_width))
                    ListOutput._dump_line(io, line, indent)
                    self._dump_object(io, obj[key] if obj[key] else 'None', indent+1)
                else:
                    # non-complex so write it
                    line = '%s : %s' % (self._get_formatted_key(key).ljust(key_width),
                                        'None' if obj[key] is None else obj[key])
                    ListOutput._dump_line(io, line, indent)
        else:
            ListOutput._dump_line(io, obj, indent)

    def dump(self, data):
        io = StringIO()
        for obj in data:
            self._dump_object(io, obj, 0)
            io.write('\n')
        result = io.getvalue()
        io.close()
        return result

class TableOutput(object):
    def __init__(self):
        self._rows = [{}]
        self._columns = {}
        self._column_order = []

    def dump(self):
        if len(self._rows) == 1:
            return

        io = StringIO()
        cols = [(c, self._columns[c]) for c in self._column_order]
        io.write(' | '.join(c.center(w) for c, w in cols))
        io.write('\n')
        io.write('-|-'.join('-' * w for c, w in cols))
        io.write('\n')
        for r in self._rows[:-1]:
            io.write(' | '.join(r.get(c, '-').ljust(w) for c, w in cols))
            io.write('\n')
        result = io.getvalue()
        io.close()
        return result

    @property
    def any_rows(self):
        return len(self._rows) > 1

    def cell(self, name, value):
        n = str(name)
        v = str(value)
        max_width = self._columns.get(n)
        if max_width is None:
            self._column_order.append(n)
            max_width = len(n)
        self._rows[-1][n] = v
        self._columns[n] = max(max_width, len(v))

    def end_row(self):
        self._rows.append({})

class TextOutput(object):

    def __init__(self):
        self.identifiers = {}

    def add(self, identifier, value):
        if identifier in self.identifiers:
            self.identifiers[identifier].append(value)
        else:
            self.identifiers[identifier] = [value]

    def dump(self):
        io = StringIO()
        for identifier in sorted(self.identifiers):
            io.write(identifier.upper())
            io.write('\t')
            for col in self.identifiers[identifier]:
                if isinstance(col, (list, dict)):
                    # TODO: Need to handle complex objects
                    io.write("null")
                else:
                    io.write(str(col))
                io.write('\t')
            io.write('\n')
        result = io.getvalue()
        io.close()
        return result

<commit_msg>Provide a simple tab separated output formatter<commit_after>from __future__ import print_function, unicode_literals

import sys
import json
import re
from datetime import datetime
from enum import Enum
from six import StringIO

from azure.cli.extensions import event_dispatcher

class OutputFormatException(Exception):
    pass

def format_json(obj):
    input_dict = obj.__dict__ if hasattr(obj, '__dict__') else obj
    return json.dumps(input_dict, indent=2, sort_keys=True, separators=(',', ': '))

def format_table(obj):
    obj_list = obj if isinstance(obj, list) else [obj]
    to = TableOutput()
    try:
        for item in obj_list:
            for item_key in sorted(item):
                to.cell(item_key, item[item_key])
            to.end_row()
        return to.dump()
    except TypeError:
        return ''

def format_text(obj):
    obj_list = obj if isinstance(obj, list) else [obj]
    to = TextOutput()
    try:
        for item in obj_list:
            for item_key in sorted(item):
                to.add(item_key, item[item_key])
        return to.dump()
    except TypeError:
        return ''

def format_list(obj):
    obj_list = obj if isinstance(obj, list) else [obj]
    lo = ListOutput()
    return lo.dump(obj_list)

def format_tsv(obj):
    obj_list = obj if isinstance(obj, list) else [obj]
    tsvo = TsvOutput()
    return tsvo.dump(obj_list)

class OutputProducer(object): #pylint: disable=too-few-public-methods

    format_dict = {
        'json': format_json,
        'table': format_table,
        'text': format_text,
        'list': format_list,
        'tsv': format_tsv,
    }

    KEYS_CAMELCASE_PATTERN = re.compile('(?!^)_([a-zA-Z])')

    def __init__(self, formatter=format_list, file=sys.stdout): #pylint: disable=redefined-builtin
        self.formatter = formatter
        self.file = file

    def out(self, obj):
        obj = OutputProducer.todict(obj)
        event_data = {'result': obj}
        event_dispatcher.raise_event(event_dispatcher.TRANSFORM_RESULT, event_data)
        event_dispatcher.raise_event(event_dispatcher.FILTER_RESULT, event_data)
        print(self.formatter(event_data['result']), file=self.file)

    @staticmethod
    def todict(obj): #pylint: disable=too-many-return-statements
        def to_camelcase(s):
            return re.sub(OutputProducer.KEYS_CAMELCASE_PATTERN, lambda x: x.group(1).upper(), s)

        if isinstance(obj, dict):
            return {k: OutputProducer.todict(v) for (k, v) in obj.items()}
        elif isinstance(obj, list):
            return [OutputProducer.todict(a) for a in obj]
        elif isinstance(obj, Enum):
            return obj.value
        elif isinstance(obj, datetime):
            return obj.isoformat()
        elif hasattr(obj, '_asdict'):
            return OutputProducer.todict(obj._asdict())
        elif hasattr(obj, '__dict__'):
            return dict([(to_camelcase(k), OutputProducer.todict(v))
                         for k, v in obj.__dict__.items()
                         if not callable(v) and not k.startswith('_')])
        else:
            return obj

    @staticmethod
    def get_formatter(format_type):
        return OutputProducer.format_dict.get(format_type, format_list)

class ListOutput(object): #pylint: disable=too-few-public-methods

    # Match the capital letters in a camel case string
    FORMAT_KEYS_PATTERN = re.compile('([A-Z][^A-Z]*)')

    def __init__(self):
        self._formatted_keys_cache = {}

    @staticmethod
    def _get_max_key_len(keys):
        return len(max(keys, key=len)) if keys else 0

    @staticmethod
    def _sort_key_func(key, item):
        # We want dictionaries to be last so use ASCII char 126 ~ to
        # prefix dictionary and list key names.
        if isinstance(item[key], dict):
            return '~~'+key
        elif isinstance(item[key], list):
            return '~'+key
        else:
            return key

    def _get_formatted_key(self, key):
        def _format_key(key):
            words = [word for word in re.split(ListOutput.FORMAT_KEYS_PATTERN, key) if word]
            return ' '.join(words).title()

        try:
            return self._formatted_keys_cache[key]
        except KeyError:
            self._formatted_keys_cache[key] = _format_key(key)
            return self._formatted_keys_cache[key]

    @staticmethod
    def _dump_line(io, line, indent):
        io.write('   ' * indent)
        io.write(line)
        io.write('\n')

    def _dump_object(self, io, obj, indent):
        if isinstance(obj, list):
            for array_item in obj:
                self._dump_object(io, array_item, indent)
        elif isinstance(obj, dict):
            obj_fk = {k: self._get_formatted_key(k) for k in obj}
            key_width = ListOutput._get_max_key_len(obj_fk.values())
            for key in sorted(obj, key=lambda x: ListOutput._sort_key_func(x, obj)):
                if isinstance(obj[key], dict) or isinstance(obj[key], list):
                    # complex object
                    line = '%s :' % (self._get_formatted_key(key).ljust(key_width))
                    ListOutput._dump_line(io, line, indent)
                    self._dump_object(io, obj[key] if obj[key] else 'None', indent+1)
                else:
                    # non-complex so write it
                    line = '%s : %s' % (self._get_formatted_key(key).ljust(key_width),
                                        'None' if obj[key] is None else obj[key])
                    ListOutput._dump_line(io, line, indent)
        else:
            ListOutput._dump_line(io, obj, indent)

    def dump(self, data):
        io = StringIO()
        for obj in data:
            self._dump_object(io, obj, 0)
            io.write('\n')
        result = io.getvalue()
        io.close()
        return result

class TableOutput(object):
    def __init__(self):
        self._rows = [{}]
        self._columns = {}
        self._column_order = []

    def dump(self):
        if len(self._rows) == 1:
            return

        io = StringIO()
        cols = [(c, self._columns[c]) for c in self._column_order]
        io.write(' | '.join(c.center(w) for c, w in cols))
        io.write('\n')
        io.write('-|-'.join('-' * w for c, w in cols))
        io.write('\n')
        for r in self._rows[:-1]:
            io.write(' | '.join(r.get(c, '-').ljust(w) for c, w in cols))
            io.write('\n')
        result = io.getvalue()
        io.close()
        return result

    @property
    def any_rows(self):
        return len(self._rows) > 1

    def cell(self, name, value):
        n = str(name)
        v = str(value)
        max_width = self._columns.get(n)
        if max_width is None:
            self._column_order.append(n)
            max_width = len(n)
        self._rows[-1][n] = v
        self._columns[n] = max(max_width, len(v))

    def end_row(self):
        self._rows.append({})

class TextOutput(object):

    def __init__(self):
        self.identifiers = {}

    def add(self, identifier, value):
        if identifier in self.identifiers:
            self.identifiers[identifier].append(value)
        else:
            self.identifiers[identifier] = [value]

    def dump(self):
        io = StringIO()
        for identifier in sorted(self.identifiers):
            io.write(identifier.upper())
            io.write('\t')
            for col in self.identifiers[identifier]:
                if isinstance(col, (list, dict)):
                    # TODO: Need to handle complex objects
                    io.write("null")
                else:
                    io.write(str(col))
                io.write('\t')
            io.write('\n')
        result = io.getvalue()
        io.close()
        return result

class TsvOutput(object):

    def _dump_obj(self, data, stream):
        if isinstance(data, list):
            stream.write(str(len(data)))
        elif isinstance(data, dict):
            pass
        else:
            stream.write(data)

    def _dump_row(self, data, stream):
        if isinstance(data, dict):
            separator = ''
            for _, value in data.items():
                stream.write(separator)
                self._dump_obj(value, stream)
                separator = '\t'
        else:
            self._dump_obj(data, stream)

    def dump(self, data):
        io = StringIO()
        for item in data:
            self._dump_row(item, io)
            io.write('\n')

        result = io.getvalue()
        io.close()
        return result
<|endoftext|>


<commit_before># pylint: disable=global-statement
import dash
from dash.dependencies import Input, Output, State
import dash_html_components as html
import os
import pandas as pd
import sys

sys.path.append(
    os.path.abspath(
        os.path.join(os.path.dirname(sys.argv[0]), os.pardir, os.pardir, os.pardir)
    )
)
module_names = ["dash_table"]
modules = [__import__(x) for x in module_names]
dash_table = modules[0]

url = "https://github.com/plotly/datasets/raw/master/" "26k-consumer-complaints.csv"
df = pd.read_csv(url)
df = df.values

app = dash.Dash()
app.css.config.serve_locally = True
app.scripts.config.serve_locally = True

app.layout = html.Div(
    [
        html.Div(id="container", children="Hello World"),
        dash_table.DataTable(
            id="table",
            data=df[0:250],
            columns=[
                {"id": 0, "name": "Complaint ID"},
                {"id": 1, "name": "Product"},
                {"id": 2, "name": "Sub-product"},
                {"id": 3, "name": "Issue"},
                {"id": 4, "name": "Sub-issue"},
                {"id": 5, "name": "State"},
                {"id": 6, "name": "ZIP"},
                {"id": 7, "name": "code"},
                {"id": 8, "name": "Date received"},
                {"id": 9, "name": "Date sent to company"},
                {"id": 10, "name": "Company"},
                {"id": 11, "name": "Company response"},
                {"id": 12, "name": "Timely response?"},
                {"id": 13, "name": "Consumer disputed?"},
            ],
            editable=True,
            sort_action='native',
        ),
        dash_table.DataTable(
            id="table2",
            data=df[0:10],
            columns=[
                {"id": 0, "name": "Complaint ID"},
                {"id": 1, "name": "Product"},
                {"id": 2, "name": "Sub-product"},
                {"id": 3, "name": "Issue"},
                {"id": 4, "name": "Sub-issue"},
                {"id": 5, "name": "State"},
                {"id": 6, "name": "ZIP"},
                {"id": 7, "name": "code"},
                {"id": 8, "name": "Date received"},
                {"id": 9, "name": "Date sent to company"},
                {"id": 10, "name": "Company"},
                {"id": 11, "name": "Company response"},
                {"id": 12, "name": "Timely response?"},
                {"id": 13, "name": "Consumer disputed?"},
            ],
            editable=True,
            sort_action='native',
        ),
    ]
)

@app.callback(
    Output("table", "data"),
    [Input("table", "data_timestamp")],
    [State("table", "data"), State("table", "data_previous")],
)
# pylint: disable=unused-argument
def updateData(timestamp, current, previous):
    # pylint: enable=unused-argument
    if current is None or previous is None:
        return current

    if len(current) == len(previous):
        for (i, datum) in enumerate(current):
            previous_datum = previous[i]
            if datum[0] != previous_datum[0]:
                datum[1] = "MODIFIED"

    return current

if __name__ == "__main__":
    app.run_server(port=8082, debug=False)
<commit_msg>hideable options<commit_after># pylint: disable=global-statement
import dash
from dash.dependencies import Input, Output, State
import dash_html_components as html
import os
import pandas as pd
import sys

sys.path.append(
    os.path.abspath(
        os.path.join(os.path.dirname(sys.argv[0]), os.pardir, os.pardir, os.pardir)
    )
)
module_names = ["dash_table"]
modules = [__import__(x) for x in module_names]
dash_table = modules[0]

url = "https://github.com/plotly/datasets/raw/master/" "26k-consumer-complaints.csv"
df = pd.read_csv(url)
df = df.values

app = dash.Dash()
app.css.config.serve_locally = True
app.scripts.config.serve_locally = True

app.layout = html.Div(
    [
        html.Div(id="container", children="Hello World"),
        dash_table.DataTable(
            id="table",
            data=df[0:250],
            columns=[
                {"id": 0, "name": "Complaint ID", "hideable": True},
                {"id": 1, "name": "Product"},
                {"id": 2, "name": "Sub-product"},
                {"id": 3, "name": "Issue"},
                {"id": 4, "name": "Sub-issue"},
                {"id": 5, "name": "State"},
                {"id": 6, "name": "ZIP"},
                {"id": 7, "name": "code"},
                {"id": 8, "name": "Date received"},
                {"id": 9, "name": "Date sent to company"},
                {"id": 10, "name": "Company"},
                {"id": 11, "name": "Company response"},
                {"id": 12, "name": "Timely response?"},
                {"id": 13, "name": "Consumer disputed?"},
            ],
            editable=True,
            sort_action='native',
            include_headers_on_copy_paste=True,

        ),
        dash_table.DataTable(
            id="table2",
            data=df[0:10],
            columns=[
                {"id": 0, "name": "Complaint ID", "hideable": True},
                {"id": 1, "name": "Product"},
                {"id": 2, "name": "Sub-product"},
                {"id": 3, "name": "Issue"},
                {"id": 4, "name": "Sub-issue"},
                {"id": 5, "name": "State"},
                {"id": 6, "name": "ZIP"},
                {"id": 7, "name": "code"},
                {"id": 8, "name": "Date received"},
                {"id": 9, "name": "Date sent to company"},
                {"id": 10, "name": "Company"},
                {"id": 11, "name": "Company response"},
                {"id": 12, "name": "Timely response?"},
                {"id": 13, "name": "Consumer disputed?"},
            ],
            editable=True,
            sort_action='native',
            include_headers_on_copy_paste=True,
        ),
    ]
)

@app.callback(
    Output("table", "data"),
    [Input("table", "data_timestamp")],
    [State("table", "data"), State("table", "data_previous")],
)
# pylint: disable=unused-argument
def updateData(timestamp, current, previous):
    # pylint: enable=unused-argument
    if current is None or previous is None:
        return current

    if len(current) == len(previous):
        for (i, datum) in enumerate(current):
            previous_datum = previous[i]
            if datum[0] != previous_datum[0]:
                datum[1] = "MODIFIED"

    return current

if __name__ == "__main__":
    app.run_server(port=8082, debug=False)
<|endoftext|>