<commit_before>import logging
import copy
import ibmsecurity.utilities.tools
from ibmsecurity.utilities import tools

logger = logging.getLogger(__name__)

uri = "/isam/dsc/config²"
requires_modules = None
requires_version = None

def get(isamAppliance, check_mode=False, force=False):
    """
    Retrieve the current cluster configuration
    """
    return isamAppliance.invoke_get("Retrieve the current dsc configuration", uri,
                                    requires_modules=requires_modules, requires_version=requires_version)

def set(isamAppliance, service_port=443, replication_port=444, worker_threads=64, max_session_lifetime=3600, client_grace=600,
        servers=None):
    """
    Set DSC configuration
    """
    warnings = []
    # Create a simple json with just the main client attributes
    dsc_json = {
        "worker_threads": worker_threads,
        "max_session_lifetime": maximum_session_lifetime,
        "client_grace": client_grace,
        "service_port": service_port,
        "replication_port": replication_port
    }
    if servers is not None:
        dsc_json["servers@"]=servers

    if force is True or _check(isamAppliance, dsc_json, ignore_password_for_idempotency) is False:
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True, warnings=warnings)
        else:
            return isamAppliance.invoke_post("Set dsc configuration", uri, _json,
                                             requires_modules=requires_modules, requires_version=requires_version,
                                             warnings=warnings)

    return isamAppliance.create_return_object(warnings=warnings)

def _check(isamAppliance, cluster_json, ignore_password_for_idempotency):
    """
    Check if provided json values match the configuration on appliance

    :param isamAppliance:
    :param cluster_json:
    :return:
    """
    ret_obj = get(isamAppliance)
    sorted_ret_obj = tools.json_sort(ret_obj['data'])
    sorted_json_data = tools.json_sort(cluster_json)
    logger.debug("Sorted Existing Data:{0}".format(sorted_ret_obj))
    logger.debug("Sorted Desired  Data:{0}".format(sorted_json_data))

    if ignore_password_for_idempotency:
        temp = copy.deepcopy(
            cluster_json)  # deep copy neccessary: otherwise password parameter would be removed from desired config dict 'cluster_json'. Comparison is done with temp<>ret_obj object
        for idx, x in enumerate(cluster_json):
            if "password" in x:
                logger.debug("Ignoring JSON password entry: '{0}' to satisfy idempotency.".format(x))
                del temp[x]
        logger.debug("Passwordless JSON to Apply: {0}".format(temp))
    else:
        temp = cluster_json

    for key, value in temp.iteritems():
        try:
            if isinstance(value, list):
                if ibmsecurity.utilities.tools.json_sort(
                        ret_obj['data'][key] != ibmsecurity.utilities.tools.json_sort(value)):
                    logger.debug(
                        "For key: {0}, values: {1} and {2} do not match.".format(key, value, ret_obj['data'][key]))
                    return False
            else:
                if ret_obj['data'][key] != value:
                    logger.debug(
                        "For key: {0}, values: {1} and {2} do not match.".format(key, value, ret_obj['data'][key]))
                    return False
        except:  # In case there is an error looking up the key in existing configuration (missing)
            logger.debug("Exception processing Key: {0} Value: {1} - missing key in current config?".format(key, value))
            return False

    logger.debug("JSON provided already is contained in current appliance configuration.")
    return True

def compare(isamAppliance1, isamAppliance2):
    """
    Compare cluster configuration between two appliances
    """
    ret_obj1 = get(isamAppliance1)
    ret_obj2 = get(isamAppliance2)

    return ibmsecurity.utilities.tools.json_compare(ret_obj1, ret_obj2, deleted_keys=[])
<commit_msg>add force arg<commit_after>import logging
import copy
import ibmsecurity.utilities.tools
from ibmsecurity.utilities import tools

logger = logging.getLogger(__name__)

uri = "/isam/dsc/config²"
requires_modules = None
requires_version = None

def get(isamAppliance, check_mode=False, force=False):
    """
    Retrieve the current cluster configuration
    """
    return isamAppliance.invoke_get("Retrieve the current dsc configuration", uri,
                                    requires_modules=requires_modules, requires_version=requires_version)

def set(isamAppliance, service_port=443, replication_port=444, worker_threads=64, max_session_lifetime=3600, client_grace=600,
        servers=None, force=False):
    """
    Set DSC configuration
    """
    warnings = []
    # Create a simple json with just the main client attributes
    dsc_json = {
        "worker_threads": worker_threads,
        "max_session_lifetime": maximum_session_lifetime,
        "client_grace": client_grace,
        "service_port": service_port,
        "replication_port": replication_port
    }
    if servers is not None:
        dsc_json["servers@"]=servers

    if force is True or _check(isamAppliance, dsc_json, ignore_password_for_idempotency) is False:
        if check_mode is True:
            return isamAppliance.create_return_object(changed=True, warnings=warnings)
        else:
            return isamAppliance.invoke_post("Set dsc configuration", uri, _json,
                                             requires_modules=requires_modules, requires_version=requires_version,
                                             warnings=warnings)

    return isamAppliance.create_return_object(warnings=warnings)

def _check(isamAppliance, cluster_json, ignore_password_for_idempotency):
    """
    Check if provided json values match the configuration on appliance

    :param isamAppliance:
    :param cluster_json:
    :return:
    """
    ret_obj = get(isamAppliance)
    sorted_ret_obj = tools.json_sort(ret_obj['data'])
    sorted_json_data = tools.json_sort(cluster_json)
    logger.debug("Sorted Existing Data:{0}".format(sorted_ret_obj))
    logger.debug("Sorted Desired  Data:{0}".format(sorted_json_data))

    if ignore_password_for_idempotency:
        temp = copy.deepcopy(
            cluster_json)  # deep copy neccessary: otherwise password parameter would be removed from desired config dict 'cluster_json'. Comparison is done with temp<>ret_obj object
        for idx, x in enumerate(cluster_json):
            if "password" in x:
                logger.debug("Ignoring JSON password entry: '{0}' to satisfy idempotency.".format(x))
                del temp[x]
        logger.debug("Passwordless JSON to Apply: {0}".format(temp))
    else:
        temp = cluster_json

    for key, value in temp.iteritems():
        try:
            if isinstance(value, list):
                if ibmsecurity.utilities.tools.json_sort(
                        ret_obj['data'][key] != ibmsecurity.utilities.tools.json_sort(value)):
                    logger.debug(
                        "For key: {0}, values: {1} and {2} do not match.".format(key, value, ret_obj['data'][key]))
                    return False
            else:
                if ret_obj['data'][key] != value:
                    logger.debug(
                        "For key: {0}, values: {1} and {2} do not match.".format(key, value, ret_obj['data'][key]))
                    return False
        except:  # In case there is an error looking up the key in existing configuration (missing)
            logger.debug("Exception processing Key: {0} Value: {1} - missing key in current config?".format(key, value))
            return False

    logger.debug("JSON provided already is contained in current appliance configuration.")
    return True

def compare(isamAppliance1, isamAppliance2):
    """
    Compare cluster configuration between two appliances
    """
    ret_obj1 = get(isamAppliance1)
    ret_obj2 = get(isamAppliance2)

    return ibmsecurity.utilities.tools.json_compare(ret_obj1, ret_obj2, deleted_keys=[])
<|endoftext|>


<commit_before>import warnings

from fluent_contents.admin import PlaceholderEditorAdmin
from fluent_contents.models import PlaceholderData
import logging

from icekit.admin_tools.previews import RawIdPreviewAdminMixin
from icekit.admin_tools.utils import admin_link
from icekit.utils.attributes import resolve

from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from django.contrib import admin
try:
    import json
except ImportError:
    from django.utils import simplejson as json

logger = logging.getLogger(__name__)

class FluentLayoutsMixin(PlaceholderEditorAdmin, RawIdPreviewAdminMixin):
    """
    Mixin class for models that have a ``layout`` field and fluent content.
    """

    change_form_template = 'admin/fluent_layouts_change_form.html'

    class Media:
        js = ('admin/js/fluent_layouts.js', )

    def formfield_for_foreignkey(self, db_field, *args, **kwargs):
        """
        Update queryset for ``layout`` field.
        """
        formfield = super(FluentLayoutsMixin, self).formfield_for_foreignkey(
            db_field, *args, **kwargs)
        if db_field.name == 'layout':
            formfield.queryset = formfield.queryset.for_model(self.model)
        return formfield

    def get_placeholder_data(self, request, obj):
        """
        Get placeholder data from layout.
        """
        if not obj or not obj.layout:
            data = [PlaceholderData(slot='main', role='m', title='Main')]
        else:
            data = obj.layout.get_placeholder_data()
        return data

class HeroMixinAdmin(RawIdPreviewAdminMixin):
    raw_id_fields = ('hero_image',)
    preview_fields = ('hero_image',)

    # Alas, we cannot use 'fieldsets' as it causes a recursionerror on
    # (polymorphic?) admins that use base_fieldsets.
    FIELDSETS = (
        ('Hero section', {
            'fields': (
                'hero_image',
            )
        }),
    )

class ListableMixinAdmin(admin.ModelAdmin):
    FIELDSETS = (
        ('Advanced listing options', {
            'classes': ('collapse',),
            'fields': (
                'list_image', # a filefield
                'boosted_search_terms',
            )
        }),
    )

class PreviewAdminMixin(object):
    """
    Shortcut for displaying a preview in a changelist (or inline).

    Override `preview` to control how the preview shows.

    Plays nicely with list_display_links and admin_link if you want a click-able
    preview.

    Add `'preview'` to `list_display` or `readonly_fields`, etc to
    display.
    """

    def preview(self, obj, request=None):
        if hasattr(obj, 'preview'):
            return obj.preview(request)
        else:
            return unicode(obj)
        admin_preview.allow_tags = True

    def admin_preview_link(self, obj, request=None):
        return admin_link(obj, inner_html=self.preview(obj, request))

    # TODO: this doesn't seem to run (maybe needs to extend modeladmin?)
    # def get_list_display_links(self):
    #     # avoid needing to add 'preview' to list_display_links
    #     default = set( super(PreviewAdminMixin, self).get_list_display_links() )
    #     default.add('preview')
    #     return list(default)

class ThumbnailAdminMixin(PreviewAdminMixin):
    """
    Shortcut for displaying a thumbnail in a changelist (or inline).

    Requires easy-thumbnails.

    Specify ImageField name in `thumbnail_field`, and optionally
    override `thumbnail_options` for customisation such as sizing,
    cropping, etc. If `thumbnail_show_exceptions` is truthy, exception
    messages are returned in place of a thumbnail on failure.
    """

    thumbnail_field = None # an attribute or callable on the object
    try:
        thumbnail_options = settings.THUMBNAIL_ALIASES['']['admin']
    except (KeyError, AttributeError):
        thumbnail_options = {'size': (150, 150),}
    thumbnail_show_exceptions = False

    def get_thumbnail_source(self, obj):
        """
        Obtains the source image field for the thumbnail.

        :param obj: An object with a thumbnail_field defined.
        :return: Image field for thumbnail or None if not found.
        """
        if hasattr(self, 'thumbnail_field') and self.thumbnail_field:
            return resolve(obj, self.thumbnail_field)

        # try get_list_image, from ListableMixin
        if hasattr(obj, 'get_list_image'):
            return resolve(obj, "get_list_image")

        logger.warning('ThumbnailAdminMixin.thumbnail_field unspecified')
        return None

    def preview(self, obj, request=None):
        """
        Generate the HTML to display for the image.

        :param obj: An object with a thumbnail_field defined.
        :return: HTML for image display.
        """
        source = self.get_thumbnail_source(obj)
        if source:
            try:
                from easy_thumbnails.files import get_thumbnailer
            except ImportError:
                logger.warning(
                    _(
                        '`easy_thumbnails` is not installed and required for '
                        'icekit.utils.admin.mixins.ThumbnailAdminMixin'
                    )
                )
                return ''
            try:
                thumbnailer = get_thumbnailer(source)
                thumbnail = thumbnailer.get_thumbnail(self.thumbnail_options)
                return ''.format(
                    thumbnail.url)
            except Exception as ex:
                logger.warning(
                    _(u'`easy_thumbnails` failed to generate a thumbnail image'
                      u' for {0}'.format(source)))
                if self.thumbnail_show_exceptions:
                    return 'Thumbnail exception: {0}'.format(ex)
        return ''
    preview.allow_tags = True

    def thumbnail(self, *args, **kwargs):
        warnings.warn(
            "Use of 'thumbnail' for showing previews in ThumbnailAdminMixin is "
            "deprecated. Use `preview` instead",
            DeprecationWarning,
            stacklevel=2
        )
        return self.preview(*args, **kwargs)
    thumbnail.allow_tags = True

<commit_msg>More robust code for getting placeholders on models with no ‘layout’ field.<commit_after>import warnings

from fluent_contents.admin import PlaceholderEditorAdmin
from fluent_contents.models import PlaceholderData
import logging

from icekit.admin_tools.previews import RawIdPreviewAdminMixin
from icekit.admin_tools.utils import admin_link
from icekit.utils.attributes import resolve

from django.utils.translation import ugettext_lazy as _
from django.conf import settings
from django.contrib import admin
try:
    import json
except ImportError:
    from django.utils import simplejson as json

logger = logging.getLogger(__name__)

class FluentLayoutsMixin(PlaceholderEditorAdmin, RawIdPreviewAdminMixin):
    """
    Mixin class for models that have a ``layout`` field and fluent content.
    """

    change_form_template = 'admin/fluent_layouts_change_form.html'

    class Media:
        js = ('admin/js/fluent_layouts.js', )

    def formfield_for_foreignkey(self, db_field, *args, **kwargs):
        """
        Update queryset for ``layout`` field.
        """
        formfield = super(FluentLayoutsMixin, self).formfield_for_foreignkey(
            db_field, *args, **kwargs)
        if db_field.name == 'layout':
            formfield.queryset = formfield.queryset.for_model(self.model)
        return formfield

    def get_placeholder_data(self, request, obj):
        """
        Get placeholder data from layout.
        """
        if not obj or not getattr(obj, 'layout', None):
            data = [PlaceholderData(slot='main', role='m', title='Main')]
        else:
            data = obj.layout.get_placeholder_data()
        return data

class HeroMixinAdmin(RawIdPreviewAdminMixin):
    raw_id_fields = ('hero_image',)
    preview_fields = ('hero_image',)

    # Alas, we cannot use 'fieldsets' as it causes a recursionerror on
    # (polymorphic?) admins that use base_fieldsets.
    FIELDSETS = (
        ('Hero section', {
            'fields': (
                'hero_image',
            )
        }),
    )

class ListableMixinAdmin(admin.ModelAdmin):
    FIELDSETS = (
        ('Advanced listing options', {
            'classes': ('collapse',),
            'fields': (
                'list_image', # a filefield
                'boosted_search_terms',
            )
        }),
    )

class PreviewAdminMixin(object):
    """
    Shortcut for displaying a preview in a changelist (or inline).

    Override `preview` to control how the preview shows.

    Plays nicely with list_display_links and admin_link if you want a click-able
    preview.

    Add `'preview'` to `list_display` or `readonly_fields`, etc to
    display.
    """

    def preview(self, obj, request=None):
        if hasattr(obj, 'preview'):
            return obj.preview(request)
        else:
            return unicode(obj)
        admin_preview.allow_tags = True

    def admin_preview_link(self, obj, request=None):
        return admin_link(obj, inner_html=self.preview(obj, request))

    # TODO: this doesn't seem to run (maybe needs to extend modeladmin?)
    # def get_list_display_links(self):
    #     # avoid needing to add 'preview' to list_display_links
    #     default = set( super(PreviewAdminMixin, self).get_list_display_links() )
    #     default.add('preview')
    #     return list(default)

class ThumbnailAdminMixin(PreviewAdminMixin):
    """
    Shortcut for displaying a thumbnail in a changelist (or inline).

    Requires easy-thumbnails.

    Specify ImageField name in `thumbnail_field`, and optionally
    override `thumbnail_options` for customisation such as sizing,
    cropping, etc. If `thumbnail_show_exceptions` is truthy, exception
    messages are returned in place of a thumbnail on failure.
    """

    thumbnail_field = None # an attribute or callable on the object
    try:
        thumbnail_options = settings.THUMBNAIL_ALIASES['']['admin']
    except (KeyError, AttributeError):
        thumbnail_options = {'size': (150, 150),}
    thumbnail_show_exceptions = False

    def get_thumbnail_source(self, obj):
        """
        Obtains the source image field for the thumbnail.

        :param obj: An object with a thumbnail_field defined.
        :return: Image field for thumbnail or None if not found.
        """
        if hasattr(self, 'thumbnail_field') and self.thumbnail_field:
            return resolve(obj, self.thumbnail_field)

        # try get_list_image, from ListableMixin
        if hasattr(obj, 'get_list_image'):
            return resolve(obj, "get_list_image")

        logger.warning('ThumbnailAdminMixin.thumbnail_field unspecified')
        return None

    def preview(self, obj, request=None):
        """
        Generate the HTML to display for the image.

        :param obj: An object with a thumbnail_field defined.
        :return: HTML for image display.
        """
        source = self.get_thumbnail_source(obj)
        if source:
            try:
                from easy_thumbnails.files import get_thumbnailer
            except ImportError:
                logger.warning(
                    _(
                        '`easy_thumbnails` is not installed and required for '
                        'icekit.utils.admin.mixins.ThumbnailAdminMixin'
                    )
                )
                return ''
            try:
                thumbnailer = get_thumbnailer(source)
                thumbnail = thumbnailer.get_thumbnail(self.thumbnail_options)
                return ''.format(
                    thumbnail.url)
            except Exception as ex:
                logger.warning(
                    _(u'`easy_thumbnails` failed to generate a thumbnail image'
                      u' for {0}'.format(source)))
                if self.thumbnail_show_exceptions:
                    return 'Thumbnail exception: {0}'.format(ex)
        return ''
    preview.allow_tags = True

    def thumbnail(self, *args, **kwargs):
        warnings.warn(
            "Use of 'thumbnail' for showing previews in ThumbnailAdminMixin is "
            "deprecated. Use `preview` instead",
            DeprecationWarning,
            stacklevel=2
        )
        return self.preview(*args, **kwargs)
    thumbnail.allow_tags = True

<|endoftext|>


<commit_before>import unittest
import time
from webtest import TestApp

from ichnaea import main
from ichnaea.worker import _get_db

class TestWorker(unittest.TestCase):

    def test_async_measure(self):
        try:
            import redis        # NOQA
        except ImportError:
            return

        global_config = {}
        app = main(global_config, celldb='sqlite://', measuredb='sqlite://',
                   async=True, batch_size=10, **{'redis.host': '127.0.0.1',
                                                 'redis.port': '-1'})
        app = TestApp(app)

        cell_data = [{"mcc": 123, "mnc": 1, "lac": 2, "cid": 1234}]

        from redis.client import BasePipeline

        old = BasePipeline.execute
        called = [0]

        def _execute(*args, **kw):
            called[0] += 1

        BasePipeline.execute = _execute

        try:
            for i in range(21):
                res = app.post_json('/v1/submit', {"items": [{"lat": 12.345678,
                                                   "lon": 23.456789,
                                                   "cell": cell_data}]},
                                    status=204)
        finally:
            BasePipeline.execute = old

        self.assertEqual(res.body, '')

        # 2 batches of 10
        self.assertEqual(called[0], 2)

    def test_async_old_batch(self):
        try:
            import redis        # NOQA
        except ImportError:
            return

        global_config = {}
        app = main(global_config, celldb='sqlite://', measuredb='sqlite://',
                   async=True, batch_age=.5,
                   batch_size=10, **{'redis.host': '127.0.0.1',
                                     'redis.port': '-1'})
        app = TestApp(app)

        cell_data = [{"mcc": 123, "mnc": 1, "lac": 2, "cid": 1234}]

        from redis.client import BasePipeline

        old = BasePipeline.execute
        called = [0]

        def _execute(*args, **kw):
            called[0] += 1

        BasePipeline.execute = _execute

        try:
            for i in range(4):
                res = app.post_json('/v1/submit', {"items": [{"lat": 12.345678,
                                                   "lon": 23.456789,
                                                   "cell": cell_data}]},
                                    status=204)
                time.sleep(.3)
        finally:
            BasePipeline.execute = old

        self.assertEqual(res.body, '')

        # 1 batches of 2
        self.assertEqual(called[0], 1)

    def test_get_db(self):
        db = _get_db('sqlite:///:memory:')
        self.assertTrue(db is _get_db('sqlite:///:memory:'))
<commit_msg>always run the async worker tests<commit_after>import unittest
import time
from webtest import TestApp

from ichnaea import main
from ichnaea.worker import _get_db

class TestWorker(unittest.TestCase):

    def test_async_measure(self):
        global_config = {}
        app = main(global_config, celldb='sqlite://', measuredb='sqlite://',
                   async=True, batch_size=10, **{'redis.host': '127.0.0.1',
                                                 'redis.port': '-1'})
        app = TestApp(app)

        cell_data = [{"mcc": 123, "mnc": 1, "lac": 2, "cid": 1234}]

        from redis.client import BasePipeline

        old = BasePipeline.execute
        called = [0]

        def _execute(*args, **kw):
            called[0] += 1

        BasePipeline.execute = _execute

        try:
            for i in range(21):
                res = app.post_json('/v1/submit', {"items": [{"lat": 12.345678,
                                                   "lon": 23.456789,
                                                   "cell": cell_data}]},
                                    status=204)
        finally:
            BasePipeline.execute = old

        self.assertEqual(res.body, '')

        # 2 batches of 10
        self.assertEqual(called[0], 2)

    def test_async_old_batch(self):
        global_config = {}
        app = main(global_config, celldb='sqlite://', measuredb='sqlite://',
                   async=True, batch_age=.5,
                   batch_size=10, **{'redis.host': '127.0.0.1',
                                     'redis.port': '-1'})
        app = TestApp(app)

        cell_data = [{"mcc": 123, "mnc": 1, "lac": 2, "cid": 1234}]

        from redis.client import BasePipeline

        old = BasePipeline.execute
        called = [0]

        def _execute(*args, **kw):
            called[0] += 1

        BasePipeline.execute = _execute

        try:
            for i in range(4):
                res = app.post_json('/v1/submit', {"items": [{"lat": 12.345678,
                                                   "lon": 23.456789,
                                                   "cell": cell_data}]},
                                    status=204)
                time.sleep(.3)
        finally:
            BasePipeline.execute = old

        self.assertEqual(res.body, '')

        # 1 batches of 2
        self.assertEqual(called[0], 1)

    def test_get_db(self):
        db = _get_db('sqlite:///:memory:')
        self.assertTrue(db is _get_db('sqlite:///:memory:'))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from icon_font_to_png.icon_font import IconFont  # noqa
from icon_font_to_png.icon_font_downloader import (  # noqa
    FontAwesomeDownloader, OcticonsDownloader, AVAILABLE_ICON_FONTS
)

__version__ = '0.3.6'
<commit_msg>Version bump<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from icon_font_to_png.icon_font import IconFont  # noqa
from icon_font_to_png.icon_font_downloader import (  # noqa
    FontAwesomeDownloader, OcticonsDownloader, AVAILABLE_ICON_FONTS
)

__version__ = '0.3.7'
<|endoftext|>


<commit_before># Copyright 2010 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

__version__ = '13.04.10-r780'
<commit_msg>Cut 13.04.16 release<commit_after># Copyright 2010 Canonical Ltd.  This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).

__version__ = '13.04.16-r798'
<|endoftext|>


<commit_before>__author__ = 'user'
<commit_msg>Clean up __init__.py<commit_after># There is nothing to see here!
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8; -*-
"""Field mapping.

When you have a big pile of dicts in slightly different formats, and
you want to transform them to dicts in a consistent format, this
module lets you write the field mappings in a small amount of code and
execute them reasonably quickly.

Quick start:

>>> m = FieldMapper({'age': ['Age', 'age'], 'lastname': 'sname'})
>>> [m.map(x) for x in [dict(Age=30, sname='Smith'),
...                     dict(age=49, sname='Wilson'),
...                     dict(x=3, y=4, lastname='Jones')]]
... #doctest: +NORMALIZE_WHITESPACE
[{'lastname': 'Smith', 'age': 30, 'original_data': {'Age': 30, 'sname': 'Smith'}},
 {'lastname': 'Wilson', 'age': 49, 'original_data': {'age': 49, 'sname': 'Wilson'}},
 {'original_data': {'y': 4, 'lastname': 'Jones', 'x': 3}}]

You can include functions as mappings; it looks for their argument
names in the input data.

>>> m = FieldMapper({'age': ['age', lambda birthyear: 2008 - birthyear]})
>>> m.map({'age': 27})
{'age': 27, 'original_data': {'age': 27}}
>>> m.map({'birthyear': 1970})
{'age': 38, 'original_data': {'birthyear': 1970}}
>>> m = FieldMapper({'date': lambda year, month, day:
...                          '%s-%s-%s' % (year, month, day)})
>>> m.map(dict(year=2008, month=11, day=19))['date']
'2008-11-19'

You can construct (slightly) more complicated pipelines with Reformat:

>>> def invert_name(n):
...     last, first = n.split(',')
...     return '%s %s' % (first.strip(), last.strip())
>>> m = FieldMapper({'name': Reformat(format=invert_name,
...                                   source=['name', 'fullname'])})
>>> m.map({'name': 'Smith, John'})['name']
'John Smith'
>>> m.map({'fullname': 'Smith,John'})['name']
'John Smith'

Finally, you can use CatchAllField (q.v.) to handle cases where these
tools don’t cut it.

BUGS:

- Any particular input field can be used reliably by at most one
  output field.  If you try to use the same input field for more than
  one thing, you may silently lose data.  (You can often work around
  this with CatchAllField — just set its `inputs` to non-conflicting
  input field names.)
- You can’t construct *arbitrarily* more complicated pipelines with
  Reformat.
- It’s still way too slow.

"""

import types

class Field:
    """Represents a field in the output data, and knows how to compute it.

    This is an abstract base class; most concrete subclasses can be
    constructed most conveniently with the factory function
    `as_field`, which creates a sort of tiny DSEL for describing field
    mappings.

    Two of the concrete subclasses can contain Fields themselves;
    CompositeField contains a list of Fields any of which can supply
    its value, and Transform contains a single Field whose value it
    transforms.

    """
    def get_from(self, data):
        "Simple method for testing."
        for key, func in self.inverteds().items():
            if key in data: return func(data)
    def inverteds(self):
        """Return a dictionary of ways to get this field’s value.

        The keys of the dictionary are the names of original source
        fields that must be present for the function to be applicable;
        the values are functions that take the original source data
        dictionary.  Those functions are entitled to access other
        fields in the dictionary, and to throw a KeyError if they want
        to fail.

        This is an efficiency hack; the objective is that we can avoid
        trying to look at fields that aren’t present at all in the
        source data.  It saved only about 14% of user CPU time when I
        tested it.

        """

class InputField(Field):
    """
    >>> f = InputField('bob')
    >>> f.inverteds().keys()
    ['bob']
    >>> f.inverteds()['bob']({'bob': 4, 'mel': 5})
    4
    """
    def __init__(self, name): self.name = name
    def inverteds(self):
        name = self.name
        return {name: lambda data: data[name]}

class Reformat(Field):
    """Changes the format of data in a field.

    >>> import fixed_width
    >>> Reformat(format=fixed_width.date,
    ...          source=['bob']).get_from({'bob': '20080930'})
    '2008-09-30'
    >>> Reformat(format=fixed_width.date,
    ...          source=['bob']).get_from({'dan': '20080830'})

    Note that the above test failed to return anything.

    >>> f = Reformat(format=lambda x: x, source=['bob', 'fred'])
    >>> sorted(f.inverteds().keys())
    ['bob', 'fred']
    >>> f.inverteds()['bob']({'bob': 39})
    39
    """
    def __init__(self, source, format):
        self._source = as_field(source)
        self._format = format
    def inverteds(self):
        rv = {}
        format = self._format
        for k, v in self._source.inverteds().items():
            # v=v so each lambda has its own v instead of all sharing
            # the same v; it's not intended that callers will override
            # v!
            rv[k] = lambda data, v=v: format(v(data))
        return rv

class MultiInputField(Field):
    """A field whose value is computed from more than one input field.

    Its `inverteds()` includes only one of the input fields, currently
    the shortest one.  That’s because there’s no reason to call it
    repeatedly; if one of the other input fields is missing, it will
    fail harmlessly with a KeyError.

    >>> f = MultiInputField(('a', 'b'), lambda a, b: a + ': ' + b)
    >>> f.inverteds().keys()
    ['a']

    >>> ffunc = f.inverteds().values()[0]
    >>> ffunc({'a': 'foo', 'b': 'bar'})
    'foo: bar'
    """
    def __init__(self, names, function):
        "`names` are the field names from which to get `function`'s args."
        self.names, self.function = names, function
    def inverteds(self):
        names = self.names
        def getter(data):
            return self.function(*[data[k] for k in names])
        return {self.names[0]: getter}

class CompositeField(Field):
    """A field with more than one possible source for its data.

    >>> f = CompositeField(['a', Reformat(source=['b'], format=len)])
    >>> sorted(f.inverteds().keys())
    ['a', 'b']
    >>> f.inverteds()['a']({'a': '90210'})
    '90210'
    >>> f.inverteds()['b']({'b': '90210'})
    5
    """
    def __init__(self, fields):
        self.fields = fields
    def inverteds(self):
        rv = {}
        for field in self.fields: rv.update(as_field(field).inverteds())
        return rv

class CatchAllField(Field):
    """A field for special cases.

    The `inputs` argument specifies a list of input field names that
    should cause this one to fire.  If any of those fields is present,
    `function` is called with the entire input record as an argument.

    >>> f = CatchAllField(['x', 'y'], lambda data: data.get('z'))
    >>> m = FieldMapper({'a': f})
    >>> [m.map(x) for x in [{'z': 2}, {'x': 4, 'z': 3}, {'y': 5, 'z': 6},
    ...                     {'x': 7}]]
    ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    [{'original_data': {...}}, {...'a': 3...}, {...'a': 6...},
     {...'a': None...}]

    """
    def __init__(self, inputs, function):
        self.inputs, self.function = inputs, function
    def inverteds(self):
        return dict((input, self.function) for input in self.inputs)

def argnames(func):
    "Compute a list of the names of the arguments of a Python function."
    return func.func_code.co_varnames[:func.func_code.co_argcount]

def as_field(obj):
    """Coerce obj to some kind of field."""
    if hasattr(obj, 'inverteds'):
        return obj
    elif isinstance(obj, basestring):
        return InputField(obj)
    elif isinstance(obj, types.ListType):
        return CompositeField(obj)
    elif isinstance(obj, types.FunctionType):
        return MultiInputField(argnames(obj), obj)
    raise "can't coerce to a field", obj

class FieldMapper:
    """Maps fields according to a field-mapping specification.

    Takes and returns a dict. The original dict comes out as a
    member named 'original_data'; otherwise its members are only
    copied across according to applicable field specs.

    If the field’s output name is not specifically mentioned among a
    field’s aliases, it isn’t included in the fields to copy from.

    """
    def __init__(self, fields):
        self.inverteds = {}
        for name, field in fields.items():
            field = as_field(field)
            self.inverteds.update(dict((k, (name, v))
                                       for k, v in field.inverteds().items()))
        self.inverted_keys = set(self.inverteds.keys())
    def map(self, data):
        rv = {'original_data': data}
        # Here we intersect the keys in the data with the keys we’re
        # interested in, in order to avoid doing unnecessary work in
        # Python.
        for fieldname in set(data.keys()) & self.inverted_keys:
            name, func = self.inverteds[fieldname]
            try:
                v = func(data)
            except KeyError:
                pass
            else:
                rv[name] = v
        return rv

if __name__ == '__main__':
    import doctest
    doctest.testmod()
<commit_msg>catching duplicate input field names in field_mapper instead of silently losing data<commit_after>#!/usr/bin/python
# -*- coding: utf-8; -*-
"""Field mapping.

When you have a big pile of dicts in slightly different formats, and
you want to transform them to dicts in a consistent format, this
module lets you write the field mappings in a small amount of code and
execute them reasonably quickly.

Quick start:

>>> m = FieldMapper({'age': ['Age', 'age'], 'lastname': 'sname'})
>>> [m.map(x) for x in [dict(Age=30, sname='Smith'),
...                     dict(age=49, sname='Wilson'),
...                     dict(x=3, y=4, lastname='Jones')]]
... #doctest: +NORMALIZE_WHITESPACE
[{'lastname': 'Smith', 'age': 30, 'original_data': {'Age': 30, 'sname': 'Smith'}},
 {'lastname': 'Wilson', 'age': 49, 'original_data': {'age': 49, 'sname': 'Wilson'}},
 {'original_data': {'y': 4, 'lastname': 'Jones', 'x': 3}}]

You can include functions as mappings; it looks for their argument
names in the input data.

>>> m = FieldMapper({'age': ['age', lambda birthyear: 2008 - birthyear]})
>>> m.map({'age': 27})
{'age': 27, 'original_data': {'age': 27}}
>>> m.map({'birthyear': 1970})
{'age': 38, 'original_data': {'birthyear': 1970}}
>>> m = FieldMapper({'date': lambda year, month, day:
...                          '%s-%s-%s' % (year, month, day)})
>>> m.map(dict(year=2008, month=11, day=19))['date']
'2008-11-19'

You can construct (slightly) more complicated pipelines with Reformat:

>>> def invert_name(n):
...     last, first = n.split(',')
...     return '%s %s' % (first.strip(), last.strip())
>>> m = FieldMapper({'name': Reformat(format=invert_name,
...                                   source=['name', 'fullname'])})
>>> m.map({'name': 'Smith, John'})['name']
'John Smith'
>>> m.map({'fullname': 'Smith,John'})['name']
'John Smith'

Finally, you can use CatchAllField (q.v.) to handle cases where these
tools don’t cut it.

BUGS:

-  Any particular input field can be used reliably by at most one
   output field.  If you try to use the same input field for more than
   one thing, you may get an AssertionError.  (You can often work
   around this with CatchAllField — just set its `inputs` to
   non-conflicting input field names.)

        >>> FieldMapper({'name': lambda firstname, lastname: firstname + ' ' + lastname,
        ...              'firstname': 'firstname', 'lastname': 'lastname'})
        ... #doctest: +ELLIPSIS
        Traceback (most recent call last):
          ...
        AssertionError: ...

- You can’t construct *arbitrarily* more complicated pipelines with
  Reformat.
- It’s still way too slow.

"""

import types

class Field:
    """Represents a field in the output data, and knows how to compute it.

    This is an abstract base class; most concrete subclasses can be
    constructed most conveniently with the factory function
    `as_field`, which creates a sort of tiny DSEL for describing field
    mappings.

    Two of the concrete subclasses can contain Fields themselves;
    CompositeField contains a list of Fields any of which can supply
    its value, and Transform contains a single Field whose value it
    transforms.

    """
    def get_from(self, data):
        "Simple method for testing."
        for key, func in self.inverteds().items():
            if key in data: return func(data)
    def inverteds(self):
        """Return a dictionary of ways to get this field’s value.

        The keys of the dictionary are the names of original source
        fields that must be present for the function to be applicable;
        the values are functions that take the original source data
        dictionary.  Those functions are entitled to access other
        fields in the dictionary, and to throw a KeyError if they want
        to fail.

        This is an efficiency hack; the objective is that we can avoid
        trying to look at fields that aren’t present at all in the
        source data.  It saved only about 14% of user CPU time when I
        tested it.

        """

class InputField(Field):
    """
    >>> f = InputField('bob')
    >>> f.inverteds().keys()
    ['bob']
    >>> f.inverteds()['bob']({'bob': 4, 'mel': 5})
    4
    """
    def __init__(self, name): self.name = name
    def inverteds(self):
        name = self.name
        return {name: lambda data: data[name]}

class Reformat(Field):
    """Changes the format of data in a field.

    >>> import fixed_width
    >>> Reformat(format=fixed_width.date,
    ...          source=['bob']).get_from({'bob': '20080930'})
    '2008-09-30'
    >>> Reformat(format=fixed_width.date,
    ...          source=['bob']).get_from({'dan': '20080830'})

    Note that the above test failed to return anything.

    >>> f = Reformat(format=lambda x: x, source=['bob', 'fred'])
    >>> sorted(f.inverteds().keys())
    ['bob', 'fred']
    >>> f.inverteds()['bob']({'bob': 39})
    39
    """
    def __init__(self, source, format):
        self._source = as_field(source)
        self._format = format
    def inverteds(self):
        rv = {}
        format = self._format
        for k, v in self._source.inverteds().items():
            # v=v so each lambda has its own v instead of all sharing
            # the same v; it's not intended that callers will override
            # v!
            rv[k] = lambda data, v=v: format(v(data))
        return rv

class MultiInputField(Field):
    """A field whose value is computed from more than one input field.

    Its `inverteds()` includes only one of the input fields, currently
    the shortest one.  That’s because there’s no reason to call it
    repeatedly; if one of the other input fields is missing, it will
    fail harmlessly with a KeyError.

    >>> f = MultiInputField(('a', 'b'), lambda a, b: a + ': ' + b)
    >>> f.inverteds().keys()
    ['a']

    >>> ffunc = f.inverteds().values()[0]
    >>> ffunc({'a': 'foo', 'b': 'bar'})
    'foo: bar'
    """
    def __init__(self, names, function):
        "`names` are the field names from which to get `function`'s args."
        self.names, self.function = names, function
    def inverteds(self):
        names = self.names
        def getter(data):
            return self.function(*[data[k] for k in names])
        return {self.names[0]: getter}

class CompositeField(Field):
    """A field with more than one possible source for its data.

    >>> f = CompositeField(['a', Reformat(source=['b'], format=len)])
    >>> sorted(f.inverteds().keys())
    ['a', 'b']
    >>> f.inverteds()['a']({'a': '90210'})
    '90210'
    >>> f.inverteds()['b']({'b': '90210'})
    5
    """
    def __init__(self, fields):
        self.fields = fields
    def inverteds(self):
        rv = {}
        for field in self.fields: rv.update(as_field(field).inverteds())
        return rv

class CatchAllField(Field):
    """A field for special cases.

    The `inputs` argument specifies a list of input field names that
    should cause this one to fire.  If any of those fields is present,
    `function` is called with the entire input record as an argument.

    >>> f = CatchAllField(['x', 'y'], lambda data: data.get('z'))
    >>> m = FieldMapper({'a': f})
    >>> [m.map(x) for x in [{'z': 2}, {'x': 4, 'z': 3}, {'y': 5, 'z': 6},
    ...                     {'x': 7}]]
    ... #doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
    [{'original_data': {...}}, {...'a': 3...}, {...'a': 6...},
     {...'a': None...}]

    """
    def __init__(self, inputs, function):
        self.inputs, self.function = inputs, function
    def inverteds(self):
        return dict((input, self.function) for input in self.inputs)

def argnames(func):
    "Compute a list of the names of the arguments of a Python function."
    return func.func_code.co_varnames[:func.func_code.co_argcount]

def as_field(obj):
    """Coerce obj to some kind of field."""
    if hasattr(obj, 'inverteds'):
        return obj
    elif isinstance(obj, basestring):
        return InputField(obj)
    elif isinstance(obj, types.ListType):
        return CompositeField(obj)
    elif isinstance(obj, types.FunctionType):
        return MultiInputField(argnames(obj), obj)
    raise "can't coerce to a field", obj

class FieldMapper:
    """Maps fields according to a field-mapping specification.

    Takes and returns a dict. The original dict comes out as a
    member named 'original_data'; otherwise its members are only
    copied across according to applicable field specs.

    If the field’s output name is not specifically mentioned among a
    field’s aliases, it isn’t included in the fields to copy from.

    """
    def __init__(self, fields):
        self.inverteds = {}
        for name, field in fields.items():
            field = as_field(field)
            for k, v in field.inverteds().items():
                assert k not in self.inverteds, (name, k, field,
                                                 self.inverteds[k])
                self.inverteds[k] = (name, v)
        self.inverted_keys = set(self.inverteds.keys())
    def map(self, data):
        rv = {'original_data': data}
        # Here we intersect the keys in the data with the keys we’re
        # interested in, in order to avoid doing unnecessary work in
        # Python.
        for fieldname in set(data.keys()) & self.inverted_keys:
            name, func = self.inverteds[fieldname]
            try:
                v = func(data)
            except KeyError:
                pass
            else:
                rv[name] = v
        return rv

if __name__ == '__main__':
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before>from googleapiclient.discovery import build
from googleapiclient.discovery import build_from_document

class Harvester():
    def __init__(self):
        with open("credentials.txt", "r") as creds:
            self.api_key = creds.read()

        self.service = build('youtube', 'v3', developerKey=self.api_key)

    def get_comment_threads(self):
        with open("sources.txt", "r") as videos:
            sources = videos.readlines()

            for source in sources:
                video_id = source

                results = self.service.commentThreads().list(
                    part="snippet",
                    videoId=video_id,
                    textFormat="plainText"
                ).execute()

                for item in results["items"]:
                    comment = item["snippet"]["topLevelComment"]
                    author = comment["snippet"]["authorDisplayName"]
                    text = comment["snippet"]["textDisplay"]
                    print("Comment by %s: %s" % (author, text))

                return results["items"]

    # Call the API's comments.list method to list the existing comment replies.
    def get_comments(self, parent_id):
        results = self.service.comments().list(
            part="snippet",
            parentId=parent_id,
            textFormat="plainText"
        ).execute()

        for item in results["items"]:
            author = item["snippet"]["authorDisplayName"]
            text = item["snippet"]["textDisplay"]
            print("Comment by %s: %s" % (author, text))

        return results["items"]

if __name__ == "__main__":
    harvester = Harvester()
    results = harvester.get_comment_threads()
    for res in results:
        harvester.get_comments(res['id'])<commit_msg>Dump outputs into a JSON file and skip those that already exist<commit_after>from googleapiclient.discovery import build
import json
import os.path

class Harvester():
    def __init__(self):
        """
        Setup and build a Youtube query object.
        Your youtube API token should go in credentials.txt in the importers/youtube directory
        """
        with open("credentials.txt", "r") as creds:
            self.api_key = creds.read()

        self.service = build('youtube', 'v3', developerKey=self.api_key)

    def get_comment_threads(self, video_id):
        """
        Fetches top-level comments for a video ID
        :param video_id: The ID of the video
        :return: The dictionary of comments
        """

        results = self.service.commentThreads().list(
            part="snippet",
            videoId=video_id,
            textFormat="plainText"
        ).execute()

        return results["items"]

    def get_comments(self, parent_id):
        """
        Fetches a thread of comment replies
        :param parent_id: The parent comment ID
        :return: The dictionary of reply comments
        """
        results = self.service.comments().list(
            part="snippet",
            parentId=parent_id,
            textFormat="plainText"
        ).execute()

        return results["items"]

if __name__ == "__main__":
    harvester = Harvester()

    with open("sources.txt", "r") as videos:

        sources = videos.readlines()

        for source in sources:
            video_id = source

            fname = "output/{0}.json".format(video_id)

            if os.path.isfile(fname):
               print("Skipping {0} as it already exists.".format(fname))
            else:
                print("Harvesting into {0}.".format(fname))
                results = harvester.get_comment_threads(video_id)

                json_out = {}
                json_out['thread'] = results

                for res in results:
                    json_out[res['id']] = harvester.get_comments(res['id'])

                with open(fname, "w") as json_writer:
                    json_writer.write(json.dumps(json_out, ensure_ascii=False))
<|endoftext|>


<commit_before>import abc
import uuid
import struct
import time

from bson import json_util, EPOCH_NAIVE
# Monkeypatch to not include tz_info in decoded JSON.
# Kind of a ridiculous solution, but works.
json_util.EPOCH_AWARE = EPOCH_NAIVE

from sqlalchemy import String, Text, event
from sqlalchemy.types import TypeDecorator, BINARY
from sqlalchemy.interfaces import PoolListener
from sqlalchemy.engine import Engine
from sqlalchemy.ext.mutable import Mutable
from sqlalchemy.ext.declarative import DeclarativeMeta

from inbox.util.encoding import base36encode, base36decode

from inbox.log import get_logger
log = get_logger()

SLOW_QUERY_THRESHOLD_MS = 5000
MAX_TEXT_LENGTH = 65535

@event.listens_for(Engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement,
                          parameters, context, executemany):
    context._query_start_time = time.time()

@event.listens_for(Engine, "after_cursor_execute")
def after_cursor_execute(conn, cursor, statement,
                         parameters, context, executemany):
    total = int(1000 * (time.time() - context._query_start_time))
    # We only care about slow reads here
    if total > SLOW_QUERY_THRESHOLD_MS and statement.startswith('SELECT'):
        statement = ' '.join(statement.split())
        try:
            # Turn `parameters` into a string as a quick fix for easier
            # downstream parsing (Elasticsearch has trouble with arrays of
            # heterogeneous type such as [1, 'a']).
            log.warning('slow query', query_time=total, statement=statement,
                        parameters=str(parameters))
        except UnicodeDecodeError:
            log.warning('slow query', query_time=total)
            log.error('logging UnicodeDecodeError')

class SQLAlchemyCompatibleAbstractMetaClass(DeclarativeMeta, abc.ABCMeta):
    """Declarative model classes that *also* inherit from an abstract base
    class need a metaclass like this one, in order to prevent metaclass
    conflict errors."""
    pass

class ABCMixin(object):
    """Use this if you want a mixin class which is actually an abstract base
    class, for example in order to enforce that concrete subclasses define
    particular methods or properties."""
    __metaclass__ = SQLAlchemyCompatibleAbstractMetaClass
    __abstract__ = True

# Column Types

# http://docs.sqlalchemy.org/en/rel_0_9/core/types.html#marshal-json-strings
class JSON(TypeDecorator):
    impl = Text

    def process_bind_param(self, value, dialect):
        if value is None:
            return None

        return json_util.dumps(value)

    def process_result_value(self, value, dialect):
        if not value:
            return None

        # Unfortunately loads() is strict about invalid utf-8 whereas dumps()
        # is not. This can result in ValueErrors during decoding - we simply
        # log and return None for now.
        # http://bugs.python.org/issue11489
        try:
            return json_util.loads(value)
        except ValueError:
            log.error('ValueError on decoding JSON', value=value)

def json_field_too_long(value):
    return len(json_util.dumps(value)) > MAX_TEXT_LENGTH

class LittleJSON(JSON):
    impl = String(255)

class BigJSON(JSON):
    # if all characters were 4-byte, this would fit in mysql's MEDIUMTEXT
    impl = Text(4194304)

class Base36UID(TypeDecorator):
    impl = BINARY(16)  # 128 bit unsigned integer

    def process_bind_param(self, value, dialect):
        if not value:
            return None
        return b36_to_bin(value)

    def process_result_value(self, value, dialect):
        return int128_to_b36(value)

# http://docs.sqlalchemy.org/en/rel_0_9/orm/extensions/mutable.html#sqlalchemy.ext.mutable.Mutable.as_mutable
# Can simply use this as is because though we use bson.json_util, loads()
# dumps() return standard Python dicts like the json.* equivalents
# (because these are simply called under the hood)
class MutableDict(Mutable, dict):
    @classmethod
    def coerce(cls, key, value):
        """ Convert plain dictionaries to MutableDict. """
        if not isinstance(value, MutableDict):
            if isinstance(value, dict):
                return MutableDict(value)

            # this call will raise ValueError
            return Mutable.coerce(key, value)
        else:
            return value

    def __setitem__(self, key, value):
        """ Detect dictionary set events and emit change events. """
        dict.__setitem__(self, key, value)
        self.changed()

    def __delitem__(self, key):
        """ Detect dictionary del events and emit change events. """
        dict.__delitem__(self, key)
        self.changed()

    def update(self, *args, **kwargs):
        for k, v in dict(*args, **kwargs).iteritems():
            self[k] = v

    # To support pickling:
    def __getstate__(self):
        return dict(self)

    def __setstate__(self, state):
        self.update(state)

class MutableList(Mutable, list):
    @classmethod
    def coerce(cls, key, value):
        """Convert plain list to MutableList"""
        if not isinstance(value, MutableList):
            if isinstance(value, list):
                return MutableList(value)

            # this call will raise ValueError
            return Mutable.coerce(key, value)
        else:
            return value

    def __setitem__(self, idx, value):
        list.__setitem__(self, idx, value)
        self.changed()

    def __setslice__(self, start, stop, values):
        list.__setslice__(self, start, stop, values)
        self.changed()

    def __delitem__(self, idx):
        list.__delitem__(self, idx)
        self.changed()

    def __delslice__(self, start, stop):
        list.__delslice__(self, start, stop)
        self.changed()

    def append(self, value):
        list.append(self, value)
        self.changed()

    def insert(self, idx, value):
        list.insert(self, idx, value)
        self.changed()

    def extend(self, values):
        list.extend(self, values)
        self.changed()

    def pop(self, *args, **kw):
        value = list.pop(self, *args, **kw)
        self.changed()
        return value

    def remove(self, value):
        list.remove(self, value)
        self.changed()

def int128_to_b36(int128):
    """ int128: a 128 bit unsigned integer
        returns a base-36 string representation
    """
    if not int128:
        return None
    assert len(int128) == 16, "should be 16 bytes (128 bits)"
    a, b = struct.unpack('>QQ', int128)  # uuid() is big-endian
    pub_id = (a << 64) | b
    return base36encode(pub_id).lower()

def b36_to_bin(b36_string):
    """ b36_string: a base-36 encoded string
        returns binary 128 bit unsigned integer
    """
    int128 = base36decode(b36_string)
    MAX_INT64 = 0xFFFFFFFFFFFFFFFF
    return struct.pack(
        '>QQ',
        (int128 >> 64) & MAX_INT64,
        int128 & MAX_INT64)

def generate_public_id():
    """ Returns a base-36 string UUID """
    u = uuid.uuid4().bytes
    return int128_to_b36(u)

# Other utilities

# My good old friend Enrico to the rescue:
# http://www.enricozini.org/2012/tips/sa-sqlmode-traditional/
#
# We set sql-mode=traditional on the server side as well, but enforce at the
# application level to be extra safe.
#
# Without this, MySQL will silently insert invalid values in the database if
# not running with sql-mode=traditional.
class ForceStrictMode(PoolListener):
    def connect(self, dbapi_con, connection_record):
        cur = dbapi_con.cursor()
        cur.execute("SET SESSION sql_mode='TRADITIONAL'")
        cur = None

def maybe_refine_query(query, subquery):
    if subquery is None:
        return query
    return query.join(subquery.subquery())

def safer_yield_per(query, id_field, start_id, count):
    """Incautious execution of 'for result in query.yield_per(N):' may cause
    slowness or OOMing over large tables. This is a less general but less
    dangerous alternative.

    Parameters
    ----------
    query: sqlalchemy.Query
        The query to yield windowed results from.
    id_field: A SQLAlchemy attribute to use for windowing. E.g.,
        `Transaction.id`
    start_id: The value of id_field at which to start iterating.
    count: int
        The number of results to fetch at a time.
    """
    while True:
        results = query.filter(id_field >= start_id).order_by(id_field).\
            limit(count).all()
        if not results:
            return
        for result in results:
            yield result
        start_id = start_id + count
<commit_msg>Fixing safer_yield_per script<commit_after>import abc
import uuid
import struct
import time

from bson import json_util, EPOCH_NAIVE
# Monkeypatch to not include tz_info in decoded JSON.
# Kind of a ridiculous solution, but works.
json_util.EPOCH_AWARE = EPOCH_NAIVE

from sqlalchemy import String, Text, event
from sqlalchemy.types import TypeDecorator, BINARY
from sqlalchemy.interfaces import PoolListener
from sqlalchemy.engine import Engine
from sqlalchemy.ext.mutable import Mutable
from sqlalchemy.ext.declarative import DeclarativeMeta

from inbox.util.encoding import base36encode, base36decode

from inbox.log import get_logger
log = get_logger()

SLOW_QUERY_THRESHOLD_MS = 5000
MAX_TEXT_LENGTH = 65535

@event.listens_for(Engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement,
                          parameters, context, executemany):
    context._query_start_time = time.time()

@event.listens_for(Engine, "after_cursor_execute")
def after_cursor_execute(conn, cursor, statement,
                         parameters, context, executemany):
    total = int(1000 * (time.time() - context._query_start_time))
    # We only care about slow reads here
    if total > SLOW_QUERY_THRESHOLD_MS and statement.startswith('SELECT'):
        statement = ' '.join(statement.split())
        try:
            # Turn `parameters` into a string as a quick fix for easier
            # downstream parsing (Elasticsearch has trouble with arrays of
            # heterogeneous type such as [1, 'a']).
            log.warning('slow query', query_time=total, statement=statement,
                        parameters=str(parameters))
        except UnicodeDecodeError:
            log.warning('slow query', query_time=total)
            log.error('logging UnicodeDecodeError')

class SQLAlchemyCompatibleAbstractMetaClass(DeclarativeMeta, abc.ABCMeta):
    """Declarative model classes that *also* inherit from an abstract base
    class need a metaclass like this one, in order to prevent metaclass
    conflict errors."""
    pass

class ABCMixin(object):
    """Use this if you want a mixin class which is actually an abstract base
    class, for example in order to enforce that concrete subclasses define
    particular methods or properties."""
    __metaclass__ = SQLAlchemyCompatibleAbstractMetaClass
    __abstract__ = True

# Column Types

# http://docs.sqlalchemy.org/en/rel_0_9/core/types.html#marshal-json-strings
class JSON(TypeDecorator):
    impl = Text

    def process_bind_param(self, value, dialect):
        if value is None:
            return None

        return json_util.dumps(value)

    def process_result_value(self, value, dialect):
        if not value:
            return None

        # Unfortunately loads() is strict about invalid utf-8 whereas dumps()
        # is not. This can result in ValueErrors during decoding - we simply
        # log and return None for now.
        # http://bugs.python.org/issue11489
        try:
            return json_util.loads(value)
        except ValueError:
            log.error('ValueError on decoding JSON', value=value)

def json_field_too_long(value):
    return len(json_util.dumps(value)) > MAX_TEXT_LENGTH

class LittleJSON(JSON):
    impl = String(255)

class BigJSON(JSON):
    # if all characters were 4-byte, this would fit in mysql's MEDIUMTEXT
    impl = Text(4194304)

class Base36UID(TypeDecorator):
    impl = BINARY(16)  # 128 bit unsigned integer

    def process_bind_param(self, value, dialect):
        if not value:
            return None
        return b36_to_bin(value)

    def process_result_value(self, value, dialect):
        return int128_to_b36(value)

# http://bit.ly/1LbMnqu
# Can simply use this as is because though we use bson.json_util, loads()
# dumps() return standard Python dicts like the json.* equivalents
# (because these are simply called under the hood)
class MutableDict(Mutable, dict):
    @classmethod
    def coerce(cls, key, value):
        """ Convert plain dictionaries to MutableDict. """
        if not isinstance(value, MutableDict):
            if isinstance(value, dict):
                return MutableDict(value)

            # this call will raise ValueError
            return Mutable.coerce(key, value)
        else:
            return value

    def __setitem__(self, key, value):
        """ Detect dictionary set events and emit change events. """
        dict.__setitem__(self, key, value)
        self.changed()

    def __delitem__(self, key):
        """ Detect dictionary del events and emit change events. """
        dict.__delitem__(self, key)
        self.changed()

    def update(self, *args, **kwargs):
        for k, v in dict(*args, **kwargs).iteritems():
            self[k] = v

    # To support pickling:
    def __getstate__(self):
        return dict(self)

    def __setstate__(self, state):
        self.update(state)

class MutableList(Mutable, list):
    @classmethod
    def coerce(cls, key, value):
        """Convert plain list to MutableList"""
        if not isinstance(value, MutableList):
            if isinstance(value, list):
                return MutableList(value)

            # this call will raise ValueError
            return Mutable.coerce(key, value)
        else:
            return value

    def __setitem__(self, idx, value):
        list.__setitem__(self, idx, value)
        self.changed()

    def __setslice__(self, start, stop, values):
        list.__setslice__(self, start, stop, values)
        self.changed()

    def __delitem__(self, idx):
        list.__delitem__(self, idx)
        self.changed()

    def __delslice__(self, start, stop):
        list.__delslice__(self, start, stop)
        self.changed()

    def append(self, value):
        list.append(self, value)
        self.changed()

    def insert(self, idx, value):
        list.insert(self, idx, value)
        self.changed()

    def extend(self, values):
        list.extend(self, values)
        self.changed()

    def pop(self, *args, **kw):
        value = list.pop(self, *args, **kw)
        self.changed()
        return value

    def remove(self, value):
        list.remove(self, value)
        self.changed()

def int128_to_b36(int128):
    """ int128: a 128 bit unsigned integer
        returns a base-36 string representation
    """
    if not int128:
        return None
    assert len(int128) == 16, "should be 16 bytes (128 bits)"
    a, b = struct.unpack('>QQ', int128)  # uuid() is big-endian
    pub_id = (a << 64) | b
    return base36encode(pub_id).lower()

def b36_to_bin(b36_string):
    """ b36_string: a base-36 encoded string
        returns binary 128 bit unsigned integer
    """
    int128 = base36decode(b36_string)
    MAX_INT64 = 0xFFFFFFFFFFFFFFFF
    return struct.pack(
        '>QQ',
        (int128 >> 64) & MAX_INT64,
        int128 & MAX_INT64)

def generate_public_id():
    """ Returns a base-36 string UUID """
    u = uuid.uuid4().bytes
    return int128_to_b36(u)

# Other utilities

# My good old friend Enrico to the rescue:
# http://www.enricozini.org/2012/tips/sa-sqlmode-traditional/
#
# We set sql-mode=traditional on the server side as well, but enforce at the
# application level to be extra safe.
#
# Without this, MySQL will silently insert invalid values in the database if
# not running with sql-mode=traditional.
class ForceStrictMode(PoolListener):
    def connect(self, dbapi_con, connection_record):
        cur = dbapi_con.cursor()
        cur.execute("SET SESSION sql_mode='TRADITIONAL'")
        cur = None

def maybe_refine_query(query, subquery):
    if subquery is None:
        return query
    return query.join(subquery.subquery())

def safer_yield_per(query, id_field, start_id, count):
    """Incautious execution of 'for result in query.yield_per(N):' may cause
    slowness or OOMing over large tables. This is a less general but less
    dangerous alternative.

    Parameters
    ----------
    query: sqlalchemy.Query
        The query to yield windowed results from.
    id_field: A SQLAlchemy attribute to use for windowing. E.g.,
        `Transaction.id`
    start_id: The value of id_field at which to start iterating.
    count: int
        The number of results to fetch at a time.
    """
    cur_id = start_id
    while True:
        results = query.filter(id_field >= cur_id).order_by(id_field).\
            limit(count).all()
        if not results:
            return
        for result in results:
            yield result
        cur_id = results[-1].id + 1
<|endoftext|>


<commit_before>"""
EMmix.py - cluster data using a mixture of Gaussians generative model.
Marcus Frean

Expects to read a datafile consisting of a matrix in which each row is
a training item.
"""

import sys
from pylab import *
from numpy import *
import numpy.random as rng
import numpy.linalg as linalg
from matplotlib.patches import Ellipse

def GaussianDensity(d,mu,v):
    """
    returns the prob of data d (matrix with items as rows), under the
    Gaussian given by mu (vector) and var (matrix)
    """
    normalisation = pow(2*pi,len(mu)/2) * sqrt(linalg.det(v))
    xx = d-mu
    yy = dot(linalg.inv(v), xx.transpose()).transpose()
    s1 = xx[:,0] * yy[:,0]
    s2 = xx[:,1] * yy[:,1]
    u = -0.5*(s1+s2)
    p = exp(u)/normalisation
    return p

def plotEllipse(pos,P,edge,face,transparency):
    U, s , Vh = svd(P)
    orient = math.atan2(U[1,0],U[0,0])*180/pi
    ellipsePlot = Ellipse(xy=pos, width=2.0*math.sqrt(s[0]),
                          height=2.0*math.sqrt(s[1]), angle=orient,
                          facecolor=face,edgecolor=edge,alpha=transparency,
                          zorder=10)

    ax = gca()
    ax.add_patch(ellipsePlot)
    return ellipsePlot

#-----------------------------------------------------------------------------

if __name__ == '__main__':

    # Set the random number generator's seed if you want reproducability.
    # (nb. this would be better as a command line arg...)
    # rng.seed(1112)

    if len(sys.argv) == 3:
        K = int(sys.argv[1])
        infile = str(sys.argv[2])
        print 'We are assuming %d classes' % (K)
    else:
        sys.exit('usage: python EMmix.py numClasses infile')

    data = genfromtxt(infile, float) #, unpack=True)
    print data
    (N,D) = data.shape
    print 'N is ',N, ' and D is ',D

    # Set initial guestimates for means, variances, and mixture coefficients
    means = zeros((D,K),float)   # centers start off
    for k in range(K):           # on randomly chosen data points
        n = rng.randint(N)
        means[:,k] = data[n,:]

    variances = 10*ones((D,D,K),float)     # initial variances
    for k in range(K):                     # start off spherical
        variances[:,:,k] = eye(2)
    mix_coeff = ones((K),float)/K          # mixing coefficients
    r = 1.0* ones((N,K))                   # start off all the same

    for iteration in range(50):
        # E step___________________________________________________
        # Evaluate the responsibilities using the current parameter
        # values.
        for k in range(K):
            r[:,k] = mix_coeff[k] * GaussianDensity(data, means[:,k], variances[:,:,k])
        r_sum = r.sum(1)

        # gamma is 'responsibility', or r normalised, as per Bishop
        gamma = (r.transpose() / r_sum).transpose()
        gamma_sums = gamma.sum(0) #ie. summed over the 0-th dimension

        # M step___________________________________________________
        for k in range(K):
            g = (data.transpose() * gamma[:,k]).sum(1)
            # update the means
            means[:,k] = g / gamma_sums[k]
            # update the mixing coefficient
            mix_coeff[k] = gamma_sums[k]/sum(gamma_sums)
            # update the (co)variances
            x = data - means[:,k]
            v00 = sum(x[:,0]*x[:,0] * gamma[:,k])
            v01 = sum(x[:,0]*x[:,1] * gamma[:,k])
            v10 = sum(x[:,1]*x[:,0] * gamma[:,k])
            v11 = sum(x[:,1]*x[:,1] * gamma[:,k])
            # Would be a good idea to include SOMETHING LIKE v00 =
            # min(v00,0.001) etc to prevent variances collapsing to
            # zero........
            variances[:,:,k] = [[v00,v01],[v10,v11]] / gamma_sums[k]

        logL = log(r_sum).sum() # that was easy!
        print 'iteration %3d logL %12.6f' % (iteration,logL)
    # END OF THE EM LOOP____________________________________________

    f1 = figure()
    title('%d Gaussians fit to %s using EM' % (K, infile))
    randColor = array([.2,.75, 1])
    for k in range(K):
        randColor = rng.random((3))
        randColor /= randColor.sum()

        ellipsePlot=plotEllipse(means[:,k],variances[:,:,k],'blue',
                                randColor,mix_coeff[k]/mix_coeff.max())
        """
        print '\n Mixture component ',k
        print 'Mean: ', means[:,k]
        print 'Covariance matrix: \n',variances[:,:,k]
        print 'Mixture coefficient: ',mix_coeff[k]
        """

    scatter(data[:,0], data[:,1], marker='o',s=.5,linewidths=None,alpha=0.5)
    #axis('equal')  # uncomment to give the 2 axes the same scale
    draw()

    out_stem = infile.split('.')[0]
    out_image = out_stem + '_EM.png'
    savefig(out_image)
    print '\n  saved image ',out_image

    # for fun, we can now show samples from this model, to cf. original data.
    f2 = figure()
    title('What the model captures')
    X = zeros((N,D))
    for i in range(N):
        j = sum(rng.random() > cumsum(mix_coeff))
        X[i,:] = array([rng.multivariate_normal(means[:,j],variances[:,:,j],1)])
    scatter(X[:,0], X[:,1], marker='o',s=.5,linewidths=None,alpha=0.5)
    #axis('equal')
    draw()
    out_imagename = out_stem+'_faked.png'
    savefig(out_imagename)
    print '\n  saved image ',out_imagename
<commit_msg>Better initialisation of clusters in EMmix.py, and the essentials put into functions.<commit_after>"""
EMmix.py - cluster data using a mixture of Gaussians generative model.
Marcus Frean

Expects to read a datafile consisting of a matrix in which each row is
a training item.
"""

import sys
from pylab import *
from numpy import *
import numpy.random as rng
import numpy.linalg as linalg
from matplotlib.patches import Ellipse

def GaussianDensity(d,mu,v):
    """
    returns the prob of data d (matrix with items as rows), under the
    Gaussian given by mu (vector) and var (matrix)
    """
    normalisation = pow(2*pi,len(mu)/2) * sqrt(linalg.det(v))
    xx = d-mu
    yy = dot(linalg.inv(v), xx.transpose()).transpose()
    s1 = xx[:,0] * yy[:,0]
    s2 = xx[:,1] * yy[:,1]
    u = -0.5*(s1+s2)
    p = exp(u)/normalisation
    return p

def plotEllipse(pos,P,edge,face,transparency):
    U, s , Vh = svd(P)
    orient = math.atan2(U[1,0],U[0,0])*180/pi
    ellipsePlot = Ellipse(xy=pos, width=2.0*math.sqrt(s[0]),
                          height=2.0*math.sqrt(s[1]), angle=orient,
                          facecolor=face,edgecolor=edge,alpha=transparency,
                          zorder=10)

    ax = gca()
    ax.add_patch(ellipsePlot)
    return ellipsePlot

def learnMOGmodel(data, model):
    (means, variances, mix_coeff) = model
    (N,D) = data.shape
    (D,K) = means.shape
    r = 1.0* ones((N,K))                   # start off all the same
    for iteration in range(50):
        # E step___________________________________________________
        # Evaluate the responsibilities using the current parameter
        # values.
        for k in range(K):
            r[:,k] = mix_coeff[k] * GaussianDensity(data, means[:,k], variances[:,:,k])
        r_sum = r.sum(1)

        # gamma is 'responsibility', or r normalised, as per Bishop
        gamma = (r.transpose() / r_sum).transpose()
        gamma_sums = gamma.sum(0) #ie. summed over the 0-th dimension

        # M step___________________________________________________
        for k in range(K):
            g = (data.transpose() * gamma[:,k]).sum(1)
            # update the means
            means[:,k] = g / gamma_sums[k]
            # update the mixing coefficient
            mix_coeff[k] = gamma_sums[k]/sum(gamma_sums)
            # update the (co)variances
            x = data - means[:,k]
            v00 = sum(x[:,0]*x[:,0] * gamma[:,k])
            v01 = sum(x[:,0]*x[:,1] * gamma[:,k])
            v10 = sum(x[:,1]*x[:,0] * gamma[:,k])
            v11 = sum(x[:,1]*x[:,1] * gamma[:,k])
            # Would be a good idea to include SOMETHING LIKE v00 =
            # min(v00,0.001) etc to prevent variances collapsing to
            # zero........
            variances[:,:,k] = [[v00,v01],[v10,v11]] / gamma_sums[k]

        logL = log(r_sum).sum() # that was easy!
        #print 'iteration %3d logL %12.6f' % (iteration,logL)
    # END OF THE EM LOOP____________________________________________
    model = (means, variances, mix_coeff)
    return model, logL

def randomStartPoint(N,D,K,data):
    # Set initial guestimates for means, variances, and mixture coefficients
    means = zeros((D,K),float)   # centers start off
    for k in range(K):           # on randomly chosen data points
        n = rng.randint(N)
        means[:,k] = data[n,:]
    variances = 10*ones((D,D,K),float)     # initial variances
    for k in range(K):                     # start off spherical
        variances[:,:,k] = eye(2)
    mix_coeff = ones((K),float)/K          # mixing coefficients
    model = (means, variances, mix_coeff)
    return model

def findBestMOGmodel(N,D,K,data):
    best_logL = -10000000.0
    for trial in range(20):
        model = randomStartPoint(N,D,K,data)
        m, logL = learnMOGmodel(data, model)
        print 'logL is ',logL
        if logL > best_logL:
            best_logL = logL
            model = m
    print 'best logL is ',best_logL
    return model

#-----------------------------------------------------------------------------

if __name__ == '__main__':

    # Set the random number generator's seed if you want reproducability.
    # (nb. this would be better as a command line arg...)
    # rng.seed(1112)

    if len(sys.argv) == 3:
        K = int(sys.argv[1])
        infile = str(sys.argv[2])
        print 'We are assuming %d classes' % (K)
    else:
        sys.exit('usage: python EMmix.py numClasses infile')

    data = genfromtxt(infile, float) #, unpack=True)
    (N,D) = data.shape
    print 'N is ',N, ' and D is ',D

    (means, variances, mix_coeff) = findBestMOGmodel(N,D,K,data)

    f1 = figure()
    title('%d Gaussians fit to %s using EM' % (K, infile))
    randColor = array([.2,.75, 1])
    for k in range(K):
        randColor = rng.random((3))
        randColor /= randColor.sum()
        ellipsePlot=plotEllipse(means[:,k],variances[:,:,k],'blue',
                                randColor,mix_coeff[k]/mix_coeff.max())

    scatter(data[:,0], data[:,1], marker='o',s=.5,linewidths=None,alpha=0.5)
    axis('equal')  # uncomment to give the 2 axes the same scale
    draw()

    out_stem = infile.split('.')[0]
    out_image = out_stem + '_EM.png'
    savefig(out_image)
    print '\n  saved image ',out_image

    # for fun, we can now show samples from this model, to cf. original data.
    f2 = figure()
    title('What the model captures')
    X = zeros((N,D))
    for i in range(N):
        j = sum(rng.random() > cumsum(mix_coeff))
        X[i,:] = array([rng.multivariate_normal(means[:,j],variances[:,:,j],1)])
    scatter(X[:,0], X[:,1], marker='o',s=.5,linewidths=None,alpha=0.5)
    axis('equal')
    draw()
    out_imagename = out_stem+'_faked.png'
    savefig(out_imagename)
    print '\n  saved image ',out_imagename
<|endoftext|>


<commit_before>"""
COMAOPPractice.py
"""
from ggame import App, Color, LineStyle, Sprite, RectangleAsset, CircleAsset, EllipseAsset, LineAsset
from ggame import ImageAsset, PolygonAsset, Frame, Sound, SoundAsset, TextAsset
import time

# colors
red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
orange = Color(0xffa500, 1.0)
skin =Color(0xFCD15B, 1.0)
wall=Color(0xE8E8E8, 1.0)
orange=Color(0xFFa500,1.0)
platc=Color(0xB9BDBB,1.0)
gooy=Color(0xCDF238,1.0)
white=Color(0xFFFFFF,1.0)
darkblue=Color(0x052099,1.0)

# lines
thinline= LineStyle(1, black)
thickline= LineStyle(5, black)
thickishline= LineStyle(2.5, black)
noline= LineStyle(0, black)
portalline= LineStyle(1, blue)
portalline2= LineStyle(1, orange)

# global variables changed with the classes
cpx = 0
cpy = 0
cox = -100
coy = -100
cbx = -100
cby = -100
ccx = 0
ccy = 0
hold = 1
holding = 1
win = 0
play = 0

class Sonar(Sprite):
    asset = ImageAsset("images/ChellSpriteSheet.png", Frame(0,0,205,361), 2, 'horizontal')
    def __init__(self, position):
        super().__init__(Chell.asset, position)
        self.x=0
        self.y=0
    def step(self):
        x=x
class PortalGame(App):
    def __init__(self, width, height):
        super().__init__(width, height)
    def step(self):
        x=x

myapp = PortalGame(1000,750)
myapp.run()<commit_msg>Updated from Brython Server: 11/2/2016 1:01:20 PM<commit_after>"""
COMAOPPractice.py
"""
from ggame import App, Color, LineStyle, Sprite, RectangleAsset, CircleAsset, EllipseAsset, LineAsset
from ggame import ImageAsset, PolygonAsset, Frame, Sound, SoundAsset, TextAsset
import time

# colors
red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
orange = Color(0xffa500, 1.0)
skin =Color(0xFCD15B, 1.0)
wall=Color(0xE8E8E8, 1.0)
orange=Color(0xFFa500,1.0)
platc=Color(0xB9BDBB,1.0)
gooy=Color(0xCDF238,1.0)
white=Color(0xFFFFFF,1.0)
darkblue=Color(0x052099,1.0)

# lines
thinline= LineStyle(1, black)
thickline= LineStyle(5, black)
thickishline= LineStyle(2.5, black)
noline= LineStyle(0, black)
portalline= LineStyle(1, blue)
portalline2= LineStyle(1, orange)

# global variables changed with the classes
cpx = 0
cpy = 0
cox = -100
coy = -100
cbx = -100
cby = -100
ccx = 0
ccy = 0
hold = 1
holding = 1
win = 0
play = 0

class Sonar(Sprite):
    asset = ImageAsset("images/ChellSpriteSheet.png", Frame(0,0,205,361), 2, 'horizontal')
    def __init__(self, position):
        super().__init__(Chell.asset, position)
        self.x=0
        self.y=0
    def step(self):
        x=x
class PortalGame(App):
    def __init__(self, width, height):
        super().__init__(width, height)
        Sonar((0,0))
    def step(self):
        x=x

myapp = PortalGame(1000,750)
myapp.run()<|endoftext|>


<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import absolute_import

import os
from contextlib import contextmanager

from flask import Flask, Blueprint, request
from flask.blueprints import BlueprintSetupState
from flask.ctx import _AppCtxGlobals
from flask.helpers import locked_cached_property
from flask.wrappers import Request
from flask_pluginengine import PluginFlaskMixin
from jinja2 import FileSystemLoader, TemplateNotFound
from werkzeug.datastructures import ImmutableOrderedMultiDict
from werkzeug.utils import cached_property

from indico.web.flask.session import IndicoSessionInterface
from indico.web.flask.util import make_view_func
from indico.util.json import IndicoJSONEncoder

_notset = object()

class IndicoRequest(Request):
    parameter_storage_class = ImmutableOrderedMultiDict

    @cached_property
    def id(self):
        return '{0:012x}'.format(id(self))

    @cached_property
    def relative_url(self):
        """The request's path including its query string if applicable.

        This basically `full_path` but without the ugly trailing
        questionmark if there is no query string.
        """
        return self.full_path.rstrip(u'?')

    @cached_property
    def remote_addr(self):
        ip = super(IndicoRequest, self).remote_addr
        if ip.startswith('::ffff:'):
            # convert ipv6-style ipv4 to the regular ipv4 notation
            ip = ip[7:]
        return ip

    def __repr__(self):
        rv = super(IndicoRequest, self).__repr__()
        if isinstance(rv, unicode):
            rv = rv.encode('utf-8')
        return rv

class _IndicoAppCtxGlobals(_AppCtxGlobals):
    # XXX: remove this class with the next flask version which will
    # already contain `pop` and `setdefault` in `_AppCtxGlobals`
    def pop(self, name, default=_notset):
        if default is _notset:
            return self.__dict__.pop(name)
        else:
            return self.__dict__.pop(name, default)

    def setdefault(self, name, default=None):
        return self.__dict__.setdefault(name, default)

class IndicoFlask(PluginFlaskMixin, Flask):
    json_encoder = IndicoJSONEncoder
    request_class = IndicoRequest
    session_interface = IndicoSessionInterface()
    app_ctx_globals_class = _IndicoAppCtxGlobals

    @property
    def session_cookie_name(self):
        name = super(IndicoFlask, self).session_cookie_name
        if not request.is_secure:
            name += '_http'
        return name

    def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
        from MaKaC.webinterface.rh.base import RHSimple
        # Endpoints from Flask-Multipass need to be wrapped in the RH
        # logic to get the autocommit logic and error handling for code
        # running inside the identity handler.
        if endpoint is not None and endpoint.startswith('_flaskmultipass'):
            view_func = RHSimple.wrap_function(view_func)
        return super(IndicoFlask, self).add_url_rule(rule, endpoint=endpoint, view_func=view_func, **options)

class IndicoBlueprintSetupState(BlueprintSetupState):
    @contextmanager
    def _unprefixed(self):
        prefix = self.url_prefix
        self.url_prefix = None
        yield
        self.url_prefix = prefix

    def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
        if rule.startswith('!/'):
            with self._unprefixed():
                super(IndicoBlueprintSetupState, self).add_url_rule(rule[1:], endpoint, view_func, **options)
        else:
            super(IndicoBlueprintSetupState, self).add_url_rule(rule, endpoint, view_func, **options)

class IndicoBlueprint(Blueprint):
    """A Blueprint implementation that allows prefixing URLs with `!` to
    ignore the url_prefix of the blueprint.

    It also supports automatically creating rules in two versions - with and
    without a prefix.

    :param event_feature: If set, this blueprint will raise `NotFound`
                          for all its endpoints unless the event referenced
                          by the `confId` or `event_id` URL argument has
                          the specified feature.
    """

    def __init__(self, *args, **kwargs):
        self.__prefix = None
        self.__default_prefix = ''
        self.__virtual_template_folder = kwargs.pop('virtual_template_folder', None)
        event_feature = kwargs.pop('event_feature', None)
        super(IndicoBlueprint, self).__init__(*args, **kwargs)

        if event_feature:
            @self.before_request
            def _check_event_feature():
                from indico.modules.events.features.util import require_feature
                event_id = request.view_args.get('confId') or request.view_args.get('event_id')
                if event_id is not None:
                    require_feature(event_id, event_feature)

    @locked_cached_property
    def jinja_loader(self):
        if self.template_folder is not None:
            return IndicoFileSystemLoader(os.path.join(self.root_path, self.template_folder),
                                          virtual_path=self.__virtual_template_folder)

    def make_setup_state(self, app, options, first_registration=False):
        return IndicoBlueprintSetupState(self, app, options, first_registration)

    def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
        if view_func is not None:
            # We might have a RH class here - convert it to a callable suitable as a view func.
            view_func = make_view_func(view_func)
        super(IndicoBlueprint, self).add_url_rule(self.__default_prefix + rule, endpoint, view_func, **options)
        if self.__prefix:
            super(IndicoBlueprint, self).add_url_rule(self.__prefix + rule, endpoint, view_func, **options)

    @contextmanager
    def add_prefixed_rules(self, prefix, default_prefix=''):
        """Creates prefixed rules in addition to the normal ones.
        When specifying a default_prefix, too, the normally "unprefixed" rules
        are prefixed with it."""
        assert self.__prefix is None and not self.__default_prefix
        self.__prefix = prefix
        self.__default_prefix = default_prefix
        yield
        self.__prefix = None
        self.__default_prefix = ''

class IndicoFileSystemLoader(FileSystemLoader):
    """FileSystemLoader that makes namespacing easier.

    The `virtual_path` kwarg lets you specify a path segment that's
    handled as if all templates inside the loader's `searchpath` were
    actually inside ``searchpath/virtual_path``.  That way you don't
    have to create subdirectories in your template folder.
    """

    def __init__(self, searchpath, encoding='utf-8', virtual_path=None):
        super(IndicoFileSystemLoader, self).__init__(searchpath, encoding)
        self.virtual_path = virtual_path

    def list_templates(self):
        templates = super(IndicoFileSystemLoader, self).list_templates()
        if self.virtual_path:
            templates = [os.path.join(self.virtual_path, t) for t in templates]
        return templates

    def get_source(self, environment, template):
        if self.virtual_path:
            if not template.startswith(self.virtual_path):
                raise TemplateNotFound(template)
            template = template[len(self.virtual_path):]
        return super(IndicoFileSystemLoader, self).get_source(environment, template)
<commit_msg>Use better request IDs<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import absolute_import

import os
from contextlib import contextmanager
from uuid import uuid4

from flask import Flask, Blueprint, request
from flask.blueprints import BlueprintSetupState
from flask.ctx import _AppCtxGlobals
from flask.helpers import locked_cached_property
from flask.wrappers import Request
from flask_pluginengine import PluginFlaskMixin
from jinja2 import FileSystemLoader, TemplateNotFound
from werkzeug.datastructures import ImmutableOrderedMultiDict
from werkzeug.utils import cached_property

from indico.util.json import IndicoJSONEncoder
from indico.web.flask.session import IndicoSessionInterface
from indico.web.flask.util import make_view_func

_notset = object()

class IndicoRequest(Request):
    parameter_storage_class = ImmutableOrderedMultiDict

    @cached_property
    def id(self):
        return uuid4().hex[:16]

    @cached_property
    def relative_url(self):
        """The request's path including its query string if applicable.

        This basically `full_path` but without the ugly trailing
        questionmark if there is no query string.
        """
        return self.full_path.rstrip(u'?')

    @cached_property
    def remote_addr(self):
        ip = super(IndicoRequest, self).remote_addr
        if ip.startswith('::ffff:'):
            # convert ipv6-style ipv4 to the regular ipv4 notation
            ip = ip[7:]
        return ip

    def __repr__(self):
        rv = super(IndicoRequest, self).__repr__()
        if isinstance(rv, unicode):
            rv = rv.encode('utf-8')
        return rv

class _IndicoAppCtxGlobals(_AppCtxGlobals):
    # XXX: remove this class with the next flask version which will
    # already contain `pop` and `setdefault` in `_AppCtxGlobals`
    def pop(self, name, default=_notset):
        if default is _notset:
            return self.__dict__.pop(name)
        else:
            return self.__dict__.pop(name, default)

    def setdefault(self, name, default=None):
        return self.__dict__.setdefault(name, default)

class IndicoFlask(PluginFlaskMixin, Flask):
    json_encoder = IndicoJSONEncoder
    request_class = IndicoRequest
    session_interface = IndicoSessionInterface()
    app_ctx_globals_class = _IndicoAppCtxGlobals

    @property
    def session_cookie_name(self):
        name = super(IndicoFlask, self).session_cookie_name
        if not request.is_secure:
            name += '_http'
        return name

    def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
        from MaKaC.webinterface.rh.base import RHSimple
        # Endpoints from Flask-Multipass need to be wrapped in the RH
        # logic to get the autocommit logic and error handling for code
        # running inside the identity handler.
        if endpoint is not None and endpoint.startswith('_flaskmultipass'):
            view_func = RHSimple.wrap_function(view_func)
        return super(IndicoFlask, self).add_url_rule(rule, endpoint=endpoint, view_func=view_func, **options)

class IndicoBlueprintSetupState(BlueprintSetupState):
    @contextmanager
    def _unprefixed(self):
        prefix = self.url_prefix
        self.url_prefix = None
        yield
        self.url_prefix = prefix

    def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
        if rule.startswith('!/'):
            with self._unprefixed():
                super(IndicoBlueprintSetupState, self).add_url_rule(rule[1:], endpoint, view_func, **options)
        else:
            super(IndicoBlueprintSetupState, self).add_url_rule(rule, endpoint, view_func, **options)

class IndicoBlueprint(Blueprint):
    """A Blueprint implementation that allows prefixing URLs with `!` to
    ignore the url_prefix of the blueprint.

    It also supports automatically creating rules in two versions - with and
    without a prefix.

    :param event_feature: If set, this blueprint will raise `NotFound`
                          for all its endpoints unless the event referenced
                          by the `confId` or `event_id` URL argument has
                          the specified feature.
    """

    def __init__(self, *args, **kwargs):
        self.__prefix = None
        self.__default_prefix = ''
        self.__virtual_template_folder = kwargs.pop('virtual_template_folder', None)
        event_feature = kwargs.pop('event_feature', None)
        super(IndicoBlueprint, self).__init__(*args, **kwargs)

        if event_feature:
            @self.before_request
            def _check_event_feature():
                from indico.modules.events.features.util import require_feature
                event_id = request.view_args.get('confId') or request.view_args.get('event_id')
                if event_id is not None:
                    require_feature(event_id, event_feature)

    @locked_cached_property
    def jinja_loader(self):
        if self.template_folder is not None:
            return IndicoFileSystemLoader(os.path.join(self.root_path, self.template_folder),
                                          virtual_path=self.__virtual_template_folder)

    def make_setup_state(self, app, options, first_registration=False):
        return IndicoBlueprintSetupState(self, app, options, first_registration)

    def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
        if view_func is not None:
            # We might have a RH class here - convert it to a callable suitable as a view func.
            view_func = make_view_func(view_func)
        super(IndicoBlueprint, self).add_url_rule(self.__default_prefix + rule, endpoint, view_func, **options)
        if self.__prefix:
            super(IndicoBlueprint, self).add_url_rule(self.__prefix + rule, endpoint, view_func, **options)

    @contextmanager
    def add_prefixed_rules(self, prefix, default_prefix=''):
        """Creates prefixed rules in addition to the normal ones.
        When specifying a default_prefix, too, the normally "unprefixed" rules
        are prefixed with it."""
        assert self.__prefix is None and not self.__default_prefix
        self.__prefix = prefix
        self.__default_prefix = default_prefix
        yield
        self.__prefix = None
        self.__default_prefix = ''

class IndicoFileSystemLoader(FileSystemLoader):
    """FileSystemLoader that makes namespacing easier.

    The `virtual_path` kwarg lets you specify a path segment that's
    handled as if all templates inside the loader's `searchpath` were
    actually inside ``searchpath/virtual_path``.  That way you don't
    have to create subdirectories in your template folder.
    """

    def __init__(self, searchpath, encoding='utf-8', virtual_path=None):
        super(IndicoFileSystemLoader, self).__init__(searchpath, encoding)
        self.virtual_path = virtual_path

    def list_templates(self):
        templates = super(IndicoFileSystemLoader, self).list_templates()
        if self.virtual_path:
            templates = [os.path.join(self.virtual_path, t) for t in templates]
        return templates

    def get_source(self, environment, template):
        if self.virtual_path:
            if not template.startswith(self.virtual_path):
                raise TemplateNotFound(template)
            template = template[len(self.virtual_path):]
        return super(IndicoFileSystemLoader, self).get_source(environment, template)
<|endoftext|>