<commit_before># -*- coding: utf-8 -*-
from django.db import models
from denorm import denorms
from django.conf import settings
import django.db.models

def denormalized(DBField, *args, **kwargs):
    """
    Turns a callable into model field, analogous to python's ``@property`` decorator.
    The callable will be used to compute the value of the field every time the model
    gets saved.
    If the callable has dependency information attached to it the fields value will
    also be recomputed if the dependencies require it.

    **Arguments:**

    DBField (required)
        The type of field you want to use to save the data.
        Note that you have to use the field class and not an instance
        of it.

    \*args, \*\*kwargs:
        Those will be passed unaltered into the constructor of ``DBField``
        once it gets actually created.
    """

    class DenormDBField(DBField):

        """
        Special subclass of the given DBField type, with a few extra additions.
        """

        def __init__(self, func, *args, **kwargs):
            self.func = func
            self.skip = kwargs.pop('skip', None)
            kwargs['editable'] = False
            DBField.__init__(self, *args, **kwargs)

        def contribute_to_class(self, cls, name, *args, **kwargs):
            if hasattr(settings, 'DENORM_BULK_UNSAFE_TRIGGERS') and settings.DENORM_BULK_UNSAFE_TRIGGERS:
                self.denorm = denorms.BaseCallbackDenorm(skip=self.skip)
            else:
                self.denorm = denorms.CallbackDenorm(skip=self.skip)
            self.denorm.func = self.func
            self.denorm.depend = [dcls(*dargs, **dkwargs) for (dcls, dargs, dkwargs) in getattr(self.func, 'depend', [])]
            self.denorm.model = cls
            self.denorm.fieldname = name
            self.field_args = (args, kwargs)
            models.signals.class_prepared.connect(self.denorm.setup, sender=cls)
            # Add The many to many signal for this class
            models.signals.pre_save.connect(denorms.many_to_many_pre_save, sender=cls)
            models.signals.post_save.connect(denorms.many_to_many_post_save, sender=cls)
            DBField.contribute_to_class(self, cls, name, *args, **kwargs)

        def pre_save(self, model_instance, add):
            """
            Updates the value of the denormalized field before it gets saved.
            """
            value = self.denorm.func(model_instance)
            if hasattr(self, 'related_field') and isinstance(value, self.related_field.model):
                setattr(model_instance, self.attname, None)
                setattr(model_instance, self.name, value)
                return getattr(model_instance, self.attname)
            else:
                setattr(model_instance, self.attname, value)
                return value

        def south_field_triple(self):
            """
            Because this field will be defined as a decorator, give
            South hints on how to recreate it for database use.
            """
            from south.modelsinspector import introspector
            field_class = DBField.__module__ + "." + DBField.__name__
            args, kwargs = introspector(self)
            return (field_class, args, kwargs)

        def deconstruct(self):
            name, path, args, kwargs = super(DenormDBField, self).deconstruct()
            super_name, super_path, super_args, super_kwargs = DBField(*args, **kwargs).deconstruct()
            return name, super_path, args, kwargs

    def deco(func):
        dbfield = DenormDBField(func, *args, **kwargs)
        return dbfield
    return deco

class AggregateField(models.PositiveIntegerField):

    def get_denorm(self, *args, **kwargs):
        """
        Returns denorm instance
        """
        raise NotImplemented('You need to override this method')

    def __init__(self, manager_name, **kwargs):
        """
        **Arguments:**

        manager_name:
            The name of the related manager to be counted.

        filter:
            Filter, which is applied to manager. For example:

        >>> active_item_count = CountField('item_set', filter={'active__exact':True})
        >>> adult_user_count = CountField('user_set', filter={'age__gt':18})

        exclude:
            Do not include filter in aggregation

        Any additional arguments are passed on to the contructor of
        PositiveIntegerField.
        """
        skip = kwargs.pop('skip', None)
        qs_filter = kwargs.pop('filter', {})
        if qs_filter and hasattr(django.db.backend, 'sqlite3'):
            raise NotImplementedError('filters for aggregate fields are currently not supported for sqlite')
        qs_exclude = kwargs.pop('exclude', {})
        self.denorm = self.get_denorm(skip)
        self.denorm.manager_name = manager_name
        self.denorm.filter = qs_filter
        self.denorm.exclude = qs_exclude
        self.kwargs = kwargs
        kwargs['default'] = 0
        kwargs['editable'] = False
        super(AggregateField, self).__init__(**kwargs)

    def contribute_to_class(self, cls, name, *args, **kwargs):
        self.denorm.model = cls
        self.denorm.fieldname = name
        models.signals.class_prepared.connect(self.denorm.setup)
        super(AggregateField, self).contribute_to_class(cls, name, *args, **kwargs)

    def south_field_triple(self):
        return (
            '.'.join(('django', 'db', 'models', models.PositiveIntegerField.__name__)),
            [],
            {
                'default': '0',
            },
        )

    def pre_save(self, model_instance, add):
        """
        Makes sure we never overwrite the count with an outdated value.
        This is necessary because if the count was changed by
        a trigger after this model instance was created, the value
        we would write has not been updated.
        """
        if add:
            # if this is a new instance there can't be any related objects yet
            value = 0
        else:
            # if we're updating, get the most recent value from the DB
            value = self.denorm.model.objects.filter(
                pk=model_instance.pk,
            ).values_list(
                self.attname, flat=True,
            )[0]

        setattr(model_instance, self.attname, value)
        return value

    def deconstruct(self):
        name, path, args, kwargs = super(AggregateField, self).deconstruct()
        del kwargs['editable']
        args = [self.denorm.manager_name] + args
        return name, path, args, kwargs

class CountField(AggregateField):
    """
    A ``PositiveIntegerField`` that stores the number of rows
    related to this model instance through the specified manager.
    The value will be incrementally updated when related objects
    are added and removed.

    """

    def __init__(self, manager_name, **kwargs):
        """
        **Arguments:**

        manager_name:
            The name of the related manager to be counted.

        filter:
            Filter, which is applied to manager. For example:

        >>> active_item_count = CountField('item_set', filter={'active__exact':True})
        >>> adult_user_count = CountField('user_set', filter={'age__gt':18})

        Any additional arguments are passed on to the contructor of
        PositiveIntegerField.
        """

        kwargs['editable'] = False
        super(CountField, self).__init__(manager_name, **kwargs)

    def get_denorm(self, skip):
        return denorms.CountDenorm(skip)

class SumField(AggregateField):
    """
    A ``PositiveIntegerField`` that stores sub of related field values
    to this model instance through the specified manager.
    The value will be incrementally updated when related objects
    are added and removed.

    """

    def __init__(self, manager_name, field, **kwargs):
        self.field = field
        kwargs['editable'] = False
        super(SumField, self).__init__(manager_name, **kwargs)

    def get_denorm(self, skip):
        return denorms.SumDenorm(skip, self.field)

class CopyField(AggregateField):
    """
    Field, which makes two field identical. Any change in related field will change this field
    """
    # TODO: JFDI

class CacheKeyField(models.BigIntegerField):
    """
    A ``BigIntegerField`` that gets set to a random value anytime
    the model is saved or a dependency is triggered.
    The field gets updated immediately and does not require *denorm.flush()*.
    It currently cannot detect a direct (bulk)update to the model
    it is declared in.
    """

    def __init__(self, **kwargs):
        """
        All arguments are passed on to the contructor of
        BigIntegerField.
        """
        self.dependencies = []
        kwargs['default'] = 0
        kwargs['editable'] = False
        self.kwargs = kwargs
        super(CacheKeyField, self).__init__(**kwargs)

    def depend_on_related(self, *args, **kwargs):
        """
        Add dependency information to the CacheKeyField.
        Accepts the same arguments like the *denorm.depend_on_related* decorator
        """
        from dependencies import CacheKeyDependOnRelated
        self.dependencies.append(CacheKeyDependOnRelated(*args, **kwargs))

    def contribute_to_class(self, cls, name, *args, **kwargs):
        for depend in self.dependencies:
            depend.fieldname = name
        self.denorm = denorms.BaseCacheKeyDenorm(depend_on_related=self.dependencies)
        self.denorm.model = cls
        self.denorm.fieldname = name
        models.signals.class_prepared.connect(self.denorm.setup)
        super(CacheKeyField, self).contribute_to_class(cls, name, *args, **kwargs)

    def pre_save(self, model_instance, add):
        if add:
            value = self.denorm.func(model_instance)
        else:
            value = self.denorm.model.objects.filter(
                pk=model_instance.pk,
            ).values_list(
                self.attname, flat=True,
            )[0]
        setattr(model_instance, self.attname, value)
        return value

    def south_field_triple(self):
        return (
            '.'.join(('django', 'db', 'models', models.BigIntegerField.__name__)),
            [],
            {
                'default': '0',
            },
        )

class CacheWrapper(object):
    def __init__(self, field):
        self.field = field

    def __set__(self, obj, value):
        key = 'CachedField_%s' % value
        cached = self.field.cache.get(key)
        if not cached:
            cached = self.field.func(obj)
            self.field.cache.set(key, cached, 60 * 60 * 24 * 30)
        obj.__dict__[self.field.name] = cached

class CachedField(CacheKeyField):
    def __init__(self, func, cache, *args, **kwargs):
        self.func = func
        self.cache = cache
        super(CachedField, self).__init__(*args, **kwargs)
        for c, a, kw in self.func.depend:
            self.depend_on_related(*a, **kw)

    def contribute_to_class(self, cls, name, *args, **kwargs):
        super(CachedField, self).contribute_to_class(cls, name, *args, **kwargs)
        setattr(cls, self.name, CacheWrapper(self))

def cached(cache, *args, **kwargs):
    def deco(func):
        dbfield = CachedField(func, cache, *args, **kwargs)
        return dbfield
    return deco
<commit_msg>deal with custom fields __init__ parameters in Django 1.7<commit_after># -*- coding: utf-8 -*-
from django.db import models
from denorm import denorms
from django.conf import settings
import django.db.models

def denormalized(DBField, *args, **kwargs):
    """
    Turns a callable into model field, analogous to python's ``@property`` decorator.
    The callable will be used to compute the value of the field every time the model
    gets saved.
    If the callable has dependency information attached to it the fields value will
    also be recomputed if the dependencies require it.

    **Arguments:**

    DBField (required)
        The type of field you want to use to save the data.
        Note that you have to use the field class and not an instance
        of it.

    \*args, \*\*kwargs:
        Those will be passed unaltered into the constructor of ``DBField``
        once it gets actually created.
    """

    class DenormDBField(DBField):

        """
        Special subclass of the given DBField type, with a few extra additions.
        """

        def __init__(self, func, *args, **kwargs):
            self.func = func
            self.skip = kwargs.pop('skip', None)
            kwargs['editable'] = False
            DBField.__init__(self, *args, **kwargs)

        def contribute_to_class(self, cls, name, *args, **kwargs):
            if hasattr(settings, 'DENORM_BULK_UNSAFE_TRIGGERS') and settings.DENORM_BULK_UNSAFE_TRIGGERS:
                self.denorm = denorms.BaseCallbackDenorm(skip=self.skip)
            else:
                self.denorm = denorms.CallbackDenorm(skip=self.skip)
            self.denorm.func = self.func
            self.denorm.depend = [dcls(*dargs, **dkwargs) for (dcls, dargs, dkwargs) in getattr(self.func, 'depend', [])]
            self.denorm.model = cls
            self.denorm.fieldname = name
            self.field_args = (args, kwargs)
            models.signals.class_prepared.connect(self.denorm.setup, sender=cls)
            # Add The many to many signal for this class
            models.signals.pre_save.connect(denorms.many_to_many_pre_save, sender=cls)
            models.signals.post_save.connect(denorms.many_to_many_post_save, sender=cls)
            DBField.contribute_to_class(self, cls, name, *args, **kwargs)

        def pre_save(self, model_instance, add):
            """
            Updates the value of the denormalized field before it gets saved.
            """
            value = self.denorm.func(model_instance)
            if hasattr(self, 'related_field') and isinstance(value, self.related_field.model):
                setattr(model_instance, self.attname, None)
                setattr(model_instance, self.name, value)
                return getattr(model_instance, self.attname)
            else:
                setattr(model_instance, self.attname, value)
                return value

        def south_field_triple(self):
            """
            Because this field will be defined as a decorator, give
            South hints on how to recreate it for database use.
            """
            from south.modelsinspector import introspector
            field_class = DBField.__module__ + "." + DBField.__name__
            args, kwargs = introspector(self)
            return (field_class, args, kwargs)

        def deconstruct(self):
            name, path, args, kwargs = super(DenormDBField, self).deconstruct()
            super_name, super_path, super_args, super_kwargs = DBField(*args, **kwargs).deconstruct()
            return name, super_path, args, kwargs

    def deco(func):
        dbfield = DenormDBField(func, *args, **kwargs)
        return dbfield
    return deco

class AggregateField(models.PositiveIntegerField):

    def get_denorm(self, *args, **kwargs):
        """
        Returns denorm instance
        """
        raise NotImplemented('You need to override this method')

    def __init__(self, manager_name=None, **kwargs):
        """
        **Arguments:**

        manager_name:
            The name of the related manager to be counted.

        filter:
            Filter, which is applied to manager. For example:

        >>> active_item_count = CountField('item_set', filter={'active__exact':True})
        >>> adult_user_count = CountField('user_set', filter={'age__gt':18})

        exclude:
            Do not include filter in aggregation

        Any additional arguments are passed on to the contructor of
        PositiveIntegerField.
        """
        skip = kwargs.pop('skip', None)
        qs_filter = kwargs.pop('filter', {})
        if qs_filter and hasattr(django.db.backend, 'sqlite3'):
            raise NotImplementedError('filters for aggregate fields are currently not supported for sqlite')
        qs_exclude = kwargs.pop('exclude', {})
        self.denorm = self.get_denorm(skip)
        self.denorm.manager_name = manager_name
        self.denorm.filter = qs_filter
        self.denorm.exclude = qs_exclude
        self.kwargs = kwargs
        kwargs['default'] = 0
        kwargs['editable'] = False
        super(AggregateField, self).__init__(**kwargs)

    def contribute_to_class(self, cls, name, *args, **kwargs):
        self.denorm.model = cls
        self.denorm.fieldname = name
        models.signals.class_prepared.connect(self.denorm.setup)
        super(AggregateField, self).contribute_to_class(cls, name, *args, **kwargs)

    def south_field_triple(self):
        return (
            '.'.join(('django', 'db', 'models', models.PositiveIntegerField.__name__)),
            [],
            {
                'default': '0',
            },
        )

    def pre_save(self, model_instance, add):
        """
        Makes sure we never overwrite the count with an outdated value.
        This is necessary because if the count was changed by
        a trigger after this model instance was created, the value
        we would write has not been updated.
        """
        if add:
            # if this is a new instance there can't be any related objects yet
            value = 0
        else:
            # if we're updating, get the most recent value from the DB
            value = self.denorm.model.objects.filter(
                pk=model_instance.pk,
            ).values_list(
                self.attname, flat=True,
            )[0]

        setattr(model_instance, self.attname, value)
        return value

    def deconstruct(self):
        name, path, args, kwargs = super(AggregateField, self).deconstruct()
        del kwargs['editable']
        args = [self.denorm.manager_name] + args
        return name, path, args, kwargs

class CountField(AggregateField):
    """
    A ``PositiveIntegerField`` that stores the number of rows
    related to this model instance through the specified manager.
    The value will be incrementally updated when related objects
    are added and removed.

    """

    def __init__(self, manager_name=None, **kwargs):
        """
        **Arguments:**

        manager_name:
            The name of the related manager to be counted.

        filter:
            Filter, which is applied to manager. For example:

        >>> active_item_count = CountField('item_set', filter={'active__exact':True})
        >>> adult_user_count = CountField('user_set', filter={'age__gt':18})

        Any additional arguments are passed on to the contructor of
        PositiveIntegerField.
        """

        kwargs['editable'] = False
        super(CountField, self).__init__(manager_name, **kwargs)

    def get_denorm(self, skip):
        return denorms.CountDenorm(skip)

class SumField(AggregateField):
    """
    A ``PositiveIntegerField`` that stores sub of related field values
    to this model instance through the specified manager.
    The value will be incrementally updated when related objects
    are added and removed.

    """

    def __init__(self, manager_name=None, field=None, **kwargs):
        self.field = field
        kwargs['editable'] = False
        super(SumField, self).__init__(manager_name, **kwargs)

    def get_denorm(self, skip):
        return denorms.SumDenorm(skip, self.field)

class CopyField(AggregateField):
    """
    Field, which makes two field identical. Any change in related field will change this field
    """
    # TODO: JFDI

class CacheKeyField(models.BigIntegerField):
    """
    A ``BigIntegerField`` that gets set to a random value anytime
    the model is saved or a dependency is triggered.
    The field gets updated immediately and does not require *denorm.flush()*.
    It currently cannot detect a direct (bulk)update to the model
    it is declared in.
    """

    def __init__(self, **kwargs):
        """
        All arguments are passed on to the contructor of
        BigIntegerField.
        """
        self.dependencies = []
        kwargs['default'] = 0
        kwargs['editable'] = False
        self.kwargs = kwargs
        super(CacheKeyField, self).__init__(**kwargs)

    def depend_on_related(self, *args, **kwargs):
        """
        Add dependency information to the CacheKeyField.
        Accepts the same arguments like the *denorm.depend_on_related* decorator
        """
        from dependencies import CacheKeyDependOnRelated
        self.dependencies.append(CacheKeyDependOnRelated(*args, **kwargs))

    def contribute_to_class(self, cls, name, *args, **kwargs):
        for depend in self.dependencies:
            depend.fieldname = name
        self.denorm = denorms.BaseCacheKeyDenorm(depend_on_related=self.dependencies)
        self.denorm.model = cls
        self.denorm.fieldname = name
        models.signals.class_prepared.connect(self.denorm.setup)
        super(CacheKeyField, self).contribute_to_class(cls, name, *args, **kwargs)

    def pre_save(self, model_instance, add):
        if add:
            value = self.denorm.func(model_instance)
        else:
            value = self.denorm.model.objects.filter(
                pk=model_instance.pk,
            ).values_list(
                self.attname, flat=True,
            )[0]
        setattr(model_instance, self.attname, value)
        return value

    def south_field_triple(self):
        return (
            '.'.join(('django', 'db', 'models', models.BigIntegerField.__name__)),
            [],
            {
                'default': '0',
            },
        )

class CacheWrapper(object):
    def __init__(self, field):
        self.field = field

    def __set__(self, obj, value):
        key = 'CachedField_%s' % value
        cached = self.field.cache.get(key)
        if not cached:
            cached = self.field.func(obj)
            self.field.cache.set(key, cached, 60 * 60 * 24 * 30)
        obj.__dict__[self.field.name] = cached

class CachedField(CacheKeyField):
    def __init__(self, func=None, cache=None, *args, **kwargs):
        self.func = func
        self.cache = cache
        super(CachedField, self).__init__(*args, **kwargs)
        if func and cache:
            for c, a, kw in self.func.depend:
                self.depend_on_related(*a, **kw)

    def contribute_to_class(self, cls, name, *args, **kwargs):
        super(CachedField, self).contribute_to_class(cls, name, *args, **kwargs)
        setattr(cls, self.name, CacheWrapper(self))

def cached(cache, *args, **kwargs):
    def deco(func):
        dbfield = CachedField(func, cache, *args, **kwargs)
        return dbfield
    return deco
<|endoftext|>


<commit_before>#  Licensed under the Apache License, Version 2.0 (the "License"); you may
#  not use this file except in compliance with the License. You may obtain
#  a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#  License for the specific language governing permissions and limitations
#  under the License.

from oslo_policy import policy

_SYSTEM_ADMIN = "role:admin and system_scope:all"

rules = [
    policy.DocumentedRuleDefault(
        name='transport_key:get',
        check_str='rule:all_users',
        scope_types=[],
        description='Get a specific transport key.',
        operations=[
            {
                'path': '/v1/transport_keys/{key-id}}',
                'method': 'GET'
            }
        ]
    ),
    policy.DocumentedRuleDefault(
        name='transport_key:delete',
        check_str=f'{_SYSTEM_ADMIN}',
        scope_types=['system'],
        description='Delete a specific transport key.',
        operations=[
            {
                'path': '/v1/transport_keys/{key-id}',
                'method': 'DELETE'
            }
        ]
    ),
    policy.DocumentedRuleDefault(
        name='transport_keys:get',
        check_str='rule:all_users',
        scope_types=[],
        description='Get a list of all transport keys.',
        operations=[
            {
                'path': '/v1/transport_keys',
                'method': 'GET'
            }
        ]
    ),
    policy.DocumentedRuleDefault(
        name='transport_keys:post',
        check_str=f'{_SYSTEM_ADMIN}',
        scope_types=['system'],
        description='Create a new transport key.',
        operations=[
            {
                'path': '/v1/transport_keys',
                'method': 'POST'
            }
        ]
    ),

]

def list_rules():
    return rules
<commit_msg>Fix RBAC for transportkeys resource<commit_after>#  Licensed under the Apache License, Version 2.0 (the "License"); you may
#  not use this file except in compliance with the License. You may obtain
#  a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#  License for the specific language governing permissions and limitations
#  under the License.

from oslo_policy import policy

_READER = "role:reader"
_SYSTEM_ADMIN = "role:admin and system_scope:all"

rules = [
    policy.DocumentedRuleDefault(
        name='transport_key:get',
        check_str='rule:all_users or {_READER}',
        scope_types=['project', 'system'],
        description='Get a specific transport key.',
        operations=[
            {
                'path': '/v1/transport_keys/{key-id}}',
                'method': 'GET'
            }
        ]
    ),
    policy.DocumentedRuleDefault(
        name='transport_key:delete',
        check_str=f'{_SYSTEM_ADMIN}',
        scope_types=['system'],
        description='Delete a specific transport key.',
        operations=[
            {
                'path': '/v1/transport_keys/{key-id}',
                'method': 'DELETE'
            }
        ]
    ),
    policy.DocumentedRuleDefault(
        name='transport_keys:get',
        check_str='rule:all_users or {_READER}',
        scope_types=['project', 'system'],
        description='Get a list of all transport keys.',
        operations=[
            {
                'path': '/v1/transport_keys',
                'method': 'GET'
            }
        ]
    ),
    policy.DocumentedRuleDefault(
        name='transport_keys:post',
        check_str=f'{_SYSTEM_ADMIN}',
        scope_types=['system'],
        description='Create a new transport key.',
        operations=[
            {
                'path': '/v1/transport_keys',
                'method': 'POST'
            }
        ]
    ),

]

def list_rules():
    return rules
<|endoftext|>


<commit_before>import urllib
import sys
import os
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from django.conf import settings
from django.contrib.auth import authenticate, login
from django.core.handlers.base import BaseHandler
from django.core.handlers.wsgi import WSGIRequest
from django.core.signals import got_request_exception
from django.http import SimpleCookie, HttpRequest
from django.template import TemplateDoesNotExist
from django.test import signals
from django.utils.functional import curry
from django.utils.encoding import smart_str
from django.utils.http import urlencode
from django.utils.itercompat import is_iterable

BOUNDARY = 'BoUnDaRyStRiNg'
MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY

class FakePayload(object):
    """
    A wrapper around StringIO that restricts what can be read since data from
    the network can't be seeked and cannot be read outside of its content
    length. This makes sure that views can't do anything under the test client
    that wouldn't work in Real Life.
    """
    def __init__(self, content):
        self.__content = StringIO(content)
        self.__len = len(content)

    def read(self, num_bytes=None):
        if num_bytes is None:
            num_bytes = self.__len or 1
        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
        content = self.__content.read(num_bytes)
        self.__len -= num_bytes
        return content

class ClientHandler(BaseHandler):
    """
    A HTTP Handler that can be used for testing purposes.
    Uses the WSGI interface to compose requests, but returns
    the raw HttpResponse object
    """
    def __call__(self, environ):
        from django.conf import settings
        from django.core import signals

        # Set up middleware if needed. We couldn't do this earlier, because
        # settings weren't available.
        if self._request_middleware is None:
            self.load_middleware()

        signals.request_started.send(sender=self.__class__)
        try:
            request = WSGIRequest(environ)
            response = self.get_response(request)

            # Apply response middleware.
            for middleware_method in self._response_middleware:
                response = middleware_method(request, response)
            response = self.apply_response_fixes(request, response)
        finally:
            signals.request_finished.send(sender=self.__class__)

        return response

def store_rendered_templates(store, signal, sender, template, context, **kwargs):
    """
    Stores templates and contexts that are rendered.
    """
    store.setdefault('template',[]).append(template)
    store.setdefault('context',[]).append(context)

def encode_multipart(boundary, data):
    """
    Encodes multipart POST data from a dictionary of form values.

    The key will be used as the form data name; the value will be transmitted
    as content. If the value is a file, the contents of the file will be sent
    as an application/octet-stream; otherwise, str(value) will be sent.
    """
    lines = []
    to_str = lambda s: smart_str(s, settings.DEFAULT_CHARSET)

    # Not by any means perfect, but good enough for our purposes.
    is_file = lambda thing: hasattr(thing, "read") and callable(thing.read)

    # Each bit of the multipart form data could be either a form value or a
    # file, or a *list* of form values and/or files. Remember that HTTP field
    # names can be duplicated!
    for (key, value) in data.items():
        if is_file(value):
            lines.extend(encode_file(boundary, key, value))
        elif not isinstance(value, basestring) and is_iterable(value):
            for item in value:
                if is_file(item):
                    lines.extend(encode_file(boundary, key, item))
                else:
                    lines.extend([
                        '--' + boundary,
                        'Content-Disposition: form-data; name="%s"' % to_str(key),
                        '',
                        to_str(item)
                    ])
        else:
            lines.extend([
                '--' + boundary,
                'Content-Disposition: form-data; name="%s"' % to_str(key),
                '',
                to_str(value)
            ])

    lines.extend([
        '--' + boundary + '--',
        '',
    ])
    return '\r\n'.join(lines)

def encode_file(boundary, key, file):
    to_str = lambda s: smart_str(s, settings.DEFAULT_CHARSET)
    return [
        '--' + boundary,
        'Content-Disposition: form-data; name="%s"; filename="%s"' \
            % (to_str(key), to_str(os.path.basename(file.name))),
        'Content-Type: application/octet-stream',
        '',
        file.read()
    ]

class Client:
    """
    A class that can act as a client for testing purposes.

    It allows the user to compose GET and POST requests, and
    obtain the response that the server gave to those requests.
    The server Response objects are annotated with the details
    of the contexts and templates that were rendered during the
    process of serving the request.

    Client objects are stateful - they will retain cookie (and
    thus session) details for the lifetime of the Client instance.

    This is not intended as a replacement for Twill/Selenium or
    the like - it is here to allow testing against the
    contexts and templates produced by a view, rather than the
    HTML rendered to the end-user.
    """
    def __init__(self, **defaults):
        self.handler = ClientHandler()
        self.defaults = defaults
        self.cookies = SimpleCookie()
        self.exc_info = None

    def store_exc_info(self, **kwargs):
        """
        Stores exceptions when they are generated by a view.
        """
        self.exc_info = sys.exc_info()

    def _session(self):
        """
        Obtains the current session variables.
        """
        if 'django.contrib.sessions' in settings.INSTALLED_APPS:
            engine = __import__(settings.SESSION_ENGINE, {}, {}, [''])
            cookie = self.cookies.get(settings.SESSION_COOKIE_NAME, None)
            if cookie:
                return engine.SessionStore(cookie.value)
        return {}
    session = property(_session)

    def request(self, **request):
        """
        The master request method. Composes the environment dictionary
        and passes to the handler, returning the result of the handler.
        Assumes defaults for the query environment, which can be overridden
        using the arguments to the request.
        """
        environ = {
            'HTTP_COOKIE':      self.cookies,
            'PATH_INFO':         '/',
            'QUERY_STRING':      '',
            'REQUEST_METHOD':    'GET',
            'SCRIPT_NAME':       '',
            'SERVER_NAME':       'testserver',
            'SERVER_PORT':       80,
            'SERVER_PROTOCOL':   'HTTP/1.1',
        }
        environ.update(self.defaults)
        environ.update(request)

        # Curry a data dictionary into an instance of the template renderer
        # callback function.
        data = {}
        on_template_render = curry(store_rendered_templates, data)
        signals.template_rendered.connect(on_template_render)

        # Capture exceptions created by the handler.
        got_request_exception.connect(self.store_exc_info)

        try:
            response = self.handler(environ)
        except TemplateDoesNotExist, e:
            # If the view raises an exception, Django will attempt to show
            # the 500.html template. If that template is not available,
            # we should ignore the error in favor of re-raising the
            # underlying exception that caused the 500 error. Any other
            # template found to be missing during view error handling
            # should be reported as-is.
            if e.args != ('500.html',):
                raise

        # Look for a signalled exception, clear the current context
        # exception data, then re-raise the signalled exception.
        # Also make sure that the signalled exception is cleared from
        # the local cache!
        if self.exc_info:
            exc_info = self.exc_info
            self.exc_info = None
            raise exc_info[1], None, exc_info[2]

        # Save the client and request that stimulated the response.
        response.client = self
        response.request = request

        # Add any rendered template detail to the response.
        # If there was only one template rendered (the most likely case),
        # flatten the list to a single element.
        for detail in ('template', 'context'):
            if data.get(detail):
                if len(data[detail]) == 1:
                    setattr(response, detail, data[detail][0]);
                else:
                    setattr(response, detail, data[detail])
            else:
                setattr(response, detail, None)

        # Update persistent cookie data.
        if response.cookies:
            self.cookies.update(response.cookies)

        return response

    def get(self, path, data={}, **extra):
        """
        Requests a response from the server using GET.
        """
        r = {
            'CONTENT_LENGTH':  None,
            'CONTENT_TYPE':    'text/html; charset=utf-8',
            'PATH_INFO':       urllib.unquote(path),
            'QUERY_STRING':    urlencode(data, doseq=True),
            'REQUEST_METHOD': 'GET',
        }
        r.update(extra)

        return self.request(**r)

    def post(self, path, data={}, content_type=MULTIPART_CONTENT, **extra):
        """
        Requests a response from the server using POST.
        """
        if content_type is MULTIPART_CONTENT:
            post_data = encode_multipart(BOUNDARY, data)
        else:
            post_data = data

        r = {
            'CONTENT_LENGTH': len(post_data),
            'CONTENT_TYPE':   content_type,
            'PATH_INFO':      urllib.unquote(path),
            'REQUEST_METHOD': 'POST',
            'wsgi.input':     FakePayload(post_data),
        }
        r.update(extra)

        return self.request(**r)

    def login(self, **credentials):
        """
        Sets the Client to appear as if it has successfully logged into a site.

        Returns True if login is possible; False if the provided credentials
        are incorrect, or the user is inactive, or if the sessions framework is
        not available.
        """
        user = authenticate(**credentials)
        if user and user.is_active \
                and 'django.contrib.sessions' in settings.INSTALLED_APPS:
            engine = __import__(settings.SESSION_ENGINE, {}, {}, [''])

            # Create a fake request to store login details.
            request = HttpRequest()
            if self.session:
                request.session = self.session
            else:
                request.session = engine.SessionStore()
            login(request, user)

            # Set the cookie to represent the session.
            session_cookie = settings.SESSION_COOKIE_NAME
            self.cookies[session_cookie] = request.session.session_key
            cookie_data = {
                'max-age': None,
                'path': '/',
                'domain': settings.SESSION_COOKIE_DOMAIN,
                'secure': settings.SESSION_COOKIE_SECURE or None,
                'expires': None,
            }
            self.cookies[session_cookie].update(cookie_data)

            # Save the session values.
            request.session.save()

            return True
        else:
            return False

    def logout(self):
        """
        Removes the authenticated user's cookies.

        Causes the authenticated user to be logged out.
        """
        session = __import__(settings.SESSION_ENGINE, {}, {}, ['']).SessionStore()
        session.delete(session_key=self.cookies[settings.SESSION_COOKIE_NAME].value)
        self.cookies = SimpleCookie()
<commit_msg>Fixed #8636: Corrected the test client to use the string '80' in the request META, rather than the integer 80. Thanks to Adam Lofts for the report and fix.<commit_after>import urllib
import sys
import os
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

from django.conf import settings
from django.contrib.auth import authenticate, login
from django.core.handlers.base import BaseHandler
from django.core.handlers.wsgi import WSGIRequest
from django.core.signals import got_request_exception
from django.http import SimpleCookie, HttpRequest
from django.template import TemplateDoesNotExist
from django.test import signals
from django.utils.functional import curry
from django.utils.encoding import smart_str
from django.utils.http import urlencode
from django.utils.itercompat import is_iterable

BOUNDARY = 'BoUnDaRyStRiNg'
MULTIPART_CONTENT = 'multipart/form-data; boundary=%s' % BOUNDARY

class FakePayload(object):
    """
    A wrapper around StringIO that restricts what can be read since data from
    the network can't be seeked and cannot be read outside of its content
    length. This makes sure that views can't do anything under the test client
    that wouldn't work in Real Life.
    """
    def __init__(self, content):
        self.__content = StringIO(content)
        self.__len = len(content)

    def read(self, num_bytes=None):
        if num_bytes is None:
            num_bytes = self.__len or 1
        assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
        content = self.__content.read(num_bytes)
        self.__len -= num_bytes
        return content

class ClientHandler(BaseHandler):
    """
    A HTTP Handler that can be used for testing purposes.
    Uses the WSGI interface to compose requests, but returns
    the raw HttpResponse object
    """
    def __call__(self, environ):
        from django.conf import settings
        from django.core import signals

        # Set up middleware if needed. We couldn't do this earlier, because
        # settings weren't available.
        if self._request_middleware is None:
            self.load_middleware()

        signals.request_started.send(sender=self.__class__)
        try:
            request = WSGIRequest(environ)
            response = self.get_response(request)

            # Apply response middleware.
            for middleware_method in self._response_middleware:
                response = middleware_method(request, response)
            response = self.apply_response_fixes(request, response)
        finally:
            signals.request_finished.send(sender=self.__class__)

        return response

def store_rendered_templates(store, signal, sender, template, context, **kwargs):
    """
    Stores templates and contexts that are rendered.
    """
    store.setdefault('template',[]).append(template)
    store.setdefault('context',[]).append(context)

def encode_multipart(boundary, data):
    """
    Encodes multipart POST data from a dictionary of form values.

    The key will be used as the form data name; the value will be transmitted
    as content. If the value is a file, the contents of the file will be sent
    as an application/octet-stream; otherwise, str(value) will be sent.
    """
    lines = []
    to_str = lambda s: smart_str(s, settings.DEFAULT_CHARSET)

    # Not by any means perfect, but good enough for our purposes.
    is_file = lambda thing: hasattr(thing, "read") and callable(thing.read)

    # Each bit of the multipart form data could be either a form value or a
    # file, or a *list* of form values and/or files. Remember that HTTP field
    # names can be duplicated!
    for (key, value) in data.items():
        if is_file(value):
            lines.extend(encode_file(boundary, key, value))
        elif not isinstance(value, basestring) and is_iterable(value):
            for item in value:
                if is_file(item):
                    lines.extend(encode_file(boundary, key, item))
                else:
                    lines.extend([
                        '--' + boundary,
                        'Content-Disposition: form-data; name="%s"' % to_str(key),
                        '',
                        to_str(item)
                    ])
        else:
            lines.extend([
                '--' + boundary,
                'Content-Disposition: form-data; name="%s"' % to_str(key),
                '',
                to_str(value)
            ])

    lines.extend([
        '--' + boundary + '--',
        '',
    ])
    return '\r\n'.join(lines)

def encode_file(boundary, key, file):
    to_str = lambda s: smart_str(s, settings.DEFAULT_CHARSET)
    return [
        '--' + boundary,
        'Content-Disposition: form-data; name="%s"; filename="%s"' \
            % (to_str(key), to_str(os.path.basename(file.name))),
        'Content-Type: application/octet-stream',
        '',
        file.read()
    ]

class Client:
    """
    A class that can act as a client for testing purposes.

    It allows the user to compose GET and POST requests, and
    obtain the response that the server gave to those requests.
    The server Response objects are annotated with the details
    of the contexts and templates that were rendered during the
    process of serving the request.

    Client objects are stateful - they will retain cookie (and
    thus session) details for the lifetime of the Client instance.

    This is not intended as a replacement for Twill/Selenium or
    the like - it is here to allow testing against the
    contexts and templates produced by a view, rather than the
    HTML rendered to the end-user.
    """
    def __init__(self, **defaults):
        self.handler = ClientHandler()
        self.defaults = defaults
        self.cookies = SimpleCookie()
        self.exc_info = None

    def store_exc_info(self, **kwargs):
        """
        Stores exceptions when they are generated by a view.
        """
        self.exc_info = sys.exc_info()

    def _session(self):
        """
        Obtains the current session variables.
        """
        if 'django.contrib.sessions' in settings.INSTALLED_APPS:
            engine = __import__(settings.SESSION_ENGINE, {}, {}, [''])
            cookie = self.cookies.get(settings.SESSION_COOKIE_NAME, None)
            if cookie:
                return engine.SessionStore(cookie.value)
        return {}
    session = property(_session)

    def request(self, **request):
        """
        The master request method. Composes the environment dictionary
        and passes to the handler, returning the result of the handler.
        Assumes defaults for the query environment, which can be overridden
        using the arguments to the request.
        """
        environ = {
            'HTTP_COOKIE':      self.cookies,
            'PATH_INFO':         '/',
            'QUERY_STRING':      '',
            'REQUEST_METHOD':    'GET',
            'SCRIPT_NAME':       '',
            'SERVER_NAME':       'testserver',
            'SERVER_PORT':       '80',
            'SERVER_PROTOCOL':   'HTTP/1.1',
        }
        environ.update(self.defaults)
        environ.update(request)

        # Curry a data dictionary into an instance of the template renderer
        # callback function.
        data = {}
        on_template_render = curry(store_rendered_templates, data)
        signals.template_rendered.connect(on_template_render)

        # Capture exceptions created by the handler.
        got_request_exception.connect(self.store_exc_info)

        try:
            response = self.handler(environ)
        except TemplateDoesNotExist, e:
            # If the view raises an exception, Django will attempt to show
            # the 500.html template. If that template is not available,
            # we should ignore the error in favor of re-raising the
            # underlying exception that caused the 500 error. Any other
            # template found to be missing during view error handling
            # should be reported as-is.
            if e.args != ('500.html',):
                raise

        # Look for a signalled exception, clear the current context
        # exception data, then re-raise the signalled exception.
        # Also make sure that the signalled exception is cleared from
        # the local cache!
        if self.exc_info:
            exc_info = self.exc_info
            self.exc_info = None
            raise exc_info[1], None, exc_info[2]

        # Save the client and request that stimulated the response.
        response.client = self
        response.request = request

        # Add any rendered template detail to the response.
        # If there was only one template rendered (the most likely case),
        # flatten the list to a single element.
        for detail in ('template', 'context'):
            if data.get(detail):
                if len(data[detail]) == 1:
                    setattr(response, detail, data[detail][0]);
                else:
                    setattr(response, detail, data[detail])
            else:
                setattr(response, detail, None)

        # Update persistent cookie data.
        if response.cookies:
            self.cookies.update(response.cookies)

        return response

    def get(self, path, data={}, **extra):
        """
        Requests a response from the server using GET.
        """
        r = {
            'CONTENT_LENGTH':  None,
            'CONTENT_TYPE':    'text/html; charset=utf-8',
            'PATH_INFO':       urllib.unquote(path),
            'QUERY_STRING':    urlencode(data, doseq=True),
            'REQUEST_METHOD': 'GET',
        }
        r.update(extra)

        return self.request(**r)

    def post(self, path, data={}, content_type=MULTIPART_CONTENT, **extra):
        """
        Requests a response from the server using POST.
        """
        if content_type is MULTIPART_CONTENT:
            post_data = encode_multipart(BOUNDARY, data)
        else:
            post_data = data

        r = {
            'CONTENT_LENGTH': len(post_data),
            'CONTENT_TYPE':   content_type,
            'PATH_INFO':      urllib.unquote(path),
            'REQUEST_METHOD': 'POST',
            'wsgi.input':     FakePayload(post_data),
        }
        r.update(extra)

        return self.request(**r)

    def login(self, **credentials):
        """
        Sets the Client to appear as if it has successfully logged into a site.

        Returns True if login is possible; False if the provided credentials
        are incorrect, or the user is inactive, or if the sessions framework is
        not available.
        """
        user = authenticate(**credentials)
        if user and user.is_active \
                and 'django.contrib.sessions' in settings.INSTALLED_APPS:
            engine = __import__(settings.SESSION_ENGINE, {}, {}, [''])

            # Create a fake request to store login details.
            request = HttpRequest()
            if self.session:
                request.session = self.session
            else:
                request.session = engine.SessionStore()
            login(request, user)

            # Set the cookie to represent the session.
            session_cookie = settings.SESSION_COOKIE_NAME
            self.cookies[session_cookie] = request.session.session_key
            cookie_data = {
                'max-age': None,
                'path': '/',
                'domain': settings.SESSION_COOKIE_DOMAIN,
                'secure': settings.SESSION_COOKIE_SECURE or None,
                'expires': None,
            }
            self.cookies[session_cookie].update(cookie_data)

            # Save the session values.
            request.session.save()

            return True
        else:
            return False

    def logout(self):
        """
        Removes the authenticated user's cookies.

        Causes the authenticated user to be logged out.
        """
        session = __import__(settings.SESSION_ENGINE, {}, {}, ['']).SessionStore()
        session.delete(session_key=self.cookies[settings.SESSION_COOKIE_NAME].value)
        self.cookies = SimpleCookie()
<|endoftext|>


<commit_before># TODO: add quicker lookup using directories as a tree like structure

from google.appengine.ext import ndb
import guido
import re

class FunctionMetadata(ndb.Model):
  name = ndb.StringProperty()
  args = ndb.JsonProperty()
  num_of_args = ndb.IntegerProperty()
  def_args = ndb.JsonProperty()
  num_of_def_args = ndb.IntegerProperty()
  filepath = ndb.StringProperty()
  location = ndb.IntegerProperty()

class DatastoreGuido(guido.Guido):
  def __init__(self, buffer_limit = 10):
    self.buffer_limit = buffer_limit
    self.buffer = []

  def add_lazy(self, name, args, def_args, filepath, location):
    f = FunctionMetadata(
        name=name,
        args=args,
        num_of_args=len(args),
        def_args=def_args,
        num_of_def_args=len(def_args),
        filepath=filepath,
        location=location)
    self.buffer.append(f)

    if len(self.buffer) >= self.buffer_limit:
      self.flush()

  def add(self, name, args, def_args, filepath, location):
    self.add_lazy(name, args, def_args, filepath, location)
    self.flush()

  def flush(self):
    for f in self.buffer:
      q = FunctionMetadata.query(FunctionMetadata.name == f.name)
      q = q.filter(FunctionMetadata.filepath == f.filepath).get()
      if q:
        continue
      f.put()

    self.buffer = []

  def search(self, num_args, file_regex=None):
    q = FunctionMetadata.query(FunctionMetadata.num_of_args <= num_args)

    dir_regex = re.compile(file_regex or ".*")
    dir_matcher = lambda f: dir_regex.match(f.filepath)

    args_matcher = lambda f: f.num_of_args + f.num_of_def_args >= num_args

    return [f for f in q.fetch() if dir_matcher(f) and args_matcher(f)]
<commit_msg>datastore guido is now consistent<commit_after># TODO: add quicker lookup using directories as a tree like structure

from google.appengine.ext import ndb
import guido
import re

class FunctionMetadata(ndb.Model):
  name = ndb.StringProperty()
  args = ndb.JsonProperty()
  num_of_args = ndb.IntegerProperty()
  def_args = ndb.JsonProperty()
  num_of_def_args = ndb.IntegerProperty()
  filepath = ndb.StringProperty()
  location = ndb.IntegerProperty()

class DatastoreGuido(guido.Guido):
  key = ndb.Key("FunctionsAncestor", "global")

  def __init__(self, buffer_limit = 10):
    self.buffer_limit = buffer_limit
    self.buffer = []

  def add_lazy(self, name, args, def_args, filepath, location):
    f = FunctionMetadata(
        parent=self.key,
        name=name,
        args=args,
        num_of_args=len(args),
        def_args=def_args,
        num_of_def_args=len(def_args),
        filepath=filepath,
        location=location)
    self.buffer.append(f)

    if len(self.buffer) >= self.buffer_limit:
      self.flush()

  def add(self, name, args, def_args, filepath, location):
    self.add_lazy(name, args, def_args, filepath, location)
    self.flush()

  def flush(self):
    for f in self.buffer:
      q = FunctionMetadata.query(ancestor=self.key)
      q = q.filter(FunctionMetadata.name == f.name)
      q = q.filter(FunctionMetadata.filepath == f.filepath).get()
      if q:
        continue
      f.put()

    self.buffer = []

  def search(self, num_args, file_regex=None):
    q = FunctionMetadata.query(ancestor=self.key)
    q = q.filter(FunctionMetadata.num_of_args <= num_args)

    dir_regex = re.compile(file_regex or ".*")
    dir_matcher = lambda f: dir_regex.match(f.filepath)

    args_matcher = lambda f: f.num_of_args + f.num_of_def_args >= num_args

    return [f for f in q.fetch() if dir_matcher(f) and args_matcher(f)]

<|endoftext|>


<commit_before>import os
import sys

sys.path.append(os.path.dirname(__name__))

from app import app
from settings import DEFAULT_WEB_SERVER

app.run(host=DEFAULT_WEB_SERVER['host'], port=DEFAULT_WEB_SERVER['port'], debug=False)
<commit_msg>webfrontend: enable debug<commit_after>import os
import sys

sys.path.append(os.path.dirname(__name__))

from app import app
from settings import DEFAULT_WEB_SERVER

app.run(host=DEFAULT_WEB_SERVER['host'], port=DEFAULT_WEB_SERVER['port'], debug=True)
<|endoftext|>


<commit_before>import json

from django.core.urlresolvers import reverse_lazy

from django.db.models import Q

from django.http import JsonResponse

from django.views.decorators.http import require_POST
from django.views.decorators.csrf import csrf_exempt

from django.views.generic import ListView, TemplateView
from django.views.generic.edit import FormView, CreateView

from django.utils.translation import ugettext_lazy as _

from .models import Alphaname, Project, Task
from .forms import AlphanameForm, OneTimeTaskForm, TaskForm, RecurringTaskForm,\
    EventDrivenTaskForm

from .tasks import submitAlphanameInstantly, addNewCampaignTask

class IndexView(TemplateView):
    """Shows all menu entries of an app"""
    template_name = 'main.html'

class AlphanameIndexView(ListView):
    """Shows all alphaname list available along with registrar and registration date"""
    template_name = 'alphanames-list.html'
    context_object_name = 'alphanames'
    model = Alphaname

class AlphanameRegisterView(CreateView):
    """Sends new alphaname register request"""
    template_name = 'alphaname-new.html'
    form_class = AlphanameForm
    success_url = reverse_lazy('alphanames-root')

    def get_form(self, form_class=None):
        return (form_class or self.form_class)(self.request, **self.get_form_kwargs())

    def form_valid(self, form):
        # Add job for sending request to register a new alphanumeric name
        submitAlphanameInstantly.delay(form.cleaned_data['name'])
        return super().form_valid(form)

class CampaignBaseView(ListView):
    """Base for tabbed list views"""

    active_menu_item = None

    def get_tab_menu(self, active_item=None):
        menu = (
            {
                'url': reverse_lazy('campaigns-root'),
                'title': _('Актуальные'),
            },
            {
                'url': reverse_lazy('campaigns-archive'),
                'title': _('Архив'),
            },
        )

        try:
            menu[active_item or self.active_menu_item]['class'] = 'active'
        except KeyError:
            pass

        return menu

    def get_context_data(self):
        context = super().get_context_data()
        context['tab_menu'] = self.get_tab_menu()
        return context

class CampaignIndexView(CampaignBaseView):
    """Lists all active campaigns currently in progress (scheduled)"""
    template_name = 'campaigns-list.html'
    active_menu_item = 0
    context_object_name = 'tasks'
    queryset = Task.objects.filter(state=0)

class CampaignArchiveView(CampaignBaseView):
    """Keeps a history of campaigns, which are inactive"""
    template_name = 'campaigns-list.html'
    active_menu_item = 1
    context_object_name = 'tasks'
    queryset = Task.objects.filter(Q(state=2) | Q(state=1))

class CampaignNewView(CreateView):
    """Helps schedule a new campaign or send new one instantly"""
    template_name = 'campaign-edit.html'
    form_class = TaskForm
    success_url = reverse_lazy('campaigns-root')

    def get_form(self, form_class=None):
        return (form_class or self.form_class)(self.request, **self.get_form_kwargs())

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['range30'] = range(1, 31)
        return context

    def form_valid(self, form):
        # Save new campaign and notify everyone about it, add job into queue if needed
        addNewCampaignTask.delay()
        return super().form_valid(form)

class CampaignNewRecurringView(CampaignNewView):
    form_class = RecurringTaskForm
    template_name = 'campaign-recurring-edit.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['weekdays'] = (
            (1, 'пн'),
            (2, 'вт'),
            (3, 'ср'),
            (4, 'чт'),
            (5, 'пт'),
            (6, 'сб'),
            (7, 'нд'),
        )
        return context

class CampaignNewEventDrivenView(CampaignNewView):
    form_class = EventDrivenTaskForm
    template_name = 'campaign-event-driven-edit.html'

class CampaignEditView(FormView):
    """Makes modifications on an active campaign"""
    template_name = 'campaign-edit.html'
    form_class = OneTimeTaskForm
    success_url = ''

    def form_valid(self, form):
        # Change campaign settings and notify everyone about its change, which involves changing DB and rescheduling job
        return super().form_valid(form)

class CampaignMessagesView(ListView):
    """Keeps a history of messages"""
    template_name = 'sent-messages.html'
    queryset = []  # TODO: replace fake queryset with an existing model

class CampaignStatsView(ListView):
    """Shows stats on campaigns"""
    template_name = 'campaigns-stats.html'
    queryset = []  # TODO: replace fake queryset with an existing model

@require_POST
@csrf_exempt
def webhook_crm_event(request, crm_event, crm_user_id):
    crm_user_id = int(crm_user_id)
    json_res = {}
    try:
        json_req = json.loads(request.body.decode())
        'sms_view_projects.project_id'
        'sms_view_follower_status.follower_status_id'
        'sms_view_project_contacts.contact_id'
        'sms_view_candidates.candidate_id'
    except json.decoder.JSONDecodeError:
        json_res = {
            'result': 'Client Error',
            'status': 400,
            'message': 'The trigger processing has failed',
        }
    else:
        json_res = {
            'result': 'OK',
            'status': 200,
            'message': 'The trigger processing has been queued',
            'data': Project.objects.for_user(crm_user_id).filter(project_id=json_req['project_id']).all()[0].
            project_name,
        }
    finally:
        return JsonResponse(json_res)

@require_POST
@csrf_exempt
def webhook_smsfly_status(request):
    return 'SMSFly webhook here'
<commit_msg>Pass task id into rq job<commit_after>import json

from django.core.urlresolvers import reverse_lazy

from django.db.models import Q

from django.http import JsonResponse

from django.views.decorators.http import require_POST
from django.views.decorators.csrf import csrf_exempt

from django.views.generic import ListView, TemplateView
from django.views.generic.edit import FormView, CreateView

from django.utils.translation import ugettext_lazy as _

from .models import Alphaname, Project, Task
from .forms import AlphanameForm, OneTimeTaskForm, TaskForm, RecurringTaskForm,\
    EventDrivenTaskForm

from .tasks import submitAlphanameInstantly, addNewCampaignTask

class IndexView(TemplateView):
    """Shows all menu entries of an app"""
    template_name = 'main.html'

class AlphanameIndexView(ListView):
    """Shows all alphaname list available along with registrar and registration date"""
    template_name = 'alphanames-list.html'
    context_object_name = 'alphanames'
    model = Alphaname

class AlphanameRegisterView(CreateView):
    """Sends new alphaname register request"""
    template_name = 'alphaname-new.html'
    form_class = AlphanameForm
    success_url = reverse_lazy('alphanames-root')

    def get_form(self, form_class=None):
        return (form_class or self.form_class)(self.request, **self.get_form_kwargs())

    def form_valid(self, form):
        # Add job for sending request to register a new alphanumeric name
        submitAlphanameInstantly.delay(form.cleaned_data['name'])
        return super().form_valid(form)

class CampaignBaseView(ListView):
    """Base for tabbed list views"""

    active_menu_item = None

    def get_tab_menu(self, active_item=None):
        menu = (
            {
                'url': reverse_lazy('campaigns-root'),
                'title': _('Актуальные'),
            },
            {
                'url': reverse_lazy('campaigns-archive'),
                'title': _('Архив'),
            },
        )

        try:
            menu[active_item or self.active_menu_item]['class'] = 'active'
        except KeyError:
            pass

        return menu

    def get_context_data(self):
        context = super().get_context_data()
        context['tab_menu'] = self.get_tab_menu()
        return context

class CampaignIndexView(CampaignBaseView):
    """Lists all active campaigns currently in progress (scheduled)"""
    template_name = 'campaigns-list.html'
    active_menu_item = 0
    context_object_name = 'tasks'
    queryset = Task.objects.filter(state=0)

class CampaignArchiveView(CampaignBaseView):
    """Keeps a history of campaigns, which are inactive"""
    template_name = 'campaigns-list.html'
    active_menu_item = 1
    context_object_name = 'tasks'
    queryset = Task.objects.filter(Q(state=2) | Q(state=1))

class CampaignNewView(CreateView):
    """Helps schedule a new campaign or send new one instantly"""
    template_name = 'campaign-edit.html'
    form_class = TaskForm
    success_url = reverse_lazy('campaigns-root')

    def get_form(self, form_class=None):
        return (form_class or self.form_class)(self.request, **self.get_form_kwargs())

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['range30'] = range(1, 31)
        return context

    def form_valid(self, form):
        # Save new campaign and notify everyone about it, add job into queue if needed
        addNewCampaignTask.delay(form.instance.id)
        return super().form_valid(form)

class CampaignNewRecurringView(CampaignNewView):
    form_class = RecurringTaskForm
    template_name = 'campaign-recurring-edit.html'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['weekdays'] = (
            (1, 'пн'),
            (2, 'вт'),
            (3, 'ср'),
            (4, 'чт'),
            (5, 'пт'),
            (6, 'сб'),
            (7, 'нд'),
        )
        return context

class CampaignNewEventDrivenView(CampaignNewView):
    form_class = EventDrivenTaskForm
    template_name = 'campaign-event-driven-edit.html'

class CampaignEditView(FormView):
    """Makes modifications on an active campaign"""
    template_name = 'campaign-edit.html'
    form_class = OneTimeTaskForm
    success_url = ''

    def form_valid(self, form):
        # Change campaign settings and notify everyone about its change, which involves changing DB and rescheduling job
        return super().form_valid(form)

class CampaignMessagesView(ListView):
    """Keeps a history of messages"""
    template_name = 'sent-messages.html'
    queryset = []  # TODO: replace fake queryset with an existing model

class CampaignStatsView(ListView):
    """Shows stats on campaigns"""
    template_name = 'campaigns-stats.html'
    queryset = []  # TODO: replace fake queryset with an existing model

@require_POST
@csrf_exempt
def webhook_crm_event(request, crm_event, crm_user_id):
    crm_user_id = int(crm_user_id)
    json_res = {}
    try:
        json_req = json.loads(request.body.decode())
        'sms_view_projects.project_id'
        'sms_view_follower_status.follower_status_id'
        'sms_view_project_contacts.contact_id'
        'sms_view_candidates.candidate_id'
    except json.decoder.JSONDecodeError:
        json_res = {
            'result': 'Client Error',
            'status': 400,
            'message': 'The trigger processing has failed',
        }
    else:
        json_res = {
            'result': 'OK',
            'status': 200,
            'message': 'The trigger processing has been queued',
            'data': Project.objects.for_user(crm_user_id).filter(project_id=json_req['project_id']).all()[0].
            project_name,
        }
    finally:
        return JsonResponse(json_res)

@require_POST
@csrf_exempt
def webhook_smsfly_status(request):
    return 'SMSFly webhook here'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

try:
    from django.contrib.auth import get_user_model

    User = get_user_model()
except ImportError:
    from django.contrib.auth.models import User

class SuBackend(object):
    supports_inactive_user = True

    def authenticate(self, su=False, user_id=None, **kwargs):
        if not su:
            return None

        try:
            user = User._default_manager.get(pk=user_id)  # pylint: disable=W0212
        except (User.DoesNotExist, ValueError):
            return None

        return user

    def get_user(self, user_id):
        try:
            return User._default_manager.get(pk=user_id)  # pylint: disable=W0212
        except User.DoesNotExist:
            return None
<commit_msg>Setting ``supports_inactive_user`` to False<commit_after># -*- coding: utf-8 -*-

try:
    from django.contrib.auth import get_user_model

    User = get_user_model()
except ImportError:
    from django.contrib.auth.models import User

class SuBackend(object):
    supports_inactive_user = False

    def authenticate(self, su=False, user_id=None, **kwargs):
        if not su:
            return None

        try:
            user = User._default_manager.get(pk=user_id)  # pylint: disable=W0212
        except (User.DoesNotExist, ValueError):
            return None

        return user

    def get_user(self, user_id):
        try:
            return User._default_manager.get(pk=user_id)  # pylint: disable=W0212
        except User.DoesNotExist:
            return None
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Reads AppStream XML metadata and metadata from
XDG .desktop files.
"""

# Copyright (c) 2014 Abhishek Bhattacharjee <abhishek.bhattacharjee11@gmail.com>
# Copyright (c) 2014 Matthias Klumpp <mak@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program.

import re
from configparser import RawConfigParser
import lxml.etree as et
from xml.sax.saxutils import escape
from io import StringIO

from dep11.component import DEP11Component, ProvidedItemType
from dep11.utils import str_enc_dec

def read_desktop_data(cpt, dcontent):
    '''
    Parses a .desktop file and sets ComponentData properties
    '''
    df = RawConfigParser(allow_no_value=True)

    items = None
    try:
        df.readfp(StringIO(dcontent))

        items = df.items("Desktop Entry")
        if df.get("Desktop Entry", "Type") != "Application":
            # ignore this file, isn't an application
            cpt.add_hint("not-an-application")
            return False
        try:
            if df.get("Desktop Entry", "NoDisplay") == "True":
                # we ignore this .desktop file, shouldn't be displayed
                cpt.add_hint("invisible-application")
                return False
            if df.get("Desktop Entry", "X-AppStream-Ignore") == "True":
                # this .desktop file should be excluded from AppStream metadata
                cpt.add_hint("invisible-application")
                return False
        except:
            # we don't care if the NoDisplay variable doesn't exist
            # if it isn't there, the file should be processed
            pass
    except Exception as e:
        # this .desktop file is not interesting
        cpt.add_hint("desktop-file-read-error", str(e))
        return True

    # if we reached this step, we are dealing with a GUI desktop app
    cpt.kind = 'desktop-app'

    for item in items:
        if len(item) != 2:
            continue
        key = item[0]
        value = str_enc_dec(item[1])
        if not value:
            continue
        if key.startswith("name"):
            if key == 'name':
                cpt.name['C'] = value
            else:
                cpt.name[key[5:-1]] = value
        elif key == 'categories':
            value = value.split(';')
            value.pop()
            cpt.categories = value
        elif key.startswith('comment'):
            if key == 'comment':
                cpt.summary['C'] = value
            else:
                cpt.summary[key[8:-1]] = value
        elif key.startswith('keywords'):
            value = re.split(';|,', value)
            if not value[-1]:
                value.pop()
            if key[8:] == '':
                if cpt.keywords:
                    if set(value) not in \
                        [set(val) for val in
                            cpt.keywords.values()]:
                        cpt.keywords.update(
                            {'C': list(map(str_enc_dec, value))}
                        )
                else:
                    cpt.keywords = {
                        'C': list(map(str_enc_dec, value))
                    }
            else:
                if cpt.keywords:
                    if set(value) not in \
                        [set(val) for val in
                            cpt.keywords.values()]:
                        cpt.keywords.update(
                            {key[9:-1]: list(map(str_enc_dec, value))}
                        )
                else:
                    cpt.keywords = {
                        key[9:-1]: list(map(str_enc_dec, value))
                    }
        elif key == 'mimetype':
            value = value.split(';')
            if len(value) > 1:
                value.pop()
            for val in value:
                cpt.add_provided_item(
                    ProvidedItemType.MIMETYPE, val
                )
        elif key == 'icon':
            cpt.icon = value
    return True

def _get_tag_locale(subs):
    attr_dic = subs.attrib
    if attr_dic:
        locale = attr_dic.get('{http://www.w3.org/XML/1998/namespace}lang')
        if locale:
            return locale
    return "C"

def _parse_description_tag(subs):
    '''
    Handles the description tag
    '''

    def prepare_desc_string(s):
        '''
        Clears linebreaks and XML-escapes the resulting string
        '''

        if not s:
            return ""
        s = s.strip()
        s = " ".join(s.split())
        return escape(s)

    ddict = dict()

    # The description tag translation is combined per language,
    # for faster parsing on the client side.
    # In case no translation is found, the untranslated version is used instead.
    # the DEP-11 YAML stores the description as HTML

    for usubs in subs:
        locale = _get_tag_locale(usubs)

        if usubs.tag == 'p':
            if not locale in ddict:
                ddict[locale] = ""
            ddict[locale] += "
%s
" % str_enc_dec(prepare_desc_string(usubs.text))
        elif usubs.tag == 'ul' or usubs.tag == 'ol':
            tmp_dict = dict()
            # find the right locale, or fallback to untranslated
            for u_usubs in usubs:
                locale = _get_tag_locale(u_usubs)

                if not locale in tmp_dict:
                    tmp_dict[locale] = ""

                if u_usubs.tag == 'li':
                    tmp_dict[locale] += "
%s
" % str_enc_dec(prepare_desc_string(u_usubs.text))

            for locale, value in tmp_dict.items():
                if not locale in ddict:
                    # This should not happen (but better be prepared)
                    ddict[locale] = ""
                ddict[locale] += "<%s>%s</%s>" % (usubs.tag, value, usubs.tag)
    return ddict

def _parse_screenshots_tag(subs):
    '''
    Handles screenshots, caption, source-image etc.
    '''
    shots = []
    for usubs in subs:
        # for one screeshot tag
        if usubs.tag == 'screenshot':
            screenshot = dict()
            attr_dic = usubs.attrib
            if attr_dic.get('type'):
                if attr_dic['type'] == 'default':
                    screenshot['default'] = True
            # in case of old styled xmls
            url = usubs.text
            if url:
                url = url.strip()
                screenshot['source-image'] = {'url': url}
                shots.append(screenshot)
                continue

            # else look for captions and image tag
            for tags in usubs:
                if tags.tag == 'caption':
                    # for localisation
                    attr_dic = tags.attrib
                    if attr_dic:
                        for v in attr_dic.values():
                            key = v
                    else:
                        key = 'C'

                    if screenshot.get('caption'):
                        screenshot['caption'][key] = str_enc_dec(tags.text)
                    else:
                        screenshot['caption'] = {key: str_enc_dec(tags.text)}
                if tags.tag == 'image':
                    screenshot['source-image'] = {'url': tags.text}

            # only add the screenshot if we have a source image
            if screenshot.get ('source-image'):
                shots.append(screenshot)

    return shots

def _parse_releases_tag(relstag):
    '''
    Parses a releases tag and returns the last three releases
    '''
    rels = list()
    for subs in relstag:
        # for one screeshot tag
        if subs.tag != 'release':
            continue

        release = dict()
        attr_dic = subs.attrib
        if attr_dic.get('version'):
            release['version'] = attr_dic['version']

        if attr_dic.get('timestamp'):
            try:
                release['unix-timestamp'] = int(attr_dic['timestamp'])
            except:
                # the timestamp was wrong - we silently ignore the error
                # TODO: Emit warning hint
                continue
        else:
            # we can't use releases which don't have a timestamp
            # TODO: Emit a warning hint here
            continue

        # else look for captions and image tag
        for usubs in subs:
            if usubs.tag == 'description':
                release['description'] = _parse_description_tag(usubs)

        rels.append(release)

    # sort releases, newest first
    rels = sorted(rels, key=lambda k: k['unix-timestamp'], reverse=True)

    if len(rels) > 3:
        return rels[:3]

    return rels

def read_appstream_upstream_xml(cpt, xml_content):
    '''
    Reads the appdata from the xml file in usr/share/appdata.
    Sets ComponentData properties
    '''
    root = None
    try:
        root = et.fromstring(bytes(xml_content, 'utf-8'))
    except Exception as e:
        cpt.add_hint("metainfo-parse-error", str(e))
        return
    if root is None:
        cpt.add_hint("metainfo-parse-error", "Error is unknown, the root node was null.")

    if root.tag == 'application':
        # we parse ancient AppStream XML, but it is a good idea to update it to make use of newer features, remove some ancient
        # oddities and to simplify the parser in future. So we add a hint for that.
        cpt.add_hint("ancient-metadata")

    key = root.attrib.get('type')
    if key:
        if key == 'desktop':
            cpt.kind = 'desktop-app'
        else:
            # for other components like addon,codec, inputmethod etc
            cpt.kind = root.attrib['type']

    for subs in root:
        locale = _get_tag_locale(subs)

        if subs.tag == 'id':
            cpt.cid = subs.text
            # legacy support
            key = subs.attrib.get('type')
            if key and not cpt.kind:
                if key == 'desktop':
                    cpt.kind = 'desktop-app'
                else:
                    cpt.kind = key

        elif subs.tag == "name":
            cpt.name[locale] = subs.text

        elif subs.tag == "summary":
            cpt.summary[locale] = subs.text

        elif subs.tag == "description":
            desc = _parse_description_tag(subs)
            cpt.description = desc

        elif subs.tag == "screenshots":
            screen = _parse_screenshots_tag(subs)
            cpt.screenshots = screen

        elif subs.tag == "provides":
            for bins in subs:
                if bins.tag == "binary":
                    cpt.add_provided_item(
                        ProvidedItemType.BINARY, bins.text
                    )
                if bins.tag == 'library':
                    cpt.add_provided_item(
                        ProvidedItemType.LIBRARY, bins.text
                    )
                if bins.tag == 'dbus':
                    if not cpt.provides.get(ProvidedItemType.DBUS):
                        cpt.provides[ProvidedItemType.DBUS] = list()

                    bus_kind = bins.attrib.get('type')
                    if bus_kind == "session":
                        bus_kind = "user"
                    if bus_kind:
                        cpt.provides[ProvidedItemType.DBUS].append({'type': bus_kind, 'service': bins.text})
                if bins.tag == 'firmware':
                    if not cpt.provides.get(ProvidedItemType.FIRMWARE):
                        cpt.provides[ProvidedItemType.FIRMWARE] = list()

                    fw_type = bins.attrib.get('type')
                    fw_data = {'type': fw_type}

                    _valid = True
                    if fw_type == "flashed":
                        fw_data['guid'] = bins.text
                    elif fw_type == "runtime":
                        fw_data['fname'] = bins.text
                    else:
                        _valid = False

                    if _valid:
                        cpt.provides[ProvidedItemType.FIRMWARE].append(fw_data)
                if bins.tag == 'python2':
                    cpt.add_provided_item(
                        ProvidedItemType.PYTHON_2, bins.text
                    )
                if bins.tag == 'python3':
                    cpt.add_provided_item(
                        ProvidedItemType.PYTHON_3, bins.text
                    )
                if bins.tag == 'codec':
                    cpt.add_provided_item(
                        ProvidedItemType.CODEC, bins.text
                    )

        elif subs.tag == "url":
            if cpt.url:
                cpt.url.update({subs.attrib['type']: subs.text})
            else:
                cpt.url = {subs.attrib['type']: subs.text}

        elif subs.tag == "project_license":
            cpt.project_license = subs.text

        elif subs.tag == "project_group":
            cpt.project_group = subs.text

        elif subs.tag == "developer_name":
            cpt.developer_name[locale] = subs.text

        elif subs.tag == "extends":
            cpt.extends.append(subs.text)

        elif subs.tag == "compulsory_for_desktop":
            cpt.compulsory_for_desktops.append(subs.text)

        elif subs.tag == "releases":
            releases = _parse_releases_tag(subs)
            cpt.releases = releases
<commit_msg>Don't be case-sensitive when checking for invisible apps<commit_after>#!/usr/bin/env python3

"""
Reads AppStream XML metadata and metadata from
XDG .desktop files.
"""

# Copyright (c) 2014 Abhishek Bhattacharjee <abhishek.bhattacharjee11@gmail.com>
# Copyright (c) 2014 Matthias Klumpp <mak@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program.

import re
from configparser import RawConfigParser
import lxml.etree as et
from xml.sax.saxutils import escape
from io import StringIO

from dep11.component import DEP11Component, ProvidedItemType
from dep11.utils import str_enc_dec

def read_desktop_data(cpt, dcontent):
    '''
    Parses a .desktop file and sets ComponentData properties
    '''
    df = RawConfigParser(allow_no_value=True)

    items = None
    try:
        df.readfp(StringIO(dcontent))

        items = df.items("Desktop Entry")
        if df.get("Desktop Entry", "Type") != "Application":
            # ignore this file, isn't an application
            cpt.add_hint("not-an-application")
            return False

        try:
            if df.get("Desktop Entry", "NoDisplay", "false").lower() == "true":
                # we ignore this .desktop file, shouldn't be displayed
                cpt.add_hint("invisible-application")
                return False
        except:
            # we don't care if the NoDisplay variable doesn't exist
            # if it isn't there, the file should be processed
            pass

        try:
            if df.get("Desktop Entry", "X-AppStream-Ignore", "false").lower() == "true":
                # this .desktop file should be excluded from AppStream metadata
                cpt.add_hint("invisible-application")
                return False
        except:
            # we don't care if the AppStream-Ignore variable doesn't exist
            # if it isn't there, the file should be processed
            pass

    except Exception as e:
        # this .desktop file is not interesting
        cpt.add_hint("desktop-file-read-error", str(e))
        return True

    # if we reached this step, we are dealing with a GUI desktop app
    cpt.kind = 'desktop-app'

    for item in items:
        if len(item) != 2:
            continue
        key = item[0]
        value = str_enc_dec(item[1])
        if not value:
            continue
        if key.startswith("name"):
            if key == 'name':
                cpt.name['C'] = value
            else:
                cpt.name[key[5:-1]] = value
        elif key == 'categories':
            value = value.split(';')
            value.pop()
            cpt.categories = value
        elif key.startswith('comment'):
            if key == 'comment':
                cpt.summary['C'] = value
            else:
                cpt.summary[key[8:-1]] = value
        elif key.startswith('keywords'):
            value = re.split(';|,', value)
            if not value[-1]:
                value.pop()
            if key[8:] == '':
                if cpt.keywords:
                    if set(value) not in \
                        [set(val) for val in
                            cpt.keywords.values()]:
                        cpt.keywords.update(
                            {'C': list(map(str_enc_dec, value))}
                        )
                else:
                    cpt.keywords = {
                        'C': list(map(str_enc_dec, value))
                    }
            else:
                if cpt.keywords:
                    if set(value) not in \
                        [set(val) for val in
                            cpt.keywords.values()]:
                        cpt.keywords.update(
                            {key[9:-1]: list(map(str_enc_dec, value))}
                        )
                else:
                    cpt.keywords = {
                        key[9:-1]: list(map(str_enc_dec, value))
                    }
        elif key == 'mimetype':
            value = value.split(';')
            if len(value) > 1:
                value.pop()
            for val in value:
                cpt.add_provided_item(
                    ProvidedItemType.MIMETYPE, val
                )
        elif key == 'icon':
            cpt.icon = value
    return True

def _get_tag_locale(subs):
    attr_dic = subs.attrib
    if attr_dic:
        locale = attr_dic.get('{http://www.w3.org/XML/1998/namespace}lang')
        if locale:
            return locale
    return "C"

def _parse_description_tag(subs):
    '''
    Handles the description tag
    '''

    def prepare_desc_string(s):
        '''
        Clears linebreaks and XML-escapes the resulting string
        '''

        if not s:
            return ""
        s = s.strip()
        s = " ".join(s.split())
        return escape(s)

    ddict = dict()

    # The description tag translation is combined per language,
    # for faster parsing on the client side.
    # In case no translation is found, the untranslated version is used instead.
    # the DEP-11 YAML stores the description as HTML

    for usubs in subs:
        locale = _get_tag_locale(usubs)

        if usubs.tag == 'p':
            if not locale in ddict:
                ddict[locale] = ""
            ddict[locale] += "
%s
" % str_enc_dec(prepare_desc_string(usubs.text))
        elif usubs.tag == 'ul' or usubs.tag == 'ol':
            tmp_dict = dict()
            # find the right locale, or fallback to untranslated
            for u_usubs in usubs:
                locale = _get_tag_locale(u_usubs)

                if not locale in tmp_dict:
                    tmp_dict[locale] = ""

                if u_usubs.tag == 'li':
                    tmp_dict[locale] += "
%s
" % str_enc_dec(prepare_desc_string(u_usubs.text))

            for locale, value in tmp_dict.items():
                if not locale in ddict:
                    # This should not happen (but better be prepared)
                    ddict[locale] = ""
                ddict[locale] += "<%s>%s</%s>" % (usubs.tag, value, usubs.tag)
    return ddict

def _parse_screenshots_tag(subs):
    '''
    Handles screenshots, caption, source-image etc.
    '''
    shots = []
    for usubs in subs:
        # for one screeshot tag
        if usubs.tag == 'screenshot':
            screenshot = dict()
            attr_dic = usubs.attrib
            if attr_dic.get('type'):
                if attr_dic['type'] == 'default':
                    screenshot['default'] = True
            # in case of old styled xmls
            url = usubs.text
            if url:
                url = url.strip()
                screenshot['source-image'] = {'url': url}
                shots.append(screenshot)
                continue

            # else look for captions and image tag
            for tags in usubs:
                if tags.tag == 'caption':
                    # for localisation
                    attr_dic = tags.attrib
                    if attr_dic:
                        for v in attr_dic.values():
                            key = v
                    else:
                        key = 'C'

                    if screenshot.get('caption'):
                        screenshot['caption'][key] = str_enc_dec(tags.text)
                    else:
                        screenshot['caption'] = {key: str_enc_dec(tags.text)}
                if tags.tag == 'image':
                    screenshot['source-image'] = {'url': tags.text}

            # only add the screenshot if we have a source image
            if screenshot.get ('source-image'):
                shots.append(screenshot)

    return shots

def _parse_releases_tag(relstag):
    '''
    Parses a releases tag and returns the last three releases
    '''
    rels = list()
    for subs in relstag:
        # for one screeshot tag
        if subs.tag != 'release':
            continue

        release = dict()
        attr_dic = subs.attrib
        if attr_dic.get('version'):
            release['version'] = attr_dic['version']

        if attr_dic.get('timestamp'):
            try:
                release['unix-timestamp'] = int(attr_dic['timestamp'])
            except:
                # the timestamp was wrong - we silently ignore the error
                # TODO: Emit warning hint
                continue
        else:
            # we can't use releases which don't have a timestamp
            # TODO: Emit a warning hint here
            continue

        # else look for captions and image tag
        for usubs in subs:
            if usubs.tag == 'description':
                release['description'] = _parse_description_tag(usubs)

        rels.append(release)

    # sort releases, newest first
    rels = sorted(rels, key=lambda k: k['unix-timestamp'], reverse=True)

    if len(rels) > 3:
        return rels[:3]

    return rels

def read_appstream_upstream_xml(cpt, xml_content):
    '''
    Reads the appdata from the xml file in usr/share/appdata.
    Sets ComponentData properties
    '''
    root = None
    try:
        root = et.fromstring(bytes(xml_content, 'utf-8'))
    except Exception as e:
        cpt.add_hint("metainfo-parse-error", str(e))
        return
    if root is None:
        cpt.add_hint("metainfo-parse-error", "Error is unknown, the root node was null.")

    if root.tag == 'application':
        # we parse ancient AppStream XML, but it is a good idea to update it to make use of newer features, remove some ancient
        # oddities and to simplify the parser in future. So we add a hint for that.
        cpt.add_hint("ancient-metadata")

    key = root.attrib.get('type')
    if key:
        if key == 'desktop':
            cpt.kind = 'desktop-app'
        else:
            # for other components like addon,codec, inputmethod etc
            cpt.kind = root.attrib['type']

    for subs in root:
        locale = _get_tag_locale(subs)

        if subs.tag == 'id':
            cpt.cid = subs.text
            # legacy support
            key = subs.attrib.get('type')
            if key and not cpt.kind:
                if key == 'desktop':
                    cpt.kind = 'desktop-app'
                else:
                    cpt.kind = key

        elif subs.tag == "name":
            cpt.name[locale] = subs.text

        elif subs.tag == "summary":
            cpt.summary[locale] = subs.text

        elif subs.tag == "description":
            desc = _parse_description_tag(subs)
            cpt.description = desc

        elif subs.tag == "screenshots":
            screen = _parse_screenshots_tag(subs)
            cpt.screenshots = screen

        elif subs.tag == "provides":
            for bins in subs:
                if bins.tag == "binary":
                    cpt.add_provided_item(
                        ProvidedItemType.BINARY, bins.text
                    )
                if bins.tag == 'library':
                    cpt.add_provided_item(
                        ProvidedItemType.LIBRARY, bins.text
                    )
                if bins.tag == 'dbus':
                    if not cpt.provides.get(ProvidedItemType.DBUS):
                        cpt.provides[ProvidedItemType.DBUS] = list()

                    bus_kind = bins.attrib.get('type')
                    if bus_kind == "session":
                        bus_kind = "user"
                    if bus_kind:
                        cpt.provides[ProvidedItemType.DBUS].append({'type': bus_kind, 'service': bins.text})
                if bins.tag == 'firmware':
                    if not cpt.provides.get(ProvidedItemType.FIRMWARE):
                        cpt.provides[ProvidedItemType.FIRMWARE] = list()

                    fw_type = bins.attrib.get('type')
                    fw_data = {'type': fw_type}

                    _valid = True
                    if fw_type == "flashed":
                        fw_data['guid'] = bins.text
                    elif fw_type == "runtime":
                        fw_data['fname'] = bins.text
                    else:
                        _valid = False

                    if _valid:
                        cpt.provides[ProvidedItemType.FIRMWARE].append(fw_data)
                if bins.tag == 'python2':
                    cpt.add_provided_item(
                        ProvidedItemType.PYTHON_2, bins.text
                    )
                if bins.tag == 'python3':
                    cpt.add_provided_item(
                        ProvidedItemType.PYTHON_3, bins.text
                    )
                if bins.tag == 'codec':
                    cpt.add_provided_item(
                        ProvidedItemType.CODEC, bins.text
                    )

        elif subs.tag == "url":
            if cpt.url:
                cpt.url.update({subs.attrib['type']: subs.text})
            else:
                cpt.url = {subs.attrib['type']: subs.text}

        elif subs.tag == "project_license":
            cpt.project_license = subs.text

        elif subs.tag == "project_group":
            cpt.project_group = subs.text

        elif subs.tag == "developer_name":
            cpt.developer_name[locale] = subs.text

        elif subs.tag == "extends":
            cpt.extends.append(subs.text)

        elif subs.tag == "compulsory_for_desktop":
            cpt.compulsory_for_desktops.append(subs.text)

        elif subs.tag == "releases":
            releases = _parse_releases_tag(subs)
            cpt.releases = releases
<|endoftext|>


<commit_before>from __future__ import print_function
import os
import sys
import subprocess as sp
import pytest
import yaml
import requests
import argparse
import re
from ggd import utils

#---------------------------------------------------------------------
## Clone repos
#---------------------------------------------------------------------

## Update local repo
utils.update_local_repo()

##Uupdate metadata repo
utils.update_metadata_local_repo()

## Get species
utils.get_species(update_repo=True)

## get channels
channels = utils.get_channels()

## get channel data
for x in channels:
    utils.get_channel_data(x)

<commit_msg>Fix function calling<commit_after>from __future__ import print_function
import os
import sys
import subprocess as sp
import pytest
import yaml
import requests
import argparse
import re
from ggd import utils

#---------------------------------------------------------------------
## Clone repos
#---------------------------------------------------------------------

## Update local repo
utils.update_local_repo()

##Uupdate metadata repo
utils.update_metadata_local_repo()

## Get species
utils.get_species(update_repo=True)

## get channels
channels = utils.get_ggd_channels()

## get channel data
for x in channels:
    utils.get_channel_data(x)

<|endoftext|>