<commit_before>#!/usr/bin/env python3.5
# coding=utf-8
"""Threaded gps client"""
from threading import Thread
from time import sleep

try: # This kludge to get around imports with files and directories the same name.
    from gps3 import agps3  # Python 3
except ImportError:
    from . import agps3  # Python 2

__author__ = 'Moe'
__copyright__ = 'Copyright 2016  Moe'
__license__ = 'MIT'
__version__ = '0.1'

HOST = '127.0.0.1'  # gpsd
GPSD_PORT = 2947  # defaults
PROTOCOL = 'json'  # "

class AGPS3mechanism(object):
    """Create threaded data stream as updated object attributes
    """

    def __init__(self):
        self.socket = agps3.GPSDSocket()
        self.data_stream = agps3.Dot()

    def stream_data(self, host=HOST, port=GPSD_PORT, enable=True, gpsd_protocol=PROTOCOL, devicepath=None):
        """ Connect and command, point and shoot, flail and bail
        """
        self.socket.connect(host, port)
        self.socket.watch(enable, gpsd_protocol, devicepath)

    def unpack_data(self, usnap=.2):  # 2/10th second sleep between empty requests
        """ Iterates over socket response and refreshes values of object attributes.
        Sleeping here has the greatest response to cpu cycles short of blocking sockets
        """
        for new_data in self.socket:
            if new_data:
                self.data_stream.unpack(new_data)
            else:
                sleep(usnap)  # Sleep in seconds after an empty look up.

    def run_thread(self, usnap=.2):
        """run thread with data
        """
        # self.stream_data() # Unless other changes are made this would limit to localhost only.
        gps3_data_thread = Thread(group=None, target=self.unpack_data, args={usnap: usnap})
        gps3_data_thread.start()

    def stop(self):
        """ Stop as much as possible, as gracefully as possible, if possible.
        """
        self.socket.close()  # Close socket, thread is on its own so far.
        print('Process stopped by user')
        print('Good bye.')

if __name__ == '__main__':
    from gps3.misc import add_args
    args = add_args()

    agps3_thread = AGPS3mechanism()  # The thread triumvirate
    agps3_thread.stream_data(host=args.host, port=args.port, gpsd_protocol=args.gpsd_protocol)
    agps3_thread.run_thread(usnap=.2)  # Throttle sleep between empty lookups in seconds defaults = 0.2 of a second.

    nod = 0
    seconds_nap = int(args.seconds_nap)  # Threaded Demo loop 'seconds_nap' is not the same as 'usnap'
    while nod <= seconds_nap - 1:
        for nod in range(1, seconds_nap):
            print('{}% completed wait period'.format(round(100 * (nod / seconds_nap), 2)))
            nod += 1
            sleep(1)

        print('\nGPS3 Thread is still functioning at {}'.format(agps3_thread.data_stream.time))
        print('Lat:{}  Lon:{}  Speed:{}  Course:{}\n'.format(agps3_thread.data_stream.lat, agps3_thread.data_stream.lon,
                                                          agps3_thread.data_stream.speed,
                                                          agps3_thread.data_stream.track))
        nod = 0
<commit_msg>Added default daemon thread/ import kludge<commit_after>#!/usr/bin/env python3.5
# coding=utf-8
"""Threaded gps client"""
from threading import Thread
from time import sleep

try: # This kludge to get around imports with files and directories the same name.
    from gps3 import agps3  # Python 3
except ImportError:
    from . import agps3  # Python 2

__author__ = 'Moe'
__copyright__ = 'Copyright 2016  Moe'
__license__ = 'MIT'
__version__ = '0.2'

HOST = '127.0.0.1'  # gpsd
GPSD_PORT = 2947  # defaults
PROTOCOL = 'json'  # "

class AGPS3mechanism(object):
    """Create threaded data stream as updated object attributes
    """

    def __init__(self):
        self.socket = agps3.GPSDSocket()
        self.data_stream = agps3.Dot()

    def stream_data(self, host=HOST, port=GPSD_PORT, enable=True, gpsd_protocol=PROTOCOL, devicepath=None):
        """ Connect and command, point and shoot, flail and bail
        """
        self.socket.connect(host, port)
        self.socket.watch(enable, gpsd_protocol, devicepath)

    def unpack_data(self, usnap=.2):  # 2/10th second sleep between empty requests
        """ Iterates over socket response and refreshes values of object attributes.
        Sleeping here has the greatest response to cpu cycles short of blocking sockets
        """
        for new_data in self.socket:
            if new_data:
                self.data_stream.unpack(new_data)
            else:
                sleep(usnap)  # Sleep in seconds after an empty look up.

    def run_thread(self, usnap=.2, daemon=True):
        """run thread with data
        """
        # self.stream_data() # Unless other changes are made this would limit to localhost only.
        gps3_data_thread = Thread(target=self.unpack_data, args={usnap: usnap}, daemon=True)
        gps3_data_thread.start()

    def stop(self):
        """ Stop as much as possible, as gracefully as possible, if possible.
        """
        self.socket.close()  # Close socket, thread is on its own so far.
        self.
        print('Process stopped by user')
        print('Good bye.')

    def join(self, timeout=None):
        """ Stop the thread and wait for it to end. """
        self._stopevent.set()
        threading.Thread.join(self, timeout)

if __name__ == '__main__':
    from gps3.misc import add_args
    args = add_args()

    agps3_thread = AGPS3mechanism()  # The thread triumvirate
    agps3_thread.stream_data(host=args.host, port=args.port, gpsd_protocol=args.gpsd_protocol)
    agps3_thread.run_thread(usnap=.2)  # Throttle sleep between empty lookups in seconds defaults = 0.2 of a second.

    nod = 0
    seconds_nap = int(args.seconds_nap)  # Threaded Demo loop 'seconds_nap' is not the same as 'usnap'
    while nod <= seconds_nap - 1:
        for nod in range(1, seconds_nap):
            print('{}% completed wait period'.format(round(100 * (nod / seconds_nap), 2)))
            nod += 1
            sleep(1)

        print('\nGPS3 Thread is still functioning at {}'.format(agps3_thread.data_stream.time))
        print('Lat:{}  Lon:{}  Speed:{}  Course:{}\n'.format(agps3_thread.data_stream.lat, agps3_thread.data_stream.lon,
                                                          agps3_thread.data_stream.speed,
                                                          agps3_thread.data_stream.track))
        nod = 0
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import warnings
from django.template import Library
from django.template.base import Node, NodeList, TextNode, VariableNode
from django.core.exceptions import ImproperlyConfigured
from django.utils.safestring import mark_safe
from djng.core.urlresolvers import get_all_remote_methods, get_current_remote_methods

register = Library()

class CsrfValueNode(Node):
    def render(self, context):
        warnings.warn("Instead of {% csrf_value %}, please use {{ csrf_token }}")
        csrf_token = context.get('csrf_token', None)
        if not csrf_token:
            raise ImproperlyConfigured('Template must be rendered using a RequestContext')
        if csrf_token == 'NOTPROVIDED':
            return mark_safe('')
        else:
            return mark_safe(csrf_token)

@register.tag(name='csrf_value')
def render_csrf_value(parser, token):
    return CsrfValueNode()

@register.simple_tag(name='djng_all_rmi')
def djng_all_rmi():
    """
    Returns a dictionary of all methods for all Views available for this project, marked with the
    ``@allow_remote_invocation`` decorator. The return string can be used directly to initialize
    the AngularJS provider, such as ``djangoRMIProvider.configure({% djng_rmi_configs %});``
    """
    return mark_safe(json.dumps(get_all_remote_methods()))

@register.simple_tag(name='djng_current_rmi', takes_context=True)
def djng_current_rmi(context):
    """
    Returns a dictionary of all methods for the current View of this request, marked with the
    @allow_remote_invocation decorator. The return string can be used directly to initialize
    the AngularJS provider, such as ``djangoRMIProvider.configure({% djng_current_rmi %});``
    """
    return mark_safe(json.dumps(get_current_remote_methods(context.get('view'))))

@register.simple_tag(name='load_djng_urls', takes_context=True)
def djng_urls(context, *namespaces):
    raise DeprecationWarning(
        "load_djng_urls templatetag is deprecated and has been removed from this version of django-angular."
        "Please refer to documentation for updated way to manage django urls in angular.")

class AngularJsNode(Node):
    def __init__(self, django_nodelist, angular_nodelist, variable):
        self.django_nodelist = django_nodelist
        self.angular_nodelist = angular_nodelist
        self.variable = variable

    def render(self, context):
        if self.variable.resolve(context):
            return self.angular_nodelist.render(context)
        return self.django_nodelist.render(context)

@register.tag
def angularjs(parser, token):
    """
    Conditionally switch between AngularJS and Django variable expansion for ``{{`` and ``}}``
    keeping Django's expansion for ``{%`` and ``%}``

    Usage::

        {% angularjs 1 %} or simply {% angularjs %}
            {% process variables through the AngularJS template engine %}
        {% endangularjs %}

        {% angularjs 0 %}
            {% process variables through the Django template engine %}
        {% endangularjs %}

        Instead of 0 and 1, it is possible to use a context variable.
    """
    bits = token.contents.split()
    if len(bits) < 2:
        bits.append('1')
    values = [parser.compile_filter(bit) for bit in bits[1:]]
    django_nodelist = parser.parse(('endangularjs',))
    angular_nodelist = NodeList()
    for node in django_nodelist:
        # convert all occurrences of VariableNode into a TextNode using the
        # AngularJS double curly bracket notation
        if isinstance(node, VariableNode):
            # convert Django's array notation into JS array notation
            tokens = node.filter_expression.token.split('.')
            token = tokens[0]
            for part in tokens[1:]:
                if part.isdigit():
                    token += '[%s]' % part
                else:
                    token += '.%s' % part
            node = TextNode('{{ %s }}' % token)
        angular_nodelist.append(node)
    parser.delete_first_token()
    return AngularJsNode(django_nodelist, angular_nodelist, values[0])
<commit_msg>remove csrf_value templatetag<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import json

from django.template import Library
from django.template.base import Node, NodeList, TextNode, VariableNode
from django.utils.safestring import mark_safe

from djng.core.urlresolvers import get_all_remote_methods, get_current_remote_methods

register = Library()

@register.simple_tag(name='djng_all_rmi')
def djng_all_rmi():
    """
    Returns a dictionary of all methods for all Views available for this project, marked with the
    ``@allow_remote_invocation`` decorator. The return string can be used directly to initialize
    the AngularJS provider, such as ``djangoRMIProvider.configure({% djng_rmi_configs %});``
    """
    return mark_safe(json.dumps(get_all_remote_methods()))

@register.simple_tag(name='djng_current_rmi', takes_context=True)
def djng_current_rmi(context):
    """
    Returns a dictionary of all methods for the current View of this request, marked with the
    @allow_remote_invocation decorator. The return string can be used directly to initialize
    the AngularJS provider, such as ``djangoRMIProvider.configure({% djng_current_rmi %});``
    """
    return mark_safe(json.dumps(get_current_remote_methods(context.get('view'))))

@register.simple_tag(name='load_djng_urls', takes_context=True)
def djng_urls(context, *namespaces):
    raise DeprecationWarning(
        "load_djng_urls templatetag is deprecated and has been removed from this version of django-angular."
        "Please refer to documentation for updated way to manage django urls in angular.")

class AngularJsNode(Node):
    def __init__(self, django_nodelist, angular_nodelist, variable):
        self.django_nodelist = django_nodelist
        self.angular_nodelist = angular_nodelist
        self.variable = variable

    def render(self, context):
        if self.variable.resolve(context):
            return self.angular_nodelist.render(context)
        return self.django_nodelist.render(context)

@register.tag
def angularjs(parser, token):
    """
    Conditionally switch between AngularJS and Django variable expansion for ``{{`` and ``}}``
    keeping Django's expansion for ``{%`` and ``%}``

    Usage::

        {% angularjs 1 %} or simply {% angularjs %}
            {% process variables through the AngularJS template engine %}
        {% endangularjs %}

        {% angularjs 0 %}
            {% process variables through the Django template engine %}
        {% endangularjs %}

        Instead of 0 and 1, it is possible to use a context variable.
    """
    bits = token.contents.split()
    if len(bits) < 2:
        bits.append('1')
    values = [parser.compile_filter(bit) for bit in bits[1:]]
    django_nodelist = parser.parse(('endangularjs',))
    angular_nodelist = NodeList()
    for node in django_nodelist:
        # convert all occurrences of VariableNode into a TextNode using the
        # AngularJS double curly bracket notation
        if isinstance(node, VariableNode):
            # convert Django's array notation into JS array notation
            tokens = node.filter_expression.token.split('.')
            token = tokens[0]
            for part in tokens[1:]:
                if part.isdigit():
                    token += '[%s]' % part
                else:
                    token += '.%s' % part
            node = TextNode('{{ %s }}' % token)
        angular_nodelist.append(node)
    parser.delete_first_token()
    return AngularJsNode(django_nodelist, angular_nodelist, values[0])
<|endoftext|>


<commit_before>from rest_framework.generics import (ListCreateAPIView,
                                     RetrieveUpdateDestroyAPIView)
from rest_framework.permissions import (IsAuthenticated,
                                        IsAuthenticatedOrReadOnly)

from utils.mixins import MultipleFieldLookupMixin

from .models import Post
from .serializers import PostSerializer

class ThreadList(ListCreateAPIView):
    serializer_class = PostSerializer
    permission_classes = (IsAuthenticatedOrReadOnly,)

    def get_queryset(self):
        board = self.kwargs['board']
        return Post.objects.filter(board__slug=board)

class ThreadDetail(MultipleFieldLookupMixin, RetrieveUpdateDestroyAPIView):
    queryset = Post.objects.all()
    serializer_class = PostSerializer
    permission_classes = (IsAuthenticatedOrReadOnly,)
    lookup_fields = ['board__slug', 'key', 'slug']
    lookup_url_kwargs = ['board', 'key', 'thread']
    model = Post

    def check_object_permissions(self, request, obj):
        super(ThreadDetail, self).check_object_permissions(request, obj)
        # Check for object permissions
        if request.method.lower() in ['delete', 'put', 'patch'] and request.user != obj.author:
            self.permission_denied(request,
                                   message='User cannot edit this object.')
<commit_msg>use SAFE_METHODS constant.<commit_after>from rest_framework.generics import (ListCreateAPIView,
                                     RetrieveUpdateDestroyAPIView)
from rest_framework.permissions import (SAFE_METHODS, IsAuthenticated,
                                        IsAuthenticatedOrReadOnly)

from utils.mixins import MultipleFieldLookupMixin

from .models import Post
from .serializers import PostSerializer

class ThreadList(ListCreateAPIView):
    serializer_class = PostSerializer
    permission_classes = (IsAuthenticatedOrReadOnly,)

    def get_queryset(self):
        board = self.kwargs['board']
        return Post.objects.filter(board__slug=board)

class ThreadDetail(MultipleFieldLookupMixin, RetrieveUpdateDestroyAPIView):
    queryset = Post.objects.all()
    serializer_class = PostSerializer
    permission_classes = (IsAuthenticatedOrReadOnly,)
    lookup_fields = ['board__slug', 'key', 'slug']
    lookup_url_kwargs = ['board', 'key', 'thread']
    model = Post

    def check_object_permissions(self, request, obj):
        super(ThreadDetail, self).check_object_permissions(request, obj)
        if request.method not in SAFE_METHODS and request.user != obj.author:
            self.permission_denied(request,
                                   message='User cannot edit this object.')
<|endoftext|>


<commit_before>from django.contrib.contenttypes.models import ContentType
from permission.conf import settings
from permission.logics.base import PermissionLogic

from rest_framework.permissions import BasePermission, SAFE_METHODS

def has_user_permission_for_instance(user, instance, permission='add'):
    if user.is_staff:
        return True
    content_type = ContentType.objects.get_for_model(instance)
    permission_name = '%s.%s_%s' % (content_type.app_label, permission, content_type.model)
    return user.has_perm(permission_name, instance)

class IsAdminOrReadOnly(BasePermission):
    def has_permission(self, request, view):
        return (
            request.method in SAFE_METHODS or
            (request.user.is_authenticated() and request.user.is_staff)
        )

class IsAdminOrOwner(IsAdminOrReadOnly):
    """
    Allows access to admin users or account's owner.
    For other users read-only access.
    """

    def has_permission(self, request, view):
        user = request.user

        if user.is_staff or request.method in SAFE_METHODS:
            return True
        elif view.suffix == 'List' or request.method == 'DELETE':
            return False

        return user == view.get_object()

class FilteredCollaboratorsPermissionLogic(PermissionLogic):
    """
    Permission logic class for collaborators based permission system.
    For users with is_staff flag everything is allowed.
    """

    def __init__(self,
                 user_field=None,
                 collaborators_query=None,
                 collaborators_filter=None,
                 any_permission=False,
                 add_permission=False,
                 change_permission=False,
                 delete_permission=False):
        """
        Constructor

        Parameters
        ^^^^^^^^^^
        collaborators_query : string
            Django queryset filter-like expression or list of expressions
            to fetch collaborators based on current object.
            Default is False.
        collaborators_filter : dict
            A filter or list of filters to apply to collaborators.
            Default is {}.
        user_field : string
            A name of a field with ForeignKey to User object;
            used to determine straight objects' permission.
            Default is None.
        any_permission : boolean
            True to give any permission of the specified object to the
            collaborators.
            Default is False.
        add_permission : boolean
            True to give add permission of the specified object to the
            collaborators.
            It will be ignored if :attr:`any_permission` is True.
            Default is False.
        change_permission : boolean
            True to give change permission of the specified object to the
            collaborators.
            It will be ignored if :attr:`any_permission` is True.
            Default is False.
        delete_permission : boolean
            True to give delete permission of the specified object to the
            collaborators.
            It will be ignored if :attr:`any_permission` is True.
            Default is False.
        """
        if isinstance(collaborators_query, basestring):
            self.collaborators_queries = [collaborators_query]
        else:
            self.collaborators_queries = collaborators_query
        if isinstance(collaborators_filter, dict):
            self.collaborators_filters = [collaborators_filter] or [{}]
        else:
            self.collaborators_filters = collaborators_filter
        self.any_permission = any_permission
        self.add_permission = add_permission
        self.change_permission = change_permission
        self.delete_permission = delete_permission
        self.user_field = user_field

    def is_permission_allowed(self, perm):
        add_permission = self.get_full_permission_string('add')
        change_permission = self.get_full_permission_string('change')
        delete_permission = self.get_full_permission_string('delete')

        if self.any_permission:
            return True
        if self.add_permission and perm == add_permission:
            return True
        if self.change_permission and perm == change_permission:
            return True
        if self.delete_permission and perm == delete_permission:
            return True
        return False

    def has_perm(self, user_obj, perm, obj=None):
        """
        Check if user has permission (of object)

        If the user_obj is not authenticated, it return ``False``.

        If no object is specified, it return ``True`` when the corresponding
        permission was specified to ``True`` (changed from v0.7.0).
        This behavior is based on the django system.
        https://code.djangoproject.com/wiki/RowLevelPermissions

        If an object is specified, it will return ``True`` if the user is
        found in ``field_name`` of the object (e.g. ``obj.collaborators``).
        So once the object store the user as a collaborator in
        ``field_name`` attribute (default: ``collaborators``), the collaborator
        can change or delete the object (you can change this behavior to set
        ``any_permission``, ``change_permission`` or ``delete_permission``
        attributes of this instance).

        Parameters
        ^^^^^^^^^^
        user_obj : django user model instance
            A django user model instance which be checked
        perm : string
            `app_label.codename` formatted permission string
        obj : None or django model instance
            None or django model instance for object permission

        Returns
        ^^^^^^^
        boolean
            Whether the user specified has specified permission (of specified
            object).
        """
        if not user_obj.is_authenticated():
            return False
        # construct the permission full name
        if obj is None:
            # object permission without obj should return True
            # Ref: https://code.djangoproject.com/wiki/RowLevelPermissions
            return self.is_permission_allowed(perm)
        elif user_obj.is_active:
            # if the user is staff, allow everything
            if user_obj.is_staff:
                return True

        if self.user_field:
            user = getattr(obj, self.user_field)
            if user and user == user_obj:
                return True

        if not self.collaborators_queries or not self.collaborators_filters:
            return False

        for query, filt in zip(self.collaborators_queries, self.collaborators_filters):
            kwargs = {query: user_obj, 'pk': obj.pk}
            kwargs.update(filt)

            if obj._meta.model._default_manager.filter(**kwargs).exists():
                return self.is_permission_allowed(perm)
        return False

class StaffPermissionLogic(PermissionLogic):
    """
    Permission logic class for is_staff authority based permission system
    """
    def __init__(self,
                 any_permission=None,
                 add_permission=None,
                 change_permission=None,
                 delete_permission=None):
        """
        Constructor

        Parameters
        ^^^^^^^^^^
        any_permission : boolean
            True for give any permission of the specified object to the staff
            user. Default value will be taken from
            ``PERMISSION_DEFAULT_SPL_ANY_PERMISSION`` in
            settings.
        add_permission : boolean
            True for give change permission of the specified object to the
            staff user.
            It will be ignored if :attr:`any_permission` is True.
            Default value will be taken from
            ``PERMISSION_DEFAULT_SPL_ADD_PERMISSION`` in
            settings.
        change_permission : boolean
            True for give change permission of the specified object to the
            staff user.
            It will be ignored if :attr:`any_permission` is True.
            Default value will be taken from
            ``PERMISSION_DEFAULT_SPL_CHANGE_PERMISSION`` in
            settings.
        delete_permission : boolean
            True for give delete permission of the specified object to the
            staff user.
            It will be ignored if :attr:`any_permission` is True.
            Default value will be taken from
            ``PERMISSION_DEFAULT_SPL_DELETE_PERMISSION`` in
            settings.
        """
        self.any_permission = any_permission
        self.add_permission = add_permission
        self.change_permission = change_permission
        self.delete_permission = delete_permission

        if self.any_permission is None:
            self.any_permission = \
                settings.PERMISSION_DEFAULT_SPL_ANY_PERMISSION
        if self.add_permission is None:
            self.add_permission = \
                settings.PERMISSION_DEFAULT_SPL_ADD_PERMISSION
        if self.change_permission is None:
            self.change_permission = \
                settings.PERMISSION_DEFAULT_SPL_CHANGE_PERMISSION
        if self.delete_permission is None:
            self.delete_permission = \
                settings.PERMISSION_DEFAULT_SPL_DELETE_PERMISSION

    def has_perm(self, user_obj, perm, obj=None):
        """
        Check if user have permission (of object)

        If the user_obj is not authenticated, it return ``False``.

        If no object is specified, it return ``True`` when the corresponding
        permission was specified to ``True`` (changed from v0.7.0).
        This behavior is based on the django system.
        https://code.djangoproject.com/wiki/RowLevelPermissions

        If an object is specified, it will return ``True`` if the user is
        staff. The staff can add, change or delete the object (you can change
        this behavior to set ``any_permission``, ``add_permission``,
        ``change_permission``, or ``delete_permission`` attributes of this
        instance).

        Parameters
        ^^^^^^^^^^
        user_obj : django user model instance
            A django user model instance which be checked
        perm : string
            `app_label.codename` formatted permission string
        obj : None or django model instance
            None or django model instance for object permission

        Returns
        ^^^^^^^
        boolean
            Weather the specified user have specified permission (of specified
            object).
        """
        if not user_obj.is_authenticated():
            return False
        # construct the permission full name
        add_permission = self.get_full_permission_string('add')
        change_permission = self.get_full_permission_string('change')
        delete_permission = self.get_full_permission_string('delete')
        if obj is None:
            if perm == add_permission:
                return (self.add_permission or self.any_permission) and user_obj.is_staff
            return True
        elif user_obj.is_active:
            if user_obj.is_staff:
                if self.any_permission:
                    # have any kind of permissions to the obj
                    return True
                if (self.add_permission and
                        perm == add_permission):
                    return True
                if (self.change_permission and
                        perm == change_permission):
                    return True
                if (self.delete_permission and
                        perm == delete_permission):
                    return True
        return False

def detect_group_type(permission_group):
    perm_group = permission_group.group
    if hasattr(perm_group, 'projectrole'):
        return 'project'
    elif hasattr(perm_group, 'customerrole'):
        return 'customer'
    elif hasattr(perm_group, 'projectgrouprole'):
        return 'project_group'

class TypedCollaboratorsPermissionLogic(PermissionLogic):
    """
    Permission logic that supports definition of several user groups based on the type of the
    checked object.

    For example, it is useful for cases when an object can be accessed either by project administrators or
    by customer owners.
    """
    def __init__(self, type_to_permission_logic_mapping, discriminator_function=None):
        self.type_to_permission_logic_mapping = type_to_permission_logic_mapping
        self.discriminator_function = discriminator_function

    def has_perm(self, user_obj, perm, obj=None):
        if not user_obj.is_authenticated():
            return False

        # always true for creation
        if obj is None:
            return True

        elif user_obj.is_active:
            # if the user is staff, allow everything
            if user_obj.is_staff:
                return True

            # detect a type of collaboration
            collaboration_type = self.discriminator_function(obj)

            # disallow operation if the type is unknown
            if not collaboration_type in self.type_to_permission_logic_mapping:
                return False
            collaborators_query = self.type_to_permission_logic_mapping[collaboration_type]['query']
            collaborators_filter = self.type_to_permission_logic_mapping[collaboration_type]['filter']

            kwargs = {
                collaborators_query: user_obj,
                'pk': obj.pk,
            }
            kwargs.update(collaborators_filter)

            if obj._meta.model._default_manager.filter(**kwargs).exists():
                return True
        return False
<commit_msg>Is resource's user is none, only staff can see it (ITACLOUD-7217)<commit_after>from django.contrib.contenttypes.models import ContentType
from permission.conf import settings
from permission.logics.base import PermissionLogic

from rest_framework.permissions import BasePermission, SAFE_METHODS

def has_user_permission_for_instance(user, instance, permission='add'):
    if user.is_staff:
        return True
    content_type = ContentType.objects.get_for_model(instance)
    permission_name = '%s.%s_%s' % (content_type.app_label, permission, content_type.model)
    return user.has_perm(permission_name, instance)

class IsAdminOrReadOnly(BasePermission):
    def has_permission(self, request, view):
        return (
            request.method in SAFE_METHODS or
            (request.user.is_authenticated() and request.user.is_staff)
        )

class IsAdminOrOwner(IsAdminOrReadOnly):
    """
    Allows access to admin users or account's owner.
    For other users read-only access.
    """

    def has_permission(self, request, view):
        user = request.user

        if user.is_staff or request.method in SAFE_METHODS:
            return True
        elif view.suffix == 'List' or request.method == 'DELETE':
            return False

        return user == view.get_object()

class FilteredCollaboratorsPermissionLogic(PermissionLogic):
    """
    Permission logic class for collaborators based permission system.
    For users with is_staff flag everything is allowed.
    """

    def __init__(self,
                 user_field=None,
                 collaborators_query=None,
                 collaborators_filter=None,
                 any_permission=False,
                 add_permission=False,
                 change_permission=False,
                 delete_permission=False):
        """
        Constructor

        Parameters
        ^^^^^^^^^^
        collaborators_query : string
            Django queryset filter-like expression or list of expressions
            to fetch collaborators based on current object.
            Default is False.
        collaborators_filter : dict
            A filter or list of filters to apply to collaborators.
            Default is {}.
        user_field : string
            A name of a field with ForeignKey to User object;
            used to determine straight objects' permission.
            Default is None.
        any_permission : boolean
            True to give any permission of the specified object to the
            collaborators.
            Default is False.
        add_permission : boolean
            True to give add permission of the specified object to the
            collaborators.
            It will be ignored if :attr:`any_permission` is True.
            Default is False.
        change_permission : boolean
            True to give change permission of the specified object to the
            collaborators.
            It will be ignored if :attr:`any_permission` is True.
            Default is False.
        delete_permission : boolean
            True to give delete permission of the specified object to the
            collaborators.
            It will be ignored if :attr:`any_permission` is True.
            Default is False.
        """
        if isinstance(collaborators_query, basestring):
            self.collaborators_queries = [collaborators_query]
        else:
            self.collaborators_queries = collaborators_query
        if isinstance(collaborators_filter, dict):
            self.collaborators_filters = [collaborators_filter] or [{}]
        else:
            self.collaborators_filters = collaborators_filter
        self.any_permission = any_permission
        self.add_permission = add_permission
        self.change_permission = change_permission
        self.delete_permission = delete_permission
        self.user_field = user_field

    def is_permission_allowed(self, perm):
        add_permission = self.get_full_permission_string('add')
        change_permission = self.get_full_permission_string('change')
        delete_permission = self.get_full_permission_string('delete')

        if self.any_permission:
            return True
        if self.add_permission and perm == add_permission:
            return True
        if self.change_permission and perm == change_permission:
            return True
        if self.delete_permission and perm == delete_permission:
            return True
        return False

    def has_perm(self, user_obj, perm, obj=None):
        """
        Check if user has permission (of object)

        If the user_obj is not authenticated, it return ``False``.

        If no object is specified, it return ``True`` when the corresponding
        permission was specified to ``True`` (changed from v0.7.0).
        This behavior is based on the django system.
        https://code.djangoproject.com/wiki/RowLevelPermissions

        If an object is specified, it will return ``True`` if the user is
        found in ``field_name`` of the object (e.g. ``obj.collaborators``).
        So once the object store the user as a collaborator in
        ``field_name`` attribute (default: ``collaborators``), the collaborator
        can change or delete the object (you can change this behavior to set
        ``any_permission``, ``change_permission`` or ``delete_permission``
        attributes of this instance).

        Parameters
        ^^^^^^^^^^
        user_obj : django user model instance
            A django user model instance which be checked
        perm : string
            `app_label.codename` formatted permission string
        obj : None or django model instance
            None or django model instance for object permission

        Returns
        ^^^^^^^
        boolean
            Whether the user specified has specified permission (of specified
            object).
        """
        if not user_obj.is_authenticated():
            return False
        # construct the permission full name
        if obj is None:
            # object permission without obj should return True
            # Ref: https://code.djangoproject.com/wiki/RowLevelPermissions
            return self.is_permission_allowed(perm)
        elif user_obj.is_active:
            # if the user is staff, allow everything
            if user_obj.is_staff:
                return True

        if self.user_field:
            return getattr(obj, self.user_field) == user_obj

        if not self.collaborators_queries or not self.collaborators_filters:
            return False

        for query, filt in zip(self.collaborators_queries, self.collaborators_filters):
            kwargs = {query: user_obj, 'pk': obj.pk}
            kwargs.update(filt)

            if obj._meta.model._default_manager.filter(**kwargs).exists():
                return self.is_permission_allowed(perm)
        return False

class StaffPermissionLogic(PermissionLogic):
    """
    Permission logic class for is_staff authority based permission system
    """
    def __init__(self,
                 any_permission=None,
                 add_permission=None,
                 change_permission=None,
                 delete_permission=None):
        """
        Constructor

        Parameters
        ^^^^^^^^^^
        any_permission : boolean
            True for give any permission of the specified object to the staff
            user. Default value will be taken from
            ``PERMISSION_DEFAULT_SPL_ANY_PERMISSION`` in
            settings.
        add_permission : boolean
            True for give change permission of the specified object to the
            staff user.
            It will be ignored if :attr:`any_permission` is True.
            Default value will be taken from
            ``PERMISSION_DEFAULT_SPL_ADD_PERMISSION`` in
            settings.
        change_permission : boolean
            True for give change permission of the specified object to the
            staff user.
            It will be ignored if :attr:`any_permission` is True.
            Default value will be taken from
            ``PERMISSION_DEFAULT_SPL_CHANGE_PERMISSION`` in
            settings.
        delete_permission : boolean
            True for give delete permission of the specified object to the
            staff user.
            It will be ignored if :attr:`any_permission` is True.
            Default value will be taken from
            ``PERMISSION_DEFAULT_SPL_DELETE_PERMISSION`` in
            settings.
        """
        self.any_permission = any_permission
        self.add_permission = add_permission
        self.change_permission = change_permission
        self.delete_permission = delete_permission

        if self.any_permission is None:
            self.any_permission = \
                settings.PERMISSION_DEFAULT_SPL_ANY_PERMISSION
        if self.add_permission is None:
            self.add_permission = \
                settings.PERMISSION_DEFAULT_SPL_ADD_PERMISSION
        if self.change_permission is None:
            self.change_permission = \
                settings.PERMISSION_DEFAULT_SPL_CHANGE_PERMISSION
        if self.delete_permission is None:
            self.delete_permission = \
                settings.PERMISSION_DEFAULT_SPL_DELETE_PERMISSION

    def has_perm(self, user_obj, perm, obj=None):
        """
        Check if user have permission (of object)

        If the user_obj is not authenticated, it return ``False``.

        If no object is specified, it return ``True`` when the corresponding
        permission was specified to ``True`` (changed from v0.7.0).
        This behavior is based on the django system.
        https://code.djangoproject.com/wiki/RowLevelPermissions

        If an object is specified, it will return ``True`` if the user is
        staff. The staff can add, change or delete the object (you can change
        this behavior to set ``any_permission``, ``add_permission``,
        ``change_permission``, or ``delete_permission`` attributes of this
        instance).

        Parameters
        ^^^^^^^^^^
        user_obj : django user model instance
            A django user model instance which be checked
        perm : string
            `app_label.codename` formatted permission string
        obj : None or django model instance
            None or django model instance for object permission

        Returns
        ^^^^^^^
        boolean
            Weather the specified user have specified permission (of specified
            object).
        """
        if not user_obj.is_authenticated():
            return False
        # construct the permission full name
        add_permission = self.get_full_permission_string('add')
        change_permission = self.get_full_permission_string('change')
        delete_permission = self.get_full_permission_string('delete')
        if obj is None:
            if perm == add_permission:
                return (self.add_permission or self.any_permission) and user_obj.is_staff
            return True
        elif user_obj.is_active:
            if user_obj.is_staff:
                if self.any_permission:
                    # have any kind of permissions to the obj
                    return True
                if (self.add_permission and
                        perm == add_permission):
                    return True
                if (self.change_permission and
                        perm == change_permission):
                    return True
                if (self.delete_permission and
                        perm == delete_permission):
                    return True
        return False

def detect_group_type(permission_group):
    perm_group = permission_group.group
    if hasattr(perm_group, 'projectrole'):
        return 'project'
    elif hasattr(perm_group, 'customerrole'):
        return 'customer'
    elif hasattr(perm_group, 'projectgrouprole'):
        return 'project_group'

class TypedCollaboratorsPermissionLogic(PermissionLogic):
    """
    Permission logic that supports definition of several user groups based on the type of the
    checked object.

    For example, it is useful for cases when an object can be accessed either by project administrators or
    by customer owners.
    """
    def __init__(self, type_to_permission_logic_mapping, discriminator_function=None):
        self.type_to_permission_logic_mapping = type_to_permission_logic_mapping
        self.discriminator_function = discriminator_function

    def has_perm(self, user_obj, perm, obj=None):
        if not user_obj.is_authenticated():
            return False

        # always true for creation
        if obj is None:
            return True

        elif user_obj.is_active:
            # if the user is staff, allow everything
            if user_obj.is_staff:
                return True

            # detect a type of collaboration
            collaboration_type = self.discriminator_function(obj)

            # disallow operation if the type is unknown
            if not collaboration_type in self.type_to_permission_logic_mapping:
                return False
            collaborators_query = self.type_to_permission_logic_mapping[collaboration_type]['query']
            collaborators_filter = self.type_to_permission_logic_mapping[collaboration_type]['filter']

            kwargs = {
                collaborators_query: user_obj,
                'pk': obj.pk,
            }
            kwargs.update(collaborators_filter)

            if obj._meta.model._default_manager.filter(**kwargs).exists():
                return True
        return False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
from __future__ import division
import os
import time
import logging
from openquake.baselib import sap, general, parallel
from openquake.hazardlib import valid
from openquake.commonlib import readinput, datastore, logs
from openquake.commands import engine

@sap.Script
def run_tiles(num_tiles, job_ini, poolsize=0):
    """
    Run a hazard calculation by splitting the sites into tiles.
    WARNING: this is experimental and meant only for internal users
    """
    t0 = time.time()
    oq = readinput.get_oqparam(job_ini)
    num_sites = len(readinput.get_mesh(oq))
    task_args = [(job_ini, slc)
                 for slc in general.split_in_slices(num_sites, num_tiles)]
    if poolsize == 0:  # no pool
        Starmap = parallel.Sequential
    elif os.environ.get('OQ_DISTRIBUTE') == 'celery':
        Starmap = parallel.Processmap  # celery plays only with processes
    else:  # multiprocessing plays only with threads
        Starmap = parallel.Threadmap
    parent_child = [None, None]

    def agg(calc_ids, calc_id):
        if not calc_ids:  # first calculation
            parent_child[0] = calc_id
        parent_child[1] = calc_id
        logs.dbcmd('update_parent_child', parent_child)
        logging.warn('Finished calculation %d of %d',
                     len(calc_ids) + 1, num_tiles)
        return calc_ids + [calc_id]
    calc_ids = Starmap(engine.run_tile, task_args, poolsize).reduce(agg, [])
    datadir = datastore.get_datadir()
    for calc_id in calc_ids:
        print(os.path.join(datadir, 'calc_%d.hdf5' % calc_id))
    print('Total calculation time: %.1f h' % ((time.time() - t0) / 3600.))

run_tiles.arg('num_tiles', 'number of tiles to generate',
              type=valid.positiveint)
run_tiles.arg('job_ini', 'calculation configuration file '
              '(or files, comma-separated)')
run_tiles.opt('poolsize', 'size of the pool (default 0, no pool)',
              type=valid.positiveint)
<commit_msg>Fixed more imports [demos]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
from __future__ import division
import os
import time
import logging
from openquake.baselib import sap, general, parallel, datastore
from openquake.hazardlib import valid
from openquake.commonlib import readinput, logs
from openquake.commands import engine

@sap.Script
def run_tiles(num_tiles, job_ini, poolsize=0):
    """
    Run a hazard calculation by splitting the sites into tiles.
    WARNING: this is experimental and meant only for internal users
    """
    t0 = time.time()
    oq = readinput.get_oqparam(job_ini)
    num_sites = len(readinput.get_mesh(oq))
    task_args = [(job_ini, slc)
                 for slc in general.split_in_slices(num_sites, num_tiles)]
    if poolsize == 0:  # no pool
        Starmap = parallel.Sequential
    elif os.environ.get('OQ_DISTRIBUTE') == 'celery':
        Starmap = parallel.Processmap  # celery plays only with processes
    else:  # multiprocessing plays only with threads
        Starmap = parallel.Threadmap
    parent_child = [None, None]

    def agg(calc_ids, calc_id):
        if not calc_ids:  # first calculation
            parent_child[0] = calc_id
        parent_child[1] = calc_id
        logs.dbcmd('update_parent_child', parent_child)
        logging.warn('Finished calculation %d of %d',
                     len(calc_ids) + 1, num_tiles)
        return calc_ids + [calc_id]
    calc_ids = Starmap(engine.run_tile, task_args, poolsize).reduce(agg, [])
    datadir = datastore.get_datadir()
    for calc_id in calc_ids:
        print(os.path.join(datadir, 'calc_%d.hdf5' % calc_id))
    print('Total calculation time: %.1f h' % ((time.time() - t0) / 3600.))

run_tiles.arg('num_tiles', 'number of tiles to generate',
              type=valid.positiveint)
run_tiles.arg('job_ini', 'calculation configuration file '
              '(or files, comma-separated)')
run_tiles.opt('poolsize', 'size of the pool (default 0, no pool)',
              type=valid.positiveint)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# zip up the titanium mobile SDKs into suitable distribution formats
#
import os, types, glob, shutil, sys, platform
import zipfile, datetime, subprocess, tempfile, time

if platform.system() == 'Darwin':
	import importresolver

cur_dir = os.path.abspath(os.path.dirname(sys._getframe(0).f_code.co_filename))
top_dir = os.path.abspath(os.path.join(os.path.dirname(sys._getframe(0).f_code.co_filename),'..'))
template_dir = os.path.join(top_dir,'support')
doc_dir = os.path.abspath(os.path.join(top_dir, 'apidoc'))
all_dir = os.path.abspath(os.path.join(template_dir,'all'))
android_dir = os.path.abspath(os.path.join(template_dir,'android'))
iphone_dir = os.path.abspath(os.path.join(template_dir,'iphone'))
osx_dir = os.path.abspath(os.path.join(template_dir,'osx'))
win32_dir = os.path.abspath(os.path.join(template_dir, 'win32'))

buildtime = datetime.datetime.now()
ts = buildtime.strftime("%m/%d/%y %H:%M")

# get the githash for the build so we can always pull this build from a specific
# commit
gitCmd = "git"
if platform.system() == "Windows":
	gitCmd += ".cmd"

p = subprocess.Popen([gitCmd,"show","--abbrev-commit"],stderr=subprocess.PIPE, stdout=subprocess.PIPE)
githash = p.communicate()[0][7:].split('\n')[0].strip()

ignoreExtensions = ['.pbxuser','.perspectivev3','.pyc']
ignoreDirs = ['.DS_Store','.git','.gitignore','libTitanium.a','titanium.jar','build','bridge.txt']

def ignore(file):
	 for f in ignoreDirs:
		if file == f:
			return True
	 return False
def generate_jsca():
	 process_args = ['python', os.path.join(doc_dir, 'docgen.py'), '-f', 'jsca']
	 jsca_temp_file = tempfile.TemporaryFile()
	 try:
		 process = subprocess.Popen(process_args, stdout=jsca_temp_file, stderr=subprocess.PIPE)
		 process_return_code = process.wait()
		 if process_return_code != 0:
			 err_output = process.stderr.read()
			 print >> sys.stderr, "Failed to generate JSCA JSON.  Output:"
			 print >> sys.stderr, err_output
			 sys.exit(1)
		 jsca_temp_file.seek(0)
		 jsca_json = jsca_temp_file.read()
		 return jsca_json
	 finally:
		 jsca_temp_file.close()

def zip_dir(zf,dir,basepath,subs=None,cb=None):
	for root, dirs, files in os.walk(dir):
		for name in ignoreDirs:
			if name in dirs:
				dirs.remove(name)	# don't visit ignored directories
		for file in files:
			e = os.path.splitext(file)
			if len(e)==2 and e[1] in ignoreExtensions: continue
			from_ = os.path.join(root, file)
			to_ = from_.replace(dir, basepath, 1)
			if subs!=None:
				c = open(from_).read()
				for key in subs:
					c = c.replace(key,subs[key])
				if cb!=None:
					c = cb(file,e[1],c)
				zf.writestr(to_,c)
			else:
				zf.write(from_, to_)

def zip_android(zf,basepath):
	android_dist_dir = os.path.join(top_dir, 'dist', 'android')
	zip_dir(zf, os.path.join(cur_dir,'simplejson'), os.path.join(basepath, 'android', 'simplejson'))

	for jar in ['titanium.jar', 'kroll-apt.jar']:
		jar_path = os.path.join(android_dist_dir, jar)
		zf.write(jar_path, '%s/android/%s' % (basepath, jar))

	ant_tasks_jar = os.path.join(android_dist_dir, 'ant-tasks.jar')
	zf.write(ant_tasks_jar, '%s/module/android/ant-tasks.jar' % basepath)

	kroll_apt_lib_dir = os.path.join(top_dir, 'android', 'kroll-apt', 'lib')
	for jar in os.listdir(kroll_apt_lib_dir):
		if jar.endswith('.jar'):
			jar_path = os.path.join(kroll_apt_lib_dir, jar)
			zf.write(jar_path, '%s/android/%s' % (basepath, jar))

	android_depends = os.path.join(top_dir, 'android','dependency.json')
	zf.write(android_depends, '%s/android/dependency.json' % basepath)

	android_modules = os.path.join(android_dist_dir, 'modules.json')
	zf.write(android_modules, '%s/android/modules.json' % basepath)

	titanium_lib_dir = os.path.join(top_dir, 'android', 'titanium', 'lib')
	for thirdparty_jar in os.listdir(titanium_lib_dir):
		if thirdparty_jar == "commons-logging-1.1.1.jar": continue
		jar_path = os.path.join(top_dir, 'android', 'titanium', 'lib', thirdparty_jar)
		zf.write(jar_path, '%s/android/%s' % (basepath, thirdparty_jar))

	# include all module lib dependencies
	modules_dir = os.path.join(top_dir, 'android', 'modules')
	for module_dir in os.listdir(modules_dir):
		module_lib_dir = os.path.join(modules_dir, module_dir, 'lib')
		if os.path.exists(module_lib_dir):
			for thirdparty_jar in os.listdir(module_lib_dir):
				if thirdparty_jar.endswith('.jar'):
					jar_path = os.path.join(module_lib_dir, thirdparty_jar)
					zf.write(jar_path, '%s/android/%s' % (basepath, thirdparty_jar))

	android_module_jars = glob.glob(os.path.join(android_dist_dir, 'titanium-*.jar'))
	for android_module_jar in android_module_jars:
		 jarname = os.path.split(android_module_jar)[1]
		 zf.write(android_module_jar, '%s/android/modules/%s' % (basepath, jarname))

	android_module_res_zips = glob.glob(os.path.join(android_dist_dir, 'titanium-*.res.zip'))
	for android_module_res_zip in android_module_res_zips:
		zipname = os.path.split(android_module_res_zip)[1]
		zf.write(android_module_res_zip, '%s/android/modules/%s' % (basepath, zipname))

def resolve_source_imports(platform):
	sys.path.append(iphone_dir)
	import run,prereq
	return importresolver.resolve_source_imports(os.path.join(top_dir,platform,'Classes'))

def make_symbol(fn):
	if fn.startswith('TI') and fn!='TITANIUM' and fn!='TI':
		return fn[2:]
	return fn

def zip_iphone_ipad(zf,basepath,platform,version,version_tag):

#	zf.writestr('%s/iphone/imports.json'%basepath,resolve_source_imports(platform))

	# include our headers such that 3rd party modules can be compiled
	headers_dir=os.path.join(top_dir,'iphone','Classes')
	for f in os.listdir(headers_dir):
		path = os.path.join(headers_dir,f)
		if os.path.isfile(path) and os.path.splitext(f)[1]=='.h':
			 zf.write(path,'%s/iphone/include/%s' % (basepath,f))
		elif os.path.isdir(path):
			for df in os.listdir(path):
				dfpath = os.path.join(headers_dir,f,df)
				if os.path.isfile(dfpath) and os.path.splitext(df)[1]=='.h':
					 zf.write(dfpath,'%s/iphone/include/%s/%s' % (basepath,f,df))

	tp_headers_dir=os.path.join(top_dir,'iphone','headers','TiCore')
	for f in os.listdir(tp_headers_dir):
		if os.path.isfile(os.path.join(tp_headers_dir,f)) and os.path.splitext(f)[1]=='.h':
			 zf.write(os.path.join(tp_headers_dir,f),'%s/iphone/include/TiCore/%s' % (basepath,f))

	subs = {
		"__VERSION__":version,
		"__TIMESTAMP__":ts,
		"__GITHASH__": githash
	}

	# xcode_templates_dir =  os.path.join(top_dir,'iphone','templates','xcode')
	# zip_dir(zf,xcode_templates_dir,basepath+'/iphone/xcode/templates',subs)

	iphone_lib = os.path.join(top_dir,'iphone',platform,'build')

	zip_dir(zf,os.path.join(top_dir,'iphone','Classes'),basepath+'/iphone/Classes',subs)
	zip_dir(zf,os.path.join(top_dir,'iphone','headers'),basepath+'/iphone/headers',subs)
	zip_dir(zf,os.path.join(top_dir,'iphone','iphone'),basepath+'/iphone/iphone',subs)

	ticore_lib = os.path.join(top_dir,'iphone','lib')

	# during 1.3.3, we added a new lib to a folder that had a .gitignore
	# and we need to manually reset this
	if not os.path.exists(os.path.join(ticore_lib,'libtiverify.a')):
		os.system("git checkout iphone/lib")
		if not os.path.exists(os.path.join(ticore_lib,'libtiverify.a')):
			print "[ERROR] missing libtiverify.a!  make sure you checkout iphone/lib or edit your iphone/.gitignore and remove the lib entry"
			sys.exit(1)

	if not os.path.exists(os.path.join(ticore_lib,'libti_ios_debugger.a')):
		os.system("git checkout iphone/lib")
		if not os.path.exists(os.path.join(ticore_lib,'libti_ios_debugger.a')):
			print "[ERROR] missing libti_ios_debugger.a!  make sure you checkout iphone/lib or edit your iphone/.gitignore and remove the lib entry"
			sys.exit(1)

	if not os.path.exists(os.path.join(ticore_lib,'libTiCore.a')):
		print "[ERROR] missing libTiCore.a!"
		sys.exit(1)

	zf.write(os.path.join(ticore_lib,'libTiCore.a'),'%s/%s/libTiCore.a'%(basepath,platform))
	zf.write(os.path.join(ticore_lib,'libtiverify.a'),'%s/%s/libtiverify.a'%(basepath,platform))
	zf.write(os.path.join(ticore_lib,'libti_ios_debugger.a'),'%s/%s/libti_ios_debugger.a'%(basepath,platform))

	zip_dir(zf,osx_dir,basepath)

	modules_dir = os.path.join(top_dir,'iphone','Resources','modules')
	for f in os.listdir(modules_dir):
		if os.path.isdir(os.path.join(modules_dir,f)):
			module_images = os.path.join(modules_dir,f)
			if os.path.exists(module_images):
				module_name = f.replace('Module','').lower()
				zip_dir(zf,module_images,'%s/%s/modules/%s/images' % (basepath,platform,module_name))

def create_platform_zip(platform,dist_dir,osname,version,version_tag):
	if not os.path.exists(dist_dir):
		os.makedirs(dist_dir)
	basepath = '%s/%s/%s' % (platform,osname,version_tag)
	sdkzip = os.path.join(dist_dir,'%s-%s-%s.zip' % (platform,version_tag,osname))
	zf = zipfile.ZipFile(sdkzip, 'w', zipfile.ZIP_DEFLATED)
	return (zf,basepath)

def zip_mobilesdk(dist_dir,osname,version,android,iphone,ipad,version_tag):
	zf, basepath = create_platform_zip('mobilesdk',dist_dir,osname,version,version_tag)

	version_txt = """version=%s
timestamp=%s
githash=%s
""" % (version,ts,githash)

	zf.writestr('%s/version.txt' % basepath,version_txt)
	jsca = generate_jsca()
	zf.writestr('%s/api.jsca' % basepath, jsca)

	zip_dir(zf,all_dir,basepath)
	zip_dir(zf,template_dir,basepath)
	if android: zip_android(zf,basepath)
	if (iphone or ipad) and osname == "osx": zip_iphone_ipad(zf,basepath,'iphone',version,version_tag)
	if osname == 'win32':
		zip_dir(zf, win32_dir, basepath)

	zf.close()

def zip_it(dist_dir,osname,version,android,iphone,ipad,version_tag):
	zip_mobilesdk(dist_dir,osname,version,android,iphone,ipad,version_tag)

class Packager(object):
	def __init__(self):
		self.os_names = { "Windows":"win32", "Linux":"linux", "Darwin":"osx" }

	def build(self,dist_dir,version,android=True,iphone=True,ipad=True,version_tag=None):
		if version_tag == None:
			version_tag = version
		zip_it(dist_dir,self.os_names[platform.system()],version,android,iphone,ipad,version_tag)

	def build_all_platforms(self,dist_dir,version,android=True,iphone=True,ipad=True,version_tag=None):
		if version_tag == None:
			version_tag = version
		for os in self.os_names.values():
			zip_it(dist_dir,os,version,android,iphone,ipad,version_tag)

if __name__ == '__main__':
	Packager().build(os.path.abspath('../dist'), "1.1.0")
<commit_msg>use sys.executable instead of "python" directly. print the full JSCA generation command so it can be debugged in the future. TIMOB-5033<commit_after>#!/usr/bin/env python
#
# zip up the titanium mobile SDKs into suitable distribution formats
#
import os, types, glob, shutil, sys, platform
import zipfile, datetime, subprocess, tempfile, time

if platform.system() == 'Darwin':
	import importresolver

cur_dir = os.path.abspath(os.path.dirname(sys._getframe(0).f_code.co_filename))
top_dir = os.path.abspath(os.path.join(os.path.dirname(sys._getframe(0).f_code.co_filename),'..'))
template_dir = os.path.join(top_dir,'support')
doc_dir = os.path.abspath(os.path.join(top_dir, 'apidoc'))
all_dir = os.path.abspath(os.path.join(template_dir,'all'))
android_dir = os.path.abspath(os.path.join(template_dir,'android'))
iphone_dir = os.path.abspath(os.path.join(template_dir,'iphone'))
osx_dir = os.path.abspath(os.path.join(template_dir,'osx'))
win32_dir = os.path.abspath(os.path.join(template_dir, 'win32'))

buildtime = datetime.datetime.now()
ts = buildtime.strftime("%m/%d/%y %H:%M")

# get the githash for the build so we can always pull this build from a specific
# commit
gitCmd = "git"
if platform.system() == "Windows":
	gitCmd += ".cmd"

p = subprocess.Popen([gitCmd,"show","--abbrev-commit"],stderr=subprocess.PIPE, stdout=subprocess.PIPE)
githash = p.communicate()[0][7:].split('\n')[0].strip()

ignoreExtensions = ['.pbxuser','.perspectivev3','.pyc']
ignoreDirs = ['.DS_Store','.git','.gitignore','libTitanium.a','titanium.jar','build','bridge.txt']

def ignore(file):
	 for f in ignoreDirs:
		if file == f:
			return True
	 return False

def generate_jsca():
	 process_args = [sys.executable, os.path.join(doc_dir, 'docgen.py'), '-f', 'jsca']
	 print "Generating JSCA..."
	 print " ".join(process_args)

	 jsca_temp_file = tempfile.TemporaryFile()
	 try:
		 process = subprocess.Popen(process_args, stdout=jsca_temp_file, stderr=subprocess.PIPE)
		 process_return_code = process.wait()
		 if process_return_code != 0:
			 err_output = process.stderr.read()
			 print >> sys.stderr, "Failed to generate JSCA JSON.  Output:"
			 print >> sys.stderr, err_output
			 sys.exit(1)
		 jsca_temp_file.seek(0)
		 jsca_json = jsca_temp_file.read()
		 return jsca_json
	 finally:
		 jsca_temp_file.close()

def zip_dir(zf,dir,basepath,subs=None,cb=None):
	for root, dirs, files in os.walk(dir):
		for name in ignoreDirs:
			if name in dirs:
				dirs.remove(name)	# don't visit ignored directories
		for file in files:
			e = os.path.splitext(file)
			if len(e)==2 and e[1] in ignoreExtensions: continue
			from_ = os.path.join(root, file)
			to_ = from_.replace(dir, basepath, 1)
			if subs!=None:
				c = open(from_).read()
				for key in subs:
					c = c.replace(key,subs[key])
				if cb!=None:
					c = cb(file,e[1],c)
				zf.writestr(to_,c)
			else:
				zf.write(from_, to_)

def zip_android(zf,basepath):
	android_dist_dir = os.path.join(top_dir, 'dist', 'android')
	zip_dir(zf, os.path.join(cur_dir,'simplejson'), os.path.join(basepath, 'android', 'simplejson'))

	for jar in ['titanium.jar', 'kroll-apt.jar']:
		jar_path = os.path.join(android_dist_dir, jar)
		zf.write(jar_path, '%s/android/%s' % (basepath, jar))

	ant_tasks_jar = os.path.join(android_dist_dir, 'ant-tasks.jar')
	zf.write(ant_tasks_jar, '%s/module/android/ant-tasks.jar' % basepath)

	kroll_apt_lib_dir = os.path.join(top_dir, 'android', 'kroll-apt', 'lib')
	for jar in os.listdir(kroll_apt_lib_dir):
		if jar.endswith('.jar'):
			jar_path = os.path.join(kroll_apt_lib_dir, jar)
			zf.write(jar_path, '%s/android/%s' % (basepath, jar))

	android_depends = os.path.join(top_dir, 'android','dependency.json')
	zf.write(android_depends, '%s/android/dependency.json' % basepath)

	android_modules = os.path.join(android_dist_dir, 'modules.json')
	zf.write(android_modules, '%s/android/modules.json' % basepath)

	titanium_lib_dir = os.path.join(top_dir, 'android', 'titanium', 'lib')
	for thirdparty_jar in os.listdir(titanium_lib_dir):
		if thirdparty_jar == "commons-logging-1.1.1.jar": continue
		jar_path = os.path.join(top_dir, 'android', 'titanium', 'lib', thirdparty_jar)
		zf.write(jar_path, '%s/android/%s' % (basepath, thirdparty_jar))

	# include all module lib dependencies
	modules_dir = os.path.join(top_dir, 'android', 'modules')
	for module_dir in os.listdir(modules_dir):
		module_lib_dir = os.path.join(modules_dir, module_dir, 'lib')
		if os.path.exists(module_lib_dir):
			for thirdparty_jar in os.listdir(module_lib_dir):
				if thirdparty_jar.endswith('.jar'):
					jar_path = os.path.join(module_lib_dir, thirdparty_jar)
					zf.write(jar_path, '%s/android/%s' % (basepath, thirdparty_jar))

	android_module_jars = glob.glob(os.path.join(android_dist_dir, 'titanium-*.jar'))
	for android_module_jar in android_module_jars:
		 jarname = os.path.split(android_module_jar)[1]
		 zf.write(android_module_jar, '%s/android/modules/%s' % (basepath, jarname))

	android_module_res_zips = glob.glob(os.path.join(android_dist_dir, 'titanium-*.res.zip'))
	for android_module_res_zip in android_module_res_zips:
		zipname = os.path.split(android_module_res_zip)[1]
		zf.write(android_module_res_zip, '%s/android/modules/%s' % (basepath, zipname))

def resolve_source_imports(platform):
	sys.path.append(iphone_dir)
	import run,prereq
	return importresolver.resolve_source_imports(os.path.join(top_dir,platform,'Classes'))

def make_symbol(fn):
	if fn.startswith('TI') and fn!='TITANIUM' and fn!='TI':
		return fn[2:]
	return fn

def zip_iphone_ipad(zf,basepath,platform,version,version_tag):

#	zf.writestr('%s/iphone/imports.json'%basepath,resolve_source_imports(platform))

	# include our headers such that 3rd party modules can be compiled
	headers_dir=os.path.join(top_dir,'iphone','Classes')
	for f in os.listdir(headers_dir):
		path = os.path.join(headers_dir,f)
		if os.path.isfile(path) and os.path.splitext(f)[1]=='.h':
			 zf.write(path,'%s/iphone/include/%s' % (basepath,f))
		elif os.path.isdir(path):
			for df in os.listdir(path):
				dfpath = os.path.join(headers_dir,f,df)
				if os.path.isfile(dfpath) and os.path.splitext(df)[1]=='.h':
					 zf.write(dfpath,'%s/iphone/include/%s/%s' % (basepath,f,df))

	tp_headers_dir=os.path.join(top_dir,'iphone','headers','TiCore')
	for f in os.listdir(tp_headers_dir):
		if os.path.isfile(os.path.join(tp_headers_dir,f)) and os.path.splitext(f)[1]=='.h':
			 zf.write(os.path.join(tp_headers_dir,f),'%s/iphone/include/TiCore/%s' % (basepath,f))

	subs = {
		"__VERSION__":version,
		"__TIMESTAMP__":ts,
		"__GITHASH__": githash
	}

	# xcode_templates_dir =  os.path.join(top_dir,'iphone','templates','xcode')
	# zip_dir(zf,xcode_templates_dir,basepath+'/iphone/xcode/templates',subs)

	iphone_lib = os.path.join(top_dir,'iphone',platform,'build')

	zip_dir(zf,os.path.join(top_dir,'iphone','Classes'),basepath+'/iphone/Classes',subs)
	zip_dir(zf,os.path.join(top_dir,'iphone','headers'),basepath+'/iphone/headers',subs)
	zip_dir(zf,os.path.join(top_dir,'iphone','iphone'),basepath+'/iphone/iphone',subs)

	ticore_lib = os.path.join(top_dir,'iphone','lib')

	# during 1.3.3, we added a new lib to a folder that had a .gitignore
	# and we need to manually reset this
	if not os.path.exists(os.path.join(ticore_lib,'libtiverify.a')):
		os.system("git checkout iphone/lib")
		if not os.path.exists(os.path.join(ticore_lib,'libtiverify.a')):
			print "[ERROR] missing libtiverify.a!  make sure you checkout iphone/lib or edit your iphone/.gitignore and remove the lib entry"
			sys.exit(1)

	if not os.path.exists(os.path.join(ticore_lib,'libti_ios_debugger.a')):
		os.system("git checkout iphone/lib")
		if not os.path.exists(os.path.join(ticore_lib,'libti_ios_debugger.a')):
			print "[ERROR] missing libti_ios_debugger.a!  make sure you checkout iphone/lib or edit your iphone/.gitignore and remove the lib entry"
			sys.exit(1)

	if not os.path.exists(os.path.join(ticore_lib,'libTiCore.a')):
		print "[ERROR] missing libTiCore.a!"
		sys.exit(1)

	zf.write(os.path.join(ticore_lib,'libTiCore.a'),'%s/%s/libTiCore.a'%(basepath,platform))
	zf.write(os.path.join(ticore_lib,'libtiverify.a'),'%s/%s/libtiverify.a'%(basepath,platform))
	zf.write(os.path.join(ticore_lib,'libti_ios_debugger.a'),'%s/%s/libti_ios_debugger.a'%(basepath,platform))

	zip_dir(zf,osx_dir,basepath)

	modules_dir = os.path.join(top_dir,'iphone','Resources','modules')
	for f in os.listdir(modules_dir):
		if os.path.isdir(os.path.join(modules_dir,f)):
			module_images = os.path.join(modules_dir,f)
			if os.path.exists(module_images):
				module_name = f.replace('Module','').lower()
				zip_dir(zf,module_images,'%s/%s/modules/%s/images' % (basepath,platform,module_name))

def create_platform_zip(platform,dist_dir,osname,version,version_tag):
	if not os.path.exists(dist_dir):
		os.makedirs(dist_dir)
	basepath = '%s/%s/%s' % (platform,osname,version_tag)
	sdkzip = os.path.join(dist_dir,'%s-%s-%s.zip' % (platform,version_tag,osname))
	zf = zipfile.ZipFile(sdkzip, 'w', zipfile.ZIP_DEFLATED)
	return (zf,basepath)

def zip_mobilesdk(dist_dir,osname,version,android,iphone,ipad,version_tag):
	zf, basepath = create_platform_zip('mobilesdk',dist_dir,osname,version,version_tag)

	version_txt = """version=%s
timestamp=%s
githash=%s
""" % (version,ts,githash)

	zf.writestr('%s/version.txt' % basepath,version_txt)
	jsca = generate_jsca()
	zf.writestr('%s/api.jsca' % basepath, jsca)

	zip_dir(zf,all_dir,basepath)
	zip_dir(zf,template_dir,basepath)
	if android: zip_android(zf,basepath)
	if (iphone or ipad) and osname == "osx": zip_iphone_ipad(zf,basepath,'iphone',version,version_tag)
	if osname == 'win32':
		zip_dir(zf, win32_dir, basepath)

	zf.close()

def zip_it(dist_dir,osname,version,android,iphone,ipad,version_tag):
	zip_mobilesdk(dist_dir,osname,version,android,iphone,ipad,version_tag)

class Packager(object):
	def __init__(self):
		self.os_names = { "Windows":"win32", "Linux":"linux", "Darwin":"osx" }

	def build(self,dist_dir,version,android=True,iphone=True,ipad=True,version_tag=None):
		if version_tag == None:
			version_tag = version
		zip_it(dist_dir,self.os_names[platform.system()],version,android,iphone,ipad,version_tag)

	def build_all_platforms(self,dist_dir,version,android=True,iphone=True,ipad=True,version_tag=None):
		if version_tag == None:
			version_tag = version
		for os in self.os_names.values():
			zip_it(dist_dir,os,version,android,iphone,ipad,version_tag)

if __name__ == '__main__':
	Packager().build(os.path.abspath('../dist'), "1.1.0")
<|endoftext|>


<commit_before>from django.contrib.sitemaps import Sitemap
from web.nodes.models import Node
from datetime import datetime
import re
from django.conf import settings
from django.core.urlresolvers import reverse

class HttpsSitemap(Sitemap):
  http_match = re.compile(r"^http://")

  def get_urls(self, page=1):
    urls = super(HttpsSitemap, self).get_urls(page)
    if getattr(settings, 'SITEMAPS_USE_HTTPS', None):
      for url in urls:
        url['location'] = self.http_match.sub("https://", url['location'])
    return urls

class NodeSitemap(HttpsSitemap):
  priority = "0.8"

  def items(self):
    return Node.objects.all()

  def lastmod(self, node):
    return node.last_seen

  def location(self, node):
    return node.get_absolute_url()

class StaticSitemap(HttpsSitemap):
  priority = "0.9"
  locations = []

  def __init__(self, *args, **kwargs):
    super(StaticSitemap, self).__init__(*args, **kwargs)
    self.locations = [
      reverse('network_statistics'),
      reverse('network_clients'),
      reverse('network_map')
    ]

  def items(self):
    return self.locations

  def lastmod(self, obj):
    return datetime.now()

  def location(self, obj):
    return obj

class RootPageSitemap(StaticSitemap):
  priority = "1.0"
  locations = [
    '/'
  ]

  def items(self):
    return self.locations

  def lastmod(self, obj):
    return datetime.now()

  def location(self, obj):
    return obj
<commit_msg>Fixed parameter passing to get_urls function of sitemap superclass.<commit_after>from django.contrib.sitemaps import Sitemap
from web.nodes.models import Node
from datetime import datetime
import re
from django.conf import settings
from django.core.urlresolvers import reverse

class HttpsSitemap(Sitemap):
  http_match = re.compile(r"^http://")

  def get_urls(self, *args, **kwargs):
    urls = super(HttpsSitemap, self).get_urls(*args, **kwargs)
    if getattr(settings, 'SITEMAPS_USE_HTTPS', None):
      for url in urls:
        url['location'] = self.http_match.sub("https://", url['location'])
    return urls

class NodeSitemap(HttpsSitemap):
  priority = "0.8"

  def items(self):
    return Node.objects.all()

  def lastmod(self, node):
    return node.last_seen

  def location(self, node):
    return node.get_absolute_url()

class StaticSitemap(HttpsSitemap):
  priority = "0.9"
  locations = []

  def __init__(self, *args, **kwargs):
    super(StaticSitemap, self).__init__(*args, **kwargs)
    self.locations = [
      reverse('network_statistics'),
      reverse('network_clients'),
      reverse('network_map')
    ]

  def items(self):
    return self.locations

  def lastmod(self, obj):
    return datetime.now()

  def location(self, obj):
    return obj

class RootPageSitemap(StaticSitemap):
  priority = "1.0"
  locations = [
    '/'
  ]

  def items(self):
    return self.locations

  def lastmod(self, obj):
    return datetime.now()

  def location(self, obj):
    return obj
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""Utility functions related to splitting work into tasks."""

import sys
import cPickle
import traceback

from celery.task.sets import TaskSet
from celery.task import task

from openquake.engine import logs, no_distribute
from openquake.engine.db import models
from openquake.engine.utils import config
from openquake.engine.writer import CacheInserter
from openquake.engine.performance import EnginePerformanceMonitor, \
    LightMonitor

class Pickled(object):
    """
    An utility to manually pickling/unpickling objects.
    The reason is that celery does not use the HIGHEST_PROTOCOL,
    so relying on celery is slower. Moreover Pickled instances
    have a nice string representation and length giving the size
    of the pickled bytestring.
    """
    def __init__(self, obj):
        self.clsname = obj.__class__.__name__
        self.pik = cPickle.dumps(obj, cPickle.HIGHEST_PROTOCOL)

    def __repr__(self):
        """String representation of the pickled object"""
        return '<Pickled %s %dK>' % (self.clsname, len(self) / 1024)

    def __len__(self):
        """Length of the pickled bytestring"""
        return len(self.pik)

    def unpickle(self):
        """Unpickle the underlying object"""
        return cPickle.loads(self.pik)

def pickle_sequence(objects):
    """
    Convert an iterable of objects into a list of pickled objects.
    If the iterable contains copies, the pickling will be done only once.
    If the iterable contains objects already pickled, they will not be
    pickled again.
    """
    cache = {}
    out = []
    for obj in objects:
        obj_id = id(obj)
        if obj_id not in cache:
            if isinstance(obj, Pickled):  # already pickled
                cache[obj_id] = obj
            else:  # pickle the object
                cache[obj_id] = Pickled(obj)
        out.append(cache[obj_id])
    return out

def safely_call(func, args):
    """
    Call the given function with the given arguments safely, i.e.
    by trapping the exceptions. Return a pair (result, exc_type)
    where exc_type is None if no exceptions occur, otherwise it
    is the exception class and the result is a string containing
    error message and traceback.
    """
    try:
        return func(*args), None
    except:
        etype, exc, tb = sys.exc_info()
        tb_str = ''.join(traceback.format_tb(tb))
        return '\n%s%s: %s' % (tb_str, etype.__name__, exc), etype

def map_reduce(task, task_args, agg, acc):
    """
    Given a task and an iterable of positional arguments, apply the
    task function to the arguments in parallel and return an aggregate
    result depending on the initial value of the accumulator
    and on the aggregation function. To save memory, the order is
    not preserved and there is no list with the intermediated results:
    the accumulator is incremented as soon as a task result comes.

    :param task: a `celery` task callable.
    :param task_args: an iterable over positional arguments
    :param agg: the aggregation function, (acc, val) -> new acc
    :param acc: the initial value of the accumulator
    :returns: the final value of the accumulator

    NB: if the environment variable OQ_NO_DISTRIBUTE is set the
    tasks are run sequentially in the current process and then
    map_reduce(task, task_args, agg, acc) is the same as
    reduce(agg, itertools.starmap(task, task_args), acc).
    Users of map_reduce should be aware of the fact that when
    thousands of tasks are spawned and large arguments are passed
    or large results are returned they may incur in memory issue:
    this is way the calculators limit the queue with the
    `concurrent_task` concept.
    """
    if no_distribute():
        for the_args in task_args:
            result, exctype = safely_call(task.task_func, the_args)
            if exctype:
                raise exctype(result)
            acc = agg(acc, result)
    else:
        unpik = 0
        job_id = task_args[0][0]
        taskname = task.__name__
        mon = LightMonitor('unpickling %s' % taskname, job_id, task)
        to_send = 0
        pickled_args = []
        for args in task_args:
            piks = pickle_sequence(args)
            pickled_args.append(piks)
            to_send += sum(len(p) for p in piks)
        logs.LOG.info('Sending %d K', to_send / 1024)
        taskset = TaskSet(tasks=map(task.subtask, pickled_args))
        for task_id, result_dict in taskset.apply_async().iter_native():
            result_pik = result_dict['result']
            with mon:
                result, exctype = result_pik.unpickle()
            if exctype:
                raise exctype(result)
            unpik += len(result_pik)
            acc = agg(acc, result)
        logs.LOG.info('Unpickled %d K in %s seconds',
                      unpik / 1024, mon.duration)
    return acc

# used to implement BaseCalculator.parallelize, which takes in account
# the `concurrent_task` concept to avoid filling the Celery queue
def parallelize(task, task_args, side_effect=lambda val: None):
    """
    Given a celery task and an iterable of positional arguments, apply the
    callable to the arguments in parallel. It is possible to pass a
    function side_effect(val) which takes the return value of the
    callable and does something with it (such as saving or printing
    it). Notice that the order is not preserved. parallelize returns None.

    :param task: a celery task
    :param task_args: an iterable over positional arguments
    :param side_effect: a function val -> None

    NB: if the environment variable OQ_NO_DISTRIBUTE is set the
    tasks are run sequentially in the current process.
    """
    map_reduce(task, task_args, lambda acc, val: side_effect(val), None)

def oqtask(task_func):
    """
    Task function decorator which sets up logging and catches (and logs) any
    errors which occur inside the task. Also checks to make sure the job is
    actually still running. If it is not running, the task doesn't get
    executed, so we don't do useless computation.
    """
    def wrapped(*args):
        """
        Initialize logs, make sure the job is still running, and run the task
        code surrounded by a try-except. If any error occurs, log it as a
        critical failure.
        """
        # job_id is always assumed to be the first argument
        job_id = args[0]
        job = models.OqJob.objects.get(id=job_id)
        if job.is_running is False:
            # the job was killed, it is useless to run the task
            return

        # it is important to save the task id soon, so that
        # the revoke functionality can work
        EnginePerformanceMonitor.store_task_id(job_id, tsk)

        with EnginePerformanceMonitor(
                'total ' + task_func.__name__, job_id, tsk, flush=True):
            # tasks write on the celery log file
            logs.set_level(job.log_level)
            try:
                # run the task
                return task_func(*args)
            finally:
                # save on the db
                CacheInserter.flushall()
                # the task finished, we can remove from the performance
                # table the associated row 'storing task id'
                models.Performance.objects.filter(
                    oq_job=job,
                    operation='storing task id',
                    task_id=tsk.request.id).delete()
    celery_queue = config.get('amqp', 'celery_queue')
    f = lambda *args: Pickled(safely_call(wrapped, [a.unpickle() for a in args]))
    f.__name__ = task_func.__name__
    tsk = task(f, queue=celery_queue)
    tsk.task_func = task_func
    return tsk
<commit_msg>Improved logging<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2014, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

"""Utility functions related to splitting work into tasks."""

import sys
import cPickle
import traceback

from celery.task.sets import TaskSet
from celery.task import task

from openquake.engine import logs, no_distribute
from openquake.engine.db import models
from openquake.engine.utils import config
from openquake.engine.writer import CacheInserter
from openquake.engine.performance import EnginePerformanceMonitor, \
    LightMonitor

ONE_MB = 1024 * 1024

class Pickled(object):
    """
    An utility to manually pickling/unpickling objects.
    The reason is that celery does not use the HIGHEST_PROTOCOL,
    so relying on celery is slower. Moreover Pickled instances
    have a nice string representation and length giving the size
    of the pickled bytestring.
    """
    def __init__(self, obj):
        self.clsname = obj.__class__.__name__
        self.pik = cPickle.dumps(obj, cPickle.HIGHEST_PROTOCOL)

    def __repr__(self):
        """String representation of the pickled object"""
        return '<Pickled %s %dK>' % (self.clsname, len(self) / 1024)

    def __len__(self):
        """Length of the pickled bytestring"""
        return len(self.pik)

    def unpickle(self):
        """Unpickle the underlying object"""
        return cPickle.loads(self.pik)

def pickle_sequence(objects):
    """
    Convert an iterable of objects into a list of pickled objects.
    If the iterable contains copies, the pickling will be done only once.
    If the iterable contains objects already pickled, they will not be
    pickled again.
    """
    cache = {}
    out = []
    for obj in objects:
        obj_id = id(obj)
        if obj_id not in cache:
            if isinstance(obj, Pickled):  # already pickled
                cache[obj_id] = obj
            else:  # pickle the object
                cache[obj_id] = Pickled(obj)
        out.append(cache[obj_id])
    return out

def safely_call(func, args):
    """
    Call the given function with the given arguments safely, i.e.
    by trapping the exceptions. Return a pair (result, exc_type)
    where exc_type is None if no exceptions occur, otherwise it
    is the exception class and the result is a string containing
    error message and traceback.
    """
    try:
        return func(*args), None
    except:
        etype, exc, tb = sys.exc_info()
        tb_str = ''.join(traceback.format_tb(tb))
        return '\n%s%s: %s' % (tb_str, etype.__name__, exc), etype

def map_reduce(task, task_args, agg, acc):
    """
    Given a task and an iterable of positional arguments, apply the
    task function to the arguments in parallel and return an aggregate
    result depending on the initial value of the accumulator
    and on the aggregation function. To save memory, the order is
    not preserved and there is no list with the intermediated results:
    the accumulator is incremented as soon as a task result comes.

    :param task: a `celery` task callable.
    :param task_args: an iterable over positional arguments
    :param agg: the aggregation function, (acc, val) -> new acc
    :param acc: the initial value of the accumulator
    :returns: the final value of the accumulator

    NB: if the environment variable OQ_NO_DISTRIBUTE is set the
    tasks are run sequentially in the current process and then
    map_reduce(task, task_args, agg, acc) is the same as
    reduce(agg, itertools.starmap(task, task_args), acc).
    Users of map_reduce should be aware of the fact that when
    thousands of tasks are spawned and large arguments are passed
    or large results are returned they may incur in memory issue:
    this is way the calculators limit the queue with the
    `concurrent_task` concept.
    """
    if no_distribute():
        for the_args in task_args:
            result, exctype = safely_call(task.task_func, the_args)
            if exctype:
                raise exctype(result)
            acc = agg(acc, result)
    else:
        unpik = 0
        job_id = task_args[0][0]
        taskname = task.__name__
        mon = LightMonitor('unpickling %s' % taskname, job_id, task)
        to_send = 0
        pickled_args = []
        for args in task_args:
            piks = pickle_sequence(args)
            pickled_args.append(piks)
            to_send += sum(len(p) for p in piks)
        logs.LOG.info('Sending %dM', to_send / ONE_MB)
        taskset = TaskSet(tasks=map(task.subtask, pickled_args))
        for task_id, result_dict in taskset.apply_async().iter_native():
            result_pik = result_dict['result']
            with mon:
                result, exctype = result_pik.unpickle()
            if exctype:
                raise exctype(result)
            unpik += len(result_pik)
            acc = agg(acc, result)
        logs.LOG.info('Unpickled %dM in %s seconds',
                      unpik / ONE_MB, mon.duration)
    return acc

# used to implement BaseCalculator.parallelize, which takes in account
# the `concurrent_task` concept to avoid filling the Celery queue
def parallelize(task, task_args, side_effect=lambda val: None):
    """
    Given a celery task and an iterable of positional arguments, apply the
    callable to the arguments in parallel. It is possible to pass a
    function side_effect(val) which takes the return value of the
    callable and does something with it (such as saving or printing
    it). Notice that the order is not preserved. parallelize returns None.

    :param task: a celery task
    :param task_args: an iterable over positional arguments
    :param side_effect: a function val -> None

    NB: if the environment variable OQ_NO_DISTRIBUTE is set the
    tasks are run sequentially in the current process.
    """
    map_reduce(task, task_args, lambda acc, val: side_effect(val), None)

def oqtask(task_func):
    """
    Task function decorator which sets up logging and catches (and logs) any
    errors which occur inside the task. Also checks to make sure the job is
    actually still running. If it is not running, the task doesn't get
    executed, so we don't do useless computation.
    """
    def wrapped(*args):
        """
        Initialize logs, make sure the job is still running, and run the task
        code surrounded by a try-except. If any error occurs, log it as a
        critical failure.
        """
        # job_id is always assumed to be the first argument
        job_id = args[0]
        job = models.OqJob.objects.get(id=job_id)
        if job.is_running is False:
            # the job was killed, it is useless to run the task
            return

        # it is important to save the task id soon, so that
        # the revoke functionality can work
        EnginePerformanceMonitor.store_task_id(job_id, tsk)

        with EnginePerformanceMonitor(
                'total ' + task_func.__name__, job_id, tsk, flush=True):
            # tasks write on the celery log file
            logs.set_level(job.log_level)
            try:
                # run the task
                return task_func(*args)
            finally:
                # save on the db
                CacheInserter.flushall()
                # the task finished, we can remove from the performance
                # table the associated row 'storing task id'
                models.Performance.objects.filter(
                    oq_job=job,
                    operation='storing task id',
                    task_id=tsk.request.id).delete()
    celery_queue = config.get('amqp', 'celery_queue')
    f = lambda *args: Pickled(safely_call(wrapped, [a.unpickle() for a in args]))
    f.__name__ = task_func.__name__
    tsk = task(f, queue=celery_queue)
    tsk.task_func = task_func
    return tsk
<|endoftext|>