<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from collections import namedtuple

import requests
import urllib
from urlparse import urljoin, urlparse

from django.conf import settings
from django.contrib.auth import login as django_login
from django.contrib.auth import logout as django_logout
from django.contrib.auth.backends import ModelBackend
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseBadRequest, HttpResponseRedirect
from django.utils import simplejson
from django.http import HttpResponse
from django.views.generic.base import View
from django.utils.decorators import method_decorator

from itsdangerous import URLSafeTimedSerializer

from lizard_auth_client.client import construct_user

# used so we can login User objects we instantiated ourselves
BACKEND = ModelBackend()

class TestHomeView(View):
    '''
    Test view, only used in standalone mode.
    Display a minimal HTML page with some URLS.
    '''
    def get(self, request, *args, **kwargs):
        user = request.user
        return HttpResponse(
            'home | protected'
            '| logout | user={} | home @ client'
            ''.format(user)
        )

class TestProtectedView(View):
    '''
    Test view, only used in standalone mode.
    Display a minimal HTML page with some URLS, but is "protected" by
    a login_required decorator.
    '''
    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(TestProtectedView, self).dispatch(
            request, *args, **kwargs)

    def get(self, request, *args, **kwargs):
        user = request.user
        return HttpResponse(
            'home | protected'
            '| logout '
            '| user={} | protected @ client'
            ''.format(user)
        )

class LoginView(View):
    '''
    View that redirects the user to the SSO server.

    Requests a Request Token and then redirects the User to the the SSO Server.
    '''
    def get(self, request, *args, **kwargs):
        # store the GET parameter named 'next', so we can redirect the user
        # to the requested page, after SSO login.
        next = get_next(request)
        request.session['sso_after_login_next'] = next

        response = {302: HttpResponseRedirect,
                    503: HttpResponse}

        sso_request = get_sso_request()
        http_response = response.get(sso_request.status_code)

        return http_response(sso_request.message,
                             status=sso_request.status_code)

class LocalLoginView(View):
    '''
    Verifies the user token with the SSO server, and logs the user in.
    '''
    def get(self, request, *args, **kwargs):
        # verify the authentication token and
        # retrieve the User instance from the SSO server
        user = verify_auth_token(request.GET['message'])
        if not user:
            return HttpResponseBadRequest('Verification failed')

        # link the user instance to the default database backend
        # and call django-auth's login function
        user.backend = "%s.%s" % (BACKEND.__module__,
                                  BACKEND.__class__.__name__)
        django_login(request, user)

        # redirect the user to the stored "next" url, which is probably a
        # protected page
        sso_after_login_next = request.session.get('sso_after_login_next', '/')
        request.session.delete('sso_after_login_next')
        return HttpResponseRedirect(sso_after_login_next)

class LogoutView(View):
    '''
    Redirect user to SSO server, to log out there.
    '''
    def get(self, request, *args, **kwargs):
        # store the 'next' parameter in the session so we can
        # redirect the user afterwards
        sso_after_logout_next = get_next(request)
        request.session['sso_after_logout_next'] = sso_after_logout_next

        # construct a signed message containing the portal key
        params = {
            'action': 'logout',
            'key': settings.SSO_KEY
        }
        message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
        query_string = urllib.urlencode([('message', message),
                                         ('key', settings.SSO_KEY)])
        url = urljoin(settings.SSO_SERVER_PUBLIC_URL,
                      'sso/portal_action') + '/'
        url = '%s?%s' % (url, query_string)

        # send the redirect response
        return HttpResponseRedirect(url)

class LocalLogoutView(View):
    '''
    Log out locally. Mostly the user is redirected here by the SSO server.
    '''
    def get(self, request, *args, **kwargs):
        # call django-auth's logout function
        django_logout(request)

        # redirect the user
        next = request.session.get('sso_after_logout_next', '/')
        request.session.delete('sso_after_logout_next')
        return HttpResponseRedirect(next)

def get_request_token():
    '''
    Requests a Request Token from the SSO Server. Returns False if the request
    failed.
    '''
    # construct a signed message containing the portal key
    params = {'key': settings.SSO_KEY}

    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    url = urljoin(settings.SSO_SERVER_PRIVATE_URL,
                  'sso/api/request_token') + '/'

    # send the message to the SSO server
    response = requests.get(url, params={'key': settings.SSO_KEY,
                                         'message': message},
                            timeout=10)
    if response.status_code != 200:
        return False

    # grab the token from the response
    data = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        response.content, max_age=300)
    return data['request_token']

def verify_auth_token(untrusted_message):
    '''
    Verifies a Auth Token. Returns a
    django.contrib.auth.models.User instance if successful or False.
    '''
    # decrypt the message
    untrusted = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        untrusted_message, max_age=300)

    # do some extra validation
    if 'auth_token' not in untrusted:
        return False
    if 'request_token' not in untrusted:
        return False

    # call the SSO server to verify the token
    params = {
        'auth_token': untrusted['auth_token'],
        'key': settings.SSO_KEY
    }
    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    url = urljoin(settings.SSO_SERVER_PRIVATE_URL, 'sso/api/verify') + '/'
    response = requests.get(url, params={'key': settings.SSO_KEY,
                                         'message': message}, timeout=10)

    # ensure the response is sane
    if response.status_code != 200:
        return False

    # build a User object from the message
    data = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        response.content, max_age=300)
    user_data = simplejson.loads(data['user'])
    return construct_user(user_data)

def get_next(request):
    '''
    Given a request, returns the URL where a user should be redirected to
    after login. Defaults to '/'.
    '''
    next = request.GET.get('next', None)
    if not next:
        return '/'
    netloc = urlparse(next)[1]

    # security check -- don't allow redirection to a different host
    # taken from django.contrib.auth.views.login
    if netloc and netloc != request.get_host():
        return '/'
    return next

def get_sso_request():
    # get a request token, which is used by the SSO server to verify
    # that the user is allowed to make a login request

    sso_request = namedtuple('SSORequest', ('status_code', 'message'))

    request_token = get_request_token()
    if not request_token:
        # Status code 503 service (sso server) unavailable
        return sso_request(503, 'Unable to obtain token')

    # construct a (signed) set of GET parameters which are used to
    # redirect the user to the SSO server
    params = {'request_token': request_token,
              'key': settings.SSO_KEY}

    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    query_string = urllib.urlencode([('message', message),
                                     ('key', settings.SSO_KEY)])
    url = urljoin(settings.SSO_SERVER_PUBLIC_URL, 'sso/authorize') + '/'
    url = '%s?%s' % (url, query_string)
    return sso_request(302, url)
<commit_msg>Refactored sso logout url.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from collections import namedtuple

import requests
import urllib
from urlparse import urljoin, urlparse

from django.conf import settings
from django.contrib.auth import login as django_login
from django.contrib.auth import logout as django_logout
from django.contrib.auth.backends import ModelBackend
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseBadRequest, HttpResponseRedirect
from django.utils import simplejson
from django.http import HttpResponse
from django.views.generic.base import View
from django.utils.decorators import method_decorator

from itsdangerous import URLSafeTimedSerializer

from lizard_auth_client.client import construct_user

# used so we can login User objects we instantiated ourselves
BACKEND = ModelBackend()

class TestHomeView(View):
    '''
    Test view, only used in standalone mode.
    Display a minimal HTML page with some URLS.
    '''
    def get(self, request, *args, **kwargs):
        user = request.user
        return HttpResponse(
            'home | protected'
            '| logout | user={} | home @ client'
            ''.format(user)
        )

class TestProtectedView(View):
    '''
    Test view, only used in standalone mode.
    Display a minimal HTML page with some URLS, but is "protected" by
    a login_required decorator.
    '''
    @method_decorator(login_required)
    def dispatch(self, request, *args, **kwargs):
        return super(TestProtectedView, self).dispatch(
            request, *args, **kwargs)

    def get(self, request, *args, **kwargs):
        user = request.user
        return HttpResponse(
            'home | protected'
            '| logout '
            '| user={} | protected @ client'
            ''.format(user)
        )

class LoginView(View):
    '''
    View that redirects the user to the SSO server.

    Requests a Request Token and then redirects the User to the the SSO Server.
    '''
    def get(self, request, *args, **kwargs):
        # store the GET parameter named 'next', so we can redirect the user
        # to the requested page, after SSO login.
        next = get_next(request)
        request.session['sso_after_login_next'] = next

        response = {302: HttpResponseRedirect,
                    503: HttpResponse}

        sso_request = get_sso_request()
        http_response = response.get(sso_request.status_code)

        return http_response(sso_request.message,
                             status=sso_request.status_code)

class LocalLoginView(View):
    '''
    Verifies the user token with the SSO server, and logs the user in.
    '''
    def get(self, request, *args, **kwargs):
        # verify the authentication token and
        # retrieve the User instance from the SSO server
        user = verify_auth_token(request.GET['message'])
        if not user:
            return HttpResponseBadRequest('Verification failed')

        # link the user instance to the default database backend
        # and call django-auth's login function
        user.backend = "%s.%s" % (BACKEND.__module__,
                                  BACKEND.__class__.__name__)
        django_login(request, user)

        # redirect the user to the stored "next" url, which is probably a
        # protected page
        sso_after_login_next = request.session.get('sso_after_login_next', '/')
        request.session.delete('sso_after_login_next')
        return HttpResponseRedirect(sso_after_login_next)

class LogoutView(View):
    '''
    Redirect user to SSO server, to log out there.
    '''
    def get(self, request, *args, **kwargs):
        # store the 'next' parameter in the session so we can
        # redirect the user afterwards
        sso_after_logout_next = get_next(request)
        request.session['sso_after_logout_next'] = sso_after_logout_next

        url = get_sso_logout()
        # send the redirect response
        return HttpResponseRedirect(url)

class LocalLogoutView(View):
    '''
    Log out locally. Mostly the user is redirected here by the SSO server.
    '''
    def get(self, request, *args, **kwargs):
        # call django-auth's logout function
        django_logout(request)

        # redirect the user
        next = request.session.get('sso_after_logout_next', '/')
        request.session.delete('sso_after_logout_next')
        return HttpResponseRedirect(next)

def get_request_token():
    '''
    Requests a Request Token from the SSO Server. Returns False if the request
    failed.
    '''
    # construct a signed message containing the portal key
    params = {'key': settings.SSO_KEY}

    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    url = urljoin(settings.SSO_SERVER_PRIVATE_URL,
                  'sso/api/request_token') + '/'

    # send the message to the SSO server
    response = requests.get(url, params={'key': settings.SSO_KEY,
                                         'message': message},
                            timeout=10)
    if response.status_code != 200:
        return False

    # grab the token from the response
    data = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        response.content, max_age=300)
    return data['request_token']

def verify_auth_token(untrusted_message):
    '''
    Verifies a Auth Token. Returns a
    django.contrib.auth.models.User instance if successful or False.
    '''
    # decrypt the message
    untrusted = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        untrusted_message, max_age=300)

    # do some extra validation
    if 'auth_token' not in untrusted:
        return False
    if 'request_token' not in untrusted:
        return False

    # call the SSO server to verify the token
    params = {
        'auth_token': untrusted['auth_token'],
        'key': settings.SSO_KEY
    }
    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    url = urljoin(settings.SSO_SERVER_PRIVATE_URL, 'sso/api/verify') + '/'
    response = requests.get(url, params={'key': settings.SSO_KEY,
                                         'message': message}, timeout=10)

    # ensure the response is sane
    if response.status_code != 200:
        return False

    # build a User object from the message
    data = URLSafeTimedSerializer(settings.SSO_SECRET).loads(
        response.content, max_age=300)
    user_data = simplejson.loads(data['user'])
    return construct_user(user_data)

def get_next(request):
    '''
    Given a request, returns the URL where a user should be redirected to
    after login. Defaults to '/'.
    '''
    next = request.GET.get('next', None)
    if not next:
        return '/'
    netloc = urlparse(next)[1]

    # security check -- don't allow redirection to a different host
    # taken from django.contrib.auth.views.login
    if netloc and netloc != request.get_host():
        return '/'
    return next

def get_sso_request():
    # get a request token, which is used by the SSO server to verify
    # that the user is allowed to make a login request

    sso_request = namedtuple('SSORequest', ('status_code', 'message'))

    request_token = get_request_token()
    if not request_token:
        # Status code 503 service (sso server) unavailable
        return sso_request(503, 'Unable to obtain token')

    # construct a (signed) set of GET parameters which are used to
    # redirect the user to the SSO server
    params = {'request_token': request_token,
              'key': settings.SSO_KEY}

    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    query_string = urllib.urlencode([('message', message),
                                     ('key', settings.SSO_KEY)])
    url = urljoin(settings.SSO_SERVER_PUBLIC_URL, 'sso/authorize') + '/'
    url = '%s?%s' % (url, query_string)
    return sso_request(302, url)

def get_sso_logout():
    """
    Construct a url with a signed message containing the portal key.
    """

    params = {
        'action': 'logout',
        'key': settings.SSO_KEY
    }
    message = URLSafeTimedSerializer(settings.SSO_SECRET).dumps(params)
    query_string = urllib.urlencode([('message', message),
                                     ('key', settings.SSO_KEY)])
    url = urljoin(settings.SSO_SERVER_PUBLIC_URL,
                  'sso/portal_action') + '/'
    url = '%s?%s' % (url, query_string)
    return url
<|endoftext|>


<commit_before>import time
from PythonQt import QtCore

class TimerCallback(object):

    def __init__(self, targetFps=30):
        '''
        Construct TimerCallback.  The targetFps defines frames per second, the
        frequency for the ticks() callback method.
        '''
        self.targetFps = targetFps
        self.timer = QtCore.QTimer()
        self.timer.setSingleShot(True)
        self.timer.connect('timeout()', self._timerEvent)

    def start(self):
        '''
        Start the timer.
        '''
        self.lastTickTime = time.time()
        self.timer.start(0)

    def stop(self):
        '''
        Stop the timer.
        '''
        self.timer.stop()

    def tick(self):
        '''
        Timer event callback method.  Subclasses can override this method.
        '''
        pass

    def _schedule(self, elapsedTimeInSeconds):
        '''
        This method is given an elapsed time since the start of the last
        call to ticks().  It schedules a timer event to acheive the targetFps.
        '''
        fpsDelayMilliseconds = int(1000.0 / self.targetFps)
        elapsedMilliseconds = int(elapsedTimeInSeconds*1000.0)
        waitMilliseconds = fpsDelayMilliseconds - elapsedMilliseconds
        self.timer.start(waitMilliseconds if waitMilliseconds > 0 else 1)

    def _timerEvent(self):
        '''
        Internal timer callback method.  Calls ticks() and measures elapsed time.
        '''
        startTime = time.time()
        self.elapsed = startTime - self.lastTickTime
        self.tick()
        self.lastTickTime = startTime
        self._schedule(time.time() - startTime)

<commit_msg>add callback support to timercallback.py<commit_after>import time
from PythonQt import QtCore
import traceback

class TimerCallback(object):

    def __init__(self, targetFps=30):
        '''
        Construct TimerCallback.  The targetFps defines frames per second, the
        frequency for the ticks() callback method.
        '''
        self.targetFps = targetFps
        self.timer = QtCore.QTimer()
        self.timer.setSingleShot(True)
        self.timer.connect('timeout()', self._timerEvent)
        self.callback = None

    def start(self):
        '''
        Start the timer.
        '''
        self.lastTickTime = time.time()
        self.timer.start(0)

    def stop(self):
        '''
        Stop the timer.
        '''
        self.timer.stop()

    def tick(self):
        '''
        Timer event callback method.  Subclasses can override this method.
        '''
        if self.callback:
            try:
                self.callback()
            except:
                print traceback.format_exc()

    def _schedule(self, elapsedTimeInSeconds):
        '''
        This method is given an elapsed time since the start of the last
        call to ticks().  It schedules a timer event to acheive the targetFps.
        '''
        fpsDelayMilliseconds = int(1000.0 / self.targetFps)
        elapsedMilliseconds = int(elapsedTimeInSeconds*1000.0)
        waitMilliseconds = fpsDelayMilliseconds - elapsedMilliseconds
        self.timer.start(waitMilliseconds if waitMilliseconds > 0 else 1)

    def _timerEvent(self):
        '''
        Internal timer callback method.  Calls ticks() and measures elapsed time.
        '''
        startTime = time.time()
        self.elapsed = startTime - self.lastTickTime
        self.tick()
        self.lastTickTime = startTime
        self._schedule(time.time() - startTime)

<|endoftext|>


<commit_before>from director import mainwindowapp
from director import robotsystem
from director import applogic
from PythonQt import QtCore

def makeRobotSystem(view):
    factory = robotsystem.RobotSystemFactory()
    options = factory.getDisabledOptions()
    factory.setDependentOptions(options, usePlannerPublisher=True, useTeleop=True)
    return factory.construct(view=view, options=options)

app = mainwindowapp.MainWindowAppFactory().construct()
robotSystem = makeRobotSystem(app.view)
robotSystem.ikPlanner.planningMode = 'pydrake'

app.app.addWidgetToDock(robotSystem.teleopPanel.widget, QtCore.Qt.RightDockWidgetArea)
app.app.addWidgetToDock(robotSystem.playbackPanel.widget, QtCore.Qt.BottomDockWidgetArea)
applogic.resetCamera(viewDirection=[-1,0,0], view=app.view)

app.app.start()
<commit_msg>call setupLocalServer() on the pydrake planner<commit_after>from director import mainwindowapp
from director import robotsystem
from director import applogic
from PythonQt import QtCore

def makeRobotSystem(view):
    factory = robotsystem.RobotSystemFactory()
    options = factory.getDisabledOptions()
    factory.setDependentOptions(options, usePlannerPublisher=True, useTeleop=True)
    return factory.construct(view=view, options=options)

app = mainwindowapp.MainWindowAppFactory().construct()
robotSystem = makeRobotSystem(app.view)

app.app.addWidgetToDock(robotSystem.teleopPanel.widget, QtCore.Qt.RightDockWidgetArea)
app.app.addWidgetToDock(robotSystem.playbackPanel.widget, QtCore.Qt.BottomDockWidgetArea)

# use pydrake ik backend
ikPlanner = robotSystem.ikPlanner
ikPlanner.planningMode = 'pydrake'
ikPlanner.plannerPub._setupLocalServer()

applogic.resetCamera(viewDirection=[-1,0,0], view=app.view)
app.app.start()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import unicode_literals

from django.db import transaction
from memoized import memoized

from dimagi.utils.logging import notify_exception
from corehq.util.test_utils import unit_testing_only

class IndicatorAdapter(object):
    def __init__(self, config):
        self.config = config

    @memoized
    def get_table(self):
        raise NotImplementedError

    def rebuild_table(self, initiated_by=None, source=None):
        raise NotImplementedError

    def drop_table(self, initiated_by=None, source=None):
        raise NotImplementedError

    @unit_testing_only
    def clear_table(self):
        raise NotImplementedError

    def get_query_object(self):
        raise NotImplementedError

    def best_effort_save(self, doc, eval_context=None):
        """
        Does a best-effort save of the document. Will fail silently if the save is not successful.

        For certain known, expected errors this will do no additional logging.
        For unexpected errors it will log them.
        """
        try:
            indicator_rows = self.get_all_values(doc, eval_context)
        except Exception as e:
            self.handle_exception(doc, e)
        else:
            self._best_effort_save_rows(indicator_rows, doc)

    def _best_effort_save_rows(self, rows, doc):
        """
        Like save rows, but should catch errors and log them
        """
        raise NotImplementedError

    def handle_exception(self, doc, exception):
        from corehq.util.cache_utils import is_rate_limited
        ex_clss = exception.__class__
        key = '{domain}.{table}.{ex_mod}.{ex_name}'.format(
            domain=self.config.domain,
            table=self.config.table_id,
            ex_mod=ex_clss.__module__,
            ex_name=ex_clss.__name__
        )
        if not is_rate_limited(key):
            notify_exception(
                None,
                'unexpected error saving UCR doc',
                details={
                    'domain': self.config.domain,
                    'doc_id': doc.get('_id', '<unknown>'),
                    'table': '{} ({})'.format(self.config.display_name, self.config._id)
                }
            )

    def save(self, doc, eval_context=None):
        """
        Saves the document. Should bubble up known errors.
        """
        indicator_rows = self.get_all_values(doc, eval_context)
        self.save_rows(indicator_rows)

    def save_rows(self, rows):
        raise NotImplementedError

    def bulk_save(self, docs):
        """
        Evalutes UCR rows for given docs and saves the result in bulk.
        """
        raise NotImplementedError

    def get_all_values(self, doc, eval_context=None):
        "Gets all the values from a document to save"
        return self.config.get_all_values(doc, eval_context)

    def bulk_delete(self, doc_ids):
        for _id in doc_ids:
            self.delete({'_id': _id})

    def delete(self, doc):
        raise NotImplementedError

    @property
    def run_asynchronous(self):
        return self.config.asynchronous

    def get_distinct_values(self, column, limit):
        raise NotImplementedError

    def log_table_build(self, initiated_by, source):
        from corehq.apps.userreports.models import DataSourceActionLog
        self._log_action(initiated_by, source, DataSourceActionLog.BUILD)

    def log_table_rebuild(self, initiated_by, source, skip=False):
        from corehq.apps.userreports.models import DataSourceActionLog
        self._log_action(initiated_by, source, DataSourceActionLog.REBUILD, skip=skip)

    def log_table_drop(self, initiated_by, source, skip=False):
        from corehq.apps.userreports.models import DataSourceActionLog
        self._log_action(initiated_by, source, DataSourceActionLog.DROP, skip=skip)

    def log_table_migrate(self, source, diffs):
        from corehq.apps.userreports.models import DataSourceActionLog
        self._log_action(None, source, DataSourceActionLog.MIGRATE, diffs=diffs)

    def _log_action(self, initiated_by, source, action, diffs=None, skip=False):
        from corehq.apps.userreports.models import DataSourceActionLog
        if skip:
            return

        kwargs = {
            'domain': self.config.domain,
            'indicator_config_id': self.config.get_id,
            'action': action,
            'initiated_by': initiated_by,
            'action_source': source,
            'migration_diffs': diffs
        }

        try:
            # make this atomic so that errors to affect outer transactions
            with transaction.atomic():
                DataSourceActionLog.objects.create(**kwargs)
        except:  # noqa
            # blanket catchall to make sure errors here don't interfere with real workflows
            notify_exception(None, "Error saving UCR action log", details=kwargs)

class IndicatorAdapterLoadTracker(object):
    def __init__(self, adapter, track_load):
        self.adapter = adapter
        self._track_load = track_load

    def __getattr__(self, attr):
        return getattr(self.adapter, attr)

    def track_load(self, value=1):
        self._track_load(value)

    def save_rows(self, rows):
        self._track_load(len(rows))
        self.adapter.save_rows(rows)

    def delete(self, doc):
        self._track_load()
        self.adapter.delete(doc)

    def get_distinct_values(self, column, limit):
        distinct_values, too_many_values = self.adapter.get_distinct_values(column, limit)
        self._track_load(len(distinct_values))
        return distinct_values, too_many_values
<commit_msg>skip if config has no ID<commit_after>from __future__ import absolute_import
from __future__ import unicode_literals

from django.db import transaction
from memoized import memoized

from dimagi.utils.logging import notify_exception
from corehq.util.test_utils import unit_testing_only

class IndicatorAdapter(object):
    def __init__(self, config):
        self.config = config

    @memoized
    def get_table(self):
        raise NotImplementedError

    def rebuild_table(self, initiated_by=None, source=None):
        raise NotImplementedError

    def drop_table(self, initiated_by=None, source=None):
        raise NotImplementedError

    @unit_testing_only
    def clear_table(self):
        raise NotImplementedError

    def get_query_object(self):
        raise NotImplementedError

    def best_effort_save(self, doc, eval_context=None):
        """
        Does a best-effort save of the document. Will fail silently if the save is not successful.

        For certain known, expected errors this will do no additional logging.
        For unexpected errors it will log them.
        """
        try:
            indicator_rows = self.get_all_values(doc, eval_context)
        except Exception as e:
            self.handle_exception(doc, e)
        else:
            self._best_effort_save_rows(indicator_rows, doc)

    def _best_effort_save_rows(self, rows, doc):
        """
        Like save rows, but should catch errors and log them
        """
        raise NotImplementedError

    def handle_exception(self, doc, exception):
        from corehq.util.cache_utils import is_rate_limited
        ex_clss = exception.__class__
        key = '{domain}.{table}.{ex_mod}.{ex_name}'.format(
            domain=self.config.domain,
            table=self.config.table_id,
            ex_mod=ex_clss.__module__,
            ex_name=ex_clss.__name__
        )
        if not is_rate_limited(key):
            notify_exception(
                None,
                'unexpected error saving UCR doc',
                details={
                    'domain': self.config.domain,
                    'doc_id': doc.get('_id', '<unknown>'),
                    'table': '{} ({})'.format(self.config.display_name, self.config._id)
                }
            )

    def save(self, doc, eval_context=None):
        """
        Saves the document. Should bubble up known errors.
        """
        indicator_rows = self.get_all_values(doc, eval_context)
        self.save_rows(indicator_rows)

    def save_rows(self, rows):
        raise NotImplementedError

    def bulk_save(self, docs):
        """
        Evalutes UCR rows for given docs and saves the result in bulk.
        """
        raise NotImplementedError

    def get_all_values(self, doc, eval_context=None):
        "Gets all the values from a document to save"
        return self.config.get_all_values(doc, eval_context)

    def bulk_delete(self, doc_ids):
        for _id in doc_ids:
            self.delete({'_id': _id})

    def delete(self, doc):
        raise NotImplementedError

    @property
    def run_asynchronous(self):
        return self.config.asynchronous

    def get_distinct_values(self, column, limit):
        raise NotImplementedError

    def log_table_build(self, initiated_by, source):
        from corehq.apps.userreports.models import DataSourceActionLog
        self._log_action(initiated_by, source, DataSourceActionLog.BUILD)

    def log_table_rebuild(self, initiated_by, source, skip=False):
        from corehq.apps.userreports.models import DataSourceActionLog
        self._log_action(initiated_by, source, DataSourceActionLog.REBUILD, skip=skip)

    def log_table_drop(self, initiated_by, source, skip=False):
        from corehq.apps.userreports.models import DataSourceActionLog
        self._log_action(initiated_by, source, DataSourceActionLog.DROP, skip=skip)

    def log_table_migrate(self, source, diffs):
        from corehq.apps.userreports.models import DataSourceActionLog
        self._log_action(None, source, DataSourceActionLog.MIGRATE, diffs=diffs)

    def _log_action(self, initiated_by, source, action, diffs=None, skip=False):
        from corehq.apps.userreports.models import DataSourceActionLog
        if skip or not self.config.data_source_id:
            return

        kwargs = {
            'domain': self.config.domain,
            'indicator_config_id': self.config.get_id,
            'action': action,
            'initiated_by': initiated_by,
            'action_source': source,
            'migration_diffs': diffs
        }

        try:
            # make this atomic so that errors to affect outer transactions
            with transaction.atomic():
                DataSourceActionLog.objects.create(**kwargs)
        except:  # noqa
            # blanket catchall to make sure errors here don't interfere with real workflows
            notify_exception(None, "Error saving UCR action log", details=kwargs)

class IndicatorAdapterLoadTracker(object):
    def __init__(self, adapter, track_load):
        self.adapter = adapter
        self._track_load = track_load

    def __getattr__(self, attr):
        return getattr(self.adapter, attr)

    def track_load(self, value=1):
        self._track_load(value)

    def save_rows(self, rows):
        self._track_load(len(rows))
        self.adapter.save_rows(rows)

    def delete(self, doc):
        self._track_load()
        self.adapter.delete(doc)

    def get_distinct_values(self, column, limit):
        distinct_values, too_many_values = self.adapter.get_distinct_values(column, limit)
        self._track_load(len(distinct_values))
        return distinct_values, too_many_values
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''Generate a Python ctypes wrapper file for a header file.

Usage example::
    wrap.py -lGL -oGL.py /usr/include/GL/gl.h

    >>> from GL import *

'''

__docformat__ = 'restructuredtext'
__version__ = '$Id: wrap.py 738 2007-03-12 04:53:42Z Alex.Holkner $'

from ctypesparser import *
import textwrap
import sys, re
from ctypes import CDLL, RTLD_GLOBAL, c_byte
from ctypes.util import find_library

def load_library(name, mode=RTLD_GLOBAL):
    if os.name == "nt":
        return CDLL(name, mode=mode)
    path = find_library(name)
    if path is None:
        # Maybe 'name' is not a library name in the linker style,
        # give CDLL a last chance to find the library.
        path = name
    return CDLL(path, mode=mode)

class CtypesWrapper(CtypesParser, CtypesTypeVisitor):
    file=None
    def begin_output(self, output_file, library, link_modules=(),
                     emit_filenames=(), all_headers=False,
                     include_symbols=None, exclude_symbols=None,
                     save_preprocessed_headers=None):
        self.library = library
        self.file = output_file
        self.all_names = []
        self.known_types = {}
        self.structs = set()
        self.opaque_structs = set()
        self.enums = set()
        self.emit_filenames = emit_filenames
        self.all_headers = all_headers
        self.include_symbols = None
        self.exclude_symbols = None
        if include_symbols:
            self.include_symbols = re.compile(include_symbols)
        if exclude_symbols:
            self.exclude_symbols = re.compile(exclude_symbols)
        self.preprocessor_parser.save_preprocessed_headers = \
            save_preprocessed_headers

        self.linked_symbols = {}
        for name in link_modules:
            module = __import__(name, globals(), locals(), ['foo'])
            for symbol in dir(module):
                if symbol not in self.linked_symbols:
                    self.linked_symbols[symbol] = '%s.%s' % (name, symbol)
        self.link_modules = link_modules

        self.print_preamble()
        self.print_link_modules_imports()

    def wrap(self, filename, source=None):
        assert self.file, 'Call begin_output first'
        self.parse(filename, source)

    def end_output(self):
        self.print_epilogue()
        self.file = None

    def does_emit(self, symbol, filename, restype=None, argtypes=[]):

        # Skip any functions which mention excluded types
        if self.exclude_symbols:
            for a in [symbol, restype] + argtypes:
                if a and self.exclude_symbols.match(str(a)):
                    return

        if self.include_symbols and not re.match(self.include_symbols, symbol):
            return False

        return self.all_headers or filename in self.emit_filenames

    def print_preamble(self):
        import textwrap
        import time
        print >> self.file, textwrap.dedent("""
            '''Wrapper for %(library)s

            Generated with:
            %(argv)s

            Do not modify this file.
            '''

            __docformat__ =  'restructuredtext'
            __version__ = '$Id: wrap.py 738 2007-03-12 04:53:42Z Alex.Holkner $'

            import ctypes
            from ctypes import *
            from UserString import UserString, MutableString

            _int_types = (c_int16, c_int32)
            if hasattr(ctypes, 'c_int64'):
                # Some builds of ctypes apparently do not have c_int64
                # defined; it's a pretty good bet that these builds do not
                # have 64-bit pointers.
                _int_types += (ctypes.c_int64,)
            for t in _int_types:
                if sizeof(t) == sizeof(c_size_t):
                    c_ptrdiff_t = t

            class c_void(Structure):
                # c_void_p is a buggy return type, converting to int, so
                # POINTER(None) == c_void_p is actually written as
                # POINTER(c_void), so it can be treated as a real pointer.
                _fields_ = [('dummy', c_int)]

            class String(MutableString, Union):

                _fields_ = [('raw', POINTER(c_char)),
                            ('data', c_char_p)]

                def __init__(self, obj=""):

                    if isinstance(obj, str) or isinstance(obj, UserString):
                        self.data = str(obj)
                    else:
                        self.raw = obj

                @classmethod
                def from_param(cls, obj):

                    # Convert None or 0
                    if obj is None or obj == 0:
                        return None

                    # Convert from String
                    elif isinstance(obj, String):
                        return obj

                    # Convert from str
                    elif isinstance(obj, str):
                        return cls(obj)

                    # Convert from raw pointer
                    elif isinstance(obj, int):
                        return cls(cast(obj, POINTER(c_char)))

                    # Convert from object
                    else:
                        return String.from_param(obj._as_parameter_)

            def ReturnString(obj):
                return String.from_param(obj)

            _libs = {}

            # As of ctypes 1.0, ctypes does not support custom error-checking
            # functions on callbacks, nor does it support custom datatypes on
            # callbacks, so we must ensure that all callbacks return
            # primitive datatypes.
            #
            # Non-primitive return values wrapped with UNCHECKED won't be
            # typechecked, and will be converted to c_void_p.
            def UNCHECKED(type):
                if (hasattr(type, "_type_") and isinstance(type._type_, str)
                    and type._type_ != "P"):
                    return type
                else:
                    return c_void_p
        """ % {
            'library': str(self.library),
            'date': time.ctime(),
            'class': self.__class__.__name__,
            'argv': ' '.join(sys.argv),
        }).lstrip()
        self.loaded_libraries = []
        for library in self.library:
            lib = load_library(library)
            if lib:
                self.loaded_libraries.append(lib)
                print >>self.file, textwrap.dedent("""
                    _libs[%r] = CDLL(%r, mode=RTLD_GLOBAL)
                """ % (lib._name, lib._name))

    def print_link_modules_imports(self):
        for name in self.link_modules:
            print >> self.file, 'from %s import *' % name
        print >> self.file

    def print_epilogue(self):

        # Print out defines
        self.preprocessor_parser.emit(self.file, self.include_symbols,
                                      self.all_names, self.all_headers,
                                      self.emit_filenames,
                                      self.linked_symbols)

    def handle_ctypes_constant(self, name, value, filename, lineno):
        if self.does_emit(name, filename):
            if name not in self.linked_symbols:
                print >> self.file, '%s = %r' % (name, value),
                print >> self.file, '\t# %s:%d' % (filename, lineno)
            self.all_names.append(name)

    def handle_ctypes_type_definition(self, name, ctype, filename, lineno):
        if self.does_emit(name, filename):
            if name not in self.linked_symbols:
                ctype.visit(self)
                self.emit_type(ctype)
                if name not in self.all_names:
                    print >> self.file, '%s = %s' % (name, str(ctype)),
                    print >> self.file, '\t# %s:%d' % (filename, lineno)
            self.all_names.append(name)
        else:
            self.known_types[name] = (ctype, filename, lineno)

    def emit_type(self, t):
        t.visit(self)
        for s in t.get_required_type_names():
            if s in self.known_types:
                if s not in self.linked_symbols:
                    s_ctype, s_filename, s_lineno = self.known_types[s]
                    s_ctype.visit(self)

                    self.emit_type(s_ctype)
                    print >> self.file, '%s = %s' % (s, str(s_ctype)),
                    print >> self.file, '\t# %s:%d' % (s_filename, s_lineno)
                del self.known_types[s]

    def visit_struct(self, struct):
        if struct.tag in self.linked_symbols:
            return

        # Output struct definition
        if struct.tag not in self.opaque_structs:
            base = {True: 'Union', False: 'Structure'}[struct.is_union]
            print >> self.file, 'class struct_%s(%s):' % (struct.tag, base)
            print >> self.file, '    pass'
            self.opaque_structs.add(struct.tag)

        # Set fields after completing class, so incomplete structs can be
        # referenced within struct.
        for name, typ in struct.members:
            self.emit_type(typ)

        if struct.tag not in self.structs and not struct.opaque:
            self.structs.add(struct.tag)
            print >> self.file, 'struct_%s.__slots__ = [' % struct.tag
            for m in struct.members:
                print >> self.file, "    '%s'," % m[0]
            print >> self.file, ']'
            print >> self.file, 'struct_%s._fields_ = [' % struct.tag
            for m in struct.members:
                print >> self.file, "    ('%s', %s)," % (m[0], m[1])
            print >> self.file, ']'
            print >> self.file

    def visit_enum(self, enum):
        if enum.tag in self.linked_symbols:
            return
        if enum.tag in self.enums:
            return
        self.enums.add(enum.tag)

        print >> self.file, 'enum_%s = c_int' % enum.tag
        for name, value in enum.enumerators:
            self.all_names.append(name)
            print >> self.file, '%s = %d' % (name, value)

    def handle_ctypes_function(self, name, restype, argtypes, filename, lineno):
        if name in self.linked_symbols:
            return

        if self.does_emit(name, filename, restype, argtypes):

            # Also emit any types this func requires that haven't yet been
            # written.
            self.emit_type(restype)
            for a in argtypes:
                self.emit_type(a)

            for lib in self.loaded_libraries:
                if hasattr(lib, name):
                    self.all_names.append(name)
                    print >> self.file, '# %s:%d' % (filename, lineno)
                    print >> self.file, '%s = _libs[%r].%s' % (name, lib._name, name)
                    print >> self.file, '%s.restype = %s' % (name, str(restype))
                    print >> self.file, '%s.argtypes = [%s]' % \
                        (name, ', '.join([str(a) for a in argtypes]))
                    print >> self.file
                    break

    def handle_ctypes_variable(self, name, ctype, filename, lineno):
        if name in self.linked_symbols:
            return

        if self.does_emit(name, filename):
            self.emit_type(ctype)
            for lib in self.loaded_libraries:
                try:
                    c_byte.in_dll(lib, name)
                except:
                    pass
                else:
                    print >> self.file, '%s = (%s).in_dll(_libs[%r], %r)' % \
                        (name, str(ctype), lib._name, name)
                    self.all_names.append(name)
                    break

def main(*argv):
    from tempfile import NamedTemporaryFile
    import optparse
    import sys
    import os.path

    usage = 'usage: %prog [options] <header.h>'
    op = optparse.OptionParser(usage=usage)
    op.add_option('-o', '--output', dest='output',
                  help='write wrapper to FILE', metavar='FILE')
    op.add_option('-l', '--library', dest='library', action='append',
                  help='link to LIBRARY', metavar='LIBRARY', default=[])
    op.add_option('-m', '--link-module', action='append', dest='link_modules',
                  help='use symbols from MODULE', metavar='MODULE',
                  default=[])
    op.add_option('-a', '--all-headers', action='store_true',
                  dest='all_headers',
                  help='include symbols from all headers', default=False)
    op.add_option('-i', '--include-symbols', dest='include_symbols',
                  help='regular expression for symbols to include')
    op.add_option('-x', '--exclude-symbols', dest='exclude_symbols',
                  help='regular expression for symbols to exclude')
    op.add_option('', '--cpp', dest='cpp',
                  help='The command to invoke the c preprocessor, including any necessary options (default: gcc -E)')
    op.add_option('', '--save-preprocessed-headers', dest='filename',
                  help='Save the preprocessed headers to the specified '
                       'FILENAME')

    (options, args) = op.parse_args(list(argv[1:]))
    if len(args) < 1:
        print >> sys.stderr, 'No header files specified.'
        sys.exit(1)
    headers = args

    if options.output is None:
        print >> sys.stderr, 'No output file specified.'
        sys.exit(1)

    if len(options.library) == 0:
        print >> sys.stderr, 'No libraries specified.'

    wrapper = CtypesWrapper()
    wrapper.begin_output(open(options.output, 'w'),
                         library=options.library,
                         emit_filenames=headers,
                         link_modules=options.link_modules,
                         all_headers=options.all_headers,
                         include_symbols=options.include_symbols,
                         exclude_symbols=options.exclude_symbols,
                         save_preprocessed_headers=options.filename
                         )
    wrapper.preprocessor_parser.cpp = options.cpp

    f = NamedTemporaryFile(suffix=".h")
    for header in headers:
        print >>f, '#include "%s"' % header
    f.flush()
    wrapper.wrap(f.name)
    wrapper.end_output()

    print 'Wrapped to %s' % options.output

if __name__ == '__main__':
    main(*sys.argv)
<commit_msg>* wrap.py: Work around buggy implementations of find_library by providing a   custom implementation on non-Windows platforms.<commit_after>#!/usr/bin/env python

'''Generate a Python ctypes wrapper file for a header file.

Usage example::
    wrap.py -lGL -oGL.py /usr/include/GL/gl.h

    >>> from GL import *

'''

__docformat__ = 'restructuredtext'
__version__ = '$Id: wrap.py 738 2007-03-12 04:53:42Z Alex.Holkner $'

from ctypesparser import *
import textwrap
import sys, re, os, tempfile
from ctypes import CDLL, RTLD_GLOBAL, c_byte

if os.name == "nt":
    from ctypes.util import find_library
else:
    # ctypes.util.find_library is buggy in Python 2.5, unfortunately,
    # so we have to parse the output of gcc -Wl,-t manually
    def find_library(name):
        expr = r'[^\(\)\s]*lib%s\.[^\(\)\s:]*' % re.escape(name)
        fdout, ccout = tempfile.mkstemp()
        os.close(fdout)
        cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; else CC=cc; fi;' \
              '$CC -Wl,-t -o %s 2>&1 -l %s' % (ccout, name)
        try:
            f = os.popen(cmd)
            trace = f.read()
            f.close()
        finally:
            if os.path.exists(ccout):
                os.unlink(ccout)
        res = re.search(expr, trace)
        if not res:
            return None
        return res.group(0)

def load_library(name, mode=RTLD_GLOBAL):
    if os.name == "nt":
        return CDLL(name, mode=mode)
    path = find_library(name)
    if path is None:
        # Maybe 'name' is not a library name in the linker style,
        # give CDLL a last chance to find the library.
        path = name
    return CDLL(path, mode=mode)

class CtypesWrapper(CtypesParser, CtypesTypeVisitor):
    file=None
    def begin_output(self, output_file, library, link_modules=(),
                     emit_filenames=(), all_headers=False,
                     include_symbols=None, exclude_symbols=None,
                     save_preprocessed_headers=None):
        self.library = library
        self.file = output_file
        self.all_names = []
        self.known_types = {}
        self.structs = set()
        self.opaque_structs = set()
        self.enums = set()
        self.emit_filenames = emit_filenames
        self.all_headers = all_headers
        self.include_symbols = None
        self.exclude_symbols = None
        if include_symbols:
            self.include_symbols = re.compile(include_symbols)
        if exclude_symbols:
            self.exclude_symbols = re.compile(exclude_symbols)
        self.preprocessor_parser.save_preprocessed_headers = \
            save_preprocessed_headers

        self.linked_symbols = {}
        for name in link_modules:
            module = __import__(name, globals(), locals(), ['foo'])
            for symbol in dir(module):
                if symbol not in self.linked_symbols:
                    self.linked_symbols[symbol] = '%s.%s' % (name, symbol)
        self.link_modules = link_modules

        self.print_preamble()
        self.print_link_modules_imports()

    def wrap(self, filename, source=None):
        assert self.file, 'Call begin_output first'
        self.parse(filename, source)

    def end_output(self):
        self.print_epilogue()
        self.file = None

    def does_emit(self, symbol, filename, restype=None, argtypes=[]):

        # Skip any functions which mention excluded types
        if self.exclude_symbols:
            for a in [symbol, restype] + argtypes:
                if a and self.exclude_symbols.match(str(a)):
                    return

        if self.include_symbols and not re.match(self.include_symbols, symbol):
            return False

        return self.all_headers or filename in self.emit_filenames

    def print_preamble(self):
        import textwrap
        import time
        print >> self.file, textwrap.dedent("""
            '''Wrapper for %(library)s

            Generated with:
            %(argv)s

            Do not modify this file.
            '''

            __docformat__ =  'restructuredtext'
            __version__ = '$Id: wrap.py 738 2007-03-12 04:53:42Z Alex.Holkner $'

            import ctypes
            from ctypes import *
            from UserString import UserString, MutableString

            _int_types = (c_int16, c_int32)
            if hasattr(ctypes, 'c_int64'):
                # Some builds of ctypes apparently do not have c_int64
                # defined; it's a pretty good bet that these builds do not
                # have 64-bit pointers.
                _int_types += (ctypes.c_int64,)
            for t in _int_types:
                if sizeof(t) == sizeof(c_size_t):
                    c_ptrdiff_t = t

            class c_void(Structure):
                # c_void_p is a buggy return type, converting to int, so
                # POINTER(None) == c_void_p is actually written as
                # POINTER(c_void), so it can be treated as a real pointer.
                _fields_ = [('dummy', c_int)]

            class String(MutableString, Union):

                _fields_ = [('raw', POINTER(c_char)),
                            ('data', c_char_p)]

                def __init__(self, obj=""):

                    if isinstance(obj, str) or isinstance(obj, UserString):
                        self.data = str(obj)
                    else:
                        self.raw = obj

                @classmethod
                def from_param(cls, obj):

                    # Convert None or 0
                    if obj is None or obj == 0:
                        return None

                    # Convert from String
                    elif isinstance(obj, String):
                        return obj

                    # Convert from str
                    elif isinstance(obj, str):
                        return cls(obj)

                    # Convert from raw pointer
                    elif isinstance(obj, int):
                        return cls(cast(obj, POINTER(c_char)))

                    # Convert from object
                    else:
                        return String.from_param(obj._as_parameter_)

            def ReturnString(obj):
                return String.from_param(obj)

            _libs = {}

            # As of ctypes 1.0, ctypes does not support custom error-checking
            # functions on callbacks, nor does it support custom datatypes on
            # callbacks, so we must ensure that all callbacks return
            # primitive datatypes.
            #
            # Non-primitive return values wrapped with UNCHECKED won't be
            # typechecked, and will be converted to c_void_p.
            def UNCHECKED(type):
                if (hasattr(type, "_type_") and isinstance(type._type_, str)
                    and type._type_ != "P"):
                    return type
                else:
                    return c_void_p
        """ % {
            'library': str(self.library),
            'date': time.ctime(),
            'class': self.__class__.__name__,
            'argv': ' '.join(sys.argv),
        }).lstrip()
        self.loaded_libraries = []
        for library in self.library:
            lib = load_library(library)
            if lib:
                self.loaded_libraries.append(lib)
                print >>self.file, textwrap.dedent("""
                    _libs[%r] = CDLL(%r, mode=RTLD_GLOBAL)
                """ % (lib._name, lib._name))

    def print_link_modules_imports(self):
        for name in self.link_modules:
            print >> self.file, 'from %s import *' % name
        print >> self.file

    def print_epilogue(self):

        # Print out defines
        self.preprocessor_parser.emit(self.file, self.include_symbols,
                                      self.all_names, self.all_headers,
                                      self.emit_filenames,
                                      self.linked_symbols)

    def handle_ctypes_constant(self, name, value, filename, lineno):
        if self.does_emit(name, filename):
            if name not in self.linked_symbols:
                print >> self.file, '%s = %r' % (name, value),
                print >> self.file, '\t# %s:%d' % (filename, lineno)
            self.all_names.append(name)

    def handle_ctypes_type_definition(self, name, ctype, filename, lineno):
        if self.does_emit(name, filename):
            if name not in self.linked_symbols:
                ctype.visit(self)
                self.emit_type(ctype)
                if name not in self.all_names:
                    print >> self.file, '%s = %s' % (name, str(ctype)),
                    print >> self.file, '\t# %s:%d' % (filename, lineno)
            self.all_names.append(name)
        else:
            self.known_types[name] = (ctype, filename, lineno)

    def emit_type(self, t):
        t.visit(self)
        for s in t.get_required_type_names():
            if s in self.known_types:
                if s not in self.linked_symbols:
                    s_ctype, s_filename, s_lineno = self.known_types[s]
                    s_ctype.visit(self)

                    self.emit_type(s_ctype)
                    print >> self.file, '%s = %s' % (s, str(s_ctype)),
                    print >> self.file, '\t# %s:%d' % (s_filename, s_lineno)
                del self.known_types[s]

    def visit_struct(self, struct):
        if struct.tag in self.linked_symbols:
            return

        # Output struct definition
        if struct.tag not in self.opaque_structs:
            base = {True: 'Union', False: 'Structure'}[struct.is_union]
            print >> self.file, 'class struct_%s(%s):' % (struct.tag, base)
            print >> self.file, '    pass'
            self.opaque_structs.add(struct.tag)

        # Set fields after completing class, so incomplete structs can be
        # referenced within struct.
        for name, typ in struct.members:
            self.emit_type(typ)

        if struct.tag not in self.structs and not struct.opaque:
            self.structs.add(struct.tag)
            print >> self.file, 'struct_%s.__slots__ = [' % struct.tag
            for m in struct.members:
                print >> self.file, "    '%s'," % m[0]
            print >> self.file, ']'
            print >> self.file, 'struct_%s._fields_ = [' % struct.tag
            for m in struct.members:
                print >> self.file, "    ('%s', %s)," % (m[0], m[1])
            print >> self.file, ']'
            print >> self.file

    def visit_enum(self, enum):
        if enum.tag in self.linked_symbols:
            return
        if enum.tag in self.enums:
            return
        self.enums.add(enum.tag)

        print >> self.file, 'enum_%s = c_int' % enum.tag
        for name, value in enum.enumerators:
            self.all_names.append(name)
            print >> self.file, '%s = %d' % (name, value)

    def handle_ctypes_function(self, name, restype, argtypes, filename, lineno):
        if name in self.linked_symbols:
            return

        if self.does_emit(name, filename, restype, argtypes):

            # Also emit any types this func requires that haven't yet been
            # written.
            self.emit_type(restype)
            for a in argtypes:
                self.emit_type(a)

            for lib in self.loaded_libraries:
                if hasattr(lib, name):
                    self.all_names.append(name)
                    print >> self.file, '# %s:%d' % (filename, lineno)
                    print >> self.file, '%s = _libs[%r].%s' % (name, lib._name, name)
                    print >> self.file, '%s.restype = %s' % (name, str(restype))
                    print >> self.file, '%s.argtypes = [%s]' % \
                        (name, ', '.join([str(a) for a in argtypes]))
                    print >> self.file
                    break

    def handle_ctypes_variable(self, name, ctype, filename, lineno):
        if name in self.linked_symbols:
            return

        if self.does_emit(name, filename):
            self.emit_type(ctype)
            for lib in self.loaded_libraries:
                try:
                    c_byte.in_dll(lib, name)
                except:
                    pass
                else:
                    print >> self.file, '%s = (%s).in_dll(_libs[%r], %r)' % \
                        (name, str(ctype), lib._name, name)
                    self.all_names.append(name)
                    break

def main(*argv):
    from tempfile import NamedTemporaryFile
    import optparse
    import sys
    import os.path

    usage = 'usage: %prog [options] <header.h>'
    op = optparse.OptionParser(usage=usage)
    op.add_option('-o', '--output', dest='output',
                  help='write wrapper to FILE', metavar='FILE')
    op.add_option('-l', '--library', dest='library', action='append',
                  help='link to LIBRARY', metavar='LIBRARY', default=[])
    op.add_option('-m', '--link-module', action='append', dest='link_modules',
                  help='use symbols from MODULE', metavar='MODULE',
                  default=[])
    op.add_option('-a', '--all-headers', action='store_true',
                  dest='all_headers',
                  help='include symbols from all headers', default=False)
    op.add_option('-i', '--include-symbols', dest='include_symbols',
                  help='regular expression for symbols to include')
    op.add_option('-x', '--exclude-symbols', dest='exclude_symbols',
                  help='regular expression for symbols to exclude')
    op.add_option('', '--cpp', dest='cpp',
                  help='The command to invoke the c preprocessor, including any necessary options (default: gcc -E)')
    op.add_option('', '--save-preprocessed-headers', dest='filename',
                  help='Save the preprocessed headers to the specified '
                       'FILENAME')

    (options, args) = op.parse_args(list(argv[1:]))
    if len(args) < 1:
        print >> sys.stderr, 'No header files specified.'
        sys.exit(1)
    headers = args

    if options.output is None:
        print >> sys.stderr, 'No output file specified.'
        sys.exit(1)

    if len(options.library) == 0:
        print >> sys.stderr, 'No libraries specified.'

    wrapper = CtypesWrapper()
    wrapper.begin_output(open(options.output, 'w'),
                         library=options.library,
                         emit_filenames=headers,
                         link_modules=options.link_modules,
                         all_headers=options.all_headers,
                         include_symbols=options.include_symbols,
                         exclude_symbols=options.exclude_symbols,
                         save_preprocessed_headers=options.filename
                         )
    wrapper.preprocessor_parser.cpp = options.cpp

    f = NamedTemporaryFile(suffix=".h")
    for header in headers:
        print >>f, '#include "%s"' % header
    f.flush()
    wrapper.wrap(f.name)
    wrapper.end_output()

    print 'Wrapped to %s' % options.output

if __name__ == '__main__':
    main(*sys.argv)
<|endoftext|>


<commit_before># -#- coding: utf-8 -#-

from django.db import models
from django.utils.translation import ugettext_lazy as _
from filer.fields.image import FilerImageField

from leonardo.module.web.models import Widget

class SiteHeadingWidget(Widget):
    site_title = models.CharField(max_length=255, verbose_name=_("Site Title"))
    logo = FilerImageField(
        blank=True, null=True, verbose_name=_("Logo"), related_name="site_logos")

    tagline = models.TextField(blank=True, verbose_name=_("Tagline"))

    class Meta:
        abstract = True
        verbose_name = _("site heading")
        verbose_name_plural = _('site headings')
<commit_msg>change file field to plain FK #42<commit_after># -#- coding: utf-8 -#-

from django.db import models
from django.utils.translation import ugettext_lazy as _
from filer.fields.image import FilerImageField

from leonardo.module.web.models import Widget

class SiteHeadingWidget(Widget):
    site_title = models.CharField(max_length=255, verbose_name=_("Site Title"))
    logo = models.ForeignKey('media.Image',
        blank=True, null=True, verbose_name=_("Logo"), related_name="%(app_label)s_%(class)s_images")

    tagline = models.TextField(blank=True, verbose_name=_("Tagline"))

    class Meta:
        abstract = True
        verbose_name = _("site heading")
        verbose_name_plural = _('site headings')
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import ast
import hashlib
import os
import subprocess
import sys
import uuid

from api_inference_community.batch import batch
from huggingface_hub import HfApi

class cd:
    """Context manager for changing the current working directory"""

    def __init__(self, newPath):
        self.newPath = os.path.expanduser(newPath)

    def __enter__(self):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, etype, value, traceback):
        os.chdir(self.savedPath)

class DockerPopen(subprocess.Popen):
    def __exit__(self, exc_type, exc_val, traceback):
        self.terminate()
        self.wait(5)
        return super().__exit__(exc_type, exc_val, traceback)

def create_docker(name: str) -> str:
    rand = str(uuid.uuid4())[:5]
    tag = f"{name}:{rand}"
    with cd(
        os.path.join(os.path.dirname(os.path.dirname(__file__)), "docker_images", name)
    ):
        subprocess.run(["docker", "build", ".", "-t", tag])
    return tag

def resolve_dataset(args, task: str):
    import datasets

    builder = datasets.load_dataset_builder(
        args.dataset_name, use_auth_token=args.token
    )

    if args.dataset_config is None:
        args.dataset_config = builder.config_id
        print(f"Inferred dataset_config {args.dataset_config}")

    splits = builder.info.splits
    if splits is not None:
        if args.dataset_split not in splits:
            raise ValueError(
                f"The split `{args.dataset_split}` is not a valid split, please choose from {','.join(splits.keys())}"
            )

    task_templates = builder.info.task_templates
    if task_templates is not None:
        for task_template in task_templates:
            if task_template.task == task:
                args.dataset_column = task_template.audio_file_path_column
                print(f"Inferred dataset_column {args.dataset_column}")
    return (
        args.dataset_name,
        args.dataset_config,
        args.dataset_split,
        args.dataset_column,
    )

def get_repo_name(model_id: str, dataset_name: str) -> str:
    # Hash needs to have the fully qualified name to disambiguate.
    hash_ = hashlib.md5((model_id + dataset_name).encode("utf-8")).hexdigest()

    model_name = model_id.split("/")[-1]
    dataset_name = dataset_name.split("/")[-1]
    return f"bulk-{model_name[:10]}-{dataset_name[:10]}-{hash_[:5]}"

def do_batch(args):
    model_id, task, framework = resolve_task_framework(args)
    dataset_name, dataset_config, dataset_split, dataset_column = resolve_dataset(
        args, task
    )
    repo_name = get_repo_name(model_id, dataset_name)
    api = HfApi()
    full_repo_id = api.create_repo(
        repo_name, args.token, private=True, repo_type="dataset", exist_ok=True
    )
    repo_id = "/".join(full_repo_id.split("/")[-2:])
    print(f"Created dataset for results {repo_id}")

    local_path = os.path.join(
        os.path.dirname(os.path.dirname(__file__)), "docker_images", framework
    )
    sys.path.append(local_path)
    os.environ["MODEL_ID"] = model_id
    os.environ["TASK"] = task

    from app.main import get_pipeline

    pipeline = get_pipeline()

    batch(
        dataset_name=dataset_name,
        dataset_config=dataset_config,
        dataset_split=dataset_split,
        dataset_column=dataset_column,
        token=args.token,
        repo_id=repo_id,
        use_gpu=False,
        pipeline=pipeline,
        task=task,
    )

def show(args):
    directory = os.path.join(
        os.path.dirname(os.path.dirname(__file__)), "docker_images"
    )
    for framework in sorted(os.listdir(directory)):
        print(f"{framework}")
        local_path = os.path.join(
            os.path.dirname(os.path.dirname(__file__)),
            "docker_images",
            framework,
            "app",
            "main.py",
        )
        # Using ast to prevent import issues with missing dependencies.
        # and slow loads.
        with open(local_path, "r") as source:
            tree = ast.parse(source.read())
            for item in tree.body:
                if (
                    isinstance(item, ast.AnnAssign)
                    and item.target.id == "ALLOWED_TASKS"
                ):
                    for key in item.value.keys:
                        print(" " * 4, key.value)

def resolve(model_id: str) -> [str, str]:
    info = HfApi().model_info(model_id)
    task = info.pipeline_tag
    framework = info.library_name
    return task, framework.replace("-", "_")

def resolve_task_framework(args):
    model_id = args.model_id
    task = args.task
    framework = args.framework
    if task is None or framework is None:
        rtask, rframework = resolve(model_id)
        if task is None:
            task = rtask
            print(f"Inferred task : {task}")
        if framework is None:
            framework = rframework
            print(f"Inferred framework : {framework}")
    return model_id, task, framework

def start(args):
    import uvicorn

    model_id, task, framework = resolve_task_framework(args)

    local_path = os.path.join(
        os.path.dirname(os.path.dirname(__file__)), "docker_images", framework
    )
    sys.path.append(local_path)
    os.environ["MODEL_ID"] = model_id
    os.environ["TASK"] = task
    uvicorn.run("app.main:app", host="127.0.0.1", port=8000, log_level="info")

def docker(args):
    model_id, task, framework = resolve_task_framework(args)

    tag = create_docker(framework)
    run_docker_command = [
        "docker",
        "run",
        "-p",
        "8000:80",
        "-e",
        f"TASK={task}",
        "-e",
        f"MODEL_ID={model_id}",
        "-v",
        "/tmp:/data",
        "-t",
        tag,
    ]

    with DockerPopen(run_docker_command) as proc:
        try:
            proc.wait()
        except KeyboardInterrupt:
            proc.terminate()

def main():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()
    parser_start = subparsers.add_parser(
        "start", help="Start a local version of a model inference"
    )
    parser_start.add_argument(
        "--model-id",
        type=str,
        required=True,
        help="Which model_id to start.",
    )
    parser_start.add_argument(
        "--task",
        type=str,
        help="Which task to load",
    )
    parser_start.add_argument(
        "--framework",
        type=str,
        help="Which framework to load",
    )
    parser_start.set_defaults(func=start)
    parser_docker = subparsers.add_parser(
        "docker", help="Start a docker version of a model inference"
    )
    parser_docker.add_argument(
        "--model-id",
        type=str,
        required=True,
        help="Which model_id to docker.",
    )
    parser_docker.add_argument(
        "--task",
        type=str,
        help="Which task to load",
    )
    parser_docker.add_argument(
        "--framework",
        type=str,
        help="Which framework to load",
    )
    parser_docker.set_defaults(func=docker)
    parser_show = subparsers.add_parser(
        "show", help="Show dockers and the various pipelines they implement"
    )
    parser_show.set_defaults(func=show)
    parser_batch = subparsers.add_parser("batch", help="Run a batch job")
    parser_batch.add_argument(
        "--model-id",
        type=str,
        required=True,
        help="Which model_id to batch.",
    )
    parser_batch.add_argument(
        "--task",
        type=str,
        help="Which task to load",
    )
    parser_batch.add_argument(
        "--framework",
        type=str,
        help="Which framework to load",
    )
    parser_batch.add_argument(
        "--dataset-name",
        type=str,
        required=True,
        help="Which dataset_name to batch.",
    )
    parser_batch.add_argument(
        "--dataset-config",
        type=str,
        help="Which dataset_config to batch.",
    )
    parser_batch.add_argument(
        "--dataset-split",
        type=str,
        required=True,
        help="Which dataset_split to batch.",
    )
    parser_batch.add_argument(
        "--dataset-column",
        type=str,
        help="Which column to batch.",
    )
    parser_batch.add_argument(
        "--token",
        type=str,
        help="Your API token",
    )
    parser_batch.set_defaults(func=do_batch)
    args = parser.parse_args()
    args.func(args)

if __name__ == "__main__":
    main()
<commit_msg>Batch is not used anymore. (#501)<commit_after>#!/usr/bin/env python
import argparse
import ast
import hashlib
import os
import subprocess
import sys
import uuid

from huggingface_hub import HfApi

class cd:
    """Context manager for changing the current working directory"""

    def __init__(self, newPath):
        self.newPath = os.path.expanduser(newPath)

    def __enter__(self):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, etype, value, traceback):
        os.chdir(self.savedPath)

class DockerPopen(subprocess.Popen):
    def __exit__(self, exc_type, exc_val, traceback):
        self.terminate()
        self.wait(5)
        return super().__exit__(exc_type, exc_val, traceback)

def create_docker(name: str) -> str:
    rand = str(uuid.uuid4())[:5]
    tag = f"{name}:{rand}"
    with cd(
        os.path.join(os.path.dirname(os.path.dirname(__file__)), "docker_images", name)
    ):
        subprocess.run(["docker", "build", ".", "-t", tag])
    return tag

def resolve_dataset(args, task: str):
    import datasets

    builder = datasets.load_dataset_builder(
        args.dataset_name, use_auth_token=args.token
    )

    if args.dataset_config is None:
        args.dataset_config = builder.config_id
        print(f"Inferred dataset_config {args.dataset_config}")

    splits = builder.info.splits
    if splits is not None:
        if args.dataset_split not in splits:
            raise ValueError(
                f"The split `{args.dataset_split}` is not a valid split, please choose from {','.join(splits.keys())}"
            )

    task_templates = builder.info.task_templates
    if task_templates is not None:
        for task_template in task_templates:
            if task_template.task == task:
                args.dataset_column = task_template.audio_file_path_column
                print(f"Inferred dataset_column {args.dataset_column}")
    return (
        args.dataset_name,
        args.dataset_config,
        args.dataset_split,
        args.dataset_column,
    )

def get_repo_name(model_id: str, dataset_name: str) -> str:
    # Hash needs to have the fully qualified name to disambiguate.
    hash_ = hashlib.md5((model_id + dataset_name).encode("utf-8")).hexdigest()

    model_name = model_id.split("/")[-1]
    dataset_name = dataset_name.split("/")[-1]
    return f"bulk-{model_name[:10]}-{dataset_name[:10]}-{hash_[:5]}"

def show(args):
    directory = os.path.join(
        os.path.dirname(os.path.dirname(__file__)), "docker_images"
    )
    for framework in sorted(os.listdir(directory)):
        print(f"{framework}")
        local_path = os.path.join(
            os.path.dirname(os.path.dirname(__file__)),
            "docker_images",
            framework,
            "app",
            "main.py",
        )
        # Using ast to prevent import issues with missing dependencies.
        # and slow loads.
        with open(local_path, "r") as source:
            tree = ast.parse(source.read())
            for item in tree.body:
                if (
                    isinstance(item, ast.AnnAssign)
                    and item.target.id == "ALLOWED_TASKS"
                ):
                    for key in item.value.keys:
                        print(" " * 4, key.value)

def resolve(model_id: str) -> [str, str]:
    info = HfApi().model_info(model_id)
    task = info.pipeline_tag
    framework = info.library_name
    return task, framework.replace("-", "_")

def resolve_task_framework(args):
    model_id = args.model_id
    task = args.task
    framework = args.framework
    if task is None or framework is None:
        rtask, rframework = resolve(model_id)
        if task is None:
            task = rtask
            print(f"Inferred task : {task}")
        if framework is None:
            framework = rframework
            print(f"Inferred framework : {framework}")
    return model_id, task, framework

def start(args):
    import uvicorn

    model_id, task, framework = resolve_task_framework(args)

    local_path = os.path.join(
        os.path.dirname(os.path.dirname(__file__)), "docker_images", framework
    )
    sys.path.append(local_path)
    os.environ["MODEL_ID"] = model_id
    os.environ["TASK"] = task
    uvicorn.run("app.main:app", host="127.0.0.1", port=8000, log_level="info")

def docker(args):
    model_id, task, framework = resolve_task_framework(args)

    tag = create_docker(framework)
    run_docker_command = [
        "docker",
        "run",
        "-p",
        "8000:80",
        "-e",
        f"TASK={task}",
        "-e",
        f"MODEL_ID={model_id}",
        "-v",
        "/tmp:/data",
        "-t",
        tag,
    ]

    with DockerPopen(run_docker_command) as proc:
        try:
            proc.wait()
        except KeyboardInterrupt:
            proc.terminate()

def main():
    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()
    parser_start = subparsers.add_parser(
        "start", help="Start a local version of a model inference"
    )
    parser_start.add_argument(
        "--model-id",
        type=str,
        required=True,
        help="Which model_id to start.",
    )
    parser_start.add_argument(
        "--task",
        type=str,
        help="Which task to load",
    )
    parser_start.add_argument(
        "--framework",
        type=str,
        help="Which framework to load",
    )
    parser_start.set_defaults(func=start)
    parser_docker = subparsers.add_parser(
        "docker", help="Start a docker version of a model inference"
    )
    parser_docker.add_argument(
        "--model-id",
        type=str,
        required=True,
        help="Which model_id to docker.",
    )
    parser_docker.add_argument(
        "--task",
        type=str,
        help="Which task to load",
    )
    parser_docker.add_argument(
        "--framework",
        type=str,
        help="Which framework to load",
    )
    parser_docker.set_defaults(func=docker)
    parser_show = subparsers.add_parser(
        "show", help="Show dockers and the various pipelines they implement"
    )
    parser_show.set_defaults(func=show)
    args = parser.parse_args()
    args.func(args)

if __name__ == "__main__":
    main()
<|endoftext|>