<commit_before>import signal
from functools import partial
from collections import defaultdict
import asyncio
from again.utils import unique_hex
from pinger import Pinger

class Registry:
    def __init__(self, ip, port):
        self._ip = ip
        self._port = port
        self._loop = asyncio.get_event_loop()
        self._registered_services = defaultdict(list)
        self._pending_services = defaultdict(list)
        self._client_protocols = {}
        self._service_dependencies = {}
        self._service_protocols = {}
        self._subscription_list = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(list))))
        self._pingers = {}

    def _rfactory(self):
        from jsonprotocol import RegistryProtocol

        return RegistryProtocol(self)

    def start(self):
        self._loop.add_signal_handler(getattr(signal, 'SIGINT'), partial(self._stop, 'SIGINT'))
        self._loop.add_signal_handler(getattr(signal, 'SIGTERM'), partial(self._stop, 'SIGTERM'))
        registry_coro = self._loop.create_server(self._rfactory, self._ip, self._port)
        self._server = self._loop.run_until_complete(registry_coro)
        try:
            self._loop.run_forever()
        except Exception as e:
            print(e)
        finally:
            self._server.close()
            self._loop.run_until_complete(self._server.wait_closed())
            self._loop.close()

    def _stop(self, signame:str):
        print('\ngot signal {} - exiting'.format(signame))
        self._loop.stop()

    def receive(self, packet:dict, registry_protocol):
        request_type = packet['type']
        if request_type == 'register':
            self._register_service(packet, registry_protocol)
        elif request_type == 'subscribe':
            self._handle_subscription(packet)
        elif request_type == 'pong':
            self._handle_pong(packet['node_id'], packet['count'])
        elif request_type == 'resolve_publication':
            self._resolve_publication(packet, registry_protocol)

    def deregister_service(self, node_id):
        for service, nodes in self._registered_services.items():
            service_present = False
            for node in nodes:
                if node[2] == node_id:
                    nodes.remove(node)
                    service_present = True
            if service_present:
                self._notify_consumers(service, node_id)
        self._service_protocols.pop(node_id)
        self._client_protocols.pop(node_id)

    def handle_ping_timeout(self, node_id):
        print("{} timed out".format(node_id))
        self.deregister_service(node_id)

    def _handle_pending_registrations(self):
        for service_name in self._pending_services:
            depends_on = self._service_dependencies[service_name]
            should_activate = True
            for dependency in depends_on:
                if self._registered_services.get(self._get_full_service_name(dependency["app"], dependency["service"],
                                                                             dependency["version"])) is None:
                    should_activate = False
                    break
            nodes = self._pending_services[service_name]
            for node in nodes:
                if should_activate:
                    self._send_activated_packet(self._client_protocols[node],
                                                self._service_dependencies[service_name])
                    nodes.remove(node)

    def _register_service(self, packet:dict, registry_protocol):
        params = packet['params']
        service_name = self._get_full_service_name(params['app'], params["service"], params['version'])
        dependencies = params['vendors']
        service_entry = (params['host'], params['port'], params['node_id'])
        self._registered_services[service_name].append(service_entry)
        self._pending_services[service_name].append(params['node_id'])
        self._client_protocols[params['node_id']] = registry_protocol
        self._connect_to_service(params['host'], params['port'], params['node_id'])
        if self._service_dependencies.get(service_name) is None:
            self._service_dependencies[service_name] = dependencies
        self._handle_pending_registrations()

    @staticmethod
    def _get_full_service_name(app:str, service:str, version:str):
        return "{}/{}/{}".format(app, service, version)

    def _send_activated_packet(self, protocol, dependencies):
        packet = self._make_activated_packet(dependencies)
        protocol.send(packet)
        pass

    def _make_activated_packet(self, vendors):
        vendors_packet = []
        for vendor in vendors:
            vendor_packet = defaultdict(list)
            vendor_name = self._get_full_service_name(vendor['app'], vendor['service'], vendor['version'])
            for host, port, node in self._registered_services[vendor_name]:
                vendor_node_packet = {
                    'host': host,
                    'port': port,
                    'node_id': node
                }
                vendor_packet['name'] = vendor_name
                vendor_packet['addresses'].append(vendor_node_packet)
            vendors_packet.append(vendor_packet)
        params = {
            'vendors': vendors_packet
        }
        packet = {'pid': unique_hex(),
                  'type': 'registered',
                  'params': params}
        return packet

    def _connect_to_service(self, host, port, node_id):
        coro = self._loop.create_connection(self._rfactory, host, port)
        future = asyncio.async(coro)
        future.add_done_callback(partial(self._handle_service_connection, node_id))

    def _handle_service_connection(self, node_id, future):
        transport, protocol = future.result()
        self._service_protocols[node_id] = protocol
        pinger = Pinger(self, self._loop, protocol, node_id)
        self._pingers[node_id] = pinger
        pinger.start_ping()

    def _handle_subscription(self, packet):
        params = packet['params']
        for each in params['subscribe_to']:
            self._subscription_list[each['app']][each['service']][each['version']][each['endpoint']].append(
                (params['ip'], params['port'], params['node_id']))

    def _handle_pong(self, node_id, count):
        pinger = self._pingers[node_id]
        pinger.pong_received(count)
        pinger.start_ping()

    def _resolve_publication(self, packet, protocol):
        params = packet['params']
        app, service, version, endpoint = params['app'], params['service'], params['version'], params['endpoint']
        nodes = [{'ip': ip, 'port': port, 'node_id': node} for ip, port, node in
                 self._subscription_list[app][service][version][endpoint]]
        result = {'type': 'subscription_list', 'request_id': packet['request_id'], 'nodes': nodes}
        protocol.send(result)

    def _notify_consumers(self, vendor, node_id):
        for service, vendors in self._service_dependencies.items():
            for each in vendors:
                vendor_name = self._get_full_service_name(each['app'], each['service'], each['version'])
                if vendor == vendor_name:
                    for host, port, node in self._registered_services[service]:
                        packet = self._make_deregister_packet(node_id, each)
                        protocol = self._client_protocols[node]
                        protocol.send(packet)

    def _make_deregister_packet(self, node_id, vendor):
        packet = {'type': 'deregister'}
        params = {'node_id': node_id, 'vendor': vendor}
        packet['params'] = params
        return packet

if __name__ == '__main__':
    REGISTRY_HOST = '127.0.0.1'
    REGISTRY_PORT = 4500
    registry = Registry(REGISTRY_HOST, REGISTRY_PORT)
    registry.start()
<commit_msg>Refactor to follow pep guidelines<commit_after>import signal
from functools import partial
from collections import defaultdict
import asyncio
from again.utils import unique_hex
from pinger import Pinger

class Registry:
    def __init__(self, ip, port):
        self._ip = ip
        self._port = port
        self._loop = asyncio.get_event_loop()
        self._registered_services = defaultdict(list)
        self._pending_services = defaultdict(list)
        self._client_protocols = {}
        self._service_dependencies = {}
        self._service_protocols = {}
        self._subscription_list = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(list))))
        self._pingers = {}

    def _rfactory(self):
        from jsonprotocol import RegistryProtocol

        return RegistryProtocol(self)

    def start(self):
        self._loop.add_signal_handler(getattr(signal, 'SIGINT'), partial(self._stop, 'SIGINT'))
        self._loop.add_signal_handler(getattr(signal, 'SIGTERM'), partial(self._stop, 'SIGTERM'))
        registry_coro = self._loop.create_server(self._rfactory, self._ip, self._port)
        self._server = self._loop.run_until_complete(registry_coro)
        try:
            self._loop.run_forever()
        except Exception as e:
            print(e)
        finally:
            self._server.close()
            self._loop.run_until_complete(self._server.wait_closed())
            self._loop.close()

    def _stop(self, signame:str):
        print('\ngot signal {} - exiting'.format(signame))
        self._loop.stop()

    def receive(self, packet:dict, registry_protocol):
        request_type = packet['type']
        if request_type == 'register':
            self._register_service(packet, registry_protocol)
        elif request_type == 'subscribe':
            self._handle_subscription(packet)
        elif request_type == 'pong':
            self._handle_pong(packet['node_id'], packet['count'])
        elif request_type == 'resolve_publication':
            self._resolve_publication(packet, registry_protocol)

    def deregister_service(self, node_id):
        for service, nodes in self._registered_services.items():
            service_present = False
            for node in nodes:
                if node[2] == node_id:
                    nodes.remove(node)
                    service_present = True
            if service_present:
                self._notify_consumers(service, node_id)
        self._service_protocols.pop(node_id)
        self._client_protocols.pop(node_id)

    def handle_ping_timeout(self, node_id):
        print("{} timed out".format(node_id))
        self.deregister_service(node_id)

    def _handle_pending_registrations(self):
        for service_name in self._pending_services:
            depends_on = self._service_dependencies[service_name]
            should_activate = True
            for dependency in depends_on:
                if self._registered_services.get(self._get_full_service_name(dependency["app"], dependency["service"],
                                                                             dependency["version"])) is None:
                    should_activate = False
                    break
            nodes = self._pending_services[service_name]
            for node in nodes:
                if should_activate:
                    self._send_activated_packet(self._client_protocols[node],
                                                self._service_dependencies[service_name])
                    nodes.remove(node)

    def _register_service(self, packet:dict, registry_protocol):
        params = packet['params']
        service_name = self._get_full_service_name(params['app'], params["service"], params['version'])
        dependencies = params['vendors']
        service_entry = (params['host'], params['port'], params['node_id'])
        self._registered_services[service_name].append(service_entry)
        self._pending_services[service_name].append(params['node_id'])
        self._client_protocols[params['node_id']] = registry_protocol
        self._connect_to_service(params['host'], params['port'], params['node_id'])
        if self._service_dependencies.get(service_name) is None:
            self._service_dependencies[service_name] = dependencies
        self._handle_pending_registrations()

    @staticmethod
    def _get_full_service_name(app:str, service:str, version:str):
        return "{}/{}/{}".format(app, service, version)

    def _send_activated_packet(self, protocol, dependencies):
        packet = self._make_activated_packet(dependencies)
        protocol.send(packet)
        pass

    def _make_activated_packet(self, vendors):
        vendors_packet = []
        for vendor in vendors:
            vendor_packet = defaultdict(list)
            vendor_name = self._get_full_service_name(vendor['app'], vendor['service'], vendor['version'])
            for host, port, node in self._registered_services[vendor_name]:
                vendor_node_packet = {
                    'host': host,
                    'port': port,
                    'node_id': node
                }
                vendor_packet['name'] = vendor_name
                vendor_packet['addresses'].append(vendor_node_packet)
            vendors_packet.append(vendor_packet)
        params = {
            'vendors': vendors_packet
        }
        packet = {'pid': unique_hex(),
                  'type': 'registered',
                  'params': params}
        return packet

    def _connect_to_service(self, host, port, node_id):
        coro = self._loop.create_connection(self._rfactory, host, port)
        future = asyncio.async(coro)
        future.add_done_callback(partial(self._handle_service_connection, node_id))

    def _handle_service_connection(self, node_id, future):
        transport, protocol = future.result()
        self._service_protocols[node_id] = protocol
        pinger = Pinger(self, self._loop, protocol, node_id)
        self._pingers[node_id] = pinger
        pinger.start_ping()

    def _handle_subscription(self, packet):
        params = packet['params']
        for each in params['subscribe_to']:
            self._subscription_list[each['app']][each['service']][each['version']][each['endpoint']].append(
                (params['ip'], params['port'], params['node_id']))

    def _handle_pong(self, node_id, count):
        pinger = self._pingers[node_id]
        pinger.pong_received(count)
        pinger.start_ping()

    def _resolve_publication(self, packet, protocol):
        params = packet['params']
        app, service, version, endpoint = params['app'], params['service'], params['version'], params['endpoint']
        nodes = [{'ip': ip, 'port': port, 'node_id': node} for ip, port, node in
                 self._subscription_list[app][service][version][endpoint]]
        result = {'type': 'subscription_list', 'request_id': packet['request_id'], 'nodes': nodes}
        protocol.send(result)

    def _notify_consumers(self, vendor, node_id):
        for service, vendors in self._service_dependencies.items():
            for each in vendors:
                vendor_name = self._get_full_service_name(each['app'], each['service'], each['version'])
                if vendor == vendor_name:
                    for host, port, node in self._registered_services[service]:
                        packet = self._make_deregister_packet(node_id, each)
                        protocol = self._client_protocols[node]
                        protocol.send(packet)

    def _make_deregister_packet(self, node_id, vendor):
        packet = {'type': 'deregister'}
        params = {'node_id': node_id, 'vendor': vendor}
        packet['params'] = params
        return packet

if __name__ == '__main__':
    REGISTRY_HOST = '127.0.0.1'
    REGISTRY_PORT = 4500
    registry = Registry(REGISTRY_HOST, REGISTRY_PORT)
    registry.start()
<|endoftext|>


<commit_before>import importlib
from django.shortcuts import render_to_response
from django import forms
from django import VERSION as DJANGO_VERSION
from django.template import RequestContext
from django.db.models import signals as signalmodule
from django.http import HttpResponse
from django.conf import settings
# Try to be compatible with Django 1.5+.
try:
    import json
except ImportError:
    from django.utils import simplejson as json

import datetime
import os

__all__ = ['render_to', 'signals', 'ajax_request', 'autostrip']

try:
    from functools import wraps
except ImportError:
    def wraps(wrapped, assigned=('__module__', '__name__', '__doc__'),
              updated=('__dict__',)):
        def inner(wrapper):
            for attr in assigned:
                setattr(wrapper, attr, getattr(wrapped, attr))
            for attr in updated:
                getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
            return wrapper
        return inner

def render_to(template=None, content_type=None, mimetype=None):
    """
    Decorator for Django views that sends returned dict to render_to_response
    function.

    Template name can be decorator parameter or TEMPLATE item in returned
    dictionary.  RequestContext always added as context instance.
    If view doesn't return dict then decorator simply returns output.

    Parameters:
     - template: template name to use
     - content_type: content type to send in response headers
     - mimetype: content type to send in response headers (deprecated)

    Examples:
    # 1. Template name in decorator parameters

    @render_to('template.html')
    def foo(request):
        bar = Bar.object.all()
        return {'bar': bar}

    # equals to
    def foo(request):
        bar = Bar.object.all()
        return render_to_response('template.html',
                                  {'bar': bar},
                                  context_instance=RequestContext(request))

    # 2. Template name as TEMPLATE item value in return dictionary.
         if TEMPLATE is given then its value will have higher priority
         than render_to argument.

    @render_to()
    def foo(request, category):
        template_name = '%s.html' % category
        return {'bar': bar, 'TEMPLATE': template_name}

    #equals to
    def foo(request, category):
        template_name = '%s.html' % category
        return render_to_response(template_name,
                                  {'bar': bar},
                                  context_instance=RequestContext(request))

    """
    def renderer(function):
        @wraps(function)
        def wrapper(request, *args, **kwargs):
            output = function(request, *args, **kwargs)
            if not isinstance(output, dict):
                return output
            tmpl = output.pop('TEMPLATE', template)
            if tmpl is None:
                template_dir = os.path.join(*function.__module__.split('.')[:-1])
                tmpl = os.path.join(template_dir, function.func_name + ".html")
            # Explicit version check to avoid swallowing other exceptions
            if DJANGO_VERSION[0] >= 1 and DJANGO_VERSION[1] >= 5:
                return render_to_response(tmpl, output,
                        context_instance=RequestContext(request),
                        content_type=content_type or mimetype)
            else:
                return render_to_response(tmpl, output,
                        context_instance=RequestContext(request),
                        mimetype=content_type or mimetype)
        return wrapper
    return renderer

class Signals(object):
    '''
    Convenient wrapper for working with Django's signals (or any other
    implementation using same API).

    Example of usage::

       # connect to registered signal
       @signals.post_save(sender=YourModel)
       def sighandler(instance, **kwargs):
           pass

       # connect to any signal
       signals.register_signal(siginstance, signame) # and then as in example above

       or

       @signals(siginstance, sender=YourModel)
       def sighandler(instance, **kwargs):
           pass

    In any case defined function will remain as is, without any changes.

    (c) 2008 Alexander Solovyov, new BSD License
    '''
    def __init__(self):
        self._signals = {}

        # register all Django's default signals
        for k, v in signalmodule.__dict__.items():
            # that's hardcode, but IMHO it's better than isinstance
            if not k.startswith('__') and k != 'Signal':
                self.register_signal(v, k)

    def __getattr__(self, name):
        return self._connect(self._signals[name])

    def __call__(self, signal, **kwargs):
        def inner(func):
            signal.connect(func, **kwargs)
            return func
        return inner

    def _connect(self, signal):
        def wrapper(**kwargs):
            return self(signal, **kwargs)
        return wrapper

    def register_signal(self, signal, name):
        self._signals[name] = signal

signals = Signals()

def date_time_handler(obj):
    if isinstance(obj, datetime.datetime):
        return obj.isoformat()
    else:
        raise TypeError("%r is not JSON serializable" % obj)

FORMAT_TYPES = {
    'application/json': lambda response: json.dumps(response, default=date_time_handler),
    'text/json':        lambda response: json.dumps(response, default=date_time_handler),
}

try:
    import yaml
except ImportError:
    pass
else:
    FORMAT_TYPES.update({
        'application/yaml': yaml.dump,
        'text/yaml':        yaml.dump,
    })

def ajax_request(func):
    """
    If view returned serializable dict, returns response in a format requested
    by HTTP_ACCEPT header. Defaults to JSON if none requested or match.

    Currently supports JSON or YAML (if installed), but can easily be extended.

    example:

        @ajax_request
        def my_view(request):
            news = News.objects.all()
            news_titles = [entry.title for entry in news]
            return {'news_titles': news_titles}
    """
    @wraps(func)
    def wrapper(request, *args, **kwargs):
        for accepted_type in request.META.get('HTTP_ACCEPT', '').split(','):
            if accepted_type in FORMAT_TYPES.keys():
                format_type = accepted_type
                break
        else:
            format_type = 'application/json'
        response = func(request, *args, **kwargs)
        if not isinstance(response, HttpResponse):
            if settings.hasattr('FORMAT_TYPES'):
                format_type_handler = settings.FORMAT_TYPES[format_type]
                if hasattr(format_type_handler, '__call__'):
                    data = format_type_handler(response)
                elif isinstance(format_type_handler, basestring):
                    module_name, function_name = format_type_handler.rsplit('.',1)
                    module = importlib.import_module(module_name)
                    function = getattr(module, function_name)
                    data = function(response)
            else:
                data = FORMAT_TYPES[format_type](response)
            response = HttpResponse(data, content_type=format_type)
            response['content-length'] = len(data)
        return response
    return wrapper

def autostrip(cls):
    """
    strip text fields before validation

    example:
    class PersonForm(forms.Form):
        name = forms.CharField(min_length=2, max_length=10)
        email = forms.EmailField()

    PersonForm = autostrip(PersonForm)

    #or you can use @autostrip in python >= 2.6

    Author: nail.xx
    """
    fields = [(key, value) for key, value in cls.base_fields.iteritems() if isinstance(value, forms.CharField)]
    for field_name, field_object in fields:
        def get_clean_func(original_clean):
            return lambda value: original_clean(value and value.strip())
        clean_func = get_clean_func(getattr(field_object, 'clean'))
        setattr(field_object, 'clean', clean_func)
    return cls
<commit_msg>fix. settings.hasattr => hasattr(settings, ...)<commit_after>import importlib
from django.shortcuts import render_to_response
from django import forms
from django import VERSION as DJANGO_VERSION
from django.template import RequestContext
from django.db.models import signals as signalmodule
from django.http import HttpResponse
from django.conf import settings
# Try to be compatible with Django 1.5+.
try:
    import json
except ImportError:
    from django.utils import simplejson as json

import datetime
import os

__all__ = ['render_to', 'signals', 'ajax_request', 'autostrip']

try:
    from functools import wraps
except ImportError:
    def wraps(wrapped, assigned=('__module__', '__name__', '__doc__'),
              updated=('__dict__',)):
        def inner(wrapper):
            for attr in assigned:
                setattr(wrapper, attr, getattr(wrapped, attr))
            for attr in updated:
                getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
            return wrapper
        return inner

def render_to(template=None, content_type=None, mimetype=None):
    """
    Decorator for Django views that sends returned dict to render_to_response
    function.

    Template name can be decorator parameter or TEMPLATE item in returned
    dictionary.  RequestContext always added as context instance.
    If view doesn't return dict then decorator simply returns output.

    Parameters:
     - template: template name to use
     - content_type: content type to send in response headers
     - mimetype: content type to send in response headers (deprecated)

    Examples:
    # 1. Template name in decorator parameters

    @render_to('template.html')
    def foo(request):
        bar = Bar.object.all()
        return {'bar': bar}

    # equals to
    def foo(request):
        bar = Bar.object.all()
        return render_to_response('template.html',
                                  {'bar': bar},
                                  context_instance=RequestContext(request))

    # 2. Template name as TEMPLATE item value in return dictionary.
         if TEMPLATE is given then its value will have higher priority
         than render_to argument.

    @render_to()
    def foo(request, category):
        template_name = '%s.html' % category
        return {'bar': bar, 'TEMPLATE': template_name}

    #equals to
    def foo(request, category):
        template_name = '%s.html' % category
        return render_to_response(template_name,
                                  {'bar': bar},
                                  context_instance=RequestContext(request))

    """
    def renderer(function):
        @wraps(function)
        def wrapper(request, *args, **kwargs):
            output = function(request, *args, **kwargs)
            if not isinstance(output, dict):
                return output
            tmpl = output.pop('TEMPLATE', template)
            if tmpl is None:
                template_dir = os.path.join(*function.__module__.split('.')[:-1])
                tmpl = os.path.join(template_dir, function.func_name + ".html")
            # Explicit version check to avoid swallowing other exceptions
            if DJANGO_VERSION[0] >= 1 and DJANGO_VERSION[1] >= 5:
                return render_to_response(tmpl, output,
                        context_instance=RequestContext(request),
                        content_type=content_type or mimetype)
            else:
                return render_to_response(tmpl, output,
                        context_instance=RequestContext(request),
                        mimetype=content_type or mimetype)
        return wrapper
    return renderer

class Signals(object):
    '''
    Convenient wrapper for working with Django's signals (or any other
    implementation using same API).

    Example of usage::

       # connect to registered signal
       @signals.post_save(sender=YourModel)
       def sighandler(instance, **kwargs):
           pass

       # connect to any signal
       signals.register_signal(siginstance, signame) # and then as in example above

       or

       @signals(siginstance, sender=YourModel)
       def sighandler(instance, **kwargs):
           pass

    In any case defined function will remain as is, without any changes.

    (c) 2008 Alexander Solovyov, new BSD License
    '''
    def __init__(self):
        self._signals = {}

        # register all Django's default signals
        for k, v in signalmodule.__dict__.items():
            # that's hardcode, but IMHO it's better than isinstance
            if not k.startswith('__') and k != 'Signal':
                self.register_signal(v, k)

    def __getattr__(self, name):
        return self._connect(self._signals[name])

    def __call__(self, signal, **kwargs):
        def inner(func):
            signal.connect(func, **kwargs)
            return func
        return inner

    def _connect(self, signal):
        def wrapper(**kwargs):
            return self(signal, **kwargs)
        return wrapper

    def register_signal(self, signal, name):
        self._signals[name] = signal

signals = Signals()

def date_time_handler(obj):
    if isinstance(obj, datetime.datetime):
        return obj.isoformat()
    else:
        raise TypeError("%r is not JSON serializable" % obj)

FORMAT_TYPES = {
    'application/json': lambda response: json.dumps(response, default=date_time_handler),
    'text/json':        lambda response: json.dumps(response, default=date_time_handler),
}

try:
    import yaml
except ImportError:
    pass
else:
    FORMAT_TYPES.update({
        'application/yaml': yaml.dump,
        'text/yaml':        yaml.dump,
    })

def ajax_request(func):
    """
    If view returned serializable dict, returns response in a format requested
    by HTTP_ACCEPT header. Defaults to JSON if none requested or match.

    Currently supports JSON or YAML (if installed), but can easily be extended.

    example:

        @ajax_request
        def my_view(request):
            news = News.objects.all()
            news_titles = [entry.title for entry in news]
            return {'news_titles': news_titles}
    """
    @wraps(func)
    def wrapper(request, *args, **kwargs):
        for accepted_type in request.META.get('HTTP_ACCEPT', '').split(','):
            if accepted_type in FORMAT_TYPES.keys():
                format_type = accepted_type
                break
        else:
            format_type = 'application/json'
        response = func(request, *args, **kwargs)
        if not isinstance(response, HttpResponse):
            if hasattr(settings, 'FORMAT_TYPES'):
                format_type_handler = settings.FORMAT_TYPES[format_type]
                if hasattr(format_type_handler, '__call__'):
                    data = format_type_handler(response)
                elif isinstance(format_type_handler, basestring):
                    module_name, function_name = format_type_handler.rsplit('.',1)
                    module = importlib.import_module(module_name)
                    function = getattr(module, function_name)
                    data = function(response)
            else:
                data = FORMAT_TYPES[format_type](response)
            response = HttpResponse(data, content_type=format_type)
            response['content-length'] = len(data)
        return response
    return wrapper

def autostrip(cls):
    """
    strip text fields before validation

    example:
    class PersonForm(forms.Form):
        name = forms.CharField(min_length=2, max_length=10)
        email = forms.EmailField()

    PersonForm = autostrip(PersonForm)

    #or you can use @autostrip in python >= 2.6

    Author: nail.xx
    """
    fields = [(key, value) for key, value in cls.base_fields.iteritems() if isinstance(value, forms.CharField)]
    for field_name, field_object in fields:
        def get_clean_func(original_clean):
            return lambda value: original_clean(value and value.strip())
        clean_func = get_clean_func(getattr(field_object, 'clean'))
        setattr(field_object, 'clean', clean_func)
    return cls
<|endoftext|>


<commit_before>from collections import namedtuple

import numpy
from scipy import stats
import statsmodels.api as sm

from wqio import validate

def sigFigs(x, n, expthresh=5, tex=False, pval=False, forceint=False):
    """ Formats a number with the correct number of sig figs.

    Parameters
    ----------
    x : int or float
        The number you want to format.
    n : int
        The number of sig figs it should have.
    expthresh : int, optional (default = 5)
        The absolute value of the order of magnitude at which numbers
        are formatted in exponential notation.
    tex : bool, optional (default is False)
        Toggles the scientific formatting of the number either for
        terminal output (False) or LaTeX documents (True).
    pval : bool, optional (default is False)
        Useful for formatting p-values from hypothesis tests. When True
        and x < 0.001, will return "<0.001".
    forceint : bool, optional (default is False)
        If true, simply returns int(x)

    Returns
    -------
    formatted : str
        The formatted number as a string

    Examples
    --------
    >>> print(sigFigs(1247.15, 3))
           1250
    >>> print(sigFigs(1247.15, 7))
           1247.150

    """

    # check on the number provided
    if x is not None and not numpy.isinf(x) and not numpy.isnan(x):

        # check on the sigFigs
        if n < 1:
            raise ValueError("number of sig figs must be greater than zero!")

        # return a string value unaltered
        if isinstance(x, str):
            out = x

        elif pval and x < 0.001:
            out = "<0.001"
            if tex:
                out = '${}$'.format(out)

        elif forceint:
            out = '{:,.0f}'.format(x)

        # logic to do all of the rounding
        elif x != 0.0:
            order = numpy.floor(numpy.log10(numpy.abs(x)))

            if -1.0 * expthresh <= order <= expthresh:
                decimal_places = int(n - 1 - order)

                if decimal_places <= 0:
                    out = '{0:,.0f}'.format(round(x, decimal_places))

                else:
                    fmt = '{0:,.%df}' % decimal_places
                    out = fmt.format(x)

            else:
                decimal_places = n - 1
                if tex:
                    fmt = r'$%%0.%df \times 10 ^ {%d}$' % (decimal_places, order)
                    out = fmt % round(x / 10 ** order, decimal_places)
                else:
                    fmt = '{0:.%de}' % decimal_places
                    out = fmt.format(x)

        else:
            out = str(round(x, n))

    # with NAs and INFs, just return 'NA'
    else:
        out = 'NA'

    return out

def formatResult(result, qualifier, sigfigs=3):
    """ Formats a results with its qualifier

    Parameters
    ----------
    results : float
        The concentration or particulate strength
    qualifier : string
        The result's qualifier
    sigfigs : int
        The number of significant digits to which `result` should be
        formatted

    Returns
    -------
    formatted : string

    Example
    -------
    >>> wqio.formatResult(1.23, '<', sigfigs=4)
    "<1.230"

    """

    return '{}{}'.format(qualifier, sigFigs(result, sigfigs))

def process_p_vals(pval):
    """ Processes p-values into nice strings to reporting. When the
    p-values are less than 0.001, "<0.001" is returned. Otherwise, a
    string with three decimal places is returned.

    Parameters
    ----------
    pval : float
        The p-value to be formatted. Must be between 0 and 1. Otherwise
        an error is raised.

    Returns
    -------
    processed : string

    """
    if pval is None:
        out = 'NA'
    elif 0 < pval < 0.001:
        out = formatResult(0.001, '<', sigfigs=1)
    elif pval > 1 or pval < 0:
        raise ValueError('p-values cannot be greater than 1 and less than 0')
    else:
        out = '%0.3f' % pval

    return out

def anderson_darling(data):
    """
    Compute the Anderson-Darling Statistic and p-value

    Parmeters
    ---------
    data : array-like

    Returns
    -------
    ADResult : namedtuple
        A collection of results. Keys are:

          * ``statistic``
          * ``critical_values``
          * ``significance_level``
          * ``pvalue``

    See Also
    --------
    scipy.stats.anderson

    """
    AD = stats.anderson(data)
    field_names = list(AD._fields) + ['pvalue']

    p = _anderson_darling_p_vals(AD, len(data))
    values = AD._asdict()
    values['pvalue'] = p

    ADResult = namedtuple('ADResult', field_names)
    return ADResult(**values)

def processAndersonDarlingResults(ad_results):
    """ Return a nice string of Anderson-Darling test results

    Parameters
    ----------
    ad_result : tuple or namedtuple
        The packed output from scipy.stats.anderson

    Returns
    -------
    result : str
        A string representation of the confidence in the result.

    """

    AD, crit, sig = ad_results
    try:
        ci = 100 - sig[AD > crit][-1]
        return '%0.1f%%' % (ci,)
    except IndexError:
        ci = 100 - sig[0]
        return '<%0.1f%%' % (ci,)

def _anderson_darling_p_vals(ad_results, n_points):
    """
    Computes the p-value of the Anderson-Darling Result

    Parameters
    ----------
    ad_result : tuple or namedtuple
        The packed output from scipy.stats.anderson
    n_points : int
        The number of points in the sample.

    Returns
    -------
    p : float

    References
    ----------
    R.B. D'Augostino and M.A. Stephens, Eds., 1986, Goodness-of-Fit
    Techniques, Marcel Dekker

    """

    AD, crit, sig = ad_results
    AD_star = AD * (1 + 0.75 / n_points + 2.25 / n_points**2)
    if AD_star >= 0.6:
        p = numpy.exp(1.2397 - (5.709 * AD_star) + (0.0186 * AD_star**2))
    elif 0.34 <= AD_star < 0.6:
        p = numpy.exp(0.9177 - (4.279 * AD_star) - (1.38 * AD_star**2))
    elif 0.2 < AD_star < 0.34:
        p = 1 - numpy.exp(-8.318 + (42.796 * AD_star) - (59.938 * AD_star**2))
    else:
        p = 1 - numpy.exp(-13.436 + (101.14 * AD_star) - (223.73 * AD_star**2))

    return p

def normalize_units(df, unitsmap, targetunit, paramcol='parameter',
                    rescol='res', unitcol='units', napolicy='ignore'):
    """
    Normalize units of measure in a dataframe.

    Parameters
    ----------
    df : pandas.DataFrame
        Dataframe contained results and units of measure data.
    unitsmap : dictionary
        Dictionary where keys are the units present in the df and
        values are the conversion factors required to standardize
        results to a common unit.
    targetunit : string or dict
        The desired final units of measure. Must be present in
        ``unitsmap``. If a string, all rows in df will be
        assigned ``targetunit``. If a dictionary, the units will be
        mapped from the dictionary using the values of ``paramcol`` as
        keys.
    paramcol, rescol, unitcol : string, optional
        Labels for the parameter, results, and units columns in the
        df.
    napolicy : string, optional
        Determines how mull/missing values are stored. By default,
        this is set to "ignore". Use "raise" to throw a ``ValueError``
        if when unit conversion or target units data cannot be found.

    Returns
    -------
    normalized : pandas.DataFrame
        Dataframe with normalized units of measure.

    """

    # determine the preferred units in the wqdata
    target = df[paramcol].map(targetunit.get)
    if napolicy == 'raise' and target.isnull().any():
        msg = "Some target units could not be mapped to the {} column"
        raise ValueError(msg.format(paramcol))

    # factors to normialize to standard units
    normalization = df[unitcol].map(unitsmap.get)
    if napolicy == 'raise' and normalization.isnull().any():
        msg = "Some normalization factors could not be mapped to the {} column"
        raise ValueError(msg.format(unitcol))

    # factor to convert to preferred units
    conversion = target.map(unitsmap.get)
    if napolicy == 'raise' and conversion.isnull().any():
        msg = "Some conversion factors could not be mapped to the target units"
        raise ValueError(msg)

    # convert results
    normalized = df.assign(**{
        rescol: df[rescol] * normalization / conversion,
        unitcol: target,
    })

    return normalized

def pH2concentration(pH, *args):
    """ Converts pH values to proton concentrations in mg/L

    Parameters
    ----------
    pH : int or float
        Number between 0 and 14 representing the pH

    Returns
    -------
    conc : float
        Concentration of protons in mg/L

    """

    # check that we recieved a valid input:
    if pH < 0 or pH > 14:
        raise ValueError('pH = %f but must be between 0 and 14' % pH)

    # avogadro's number (items/mole)
    avogadro = 6.0221413e+23

    # mass of a proton (kg)
    proton_mass = 1.672621777e-27

    # grams per kilogram
    kg2g = 1000

    # milligrams per gram
    g2mg = 1000

    return 10 ** (-1 * pH) * avogadro * proton_mass * kg2g * g2mg

def fit_line(x, y, xhat=None, fitprobs=None, fitlogs=None, dist=None, through_origin=False):
    """ Fits a line to x-y data in various forms (raw, log, prob scales)

    Parameters
    ----------
    x, y : array-like
        Independent and dependent data, respectively.
    xhat : array-like or None, optional
        The values at which yhat should should be estimated. If
        not provided, falls back to the sorted values of ``x``.
    fitprobs, fitlogs : str, options.
        Defines how data should be transformed. Valid values are
        'x', 'y', or 'both'. If using ``fitprobs``, variables should
        be expressed as a percentage, i.e.,
        Probablility transform = lambda x: ``dist``.ppf(x / 100.).
        Log transform = lambda x: numpy.log(x).
        Take care to not pass the same value to both ``fitlogs`` and
        ``figprobs`` as both transforms will be applied.
    dist : scipy.stats distribution or None, optional
        A fully-spec'd scipy.stats distribution such that ``dist.ppf``
        can be called. If not provided, defaults to scipt.stats.norm.
    through_origin : bool, optional (default = False)
        If True, the y-intercept of the best-fit line will be
        constrained to 0 during the regression.

    Returns
    -------
    xhat, yhat : numpy arrays
        Linear model estimates of ``x`` and ``y``.
    results : a statmodels result object
        The object returned by statsmodels.OLS.fit()

    """

    fitprobs = validate.fit_arguments(fitprobs, "fitprobs")
    fitlogs = validate.fit_arguments(fitlogs, "fitlogs")

    x = numpy.asarray(x)
    y = numpy.asarray(y)

    if xhat is None:
        xhat = numpy.array([numpy.min(x), numpy.max(x)])

    if dist is None:
        dist = stats.norm

    if fitprobs in ['x', 'both']:
        x = dist.ppf(x / 100.)
        xhat = dist.ppf(numpy.array(xhat) / 100.)

    if fitprobs in ['y', 'both']:
        y = dist.ppf(y / 100.)

    if fitlogs in ['x', 'both']:
        x = numpy.log(x)
    if fitlogs in ['y', 'both']:
        y = numpy.log(y)

    x = sm.add_constant(x)

    # fix the y-intercept at 0 if requested
    # note that this is a nuance of the statsmodels package.
    # `x[:, 0] = 5` doesn't fix the intercept at 5, for instance.
    if through_origin:
        x[:, 0] = 0

    results = sm.OLS(y, x).fit()
    yhat = estimateFromLineParams(xhat, results.params[1], results.params[0],
                                  xlog=fitlogs in ['x', 'both'],
                                  ylog=fitlogs in ['y', 'both'])

    if fitprobs in ['y', 'both']:
        yhat = 100. * dist.cdf(yhat)
    if fitprobs in ['x', 'both']:
        xhat = 100. * dist.cdf(xhat)

    return xhat, yhat, results

def estimateFromLineParams(xdata, slope, intercept, xlog=False, ylog=False):
    """ Estimate the dependent of a linear fit given x-data and linear
    parameters.

    Parameters
    ----------
    xdata : numpy array or pandas Series/DataFrame
        The input independent variable of the fit
    slope : float
        Slope of the best-fit line
    intercept : float
        y-intercept of the best-fit line
    xlog, ylog : bool (default = False)
        Toggles whether or not the logs of the x- or y- data should be
        used to perform the regression.

    Returns
    -------
    yhat : same type as xdata
        Estimate of the dependent variable.

    """

    x = numpy.array(xdata)
    if ylog:
        if xlog:
            yhat = numpy.exp(intercept) * x ** slope
        else:
            yhat = numpy.exp(intercept) * numpy.exp(slope) ** x

    else:
        if xlog:
            yhat = slope * numpy.log(x) + intercept

        else:
            yhat = slope * x + intercept

    return yhat

def checkIntervalOverlap(interval1, interval2, oneway=False):
    """ Checks if two numeric intervals overlaps.

    Parameters
    ----------
    interval1, interval2 : array like
        len = 2 sequences to compare
    oneway : bool (default = False)
        If True, only checks that ``interval1`` falls at least partially
        inside ``interval2``, but not the other way around

    Returns
    -------
    overlap : bool
        True if an overlap exists. Otherwise, False.

    """

    test1 = numpy.min(interval2) <= numpy.max(interval1) <= numpy.max(interval2)
    test2 = numpy.min(interval2) <= numpy.min(interval1) <= numpy.max(interval2)

    # TODO: try test1 or test2 or (not oneway and test3)

    if oneway:
        return test1 or test2
    else:
        test3 = checkIntervalOverlap(interval2, interval1, oneway=True)
        return test1 or test2 or test3

def winsorize_dataframe(dataframe, **limits):
    """ Winsorizes columns in a dataframe

    Parameters
    ----------
    dataframe : pandas.DataFrame
        The data to be modified (a copy is created).
    **limits : optional kwargs (floats, or two-tuples of floats)
        Optional key-value pairs of column names and (two-tuples of)
        floats that are the windsor limits to be applied to the
        respective column. Values should be between 0 and 1.

    Returns
    -------
    winsored_df : pandas.DataFrame
        The modified dataframe.

    See also
    --------
    scipy.stats.mstats.winsorize

    """
    df = dataframe.copy()
    for colname, limit in limits.items():
        df[colname] = stats.mstats.winsorize(df[colname], limits=limit)

    return df
<commit_msg>let users know which values can't be found<commit_after>from collections import namedtuple

import numpy
from scipy import stats
import statsmodels.api as sm

from wqio import validate

def sigFigs(x, n, expthresh=5, tex=False, pval=False, forceint=False):
    """ Formats a number with the correct number of sig figs.

    Parameters
    ----------
    x : int or float
        The number you want to format.
    n : int
        The number of sig figs it should have.
    expthresh : int, optional (default = 5)
        The absolute value of the order of magnitude at which numbers
        are formatted in exponential notation.
    tex : bool, optional (default is False)
        Toggles the scientific formatting of the number either for
        terminal output (False) or LaTeX documents (True).
    pval : bool, optional (default is False)
        Useful for formatting p-values from hypothesis tests. When True
        and x < 0.001, will return "<0.001".
    forceint : bool, optional (default is False)
        If true, simply returns int(x)

    Returns
    -------
    formatted : str
        The formatted number as a string

    Examples
    --------
    >>> print(sigFigs(1247.15, 3))
           1250
    >>> print(sigFigs(1247.15, 7))
           1247.150

    """

    # check on the number provided
    if x is not None and not numpy.isinf(x) and not numpy.isnan(x):

        # check on the sigFigs
        if n < 1:
            raise ValueError("number of sig figs must be greater than zero!")

        # return a string value unaltered
        if isinstance(x, str):
            out = x

        elif pval and x < 0.001:
            out = "<0.001"
            if tex:
                out = '${}$'.format(out)

        elif forceint:
            out = '{:,.0f}'.format(x)

        # logic to do all of the rounding
        elif x != 0.0:
            order = numpy.floor(numpy.log10(numpy.abs(x)))

            if -1.0 * expthresh <= order <= expthresh:
                decimal_places = int(n - 1 - order)

                if decimal_places <= 0:
                    out = '{0:,.0f}'.format(round(x, decimal_places))

                else:
                    fmt = '{0:,.%df}' % decimal_places
                    out = fmt.format(x)

            else:
                decimal_places = n - 1
                if tex:
                    fmt = r'$%%0.%df \times 10 ^ {%d}$' % (decimal_places, order)
                    out = fmt % round(x / 10 ** order, decimal_places)
                else:
                    fmt = '{0:.%de}' % decimal_places
                    out = fmt.format(x)

        else:
            out = str(round(x, n))

    # with NAs and INFs, just return 'NA'
    else:
        out = 'NA'

    return out

def formatResult(result, qualifier, sigfigs=3):
    """ Formats a results with its qualifier

    Parameters
    ----------
    results : float
        The concentration or particulate strength
    qualifier : string
        The result's qualifier
    sigfigs : int
        The number of significant digits to which `result` should be
        formatted

    Returns
    -------
    formatted : string

    Example
    -------
    >>> wqio.formatResult(1.23, '<', sigfigs=4)
    "<1.230"

    """

    return '{}{}'.format(qualifier, sigFigs(result, sigfigs))

def process_p_vals(pval):
    """ Processes p-values into nice strings to reporting. When the
    p-values are less than 0.001, "<0.001" is returned. Otherwise, a
    string with three decimal places is returned.

    Parameters
    ----------
    pval : float
        The p-value to be formatted. Must be between 0 and 1. Otherwise
        an error is raised.

    Returns
    -------
    processed : string

    """
    if pval is None:
        out = 'NA'
    elif 0 < pval < 0.001:
        out = formatResult(0.001, '<', sigfigs=1)
    elif pval > 1 or pval < 0:
        raise ValueError('p-values cannot be greater than 1 and less than 0')
    else:
        out = '%0.3f' % pval

    return out

def anderson_darling(data):
    """
    Compute the Anderson-Darling Statistic and p-value

    Parmeters
    ---------
    data : array-like

    Returns
    -------
    ADResult : namedtuple
        A collection of results. Keys are:

          * ``statistic``
          * ``critical_values``
          * ``significance_level``
          * ``pvalue``

    See Also
    --------
    scipy.stats.anderson

    """
    AD = stats.anderson(data)
    field_names = list(AD._fields) + ['pvalue']

    p = _anderson_darling_p_vals(AD, len(data))
    values = AD._asdict()
    values['pvalue'] = p

    ADResult = namedtuple('ADResult', field_names)
    return ADResult(**values)

def processAndersonDarlingResults(ad_results):
    """ Return a nice string of Anderson-Darling test results

    Parameters
    ----------
    ad_result : tuple or namedtuple
        The packed output from scipy.stats.anderson

    Returns
    -------
    result : str
        A string representation of the confidence in the result.

    """

    AD, crit, sig = ad_results
    try:
        ci = 100 - sig[AD > crit][-1]
        return '%0.1f%%' % (ci,)
    except IndexError:
        ci = 100 - sig[0]
        return '<%0.1f%%' % (ci,)

def _anderson_darling_p_vals(ad_results, n_points):
    """
    Computes the p-value of the Anderson-Darling Result

    Parameters
    ----------
    ad_result : tuple or namedtuple
        The packed output from scipy.stats.anderson
    n_points : int
        The number of points in the sample.

    Returns
    -------
    p : float

    References
    ----------
    R.B. D'Augostino and M.A. Stephens, Eds., 1986, Goodness-of-Fit
    Techniques, Marcel Dekker

    """

    AD, crit, sig = ad_results
    AD_star = AD * (1 + 0.75 / n_points + 2.25 / n_points**2)
    if AD_star >= 0.6:
        p = numpy.exp(1.2397 - (5.709 * AD_star) + (0.0186 * AD_star**2))
    elif 0.34 <= AD_star < 0.6:
        p = numpy.exp(0.9177 - (4.279 * AD_star) - (1.38 * AD_star**2))
    elif 0.2 < AD_star < 0.34:
        p = 1 - numpy.exp(-8.318 + (42.796 * AD_star) - (59.938 * AD_star**2))
    else:
        p = 1 - numpy.exp(-13.436 + (101.14 * AD_star) - (223.73 * AD_star**2))

    return p

def normalize_units(df, unitsmap, targetunit, paramcol='parameter',
                    rescol='res', unitcol='units', napolicy='ignore'):
    """
    Normalize units of measure in a dataframe.

    Parameters
    ----------
    df : pandas.DataFrame
        Dataframe contained results and units of measure data.
    unitsmap : dictionary
        Dictionary where keys are the units present in the df and
        values are the conversion factors required to standardize
        results to a common unit.
    targetunit : string or dict
        The desired final units of measure. Must be present in
        ``unitsmap``. If a string, all rows in df will be
        assigned ``targetunit``. If a dictionary, the units will be
        mapped from the dictionary using the values of ``paramcol`` as
        keys.
    paramcol, rescol, unitcol : string, optional
        Labels for the parameter, results, and units columns in the
        df.
    napolicy : string, optional
        Determines how mull/missing values are stored. By default,
        this is set to "ignore". Use "raise" to throw a ``ValueError``
        if when unit conversion or target units data cannot be found.

    Returns
    -------
    normalized : pandas.DataFrame
        Dataframe with normalized units of measure.

    """

    # determine the preferred units in the wqdata
    target = df[paramcol].map(targetunit.get)
    if napolicy == 'raise' and target.isnull().any():
        nulls = df[target.isnull()][paramcol].unique()
        msg = "Some target units could not be mapped to the {} column ({})"
        raise ValueError(msg.format(paramcol, nulls))

    # factors to normialize to standard units
    normalization = df[unitcol].map(unitsmap.get)
    if napolicy == 'raise' and normalization.isnull().any():
        nulls = df[normalization.isnull()][unitcol]
        msg = "Some normalization factors could not be mapped to the {} column ({})"
        raise ValueError(msg.format(unitcol, nulls))

    # factor to convert to preferred units
    conversion = target.map(unitsmap.get)
    if napolicy == 'raise' and conversion.isnull().any():
        nulls = target[conversion.isnull()]
        msg = "Some conversion factors could not be mapped to the target units ({})"
        raise ValueError(msg.format(nulls))

    # convert results
    normalized = df.assign(**{
        rescol: df[rescol] * normalization / conversion,
        unitcol: target,
    })

    return normalized

def pH2concentration(pH, *args):
    """ Converts pH values to proton concentrations in mg/L

    Parameters
    ----------
    pH : int or float
        Number between 0 and 14 representing the pH

    Returns
    -------
    conc : float
        Concentration of protons in mg/L

    """

    # check that we recieved a valid input:
    if pH < 0 or pH > 14:
        raise ValueError('pH = %f but must be between 0 and 14' % pH)

    # avogadro's number (items/mole)
    avogadro = 6.0221413e+23

    # mass of a proton (kg)
    proton_mass = 1.672621777e-27

    # grams per kilogram
    kg2g = 1000

    # milligrams per gram
    g2mg = 1000

    return 10 ** (-1 * pH) * avogadro * proton_mass * kg2g * g2mg

def fit_line(x, y, xhat=None, fitprobs=None, fitlogs=None, dist=None, through_origin=False):
    """ Fits a line to x-y data in various forms (raw, log, prob scales)

    Parameters
    ----------
    x, y : array-like
        Independent and dependent data, respectively.
    xhat : array-like or None, optional
        The values at which yhat should should be estimated. If
        not provided, falls back to the sorted values of ``x``.
    fitprobs, fitlogs : str, options.
        Defines how data should be transformed. Valid values are
        'x', 'y', or 'both'. If using ``fitprobs``, variables should
        be expressed as a percentage, i.e.,
        Probablility transform = lambda x: ``dist``.ppf(x / 100.).
        Log transform = lambda x: numpy.log(x).
        Take care to not pass the same value to both ``fitlogs`` and
        ``figprobs`` as both transforms will be applied.
    dist : scipy.stats distribution or None, optional
        A fully-spec'd scipy.stats distribution such that ``dist.ppf``
        can be called. If not provided, defaults to scipt.stats.norm.
    through_origin : bool, optional (default = False)
        If True, the y-intercept of the best-fit line will be
        constrained to 0 during the regression.

    Returns
    -------
    xhat, yhat : numpy arrays
        Linear model estimates of ``x`` and ``y``.
    results : a statmodels result object
        The object returned by statsmodels.OLS.fit()

    """

    fitprobs = validate.fit_arguments(fitprobs, "fitprobs")
    fitlogs = validate.fit_arguments(fitlogs, "fitlogs")

    x = numpy.asarray(x)
    y = numpy.asarray(y)

    if xhat is None:
        xhat = numpy.array([numpy.min(x), numpy.max(x)])

    if dist is None:
        dist = stats.norm

    if fitprobs in ['x', 'both']:
        x = dist.ppf(x / 100.)
        xhat = dist.ppf(numpy.array(xhat) / 100.)

    if fitprobs in ['y', 'both']:
        y = dist.ppf(y / 100.)

    if fitlogs in ['x', 'both']:
        x = numpy.log(x)
    if fitlogs in ['y', 'both']:
        y = numpy.log(y)

    x = sm.add_constant(x)

    # fix the y-intercept at 0 if requested
    # note that this is a nuance of the statsmodels package.
    # `x[:, 0] = 5` doesn't fix the intercept at 5, for instance.
    if through_origin:
        x[:, 0] = 0

    results = sm.OLS(y, x).fit()
    yhat = estimateFromLineParams(xhat, results.params[1], results.params[0],
                                  xlog=fitlogs in ['x', 'both'],
                                  ylog=fitlogs in ['y', 'both'])

    if fitprobs in ['y', 'both']:
        yhat = 100. * dist.cdf(yhat)
    if fitprobs in ['x', 'both']:
        xhat = 100. * dist.cdf(xhat)

    return xhat, yhat, results

def estimateFromLineParams(xdata, slope, intercept, xlog=False, ylog=False):
    """ Estimate the dependent of a linear fit given x-data and linear
    parameters.

    Parameters
    ----------
    xdata : numpy array or pandas Series/DataFrame
        The input independent variable of the fit
    slope : float
        Slope of the best-fit line
    intercept : float
        y-intercept of the best-fit line
    xlog, ylog : bool (default = False)
        Toggles whether or not the logs of the x- or y- data should be
        used to perform the regression.

    Returns
    -------
    yhat : same type as xdata
        Estimate of the dependent variable.

    """

    x = numpy.array(xdata)
    if ylog:
        if xlog:
            yhat = numpy.exp(intercept) * x ** slope
        else:
            yhat = numpy.exp(intercept) * numpy.exp(slope) ** x

    else:
        if xlog:
            yhat = slope * numpy.log(x) + intercept

        else:
            yhat = slope * x + intercept

    return yhat

def checkIntervalOverlap(interval1, interval2, oneway=False):
    """ Checks if two numeric intervals overlaps.

    Parameters
    ----------
    interval1, interval2 : array like
        len = 2 sequences to compare
    oneway : bool (default = False)
        If True, only checks that ``interval1`` falls at least partially
        inside ``interval2``, but not the other way around

    Returns
    -------
    overlap : bool
        True if an overlap exists. Otherwise, False.

    """

    test1 = numpy.min(interval2) <= numpy.max(interval1) <= numpy.max(interval2)
    test2 = numpy.min(interval2) <= numpy.min(interval1) <= numpy.max(interval2)

    # TODO: try test1 or test2 or (not oneway and test3)

    if oneway:
        return test1 or test2
    else:
        test3 = checkIntervalOverlap(interval2, interval1, oneway=True)
        return test1 or test2 or test3

def winsorize_dataframe(dataframe, **limits):
    """ Winsorizes columns in a dataframe

    Parameters
    ----------
    dataframe : pandas.DataFrame
        The data to be modified (a copy is created).
    **limits : optional kwargs (floats, or two-tuples of floats)
        Optional key-value pairs of column names and (two-tuples of)
        floats that are the windsor limits to be applied to the
        respective column. Values should be between 0 and 1.

    Returns
    -------
    winsored_df : pandas.DataFrame
        The modified dataframe.

    See also
    --------
    scipy.stats.mstats.winsorize

    """
    df = dataframe.copy()
    for colname, limit in limits.items():
        df[colname] = stats.mstats.winsorize(df[colname], limits=limit)

    return df
<|endoftext|>


<commit_before>import requests
import rest
import urllib

class ConjuctiveQueryProcessor(object):
	def __init__(self,request,project_name,config_fields,project_root_name,es):
		self.myargs = request.args
		self.field_names = self.myargs.get("_fields",None)
		self.num_results =  self.myargs.get("_size",20)
		self.ordering = self.myargs.get("_order-by",None)
		self.page = self.myargs.get("_page",0)
		self.verbosity = self.myargs.get("_verbosity","full")
		self.response_format = self.myargs.get("_format","json")
		self.config_fields = config_fields
		self.es = es
		self.project_root_name = project_root_name
		self.project_name = project_name
		self.nested_query = self.myargs.get("_dereference",None)
		self.KG_PREFIX = 'knowledge_graph'
		self.SOURCE = '_source'
		self.stats = self.myargs.get("_statistics","yes")

	def process(self):
		'''
		This is the main function in this class. This calls several functions to validate input, set match clauses, set filter clauses
		set sort clauses and finally resolve any nested documents if needed. Finally this function returns the data as a json or json_lines
		joined by a '\n'
		'''
		valid_input = self.validate_input()
		if not valid_input:
			err_json = {}
			err_json['message'] = "Please enter valid query params. Fields must exist for the given project. If not sure, please access http://mydigurl/projects/<project_name>/fields API for reference"
			return rest.bad_request(err_json)
		query = self._build_query("must")
		res = self.es.search(self.project_name, self.project_root_name ,query, ignore_no_index=True)
		res_filtered = self.filter_response(res,self.field_names)
		resp={}
		if self.nested_query is not None and len(res_filtered['hits']['hits']) > 0:
			res_filtered = self.setNestedDocuments(res_filtered)
		if self.response_format =="json_lines":
			return rest.ok('\n'.join(str(x) for x in res_filtered['hits']['hits']))
		else:
			if self.verbosity == "minimal":
				if self.field_names is None:
					self.field_names = ','.join(self.config_fields)
				response_docs = self.minify_response(res_filtered,self.field_names)
				resp['hits']={}
				resp['hits']['hits'] = response_docs
				if not self.stats == "no":
					resp['hit_count'] = res_filtered['hits']['total']
					resp['execution_time'] = res_filtered['took']
			else:
				resp = res_filtered

		return rest.ok(resp)

	def setNestedDocuments(self,resp):
		'''
		    This function is to query for the nested documents and retrieve them
		    It attaches only the KG component of the nested document into the parent document. Expects a json response,
		    returns a json resposne after dereferencing the required set of child documents. Can dereference multiple children
		    documents.
		    See readme for exact usage and details.
		'''
		list_of_fields = self.nested_query.split(',')
		ids_to_query = {}
		for field in list_of_fields:
			ids_to_query[field] = []
		for json_doc in resp['hits']['hits']:
			for field in list_of_fields:
				try:
					temp_id = json_doc[self.SOURCE][self.KG_PREFIX][field][0]['value']
					if temp_id not in ids_to_query[field]:
						ids_to_query[field].append(temp_id)
				except Exception as e:
					print e
					pass
		result_map = self.executeNestedQuery(ids_to_query)
		for json_doc in resp['hits']['hits']:
			for field in list_of_fields:
				try:
					temp_id = json_doc[self.SOURCE][self.KG_PREFIX][field][0]['value']
					if temp_id in result_map:
						new_list = []
						new_doc = json_doc[self.SOURCE][self.KG_PREFIX][field][0]
						new_doc['knowledge_graph'] = result_map[temp_id][self.SOURCE][self.KG_PREFIX]
						new_list.append(new_doc)
						json_doc[self.SOURCE][self.KG_PREFIX][field]= new_list
				except Exception as e:
					print e
					pass
		return resp

	def executeNestedQuery(self,idMap):
		'''
			A function to perform the mget search and retrieve the nested documents
		'''
		result_map = {}
		for key in idMap:
			query = {
    		"ids" : idMap[key]
			}
			resp = self.es.mget(index=self.project_name,body=query,doc_type=self.project_root_name)
			if len(resp['docs']) > 0:
				for json_doc in resp['docs']:
					result_map[json_doc['_source']['document_id']] = json_doc
		return result_map

	def validate_input(self):
		for key in self.myargs.keys():
			if not key.startswith("_"):
				if "/" in key:
					if key.split('/')[0] not in self.config_fields:
						return False
				elif "$" in key:
					if key.split('$')[0] not in self.config_fields:
						return False
				elif "." in key:
					continue
				elif key not in self.config_fields:
					return False

		return True

	def minify_response(self,response,myargs):
		'''
			This function takes in the response in json format and minimizes the KG Component down to simple key,value pairs
			Also minimizes the KG component inside a particular nested field as well.
		'''
		fields = myargs.split(',')
		docs = response['hits']['hits']
		minified_docs = []
		for json_doc in docs:
			minidoc = {}
			for field in fields:
				try:
					new_list = []
					new_json = {}
					if 'data' in json_doc[self.SOURCE][self.KG_PREFIX][field][0].keys():
						new_json['value'] = json_doc[self.SOURCE][self.KG_PREFIX][field][0]['data']
					else:
						new_json['value'] = json_doc[self.SOURCE][self.KG_PREFIX][field][0]['value']
					if self.KG_PREFIX in json_doc[self.SOURCE][self.KG_PREFIX][field][0].keys():
						nested_kg = json_doc[self.SOURCE][self.KG_PREFIX][field][0][self.KG_PREFIX]
						min_nested_kg = {}
						for inner_field in nested_kg.keys():
							nest_list = []
							nest_json = {}
							if 'data' in nested_kg[inner_field][0].keys():
								nest_json['value'] = nested_kg[inner_field][0]['data']
							else:
								nest_json['value'] = nested_kg[inner_field][0]['value']
							nest_list.append(nest_json)
							nested_kg[inner_field] = nest_list
						new_json[self.KG_PREFIX] = nested_kg
					new_list.append(new_json)
					minidoc[field] = new_list
				except Exception as e:
					pass
			minified_docs.append(minidoc)
		return minified_docs

	def generate_match_clause(self,term,args):
		'''
			This function generates match clauses which are inserted into the must part of the conjuctive query
		'''
		extraction = dict()
		must_clause = dict()
		if "/" in term:
			extraction['field_name'],rest = term.split('/')
			extraction['valueorkey'] = rest
		elif "." in term:
			extraction['field_name'],rest = term.split('.')
			extraction['valueorkey'] = rest
		else:
			extraction['field_name'] = term
			extraction['valueorkey'] = "value"
		if "/" in term:
			must_clause = {
				"match": {
					"knowledge_graph." + extraction['field_name'] + "." + extraction['valueorkey']: urllib.unquote(args[term])
				}
			}
		elif "." in term:
			must_clause = {
				"match": {
					"knowledge_graph." + extraction['field_name'] + "__" + extraction['valueorkey']: urllib.unquote(args[term])
				}
			}
		return must_clause

	def get_sort_order(self):
		'''
		This function generates order clauses for the query. It uses the member variables set in class to generate
		any order by clauses if needed. It looks for fields like field_name$desc or field_name$asc.
		'''
		sort_clauses = []
		field_prefix = "knowledge_graph."
		field_suffix = ".value"
		for order in self.ordering.split(','):
			order_key,order_val = order.split('$')
			sort_clause =  { field_prefix + order_key + field_suffix : {"order" : order_val}}
			sort_clauses.append(sort_clause)
		return sort_clauses

	def generate_range_clause(self,term,args):
		'''
		This function converts filter operators such as field.name$desc into a sort clause,
		which can be used in our Elastic search query
		@input a single term of the form field.value$asc or field.value$desc and set of all input argument mappings.
		@output a filter_clause with the field.value and filter type.

		'''
		conversions = { "less-than": "lt", "less-equal-than" : "lte", "greater-than": "gt","greater-equal-than": "gte"}
		extracted_term = term.split('$')
		range_clause = {
					"range" : {
           				 "knowledge_graph."+extracted_term[0]+".value" : {
               					 conversions[extracted_term[1]] : args[term]
            	}
        	}
        }
		return range_clause

	def _build_query(self,querytype):
		"""
		Builds an ElasticSearch query from a simple spec of DIG fields and constraints.
		@param field_query_terms: List of field:value pairs.
		The field can end with "/value" or "/key" to specify where to do the term query.
		@type field_query_terms:
		@return: JSON object with a bool term query in the ElasticSearch DSL.
		@rtype: dict
		"""
		self.num_results = int(self.num_results)
		self.page = int(self.page)
		clause_list = []
		for query_term in self.myargs:
			if not query_term.startswith("_") and "$" not in query_term:
				clause_list.append(self.generate_match_clause(query_term,self.myargs))
			elif not query_term.startswith("_"):
				clause_list.append(self.generate_range_clause(query_term,self.myargs))
		full_query = {
			"query": {
				"bool": {
					querytype: clause_list
				}
			}
		}
		full_query['size'] = self.num_results
		full_query['from'] = self.page*self.num_results
		if self.ordering is not None:
			full_query['sort'] = self.get_sort_order()
		print full_query
		return full_query

	def filter_response(self,resp,fields):
		'''
		This function takes a response from elasticSearch and keeps only the specified fields in the 'knowledge_graph' component
		@input : response from ES and fields specified in query
		@output : filter fields in _source.knowledge_graph basis fields specified by user
		'''
		if fields is None:
			return resp
		else:
			docs = resp['hits']['hits']
			for json_doc in docs:
				for field in json_doc['_source']['knowledge_graph'].keys():
					if field not in fields:
						del json_doc['_source']['knowledge_graph'][field]
			resp['hits']['hits'] = docs
			return resp
			<commit_msg>changed resp structure to be consistent<commit_after>import requests
import rest
import urllib
import traceback,sys

class ConjuctiveQueryProcessor(object):
    def __init__(self,request,project_name,config_fields,project_root_name,es):
        self.myargs = request.args
        self.field_names = self.myargs.get("_fields",None)
        self.num_results =  self.myargs.get("_size",20)
        self.ordering = self.myargs.get("_order-by",None)
        self.page = self.myargs.get("_page",0)
        self.verbosity = self.myargs.get("_verbosity","full")
        self.response_format = self.myargs.get("_format","json")
        self.config_fields = config_fields
        self.es = es
        self.project_root_name = project_root_name
        self.project_name = project_name
        self.nested_query = self.myargs.get("_dereference",None)
        self.KG_PREFIX = 'knowledge_graph'
        self.SOURCE = '_source'
        self.stats = self.myargs.get("_statistics","yes")

    def process(self):
        '''
        This is the main function in this class. This calls several functions to validate input, set match clauses, set filter clauses
        set sort clauses and finally resolve any nested documents if needed. Finally this function returns the data as a json or json_lines
        joined by a '\n'
        '''
        valid_input = self.validate_input()
        if not valid_input:
            err_json = {}
            err_json['message'] = "Please enter valid query params. Fields must exist for the given project. If not sure, please access http://mydigurl/projects/<project_name>/fields API for reference"
            return rest.bad_request(err_json)
        query = self._build_query("must")
        res = self.es.search(self.project_name, self.project_root_name ,query, ignore_no_index=True)
        res_filtered = self.filter_response(res,self.field_names)
        resp={}
        if self.nested_query is not None and len(res_filtered['hits']['hits']) > 0:
            res_filtered = self.setNestedDocuments(res_filtered)
        if self.response_format =="json_lines":
            return rest.ok('\n'.join(str(x) for x in res_filtered['hits']['hits']))
        else:
            if self.verbosity == "minimal":
                if self.field_names is None:
                    self.field_names = ','.join(self.config_fields)
                response_docs = self.minify_response(res_filtered,self.field_names)
                resp['hits']={}
                resp['hits']['hits'] = response_docs
                if not self.stats == "no":
                    resp['hit_count'] = res_filtered['hits']['total']
                    resp['execution_time'] = res_filtered['took']
            else:
                resp = res_filtered

        return rest.ok(resp)

    def setNestedDocuments(self,resp):
        '''
            This function is to query for the nested documents and retrieve them
            It attaches only the KG component of the nested document into the parent document. Expects a json response,
            returns a json resposne after dereferencing the required set of child documents. Can dereference multiple children
            documents.
            See readme for exact usage and details.
        '''
        list_of_fields = self.nested_query.split(',')
        ids_to_query = {}
        for field in list_of_fields:
            ids_to_query[field] = []
        for json_doc in resp['hits']['hits']:
            for field in list_of_fields:
                try:
                    temp_id = json_doc[self.SOURCE][self.KG_PREFIX][field][0]['value']
                    if temp_id not in ids_to_query[field]:
                        ids_to_query[field].append(temp_id)
                except Exception as e:
                    print e
                    pass
        result_map = self.executeNestedQuery(ids_to_query)
        for json_doc in resp['hits']['hits']:
            for field in list_of_fields:
                try:
                    temp_id = json_doc[self.SOURCE][self.KG_PREFIX][field][0]['value']
                    if temp_id in result_map:
                        new_list = []
                        new_doc = json_doc[self.SOURCE][self.KG_PREFIX][field][0]
                        new_doc['knowledge_graph'] = result_map[temp_id][self.SOURCE][self.KG_PREFIX]
                        new_list.append(new_doc)
                        json_doc[self.SOURCE][self.KG_PREFIX][field]= new_list
                except Exception as e:
                    print e
                    pass
        return resp

    def executeNestedQuery(self,idMap):
        '''
            A function to perform the mget search and retrieve the nested documents
        '''
        result_map = {}
        for key in idMap:
            query = {
            "ids" : idMap[key]
            }
            resp = self.es.mget(index=self.project_name,body=query,doc_type=self.project_root_name)
            if len(resp['docs']) > 0:
                for json_doc in resp['docs']:
                    result_map[json_doc['_source']['document_id']] = json_doc
        return result_map

    def validate_input(self):
        for key in self.myargs.keys():
            if not key.startswith("_"):
                if "/" in key:
                    if key.split('/')[0] not in self.config_fields:
                        return False
                elif "$" in key:
                    if key.split('$')[0] not in self.config_fields:
                        return False
                elif "." in key:
                    continue
                elif key not in self.config_fields:
                    return False

        return True

    def minify_response(self,response,myargs):
        '''
            This function takes in the response in json format and minimizes the KG Component down to simple key,value pairs
            Also minimizes the KG component inside a particular nested field as well.
        '''
        fields = myargs.split(',')
        docs = response['hits']['hits']
        minified_docs = []
        for json_doc in docs:
            minidoc = {}
            minidoc[self.SOURCE] = {}
            minidoc[self.SOURCE][self.KG_PREFIX] = {}
            for field in fields:
                try:
                    new_list = []
                    new_json = {}
                    print json_doc
                    print json_doc[self.SOURCE]
                    print json_doc[self.SOURCE][self.KG_PREFIX]
                    print json_doc[self.SOURCE][self.KG_PREFIX][field]
                    print json_doc[self.SOURCE][self.KG_PREFIX][field][0]
                    print json_doc[self.SOURCE][self.KG_PREFIX][field][0].keys()
                    if 'data' in json_doc[self.SOURCE][self.KG_PREFIX][field][0].keys():
                        new_json['value'] = json_doc[self.SOURCE][self.KG_PREFIX][field][0]['data']
                    else:
                        new_json['value'] = json_doc[self.SOURCE][self.KG_PREFIX][field][0]['value']
                    if self.KG_PREFIX in json_doc[self.SOURCE][self.KG_PREFIX][field][0].keys():
                        nested_kg = json_doc[self.SOURCE][self.KG_PREFIX][field][0][self.KG_PREFIX]
                        min_nested_kg = {}
                        for inner_field in nested_kg.keys():
                            nest_list = []
                            nest_json = {}
                            if 'data' in nested_kg[inner_field][0].keys():
                                nest_json['value'] = nested_kg[inner_field][0]['data']
                            else:
                                nest_json['value'] = nested_kg[inner_field][0]['value']
                            nest_list.append(nest_json)
                            nested_kg[inner_field] = nest_list
                        new_json[self.KG_PREFIX] = nested_kg
                    new_list.append(new_json)
                    minidoc[self.SOURCE][self.KG_PREFIX][field] = new_list
                except Exception as e:
                    pass
            minified_docs.append(minidoc)
        return minified_docs

    def generate_match_clause(self,term,args):
        '''
            This function generates match clauses which are inserted into the must part of the conjuctive query
        '''
        extraction = dict()
        must_clause = dict()
        if "/" in term:
            extraction['field_name'],rest = term.split('/')
            extraction['valueorkey'] = rest
        elif "." in term:
            extraction['field_name'],rest = term.split('.')
            extraction['valueorkey'] = rest
        else:
            extraction['field_name'] = term
            extraction['valueorkey'] = "value"
        if "/" in term:
            must_clause = {
                "match": {
                    "knowledge_graph." + extraction['field_name'] + "." + extraction['valueorkey']: urllib.unquote(args[term])
                }
            }
        elif "." in term:
            must_clause = {
                "match": {
                    "knowledge_graph." + extraction['field_name'] + "__" + extraction['valueorkey']: urllib.unquote(args[term])
                }
            }
        return must_clause

    def get_sort_order(self):
        '''
        This function generates order clauses for the query. It uses the member variables set in class to generate
        any order by clauses if needed. It looks for fields like field_name$desc or field_name$asc.
        '''
        sort_clauses = []
        field_prefix = "knowledge_graph."
        field_suffix = ".value"
        for order in self.ordering.split(','):
            order_key,order_val = order.split('$')
            sort_clause =  { field_prefix + order_key + field_suffix : {"order" : order_val}}
            sort_clauses.append(sort_clause)
        return sort_clauses

    def generate_range_clause(self,term,args):
        '''
        This function converts filter operators such as field.name$desc into a sort clause,
        which can be used in our Elastic search query
        @input a single term of the form field.value$asc or field.value$desc and set of all input argument mappings.
        @output a filter_clause with the field.value and filter type.

        '''
        conversions = { "less-than": "lt", "less-equal-than" : "lte", "greater-than": "gt","greater-equal-than": "gte"}
        extracted_term = term.split('$')
        range_clause = {
                    "range" : {
                         "knowledge_graph."+extracted_term[0]+".value" : {
                                 conversions[extracted_term[1]] : args[term]
                }
            }
        }
        return range_clause

    def _build_query(self,querytype):
        """
        Builds an ElasticSearch query from a simple spec of DIG fields and constraints.
        @param field_query_terms: List of field:value pairs.
        The field can end with "/value" or "/key" to specify where to do the term query.
        @type field_query_terms:
        @return: JSON object with a bool term query in the ElasticSearch DSL.
        @rtype: dict
        """
        self.num_results = int(self.num_results)
        self.page = int(self.page)
        clause_list = []
        for query_term in self.myargs:
            if not query_term.startswith("_") and "$" not in query_term:
                clause_list.append(self.generate_match_clause(query_term,self.myargs))
            elif not query_term.startswith("_"):
                clause_list.append(self.generate_range_clause(query_term,self.myargs))
        full_query = {
            "query": {
                "bool": {
                    querytype: clause_list
                }
            }
        }
        full_query['size'] = self.num_results
        full_query['from'] = self.page*self.num_results
        if self.ordering is not None:
            full_query['sort'] = self.get_sort_order()
        print full_query
        return full_query

    def filter_response(self,resp,fields):
        '''
        This function takes a response from elasticSearch and keeps only the specified fields in the 'knowledge_graph' component
        @input : response from ES and fields specified in query
        @output : filter fields in _source.knowledge_graph basis fields specified by user
        '''
        if fields is None:
            return resp
        else:
            docs = resp['hits']['hits']
            for json_doc in docs:
                for field in json_doc['_source']['knowledge_graph'].keys():
                    if field not in fields:
                        del json_doc['_source']['knowledge_graph'][field]
            resp['hits']['hits'] = docs
            return resp
            <|endoftext|>


<commit_before>import os
import re
import sys
import time
import errno
from subprocess import Popen, PIPE

from communicate import safe_communicate, OutputLimitExceeded
from error import CompileError
from .utils import test_executor
from .resource_proxy import ResourceProxy
from judgeenv import env

try:
    from winutils import max_memory, execution_time
except ImportError:
    windows = False
    max_memory, execution_time = None, None
else:
    windows = True

recomment = re.compile(r'/\*.*?\*/', re.DOTALL)
restring = re.compile(r''''(?:\\.|[^'\\])'|"(?:\\.|[^"\\])*"''', re.DOTALL)
reinline_comment = re.compile('//.*$', re.MULTILINE)
reclass = re.compile(r'\bpublic\s+class\s+([_a-zA-Z\$][_0-9a-zA-z\$]*?)\b')
repackage = re.compile(r'\bpackage\s+([^.;]+(?:\.[^.;]+)*?);')
redeunicode = re.compile(r'\\u([0-9a-f]{4})', re.I)
deunicode = lambda x: redeunicode.sub(lambda a: unichr(int(a.group(1), 16), x))
JAVA_EXECUTOR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'java_executor.jar'))

def find_class(source):
    source = reinline_comment.sub('', restring.sub('', recomment.sub('', source)))
    class_name = reclass.search(source)
    if class_name is None:
        raise CompileError('No public class: your main class must be declared as a "public class"')
    package = repackage.search(source)
    if package:
        raise CompileError('Invalid package %s: do not declare package' % package.group(1))
    return class_name

class JavaPopen(object):
    def __init__(self, args, executable, cwd, time_limit, memory_limit, statefile):
        self.process = Popen(args, executable=executable, cwd=cwd,
                             stdin=PIPE, stdout=PIPE, stderr=PIPE)
        self.execution_time, self.tle = None, None
        self.max_memory, self.mle = None, None
        self.stderr = None
        self.error_info, self.error = None, None
        self.returncode = None
        self.feedback = None
        self.time_limit = time_limit
        self.memory_limit = memory_limit
        self.statefile = statefile
        self._killed = False
        self._start = time.time()

    def communicate(self, stdin=None):
        return self._communicate(*self.process.communicate(stdin))

    def _update_windows_stats(self):
        handle = int(self.process._handle)
        self.execution_time = execution_time(handle)
        self.tie = self.execution_time > self.time_limit
        self.max_memory = max_memory(handle) / 1024.
        self.mle = self.max_memory > self.memory_limit
        if self.returncode is None:
            self.returncode = self.process.returncode

    def _fallback_unix_stats(self):
        self.execution_time = time.time() - self._start
        self.tle = self.execution_time > self.time_limit
        self.max_memory = 0
        self.mle = 0
        self.returncode = -1

    def safe_communicate(self, stdin=None, outlimit=None, errlimit=None):
        try:
            return self._communicate(*safe_communicate(self.process, stdin, outlimit, errlimit))
        except OutputLimitExceeded:
            if windows:
                self._update_windows_stats()
            else:
                self._fallback_unix_stats()
            raise

    def _communicate(self, stdout, stderr):
        try:
            with open(self.statefile, 'r') as proc:
                self.error_info = proc.read().strip()
        except:
            return stdout, stderr
        try:
            data = self.error_info.split(None, 5)
            self.execution_time, self.tle, self.max_memory, self.mle, self.returncode = map(int, data[:5])
            self.feedback = data[5]
        except:
            if not windows:
                print>> sys.stderr, self.error_info
                self._fallback_unix_stats()
                return stdout, stderr
        if windows:
            self._update_windows_stats()
        else:
            self.execution_time /= 1000.0
        if self.returncode == -1:
            self.returncode = 1
        if self.feedback == 'OK':
            self.feedback = None
        return stdout, stderr

    def kill(self):
        self._killed = True
        return self.process.kill()

    @property
    def r_execution_time(self):
        return self.execution_time

    def __getattr__(self, attr):
        return getattr(self.process, attr)

class Executor(ResourceProxy):
    JAVA = 'java'
    JAVAC = 'javac'
    name = 'JAVA'
    test_program = '''\
public class self_test {
    public static void main(String[] args) {
        System.out.println("Hello, World!");
    }
}'''

    def __init__(self, problem_id, source_code):
        super(Executor, self).__init__()
        source_code = deunicode(source_code)
        class_name = find_class(source_code)
        source_code_file = self._file('%s.java' % class_name.group(1))
        try:
            with open(source_code_file, 'wb') as fo:
                fo.write(source_code)
        except IOError as e:
            if e.errno in (errno.ENAMETOOLONG, errno.ENOENT):
                raise CompileError('Why do you need a class name so long? '
                                   'As a judge, I sentence your code to death.')
            raise
        javac_args = [env['runtime'][self.JAVAC], source_code_file]
        javac_process = Popen(javac_args, stderr=PIPE, cwd=self._dir)
        _, compile_error = javac_process.communicate()
        if javac_process.returncode != 0:
            if 'is public, should be declared in a file named' in compile_error:
                raise CompileError('You are a troll. Trolls are not welcome. '
                                   'As a judge, I sentence your code to death.')
            raise CompileError(compile_error)
        self._class_name = class_name.group(1)
        self.warning = compile_error
        self.statefile = self._file('state')

    def launch(self, *args, **kwargs):
        return JavaPopen(['java', '-client',
                          '-Xmx%sK' % kwargs.get('memory'), '-jar', JAVA_EXECUTOR, self._dir,
                          self._class_name, str(int(kwargs.get('time') * 1000)), 'state'] + list(args),
                         executable=env['runtime'][self.JAVA], cwd=self._dir,
                         time_limit=kwargs.get('time'), memory_limit=kwargs.get('memory'), statefile=self.statefile)

    def launch_unsafe(self, *args, **kwargs):
        return Popen(['java', '-client', self._class_name] + list(args),
                     executable=env['runtime'][self.JAVA],
                     cwd=self._dir,
                     **kwargs)

    @classmethod
    def initialize(cls):
        if cls.JAVA not in env['runtime'] or cls.JAVAC not in env['runtime']:
            return False
        if not os.path.isfile(env['runtime'][cls.JAVA]) or not os.path.isfile(env['runtime'][cls.JAVAC]):
            return False
        return test_executor(cls.name, cls, cls.test_program, problem='self_test')

initialize = Executor.initialize
<commit_msg>Rematch parentheses<commit_after>import os
import re
import sys
import time
import errno
from subprocess import Popen, PIPE

from communicate import safe_communicate, OutputLimitExceeded
from error import CompileError
from .utils import test_executor
from .resource_proxy import ResourceProxy
from judgeenv import env

try:
    from winutils import max_memory, execution_time
except ImportError:
    windows = False
    max_memory, execution_time = None, None
else:
    windows = True

recomment = re.compile(r'/\*.*?\*/', re.DOTALL)
restring = re.compile(r''''(?:\\.|[^'\\])'|"(?:\\.|[^"\\])*"''', re.DOTALL)
reinline_comment = re.compile('//.*$', re.MULTILINE)
reclass = re.compile(r'\bpublic\s+class\s+([_a-zA-Z\$][_0-9a-zA-z\$]*?)\b')
repackage = re.compile(r'\bpackage\s+([^.;]+(?:\.[^.;]+)*?);')
redeunicode = re.compile(r'\\u([0-9a-f]{4})', re.I)
deunicode = lambda x: redeunicode.sub(lambda a: unichr(int(a.group(1), 16)), x)
JAVA_EXECUTOR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'java_executor.jar'))

def find_class(source):
    source = reinline_comment.sub('', restring.sub('', recomment.sub('', source)))
    class_name = reclass.search(source)
    if class_name is None:
        raise CompileError('No public class: your main class must be declared as a "public class"')
    package = repackage.search(source)
    if package:
        raise CompileError('Invalid package %s: do not declare package' % package.group(1))
    return class_name

class JavaPopen(object):
    def __init__(self, args, executable, cwd, time_limit, memory_limit, statefile):
        self.process = Popen(args, executable=executable, cwd=cwd,
                             stdin=PIPE, stdout=PIPE, stderr=PIPE)
        self.execution_time, self.tle = None, None
        self.max_memory, self.mle = None, None
        self.stderr = None
        self.error_info, self.error = None, None
        self.returncode = None
        self.feedback = None
        self.time_limit = time_limit
        self.memory_limit = memory_limit
        self.statefile = statefile
        self._killed = False
        self._start = time.time()

    def communicate(self, stdin=None):
        return self._communicate(*self.process.communicate(stdin))

    def _update_windows_stats(self):
        handle = int(self.process._handle)
        self.execution_time = execution_time(handle)
        self.tie = self.execution_time > self.time_limit
        self.max_memory = max_memory(handle) / 1024.
        self.mle = self.max_memory > self.memory_limit
        if self.returncode is None:
            self.returncode = self.process.returncode

    def _fallback_unix_stats(self):
        self.execution_time = time.time() - self._start
        self.tle = self.execution_time > self.time_limit
        self.max_memory = 0
        self.mle = 0
        self.returncode = -1

    def safe_communicate(self, stdin=None, outlimit=None, errlimit=None):
        try:
            return self._communicate(*safe_communicate(self.process, stdin, outlimit, errlimit))
        except OutputLimitExceeded:
            if windows:
                self._update_windows_stats()
            else:
                self._fallback_unix_stats()
            raise

    def _communicate(self, stdout, stderr):
        try:
            with open(self.statefile, 'r') as proc:
                self.error_info = proc.read().strip()
        except:
            return stdout, stderr
        try:
            data = self.error_info.split(None, 5)
            self.execution_time, self.tle, self.max_memory, self.mle, self.returncode = map(int, data[:5])
            self.feedback = data[5]
        except:
            if not windows:
                print>> sys.stderr, self.error_info
                self._fallback_unix_stats()
                return stdout, stderr
        if windows:
            self._update_windows_stats()
        else:
            self.execution_time /= 1000.0
        if self.returncode == -1:
            self.returncode = 1
        if self.feedback == 'OK':
            self.feedback = None
        return stdout, stderr

    def kill(self):
        self._killed = True
        return self.process.kill()

    @property
    def r_execution_time(self):
        return self.execution_time

    def __getattr__(self, attr):
        return getattr(self.process, attr)

class Executor(ResourceProxy):
    JAVA = 'java'
    JAVAC = 'javac'
    name = 'JAVA'
    test_program = '''\
public class self_test {
    public static void main(String[] args) {
        System.out.println("Hello, World!");
    }
}'''

    def __init__(self, problem_id, source_code):
        super(Executor, self).__init__()
        source_code = deunicode(source_code)
        class_name = find_class(source_code)
        source_code_file = self._file('%s.java' % class_name.group(1))
        try:
            with open(source_code_file, 'wb') as fo:
                fo.write(source_code)
        except IOError as e:
            if e.errno in (errno.ENAMETOOLONG, errno.ENOENT):
                raise CompileError('Why do you need a class name so long? '
                                   'As a judge, I sentence your code to death.')
            raise
        javac_args = [env['runtime'][self.JAVAC], source_code_file]
        javac_process = Popen(javac_args, stderr=PIPE, cwd=self._dir)
        _, compile_error = javac_process.communicate()
        if javac_process.returncode != 0:
            if 'is public, should be declared in a file named' in compile_error:
                raise CompileError('You are a troll. Trolls are not welcome. '
                                   'As a judge, I sentence your code to death.')
            raise CompileError(compile_error)
        self._class_name = class_name.group(1)
        self.warning = compile_error
        self.statefile = self._file('state')

    def launch(self, *args, **kwargs):
        return JavaPopen(['java', '-client',
                          '-Xmx%sK' % kwargs.get('memory'), '-jar', JAVA_EXECUTOR, self._dir,
                          self._class_name, str(int(kwargs.get('time') * 1000)), 'state'] + list(args),
                         executable=env['runtime'][self.JAVA], cwd=self._dir,
                         time_limit=kwargs.get('time'), memory_limit=kwargs.get('memory'), statefile=self.statefile)

    def launch_unsafe(self, *args, **kwargs):
        return Popen(['java', '-client', self._class_name] + list(args),
                     executable=env['runtime'][self.JAVA],
                     cwd=self._dir,
                     **kwargs)

    @classmethod
    def initialize(cls):
        if cls.JAVA not in env['runtime'] or cls.JAVAC not in env['runtime']:
            return False
        if not os.path.isfile(env['runtime'][cls.JAVA]) or not os.path.isfile(env['runtime'][cls.JAVAC]):
            return False
        return test_executor(cls.name, cls, cls.test_program, problem='self_test')

initialize = Executor.initialize
<|endoftext|>