<commit_before># -*- encoding: utf-8 -*-
#
#  base
#  ****
#
# Implementation of BaseHandler, the Cyclone class RequestHandler extended with our
# needings.
#
# TODO - test the prepare/POST wrapper, because has never been tested

import httplib
import types
import collections
import json
import re
import sys

from cyclone.web import RequestHandler, HTTPError, HTTPAuthenticationRequired
from cyclone import escape

from globaleaks.utils import log, MailException
from globaleaks import settings
from globaleaks.rest import errors

class BaseHandler(RequestHandler):

    @staticmethod
    def validate_python_type(value, python_type):
        """
        Return True if the python class instantiates the python_type given.
        """
        if python_type == int:
            return any((isinstance(value, python_type), isinstance(value, unicode), value is None))
        else:
            return any((isinstance(value, python_type), value is None))

    @staticmethod
    def validate_GLtype(value, gl_type):
        """
        Return True if the python class matches the given regexp.
        """
        return bool(re.match(gl_type, value))

    @staticmethod
    def validate_type(value, type):
        # if it's callable, than assumes is a primitive class
        if callable(type):
            retval = BaseHandler.validate_python_type(value, type)
            if not retval:
                log.err("-- Invalid python_type, in [%s] expected %s" % (str(value), type))
            return retval
        # value as "{foo:bar}"
        elif isinstance(type, collections.Mapping):
            retval = BaseHandler.validate_jmessage(value, type)
            if not retval:
                log.err("-- Invalid JSON/dict [%s] expected %s" % (str(value), str(type)))
            return retval
        # regexp
        elif isinstance(type, str):
            retval = BaseHandler.validate_GLtype(value, type)
            if not retval:
                log.err("-- Failed Match in regexp [%s] against %s" % (str(value), str(type) ))
            return retval
        # value as "[ type ]"
        elif isinstance(type, collections.Iterable):
            # empty list is ok
            if len(value) == 0:
                return True
            else:
                retval = all(BaseHandler.validate_type(x, type[0]) for x in value)
                if not retval:
                    log.err("-- List validation failed [%s] of %s" % (str(value), str(type)))
                return retval
        else:
            raise AssertionError

    @staticmethod
    def validate_jmessage(jmessage, message_template):
        """
        Takes a string that represents a JSON messages and checks to see if it
        conforms to the message type it is supposed to be.

        This message must be either a dict or a list. This function may be called
        recursively to validate sub-parameters that are also go GLType.

        message: the message string that should be validated

        message_type: the GLType class it should match.
        """
        valid_jmessage = {}
        for key in message_template.keys():
            if key not in jmessage:
                log.debug('key %s not in %s' % (key, jmessage))
                raise errors.InvalidInputFormat('wrong schema: missing %s' % key)
            else:
                valid_jmessage[key] = jmessage[key]

        jmessage = valid_jmessage
        del valid_jmessage

        if not all(BaseHandler.validate_type(jmessage[key], value) for key, value in
                    message_template.iteritems()):
            raise errors.InvalidInputFormat('wrong content 1')

        if not all(BaseHandler.validate_type(value, message_template[key]) for key, value in
                   jmessage.iteritems()):
            raise errors.InvalidInputFormat('wrong content 2')

        return True

    @staticmethod
    def validate_message(message, message_template):
        try:
            jmessage = json.loads(message)
        except ValueError:
            raise errors.InvalidInputFormat("Invalid JSON message")

        if BaseHandler.validate_jmessage(jmessage, message_template):
            return jmessage

    def output_stripping(self, message, message_template):
        """
        @param message: the serialized dict received
        @param message_template: the answers definition
        @return: a dict or a list without the unwanted keys
        """
        pass

    requestTypes = {}
    def prepare(self):
        """
        This method is called by cyclone, and is implemented to
        handle the POST fallback, in environment where PUT and DELETE
        method may not be used.
        """
        if self.request.method.lower() == 'post':
            try:
                wrappedMethod = self.get_argument('method')[0]
                print "[^] Forwarding", wrappedMethod, "from POST"
                if wrappedMethod.lower() == 'delete' or \
                        wrappedMethod.lower() == 'put':
                    self.request.method = wrappedMethod.upper()
            except HTTPError:
                pass

    def write_error(self, status_code, **kw):
        exception = kw.get('exception')
        if exception and hasattr(exception, 'error_code'):
            self.set_status(status_code)
            self.finish({'error_message': exception.reason,
                'error_code' : exception.error_code})
        else:
            RequestHandler.write_error(self, status_code, **kw)

    def write(self, chunk):
        """
        This is a monkey patch to RequestHandler to allow us to serialize also
        json list objects.
        """
        if isinstance(chunk, types.ListType):
            chunk = escape.json_encode(chunk)
            RequestHandler.write(self, chunk)
            self.set_header("Content-Type", "application/json")
        else:
            RequestHandler.write(self, chunk)

    def get_current_user(self):
        session_id = self.request.headers.get('X-Session')
        if not session_id:
            return None

        try:
            session = settings.sessions[session_id]
        except KeyError:
            return None
        return session

    @property
    def is_whistleblower(self):
        if not self.current_user or not self.current_user.has_key('role'):
            raise errors.NotAuthenticated

        if self.current_user['role'] == 'wb':
            return True
        else:
            return False

    @property
    def is_receiver(self):
        if not self.current_user or not self.current_user.has_key('role'):
            raise errors.NotAuthenticated

        if self.current_user['role'] == 'receiver':
            return True
        else:
            return False

    def _handle_request_exception(self, e):
        try:
            if isinstance(e.value, (HTTPError, HTTPAuthenticationRequired)):
                e = e.value
        except:
            pass

        if isinstance(e, (HTTPError, HTTPAuthenticationRequired)):
            if self.settings.get("debug") is True and e.log_message:
                format = "%d %s: " + e.log_message
                args = [e.status_code, self._request_summary()] + list(e.args)
                msg = lambda *args: format % args
                log.msg(msg(*args))
            if e.status_code not in httplib.responses:
                log.msg("Bad HTTP status code: %d" % e.status_code)
                return self.send_error(500, exception=e)
            else:
                return self.send_error(e.status_code, exception=e)
        else:
            if self.settings.get("debug") is True:
                log.msg(e)
            log.msg("Uncaught exception %s :: %r" % \
                    (self._request_summary(), self.request))
            type, value, tb = sys.exc_info()
            MailException(type, value, tb)
            return self.send_error(500, exception=e)
<commit_msg>fixed some code related to exception handling and mail notification.<commit_after># -*- encoding: utf-8 -*-
#
#  base
#  ****
#
# Implementation of BaseHandler, the Cyclone class RequestHandler extended with our
# needings.
#
# TODO - test the prepare/POST wrapper, because has never been tested

import httplib
import types
import collections
import json
import re
import sys

from cyclone.web import RequestHandler, HTTPError, HTTPAuthenticationRequired
from cyclone import escape

from globaleaks.utils import log, MailException
from globaleaks import settings
from globaleaks.rest import errors

class BaseHandler(RequestHandler):

    @staticmethod
    def validate_python_type(value, python_type):
        """
        Return True if the python class instantiates the python_type given.
        """
        if python_type == int:
            return any((isinstance(value, python_type), isinstance(value, unicode), value is None))
        else:
            return any((isinstance(value, python_type), value is None))

    @staticmethod
    def validate_GLtype(value, gl_type):
        """
        Return True if the python class matches the given regexp.
        """
        return bool(re.match(gl_type, value))

    @staticmethod
    def validate_type(value, type):
        # if it's callable, than assumes is a primitive class
        if callable(type):
            retval = BaseHandler.validate_python_type(value, type)
            if not retval:
                log.err("-- Invalid python_type, in [%s] expected %s" % (str(value), type))
            return retval
        # value as "{foo:bar}"
        elif isinstance(type, collections.Mapping):
            retval = BaseHandler.validate_jmessage(value, type)
            if not retval:
                log.err("-- Invalid JSON/dict [%s] expected %s" % (str(value), str(type)))
            return retval
        # regexp
        elif isinstance(type, str):
            retval = BaseHandler.validate_GLtype(value, type)
            if not retval:
                log.err("-- Failed Match in regexp [%s] against %s" % (str(value), str(type) ))
            return retval
        # value as "[ type ]"
        elif isinstance(type, collections.Iterable):
            # empty list is ok
            if len(value) == 0:
                return True
            else:
                retval = all(BaseHandler.validate_type(x, type[0]) for x in value)
                if not retval:
                    log.err("-- List validation failed [%s] of %s" % (str(value), str(type)))
                return retval
        else:
            raise AssertionError

    @staticmethod
    def validate_jmessage(jmessage, message_template):
        """
        Takes a string that represents a JSON messages and checks to see if it
        conforms to the message type it is supposed to be.

        This message must be either a dict or a list. This function may be called
        recursively to validate sub-parameters that are also go GLType.

        message: the message string that should be validated

        message_type: the GLType class it should match.
        """
        valid_jmessage = {}
        for key in message_template.keys():
            if key not in jmessage:
                log.debug('key %s not in %s' % (key, jmessage))
                raise errors.InvalidInputFormat('wrong schema: missing %s' % key)
            else:
                valid_jmessage[key] = jmessage[key]

        jmessage = valid_jmessage
        del valid_jmessage

        if not all(BaseHandler.validate_type(jmessage[key], value) for key, value in
                    message_template.iteritems()):
            raise errors.InvalidInputFormat('wrong content 1')

        if not all(BaseHandler.validate_type(value, message_template[key]) for key, value in
                   jmessage.iteritems()):
            raise errors.InvalidInputFormat('wrong content 2')

        return True

    @staticmethod
    def validate_message(message, message_template):
        try:
            jmessage = json.loads(message)
        except ValueError:
            raise errors.InvalidInputFormat("Invalid JSON message")

        if BaseHandler.validate_jmessage(jmessage, message_template):
            return jmessage

    def output_stripping(self, message, message_template):
        """
        @param message: the serialized dict received
        @param message_template: the answers definition
        @return: a dict or a list without the unwanted keys
        """
        pass

    requestTypes = {}
    def prepare(self):
        """
        This method is called by cyclone, and is implemented to
        handle the POST fallback, in environment where PUT and DELETE
        method may not be used.
        """
        if self.request.method.lower() == 'post':
            try:
                wrappedMethod = self.get_argument('method')[0]
                print "[^] Forwarding", wrappedMethod, "from POST"
                if wrappedMethod.lower() == 'delete' or \
                        wrappedMethod.lower() == 'put':
                    self.request.method = wrappedMethod.upper()
            except HTTPError:
                pass

    def write_error(self, status_code, **kw):
        exception = kw.get('exception')
        if exception and hasattr(exception, 'error_code'):
            self.set_status(status_code)
            self.finish({'error_message': exception.reason,
                'error_code' : exception.error_code})
        else:
            RequestHandler.write_error(self, status_code, **kw)

    def write(self, chunk):
        """
        This is a monkey patch to RequestHandler to allow us to serialize also
        json list objects.
        """
        if isinstance(chunk, types.ListType):
            chunk = escape.json_encode(chunk)
            RequestHandler.write(self, chunk)
            self.set_header("Content-Type", "application/json")
        else:
            RequestHandler.write(self, chunk)

    def get_current_user(self):
        session_id = self.request.headers.get('X-Session')
        if not session_id:
            return None

        try:
            session = settings.sessions[session_id]
        except KeyError:
            return None
        return session

    @property
    def is_whistleblower(self):
        if not self.current_user or not self.current_user.has_key('role'):
            raise errors.NotAuthenticated

        if self.current_user['role'] == 'wb':
            return True
        else:
            return False

    @property
    def is_receiver(self):
        if not self.current_user or not self.current_user.has_key('role'):
            raise errors.NotAuthenticated

        if self.current_user['role'] == 'receiver':
            return True
        else:
            return False

    def _handle_request_exception(self, e):
        # exception informations must be saved here before continue.
        type, value, tb = sys.exc_info()
        try:
            if isinstance(e.value, (HTTPError, HTTPAuthenticationRequired)):
                e = e.value
        except:
            pass

        if isinstance(e, (HTTPError, HTTPAuthenticationRequired)):
            if self.settings.get("debug") is True and e.log_message:
                format = "%d %s: " + e.log_message
                args = [e.status_code, self._request_summary()] + list(e.args)
                msg = lambda *args: format % args
                log.msg(msg(*args))
            if e.status_code not in httplib.responses:
                log.msg("Bad HTTP status code: %d" % e.status_code)
                return self.send_error(500, exception=e)
            else:
                return self.send_error(e.status_code, exception=e)
        else:
            if self.settings.get("debug") is True:
                log.msg(e)
            log.msg("Uncaught exception %s :: %r" % \
                    (self._request_summary(), self.request))
            MailException(type, value, tb)
            return self.send_error(500, exception=e)
<|endoftext|>


<commit_before># -*- coding: UTF-8
#   node
#   ****
#   :copyright: 2012 Hermes No Profit Association - GlobaLeaks Project
#   :author: Claudio Agosti <vecna@globaleaks.org>, Arturo Filastò <art@globaleaks.org>
#   :license: see LICENSE
#

from twisted.internet.defer import inlineCallbacks

from globaleaks.rest import answers
from globaleaks import models

from cyclone.web import RequestHandler, asynchronous

def create_node(context_dict):
    context = models.admin.Context()
    context.name = context_dict['name']
    context.description = context_dict['description']
    context.fields = context_dict['fields']
    context.selectable_receiver = context_dict['selectable_receiver']
    context.escalation_threshold = context_dict['escalation_threshold']
    context.languages_supported = context_dict['languages_supported']

class Node(RequestHandler):

    @asynchronous
    @inlineCallbacks
    def get(self):
        from globaleaks.messages import dummy

        context = models.admin.Context()
        context_description_dicts = yield context.list_description_dicts()
        response = {"contexts": context_description_dicts,
                    "public_statistics": dummy.base.publicStatisticsDict,
                    "node_properties": dummy.base.nodePropertiesDict,
                    "public_site": "http://www.fuffa.org/",
                    "hidden_service": "cnwoecowiecnirnio23rn23io.onion",
                    "url_schema": "/",
                    "name": "don't track us: AUTOVELOX"}
        self.write(response)
        self.finish()
<commit_msg>Add node docstring. as per #24<commit_after># -*- coding: UTF-8
#   node
#   ****
#   :copyright: 2012 Hermes No Profit Association - GlobaLeaks Project
#   :author: Claudio Agosti <vecna@globaleaks.org>, Arturo Filastò <art@globaleaks.org>
#   :license: see LICENSE
#

from twisted.internet.defer import inlineCallbacks

from globaleaks.rest import answers
from globaleaks import models

from cyclone.web import RequestHandler, asynchronous

def create_node(context_dict):
    context = models.admin.Context()
    context.name = context_dict['name']
    context.description = context_dict['description']
    context.fields = context_dict['fields']
    context.selectable_receiver = context_dict['selectable_receiver']
    context.escalation_threshold = context_dict['escalation_threshold']
    context.languages_supported = context_dict['languages_supported']

class Node(RequestHandler):
    """
    Returns information on the GlobaLeaks node. This includes submission
    paramters and how information should be presented by the client side
    application.

    Follow the resource describing Node (uniq instance, opened to all)

    """
    @asynchronous
    @inlineCallbacks
    def get(self):
        """
        Returns a json object containing all the information of the node.
        * Response:
            Status Code: 200 (OK)
            {
              'name': 'string',
              'statistics': '$nodeStatisticsDict',
              'node_properties': '$nodePropertiesDict',
              'contexts': [ '$contextDescriptionDict', { }, ],
              'description': '$localizationDict',
              'public_site': 'string',
              'hidden_service': 'string',
              'url_schema': 'string'
             }

        """
        from globaleaks.messages import dummy

        context = models.admin.Context()
        context_description_dicts = yield context.list_description_dicts()
        response = {"contexts": context_description_dicts,
                    "public_statistics": dummy.base.publicStatisticsDict,
                    "node_properties": dummy.base.nodePropertiesDict,
                    "public_site": "http://www.fuffa.org/",
                    "hidden_service": "cnwoecowiecnirnio23rn23io.onion",
                    "url_schema": "/",
                    "name": "don't track us: AUTOVELOX"}
        self.write(response)
        self.finish()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) 2014, Nicolas P. Rougier
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
# -----------------------------------------------------------------------------
"""
GL Configuration settings
"""
from glumpy import defaults

# Default configuration
__configuration__ = None

def get_default():
    """
    Return the default configuration.
    """
    global __configuration__

    if __configuration__ is None:
        __configuration__ = Configuration()

    return __configuration__

class Configuration(object):
    """
    GL Configuration settings
    """

    def __init__(self):

        self._red_size            = 8
        self._green_size          = 8
        self._blue_size           = 8
        self._alpha_size          = 8

        self._double_buffer       = defaults.double_buffer()
        self._depth_size          = defaults.depth_size()
        self._stencil_size        = defaults.stencil_size()
        self._samples             = 0

        self._stereo              = defaults.stereo()
        self._srgb                = defaults.srgb()

        self._api                 = defaults.gl_api()
        self._major_version       = defaults.gl_major_version()
        self._minor_version       = defaults.gl_minor_version()
        self._profile             = defaults.gl_profile()

    # ---------------------------------------------------------------- repr ---
    def __repr__(self):
        """
        Configurarion printable description
        """

        s = ""
        s += "Color buffer size:     %d bit(s) (R:%d, G:%d, B:%d, A:%d)\n" % (
            self._red_size + self._green_size + self._blue_size + self._alpha_size,
            self._red_size,
            self._green_size,
            self._blue_size,
            self._alpha_size)
        s += "Depth buffer size:     %d bit(s)\n" % (self._depth_size)
        s += "Stencil buffer size:   %d bit(s)\n" % (self._stencil_size)
        s += "Double buffered:       %d\n" % (self._double_buffer)
        s += "Stereo mode:           %d\n" % (self._stereo)
        s += "sRGB mode:             %d\n" % (self._srgb)
        s += "Anti-aliasing samples: %d\n" % (self._samples)
        s += "GL API:                %s\n" % (self._api)
        s += "GL Version:            %d.%d\n" % (self._major_version,
                                                 self._minor_version)
        s += "GL Profile:            %s" % (self._profile)
        return s

    # ------------------------------------------------------------ red size ---
    @property
    def red_size(self):
        """
        Minimum number of bits for the red channel of the color buffer.
        """
        return self._red_size

    @red_size.setter
    def red_size(self, value):
        self._red_size = value

    # ---------------------------------------------------------- green size ---
    @property
    def green_size(self):
        """
        Minimum number of bits for the blue channel of the color buffer.
        """
        return self._green_size

    @green_size.setter
    def green_size(self, value):
        self._green_size = value

    # ----------------------------------------------------------- blue size ---
    @property
    def blue_size(self):
        """
        Minimum number of bits for the green channel of the color buffer.
        """
        return self._blue_size

    @blue_size.setter
    def blue_size(self, value):
        self._blue_size = value

    # ---------------------------------------------------------- alpha size ---
    @property
    def alpha_size(self):
        """
        Minimum number of bits for the alpha channel of the color buffer.
        """
        return self._alpha_size

    @alpha_size.setter
    def alpha_size(self, value):
        self._alpha_size = value

    # ------------------------------------------------------- double buffer ---
    @property
    def double_buffer(self):
        """
        Whether to use single or double buffered rendering.
        """
        return self._double_buffer

    @double_buffer.setter
    def double_buffer(self, value):
        self._double_buffer = value

    # ---------------------------------------------------------- depth size ---
    @property
    def depth_size(self):
        """
        Minimum number of bits in the depth buffer.
        """
        return self._depth_size

    @depth_size.setter
    def depth_size(self, value):
        self._depth_size = value

    # -------------------------------------------------------- stencil size ---
    @property
    def stencil_size(self):
        """
        Minimum number of bits in the stencil buffer.
        """
        return self._stencil_size

    @stencil_size.setter
    def stencil_size(self, value):
        self._stencil_size = value

    # -------------------------------------------------------------- stereo ---
    @property
    def stereo(self):
        """
        Whether the output is stereo.
        """
        return self._stereo

    @stereo.setter
    def stereo(self, value):
        self._stereo = value

    # ------------------------------------------------------------- samples ---
    @property
    def samples(self):
        """
        Number of samples used around the current pixel for multisample
        anti-aliasing
        """
        return self._samples

    @samples.setter
    def samples(self, value):
        self._samples = value

    # ----------------------------------------------------------------- api ---
    @property
    def api(self):
        """
        OpenGL API
        """
        return self._api

    @api.setter
    def api(self, value):
        self._api = value

    # ------------------------------------------------------- major version ---
    @property
    def major_version(self):
        """
        OpenGL context major version
        """
        return self._major_version

    @major_version.setter
    def major_version(self, value):
        self._major_version = value

    # ------------------------------------------------------- minor version ---
    @property
    def minor_version(self):
        """
        OpenGL context minor version
        """
        return self._minor_version

    @minor_version.setter
    def minor_version(self, value):
        self._minor_version = value

    # ------------------------------------------------------------- profile ---
    @property
    def profile(self):
        """
        OpenGL profile
        """
        return self._profile

    @profile.setter
    def profile(self, value):
        self._profile = value

    # ---------------------------------------------------------------- srgb ---
    @property
    def srgb(self):
        """
        Whether to user sRGB capable visuals.
        """
        return self._srgb

    @srgb.setter
    def srgb(self, value):
        self._srgb = value

# ---------------------------------------------------- gl_get_configuration ---
def gl_get_configuration():
    """
    Read gl configuration independently of backends.
    """

    import ctypes
    import OpenGL.GL as gl

    configuration =  Configuration()
    gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, 0)
    value = ctypes.c_int()
    gl.glGetFramebufferAttachmentParameteriv(
        gl.GL_FRAMEBUFFER, gl.GL_FRONT_LEFT,
        gl.GL_FRAMEBUFFER_ATTACHMENT_RED_SIZE, value )
    configuration._red_size = value.value

    gl.glGetFramebufferAttachmentParameteriv(
        gl.GL_FRAMEBUFFER, gl.GL_FRONT_LEFT,
        gl.GL_FRAMEBUFFER_ATTACHMENT_GREEN_SIZE, value )
    configuration._green_size = value.value

    gl.glGetFramebufferAttachmentParameteriv(
        gl.GL_FRAMEBUFFER, gl.GL_FRONT_LEFT,
        gl.GL_FRAMEBUFFER_ATTACHMENT_BLUE_SIZE, value )
    configuration._blue_size = value.value

    gl.glGetFramebufferAttachmentParameteriv(
        gl.GL_FRAMEBUFFER, gl.GL_FRONT_LEFT,
        gl.GL_FRAMEBUFFER_ATTACHMENT_ALPHA_SIZE, value )
    configuration._alpha_size = value.value

    gl.glGetFramebufferAttachmentParameteriv(
        gl.GL_FRAMEBUFFER, gl.GL_DEPTH,
        gl.GL_FRAMEBUFFER_ATTACHMENT_DEPTH_SIZE, value )
    configuration._depth_size = value.value

    gl.glGetFramebufferAttachmentParameteriv(
        gl.GL_FRAMEBUFFER, gl.GL_STENCIL,
        gl.GL_FRAMEBUFFER_ATTACHMENT_STENCIL_SIZE, value )
    configuration._stencil_size = value.value

    gl.glGetFramebufferAttachmentParameteriv(
        gl.GL_FRAMEBUFFER, gl.GL_FRONT_LEFT,
        gl.GL_FRAMEBUFFER_ATTACHMENT_COLOR_ENCODING, value )
    if value.value == gl.GL_LINEAR:
        configuration._srgb = False
    elif value.value == gl.GL_SRGB:
        configuration._srgb = True

    configuration._stereo        = gl.glGetInteger(gl.GL_STEREO)
    configuration._double_buffer = gl.glGetInteger(gl.GL_DOUBLEBUFFER)
    configuration._samples       = gl.glGetInteger(gl.GL_SAMPLES)

    # Dumb parsing of the GL_VERSION string
    version = gl.glGetString(gl.GL_VERSION)
    version = version.split(" ")[0]
    major,minor = version.split('.')[:2]
    configuration._version = version
    configuration._major_version = int(major)
    configuration._minor_version = int(minor)
    configuration._profile = "unknown"

    return configuration
<commit_msg>Surround reading with try/except (see issue #5)<commit_after># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) 2014, Nicolas P. Rougier
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
# -----------------------------------------------------------------------------
"""
GL Configuration settings
"""
from glumpy.log import log
from glumpy import defaults

# Default configuration
__configuration__ = None

def get_default():
    """
    Return the default configuration.
    """
    global __configuration__

    if __configuration__ is None:
        __configuration__ = Configuration()

    return __configuration__

class Configuration(object):
    """
    GL Configuration settings
    """

    def __init__(self):

        self._red_size            = 8
        self._green_size          = 8
        self._blue_size           = 8
        self._alpha_size          = 8

        self._double_buffer       = defaults.double_buffer()
        self._depth_size          = defaults.depth_size()
        self._stencil_size        = defaults.stencil_size()
        self._samples             = 0

        self._stereo              = defaults.stereo()
        self._srgb                = defaults.srgb()

        self._api                 = defaults.gl_api()
        self._major_version       = defaults.gl_major_version()
        self._minor_version       = defaults.gl_minor_version()
        self._profile             = defaults.gl_profile()

    # ---------------------------------------------------------------- repr ---
    def __repr__(self):
        """
        Configurarion printable description
        """

        s = ""
        s += "Color buffer size:     %d bit(s) (R:%d, G:%d, B:%d, A:%d)\n" % (
            self._red_size + self._green_size + self._blue_size + self._alpha_size,
            self._red_size,
            self._green_size,
            self._blue_size,
            self._alpha_size)
        s += "Depth buffer size:     %d bit(s)\n" % (self._depth_size)
        s += "Stencil buffer size:   %d bit(s)\n" % (self._stencil_size)
        s += "Double buffered:       %d\n" % (self._double_buffer)
        s += "Stereo mode:           %d\n" % (self._stereo)
        s += "sRGB mode:             %d\n" % (self._srgb)
        s += "Anti-aliasing samples: %d\n" % (self._samples)
        s += "GL API:                %s\n" % (self._api)
        s += "GL Version:            %d.%d\n" % (self._major_version,
                                                 self._minor_version)
        s += "GL Profile:            %s" % (self._profile)
        return s

    # ------------------------------------------------------------ red size ---
    @property
    def red_size(self):
        """
        Minimum number of bits for the red channel of the color buffer.
        """
        return self._red_size

    @red_size.setter
    def red_size(self, value):
        self._red_size = value

    # ---------------------------------------------------------- green size ---
    @property
    def green_size(self):
        """
        Minimum number of bits for the blue channel of the color buffer.
        """
        return self._green_size

    @green_size.setter
    def green_size(self, value):
        self._green_size = value

    # ----------------------------------------------------------- blue size ---
    @property
    def blue_size(self):
        """
        Minimum number of bits for the green channel of the color buffer.
        """
        return self._blue_size

    @blue_size.setter
    def blue_size(self, value):
        self._blue_size = value

    # ---------------------------------------------------------- alpha size ---
    @property
    def alpha_size(self):
        """
        Minimum number of bits for the alpha channel of the color buffer.
        """
        return self._alpha_size

    @alpha_size.setter
    def alpha_size(self, value):
        self._alpha_size = value

    # ------------------------------------------------------- double buffer ---
    @property
    def double_buffer(self):
        """
        Whether to use single or double buffered rendering.
        """
        return self._double_buffer

    @double_buffer.setter
    def double_buffer(self, value):
        self._double_buffer = value

    # ---------------------------------------------------------- depth size ---
    @property
    def depth_size(self):
        """
        Minimum number of bits in the depth buffer.
        """
        return self._depth_size

    @depth_size.setter
    def depth_size(self, value):
        self._depth_size = value

    # -------------------------------------------------------- stencil size ---
    @property
    def stencil_size(self):
        """
        Minimum number of bits in the stencil buffer.
        """
        return self._stencil_size

    @stencil_size.setter
    def stencil_size(self, value):
        self._stencil_size = value

    # -------------------------------------------------------------- stereo ---
    @property
    def stereo(self):
        """
        Whether the output is stereo.
        """
        return self._stereo

    @stereo.setter
    def stereo(self, value):
        self._stereo = value

    # ------------------------------------------------------------- samples ---
    @property
    def samples(self):
        """
        Number of samples used around the current pixel for multisample
        anti-aliasing
        """
        return self._samples

    @samples.setter
    def samples(self, value):
        self._samples = value

    # ----------------------------------------------------------------- api ---
    @property
    def api(self):
        """
        OpenGL API
        """
        return self._api

    @api.setter
    def api(self, value):
        self._api = value

    # ------------------------------------------------------- major version ---
    @property
    def major_version(self):
        """
        OpenGL context major version
        """
        return self._major_version

    @major_version.setter
    def major_version(self, value):
        self._major_version = value

    # ------------------------------------------------------- minor version ---
    @property
    def minor_version(self):
        """
        OpenGL context minor version
        """
        return self._minor_version

    @minor_version.setter
    def minor_version(self, value):
        self._minor_version = value

    # ------------------------------------------------------------- profile ---
    @property
    def profile(self):
        """
        OpenGL profile
        """
        return self._profile

    @profile.setter
    def profile(self, value):
        self._profile = value

    # ---------------------------------------------------------------- srgb ---
    @property
    def srgb(self):
        """
        Whether to user sRGB capable visuals.
        """
        return self._srgb

    @srgb.setter
    def srgb(self, value):
        self._srgb = value

# ---------------------------------------------------- gl_get_configuration ---
def gl_get_configuration():
    """
    Read gl configuration independently of backends.
    """

    import ctypes
    import OpenGL.GL as gl

    configuration =  Configuration()
    gl.glBindFramebuffer(gl.GL_FRAMEBUFFER, 0)
    value = ctypes.c_int()

    try:
        gl.glGetFramebufferAttachmentParameteriv(
            gl.GL_FRAMEBUFFER, gl.GL_FRONT_LEFT,
            gl.GL_FRAMEBUFFER_ATTACHMENT_RED_SIZE, value )
        configuration._red_size = value.value
    except:
        log.warn("Cannot read RED channel size from the framebuffer")
        configuration._red_size = 0

    try:
        gl.glGetFramebufferAttachmentParameteriv(
            gl.GL_FRAMEBUFFER, gl.GL_FRONT_LEFT,
            gl.GL_FRAMEBUFFER_ATTACHMENT_GREEN_SIZE, value )
        configuration._green_size = value.value
    except:
        log.warn("Cannot read GREEN channel size from the framebuffer")
        configuration._green_size = 0

    try:
        gl.glGetFramebufferAttachmentParameteriv(
            gl.GL_FRAMEBUFFER, gl.GL_FRONT_LEFT,
            gl.GL_FRAMEBUFFER_ATTACHMENT_BLUE_SIZE, value )
        configuration._blue_size = value.value
    except:
        log.warn("Cannot read BLUE channel size from the framebuffer")
        configuration._blue_size = 0

    try:
        gl.glGetFramebufferAttachmentParameteriv(
            gl.GL_FRAMEBUFFER, gl.GL_FRONT_LEFT,
            gl.GL_FRAMEBUFFER_ATTACHMENT_ALPHA_SIZE, value )
        configuration._alpha_size = value.value
    except:
        log.warn("Cannot read ALPHA channel size from the framebuffer")
        configuration._alpha_size = 0

    try:
        gl.glGetFramebufferAttachmentParameteriv(
            gl.GL_FRAMEBUFFER, gl.GL_DEPTH,
            gl.GL_FRAMEBUFFER_ATTACHMENT_DEPTH_SIZE, value )
        configuration._depth_size = value.value
    except:
        log.warn("Cannot read DEPTH size from the framebuffer")
        configuration._depth_size = 0

    try:
        gl.glGetFramebufferAttachmentParameteriv(
            gl.GL_FRAMEBUFFER, gl.GL_STENCIL,
            gl.GL_FRAMEBUFFER_ATTACHMENT_STENCIL_SIZE, value )
        configuration._stencil_size = value.value
    except:
        log.warn("Cannot read STENCIL size from the framebuffer")
        configuration._stencil_size = 0

    try:
        gl.glGetFramebufferAttachmentParameteriv(
            gl.GL_FRAMEBUFFER, gl.GL_FRONT_LEFT,
            gl.GL_FRAMEBUFFER_ATTACHMENT_COLOR_ENCODING, value )
        if value.value == gl.GL_LINEAR:
            configuration._srgb = False
        elif value.value == gl.GL_SRGB:
            configuration._srgb = True
    except:
        log.warn("Cannot read sRGB value from the framebuffer")
        configuration._srgb = False

    configuration._stereo        = gl.glGetInteger(gl.GL_STEREO)
    configuration._double_buffer = gl.glGetInteger(gl.GL_DOUBLEBUFFER)
    configuration._samples       = gl.glGetInteger(gl.GL_SAMPLES)

    # Dumb parsing of the GL_VERSION string
    version = gl.glGetString(gl.GL_VERSION)
    version = version.split(" ")[0]
    major,minor = version.split('.')[:2]
    configuration._version = version
    configuration._major_version = int(major)
    configuration._minor_version = int(minor)
    configuration._profile = "unknown"

    return configuration
<|endoftext|>


<commit_before>import json

from django.core.management.base import BaseCommand

from evostream import EvoStreamException

class BaseEvoStreamCommand(BaseCommand):
    requires_system_checks = False

    silent_keys = ()

    def get_results(self, *args, **options):
        raise NotImplementedError()

    def format_results(self, results, verbosity):
        if verbosity > 1 or len(self.silent_keys) == 0:
            self.stdout.write(json.dumps(results, indent=1, sort_keys=True))
        else:
            if not isinstance(results, list):
                results = [results]
            for result in results:
                for key in self.silent_keys:
                    if key in result:
                        self.stdout.write(key + ': ' + json.dumps(result[key]) + '\n')

    def handle(self, verbosity, *args, **options):
        if not isinstance(verbosity, int):
            verbosity = int(verbosity)

        try:
            results = self.get_results(*args, **options)
        except EvoStreamException as ex:
            self.stderr.write(str(ex) + '\n')
            return

        if results is None:
            self.stdout.write('No data\n')
            return

        self.format_results(results, verbosity=verbosity)
<commit_msg>code formatting<commit_after>import json

from django.core.management.base import BaseCommand

from evostream import EvoStreamException

class BaseEvoStreamCommand(BaseCommand):
    requires_system_checks = False

    silent_keys = ()

    def get_results(self, *args, **options):
        raise NotImplementedError()

    def format_results(self, results, verbosity):
        if verbosity > 1 or len(self.silent_keys) == 0:
            self.stdout.write(json.dumps(results, indent=1, sort_keys=True))
        else:
            if not isinstance(results, list):
                results = [results]

            for result in results:
                for key in self.silent_keys:
                    if key in result:
                        self.stdout.write(key + ': ' + json.dumps(result[key]) + '\n')

    def handle(self, verbosity, *args, **options):
        if not isinstance(verbosity, int):
            verbosity = int(verbosity)

        try:
            results = self.get_results(*args, **options)
        except EvoStreamException as ex:
            self.stderr.write(str(ex) + '\n')
            return

        if results is None:
            self.stdout.write('No data\n')
            return

        self.format_results(results, verbosity=verbosity)
<|endoftext|>


<commit_before># future
from __future__ import annotations

# stdlib
import secrets
from typing import Callable
from typing import List
from typing import TYPE_CHECKING
from typing import Tuple
from typing import Union

# third party
import jax
from jax import numpy as jnp
import numpy as np
from tqdm import tqdm

# relative
from ..tensor.fixed_precision_tensor import FixedPrecisionTensor
from ..tensor.lazy_repeat_array import lazyrepeatarray
from ..tensor.passthrough import PassthroughTensor  # type: ignore
from .data_subject_ledger import DataSubjectLedger
from .data_subject_ledger import RDPParams
from .data_subject_list import DataSubjectList

if TYPE_CHECKING:
    # relative
    from ..tensor.autodp.gamma_tensor import GammaTensor

# def calculate_bounds_for_mechanism(
#     value_array: np.ndarray, min_val_array: np.ndarray, max_val_array: np.ndarray
# ) -> Tuple[np.ndarray, np.ndarray]:
#     """Calculates the squared L2 norm values needed to create a Mechanism, and calculate
#     privacy budget + spend. If you calculate the privacy budget spend with the worst
#     case bound, you can show this number to the DS. If you calculate it with the
#     regular value (the value computed below when public_only = False, you cannot show
#     the privacy budget to the DS because this violates privacy."""

#     # TODO: Double check whether the iDPGaussianMechanism class squares its
#     # squared_l2_norm values!!

#     # min_val_array = min_val_array.astype(np.int64)
#     # max_val_array = max_val_array.astype(np.int64)

#     # using np.ones_like dtype=value_array.dtype because without it the output was
#     # of type "O" python object causing issues when doing operations against JAX
#     worst_case_l2_norm = np.sqrt(
#         np.sum(np.square(max_val_array - min_val_array))
#     ) * np.ones_like(
#         value_array
#     )  # dtype=value_array.dtype)

#     l2_norm = np.sqrt(np.sum(np.square(value_array))) * np.ones_like(value_array)
#     # dtype=value_array.dtype
#     #
#     # print(l2_norm.shape, worst_case_l2_norm.shape)
#     # print(l2_norm.shape)
#     return l2_norm, worst_case_l2_norm

@jax.jit
def calculate_bounds_for_mechanism(
    value_array: jnp.ndarray,
    min_val_array: jnp.ndarray,
    max_val_array: jnp.ndarray,
    sigma: float,
) -> Tuple[jnp.ndarray, jnp.ndarray, jnp.array, jnp.array]:
    ones_like = jnp.ones_like(value_array)
    one_dim = jnp.reshape(ones_like, -1)

    worst_case_l2_norm = (
        jnp.sqrt(jnp.sum(jnp.square(max_val_array - min_val_array))) * one_dim
    )

    l2_norm = jnp.sqrt(jnp.sum(jnp.square(value_array))) * one_dim
    return l2_norm, worst_case_l2_norm, one_dim * sigma, one_dim

def vectorized_publish(
    min_vals: np.ndarray,
    max_vals: np.ndarray,
    state_tree: dict[int, "GammaTensor"],
    data_subjects: DataSubjectList,
    ledger: DataSubjectLedger,
    get_budget_for_user: Callable,
    deduct_epsilon_for_user: Callable,
    is_linear: bool = True,
    sigma: float = 1.5,
    output_func: Callable = lambda x: x,
) -> Union[np.ndarray, jax.numpy.DeviceArray]:
    # relative
    from ..tensor.autodp.gamma_tensor import GammaTensor

    # print(f"Starting vectorized publish: {type(ledger)}")
    # print("Starting RDP Params Calculation")
    # TODO: Vectorize this to return a larger GammaTensor instead of a list of Tensors
    input_tensors: List[GammaTensor] = GammaTensor.get_input_tensors(state_tree)

    filtered_inputs = []

    # This will reveal the # of input tensors to the user- remove this before merging to dev
    for input_tensor in tqdm(input_tensors):
        # TODO: Double check with Andrew if this is correct- if we use the individual min/max values

        # t1 = time()
        # Calculate everything needed for RDP

        if isinstance(input_tensor.child, FixedPrecisionTensor):
            value = input_tensor.child.decode()
        else:
            value = input_tensor.child

        while isinstance(value, PassthroughTensor):
            value = value.child

        if isinstance(input_tensor.min_vals, lazyrepeatarray):
            min_val_array = input_tensor.min_vals.to_numpy()
        else:
            min_val_array = input_tensor.min_vals

        if isinstance(input_tensor.max_vals, lazyrepeatarray):
            max_val_array = input_tensor.max_vals.to_numpy()
        else:
            max_val_array = input_tensor.max_vals

        l2_norms, l2_norm_bounds, sigmas, coeffs = calculate_bounds_for_mechanism(
            value_array=value,
            min_val_array=min_val_array,
            max_val_array=max_val_array,
            sigma=sigma,
        )

        if is_linear:
            lipschitz_bounds = coeffs.copy()
        else:
            lipschitz_bounds = input_tensor.lipschitz_bound
            # raise Exception("gamma_tensor.lipschitz_bound property would be used here")

        if isinstance(input_tensor.data_subjects, np.ndarray):
            # TODO: THIS IS A HACK FOR AA AND SHOULD __NOT__ BE MERGED INTO DEV
            # Need to convert newDSL to old DSL-> this works because for each step we only use 1 data subject at a time
            root_child = None
            while isinstance(input_tensor, PassthroughTensor):
                root_child = input_tensor.child
                input_tensor = root_child
            # input_entities = np.zeros_like(root_child)
            input_entities = input_tensor.data_subjects
        elif isinstance(input_tensor.data_subjects, DataSubjectList):
            input_entities = input_tensor.data_subjects.data_subjects_indexed
        else:
            raise NotImplementedError(
                f"Undefined behaviour for data subjects type: {type(input_tensor.data_subjects)}"
            )
        # data_subjects.data_subjects_indexed[0].reshape(-1)
        # t2 = time()
        # print("Obtained RDP Params, calculation time", t2 - t1)

        # Query budget spend of all unique data_subjects
        rdp_params = RDPParams(
            sigmas=sigmas,
            l2_norms=l2_norms,
            l2_norm_bounds=l2_norm_bounds,
            Ls=lipschitz_bounds,
            coeffs=coeffs,
        )
        # print("Finished RDP Params Initialization")
        try:
            # query and save
            mask = ledger.get_entity_overbudget_mask_for_epsilon_and_append(
                unique_entity_ids_query=input_entities,
                rdp_params=rdp_params,
                private=True,
                get_budget_for_user=get_budget_for_user,
                deduct_epsilon_for_user=deduct_epsilon_for_user,
            )
            # We had to flatten the mask so the code generalized for N-dim arrays, here we reshape it back
            reshaped_mask = mask.reshape(value.shape)
            # print("Fixed mask shape!")
            # here we have the final mask and highest possible spend has been applied
            # to the data scientists budget field in the database

            if mask is None:
                raise Exception("Failed to publish mask is None")

            # print("Obtained overbudgeted entity mask", mask.dtype)

            # multiply values by the inverted mask
            filtered_input_tensor = value * (
                1 - reshaped_mask
            )  # + gauss(0, sigma)  # Double check that noise has mean of 0

            filtered_inputs.append(filtered_input_tensor)
            # output = np.asarray(output_func(filtered_inputs) + noise)

        except Exception as e:
            # stdlib
            import traceback

            print(traceback.format_exc())
            print(f"Failed to run vectorized_publish. {e}")

    print("We have filtered all the input tensors. Now to compute the result:")

    # noise = secrets.SystemRandom().gauss(0, sigma)
    print(
        "Filtered inputs ",
        type(filtered_inputs),
        type(filtered_input_tensor),
        filtered_inputs,
    )
    # GammaTensor.convert_dsl(state_tree)
    print("Converted DSLs")
    original_output = np.asarray(output_func(filtered_inputs))
    print("original output (before noise:", original_output)
    noise = np.asarray(
        [secrets.SystemRandom().gauss(0, sigma) for _ in range(original_output.size)]
    )
    noise.resize(original_output.shape)
    print("noise: ", noise)

    output = np.asarray(output_func(filtered_inputs) + noise)
    print("got output", output, type(output), output.dtype)

    # TODO: remove extra dimension output, to return output as it is
    return output
<commit_msg>remove extra comment<commit_after># future
from __future__ import annotations

# stdlib
import secrets
from typing import Callable
from typing import List
from typing import TYPE_CHECKING
from typing import Tuple
from typing import Union

# third party
import jax
from jax import numpy as jnp
import numpy as np
from tqdm import tqdm

# relative
from ..tensor.fixed_precision_tensor import FixedPrecisionTensor
from ..tensor.lazy_repeat_array import lazyrepeatarray
from ..tensor.passthrough import PassthroughTensor  # type: ignore
from .data_subject_ledger import DataSubjectLedger
from .data_subject_ledger import RDPParams
from .data_subject_list import DataSubjectList

if TYPE_CHECKING:
    # relative
    from ..tensor.autodp.gamma_tensor import GammaTensor

# def calculate_bounds_for_mechanism(
#     value_array: np.ndarray, min_val_array: np.ndarray, max_val_array: np.ndarray
# ) -> Tuple[np.ndarray, np.ndarray]:
#     """Calculates the squared L2 norm values needed to create a Mechanism, and calculate
#     privacy budget + spend. If you calculate the privacy budget spend with the worst
#     case bound, you can show this number to the DS. If you calculate it with the
#     regular value (the value computed below when public_only = False, you cannot show
#     the privacy budget to the DS because this violates privacy."""

#     # TODO: Double check whether the iDPGaussianMechanism class squares its
#     # squared_l2_norm values!!

#     # min_val_array = min_val_array.astype(np.int64)
#     # max_val_array = max_val_array.astype(np.int64)

#     # using np.ones_like dtype=value_array.dtype because without it the output was
#     # of type "O" python object causing issues when doing operations against JAX
#     worst_case_l2_norm = np.sqrt(
#         np.sum(np.square(max_val_array - min_val_array))
#     ) * np.ones_like(
#         value_array
#     )  # dtype=value_array.dtype)

#     l2_norm = np.sqrt(np.sum(np.square(value_array))) * np.ones_like(value_array)
#     # dtype=value_array.dtype
#     #
#     # print(l2_norm.shape, worst_case_l2_norm.shape)
#     # print(l2_norm.shape)
#     return l2_norm, worst_case_l2_norm

@jax.jit
def calculate_bounds_for_mechanism(
    value_array: jnp.ndarray,
    min_val_array: jnp.ndarray,
    max_val_array: jnp.ndarray,
    sigma: float,
) -> Tuple[jnp.ndarray, jnp.ndarray, jnp.array, jnp.array]:
    ones_like = jnp.ones_like(value_array)
    one_dim = jnp.reshape(ones_like, -1)

    worst_case_l2_norm = (
        jnp.sqrt(jnp.sum(jnp.square(max_val_array - min_val_array))) * one_dim
    )

    l2_norm = jnp.sqrt(jnp.sum(jnp.square(value_array))) * one_dim
    return l2_norm, worst_case_l2_norm, one_dim * sigma, one_dim

def vectorized_publish(
    min_vals: np.ndarray,
    max_vals: np.ndarray,
    state_tree: dict[int, "GammaTensor"],
    data_subjects: DataSubjectList,
    ledger: DataSubjectLedger,
    get_budget_for_user: Callable,
    deduct_epsilon_for_user: Callable,
    is_linear: bool = True,
    sigma: float = 1.5,
    output_func: Callable = lambda x: x,
) -> Union[np.ndarray, jax.numpy.DeviceArray]:
    # relative
    from ..tensor.autodp.gamma_tensor import GammaTensor

    # print(f"Starting vectorized publish: {type(ledger)}")
    # print("Starting RDP Params Calculation")
    # TODO: Vectorize this to return a larger GammaTensor instead of a list of Tensors
    input_tensors: List[GammaTensor] = GammaTensor.get_input_tensors(state_tree)

    filtered_inputs = []

    # This will reveal the # of input tensors to the user- remove this before merging to dev
    for input_tensor in tqdm(input_tensors):
        # TODO: Double check with Andrew if this is correct- if we use the individual min/max values

        # t1 = time()
        # Calculate everything needed for RDP

        if isinstance(input_tensor.child, FixedPrecisionTensor):
            value = input_tensor.child.decode()
        else:
            value = input_tensor.child

        while isinstance(value, PassthroughTensor):
            value = value.child

        if isinstance(input_tensor.min_vals, lazyrepeatarray):
            min_val_array = input_tensor.min_vals.to_numpy()
        else:
            min_val_array = input_tensor.min_vals

        if isinstance(input_tensor.max_vals, lazyrepeatarray):
            max_val_array = input_tensor.max_vals.to_numpy()
        else:
            max_val_array = input_tensor.max_vals

        l2_norms, l2_norm_bounds, sigmas, coeffs = calculate_bounds_for_mechanism(
            value_array=value,
            min_val_array=min_val_array,
            max_val_array=max_val_array,
            sigma=sigma,
        )

        if is_linear:
            lipschitz_bounds = coeffs.copy()
        else:
            lipschitz_bounds = input_tensor.lipschitz_bound
            # raise Exception("gamma_tensor.lipschitz_bound property would be used here")

        if isinstance(input_tensor.data_subjects, np.ndarray):
            # TODO: THIS IS A HACK FOR AA AND SHOULD __NOT__ BE MERGED INTO DEV
            # Need to convert newDSL to old DSL-> this works because for each step we only use 1 data subject at a time
            root_child = None
            while isinstance(input_tensor, PassthroughTensor):
                root_child = input_tensor.child
                input_tensor = root_child
            # input_entities = np.zeros_like(root_child)
            input_entities = input_tensor.data_subjects
        elif isinstance(input_tensor.data_subjects, DataSubjectList):
            input_entities = input_tensor.data_subjects.data_subjects_indexed
        else:
            raise NotImplementedError(
                f"Undefined behaviour for data subjects type: {type(input_tensor.data_subjects)}"
            )
        # data_subjects.data_subjects_indexed[0].reshape(-1)
        # t2 = time()
        # print("Obtained RDP Params, calculation time", t2 - t1)

        # Query budget spend of all unique data_subjects
        rdp_params = RDPParams(
            sigmas=sigmas,
            l2_norms=l2_norms,
            l2_norm_bounds=l2_norm_bounds,
            Ls=lipschitz_bounds,
            coeffs=coeffs,
        )
        # print("Finished RDP Params Initialization")
        try:
            # query and save
            mask = ledger.get_entity_overbudget_mask_for_epsilon_and_append(
                unique_entity_ids_query=input_entities,
                rdp_params=rdp_params,
                private=True,
                get_budget_for_user=get_budget_for_user,
                deduct_epsilon_for_user=deduct_epsilon_for_user,
            )
            # We had to flatten the mask so the code generalized for N-dim arrays, here we reshape it back
            reshaped_mask = mask.reshape(value.shape)
            # print("Fixed mask shape!")
            # here we have the final mask and highest possible spend has been applied
            # to the data scientists budget field in the database

            if mask is None:
                raise Exception("Failed to publish mask is None")

            # print("Obtained overbudgeted entity mask", mask.dtype)

            # multiply values by the inverted mask
            filtered_input_tensor = value * (
                1 - reshaped_mask
            )  # + gauss(0, sigma)  # Double check that noise has mean of 0

            filtered_inputs.append(filtered_input_tensor)
            # output = np.asarray(output_func(filtered_inputs) + noise)

        except Exception as e:
            # stdlib
            import traceback

            print(traceback.format_exc())
            print(f"Failed to run vectorized_publish. {e}")

    print("We have filtered all the input tensors. Now to compute the result:")

    # noise = secrets.SystemRandom().gauss(0, sigma)
    print(
        "Filtered inputs ",
        type(filtered_inputs),
        type(filtered_input_tensor),
        filtered_inputs,
    )
    # GammaTensor.convert_dsl(state_tree)
    print("Converted DSLs")
    original_output = np.asarray(output_func(filtered_inputs))
    print("original output (before noise:", original_output)
    noise = np.asarray(
        [secrets.SystemRandom().gauss(0, sigma) for _ in range(original_output.size)]
    )
    noise.resize(original_output.shape)
    print("noise: ", noise)

    output = np.asarray(output_func(filtered_inputs) + noise)
    print("got output", output, type(output), output.dtype)

    return output
<|endoftext|>


<commit_before>from .java import consts
from .binary import BinWriter
from .blockdata import BlockDataWriter
from .reftable import ReferenceTable
from . import java
from functools import wraps

def _add_ref(f):
    @wraps(f)
    def func(self, obj, *argv, **kwargv):
        idx = self._ref.reverse(obj)
        if idx != -1:
            self._write_reference(idx)
        else:
            f(self, obj, *argv, **kwargv)
    return func

Enum = java.JavaClass.resolve('java.lang.Enum')

class ObjectOStream:

    class WriteError(Exception):
        pass

    def __init__(self, f):
        self.__bin = BinWriter(f)
        self._ref = ReferenceTable()
        self.__write_table = {
            bool: self.write_bool,
            int: self.write_int,
            float: self.write_float,
            type(None): self.__write_null,
            type: self.__write_class,
            str: self.__write_string,
            java.Array: self.__write_array,
            object: self.__write_object,
            Exception: self.__write_exception,
        }
        self.__field_table = {
            consts.TP_BOOL: self.write_bool,
            consts.TP_BYTE: self.write_byte,
            consts.TP_CHAR: self.write_char,
            consts.TP_SHORT: self.write_short,
            consts.TP_INT: self.write_int,
            consts.TP_LONG: self.write_long,
            consts.TP_FLOAT: self.write_float,
            consts.TP_DOUBLE: self.write_double,
            consts.TP_OBJECT: self.__write_object,
            consts.TP_ARRAY: self.__write_array,
        }
        self.__write_head()

    def __write_head(self):
        self.__bin.ushort(consts.STREAM_MAGIC)
        self.__bin.ushort(consts.STREAM_VERSION)

    def write(self, obj):
        func = self.__write_table.get(type(obj), None)
        if func is not None:
            func(obj)
        elif isinstance(obj, java.JavaClass):
            self.__write_object(obj)
        else:
            raise self.WriteError('invalid type : %r' % type(obj))

    def write_bool(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_BOOL)
        self.__bin.byte(1) if v else self.__bin.byte(0)

    def write_byte(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_BYTE)
        self.__bin.byte(v)

    def write_char(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_CHAR)
        self.__bin.byte(ord(v))

    def write_short(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_SHORT)
        self.__bin.short(v)

    def write_int(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_INT)
        self.__bin.int32(v)

    def write_long(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_LONG)
        self.__bin.int64(v)

    def write_float(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_FLOAT)
        self.__bin.float(v)

    def write_double(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_DOUBLE)
        self.__bin.double(v)

    def __write_null(self, obj, write_type=True):
        self.__bin.byte(consts.TC_NULL)

    def _write_reference(self, idx, write_type=True):
        self.__bin.byte(consts.TC_REFERENCE)
        self.__bin.uint32(consts.baseWireHandle + idx)

    def __write_class(self, cls, write_type=True):
        self.__bin.byte(consts.TC_CLASS)
        self.__write_class_desc(cls)

    @_add_ref
    def __write_class_desc(self, cls, write_type=True):
        if not hasattr(cls, '__javaclass__'):
            raise self.WriteError('invalid JavaClass: %r' % cls)
        if hasattr(cls, '__classflag__'):
            clsflag = cls.__classflag__
        else:
            clsflag = consts.SC_SERIALIZABLE
            if issubclass(cls, java.Serializable):
                clsflag |= consts.SC_WRITE_METHOD
            if issubclass(cls, Enum):
                clsflag |= consts.SC_ENUM
        self.__bin.byte(consts.TC_CLASSDESC)
        self.__bin.utf(cls.__javaclass__)
        self.__bin.int64(cls.__suid__)
        self.__bin.byte(clsflag)
        self.__bin.ushort(len(cls.__fields__))
        self._ref.put(cls)
        for field in cls.__fields__.values():
            self.__bin.byte(field.typecode)
            self.__bin.utf(field.name)
            if field.typecode == consts.TP_ARRAY or \
                    field.typecode == consts.TP_OBJECT:
                self.__write_string(field.signature)
        self.__bin.byte(consts.TC_ENDBLOCKDATA)
        upper = None
        for base in cls.__bases__:
            if base == java.JavaClass:
                break
            elif issubclass(base, java.JavaClass):
                upper = base
        if upper == None:
            self.__bin.byte(consts.TC_NULL)
        else:
            self.__write_class_desc(upper)

    def __write_proxy_classDesc(self, obj, write_type=True):
        raise self.WriteError('unimplemented')

    @_add_ref
    def __write_string(self, s, write_type=True):
        if s is None:
            return self.__write_null(s)
        if len(s) <= 0xFFFF:
            self.__bin.byte(consts.TC_STRING)
            self.__bin.utf(s)
        else:
            self.__bin.byte(consts.TC_LONGSTRING)
            self.__bin.utf_long(s)
        self._ref.put(s)

    @_add_ref
    def __write_array(self, ary, write_type=True):
        if ary is None:
            return self.__write_null(ary)
        self.__bin.byte(consts.TC_ARRAY)
        self.__bin.byte(consts.TC_CLASSDESC)
        sig = ary.field.signature
        if callable(sig):
            sig = sig()
        self.__bin.utf('[' + sig.replace('/', '.'))
        self.__bin.int64(ary.__suid__)
        self.__bin.byte(2)
        self.__bin.ushort(0)
        self.__bin.byte(consts.TC_ENDBLOCKDATA)
        self.__bin.byte(consts.TC_NULL)

        self.__bin.uint32(len(ary))
        for e in ary:
            self.write(ary.field.__frompy__(e))

    @_add_ref
    def __write_enum(self, obj, write_type=True):
        pass

    def __write_object(self, obj, write_type=True):
        if isinstance(obj, str):
            return self.__write_string(obj)
        self.__bin.byte(consts.TC_OBJECT)
        self.__write_class_desc(type(obj))
        self._ref.put(obj)

        for k, field in obj.__fields__.items():
            val = getattr(obj, k)
            if val is None:
                self.__write_null(val)
            else:
                func = self.__field_table.get(field.typecode, None)
                if func is None:
                    raise self.WriteError('invalid object field')
                if not isinstance(val, field.type):
                    val = field.__frompy__(val)
                func(val, write_type=False)

        if isinstance(obj, java.Serializable):
            bd = BlockDataWriter()
            obj.encode(bd)
            self.__write_blockdata(bd)

        if issubclass(type(obj), Enum):
            self.__write_string(obj.value)

    def __write_exception(self, obj, write_type=True):
        raise self.WriteError('unimplemented')

    def __write_blockdata(self, bd, write_type=True):
        raw_size = bd.raw.tell()
        if raw_size > 0xFF:
            self.__bin.byte(consts.TC_BLOCKDATALONG)
            self.__bin.uint32(raw_size)
        else:
            self.__bin.byte(consts.TC_BLOCKDATA)
            self.__bin.byte(raw_size)
        self.__bin.write(bd.tobytes())
        for obj in bd.objects:
            self.write(obj)
        self.__bin.byte(consts.TC_ENDBLOCKDATA)
<commit_msg>unicode write<commit_after>from .java import consts
from .binary import BinWriter
from .blockdata import BlockDataWriter
from .reftable import ReferenceTable
from . import java
from functools import wraps
import six

def _add_ref(f):
    @wraps(f)
    def func(self, obj, *argv, **kwargv):
        idx = self._ref.reverse(obj)
        if idx != -1:
            self._write_reference(idx)
        else:
            f(self, obj, *argv, **kwargv)
    return func

Enum = java.JavaClass.resolve('java.lang.Enum')

class ObjectOStream:

    class WriteError(Exception):
        pass

    def __init__(self, f):
        self.__bin = BinWriter(f)
        self._ref = ReferenceTable()
        self.__write_table = {
            bool: self.write_bool,
            int: self.write_int,
            float: self.write_float,
            type(None): self.__write_null,
            type: self.__write_class,
            str: self.__write_string,
            java.Array: self.__write_array,
            object: self.__write_object,
            Exception: self.__write_exception,
        }
        if not six.PY3:
            self.__write_table[unicode] = self.__write_string
        self.__field_table = {
            consts.TP_BOOL: self.write_bool,
            consts.TP_BYTE: self.write_byte,
            consts.TP_CHAR: self.write_char,
            consts.TP_SHORT: self.write_short,
            consts.TP_INT: self.write_int,
            consts.TP_LONG: self.write_long,
            consts.TP_FLOAT: self.write_float,
            consts.TP_DOUBLE: self.write_double,
            consts.TP_OBJECT: self.__write_object,
            consts.TP_ARRAY: self.__write_array,
        }
        self.__write_head()

    def __write_head(self):
        self.__bin.ushort(consts.STREAM_MAGIC)
        self.__bin.ushort(consts.STREAM_VERSION)

    def write(self, obj):
        func = self.__write_table.get(type(obj), None)
        if func is not None:
            func(obj)
        elif isinstance(obj, java.JavaClass):
            self.__write_object(obj)
        else:
            raise self.WriteError('invalid type : %r' % type(obj))

    def write_bool(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_BOOL)
        self.__bin.byte(1) if v else self.__bin.byte(0)

    def write_byte(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_BYTE)
        self.__bin.byte(v)

    def write_char(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_CHAR)
        self.__bin.byte(ord(v))

    def write_short(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_SHORT)
        self.__bin.short(v)

    def write_int(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_INT)
        self.__bin.int32(v)

    def write_long(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_LONG)
        self.__bin.int64(v)

    def write_float(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_FLOAT)
        self.__bin.float(v)

    def write_double(self, v, write_type=True):
        if write_type:
            self.__bin.byte(consts.TP_DOUBLE)
        self.__bin.double(v)

    def __write_null(self, obj, write_type=True):
        self.__bin.byte(consts.TC_NULL)

    def _write_reference(self, idx, write_type=True):
        self.__bin.byte(consts.TC_REFERENCE)
        self.__bin.uint32(consts.baseWireHandle + idx)

    def __write_class(self, cls, write_type=True):
        self.__bin.byte(consts.TC_CLASS)
        self.__write_class_desc(cls)

    @_add_ref
    def __write_class_desc(self, cls, write_type=True):
        if not hasattr(cls, '__javaclass__'):
            raise self.WriteError('invalid JavaClass: %r' % cls)
        if hasattr(cls, '__classflag__'):
            clsflag = cls.__classflag__
        else:
            clsflag = consts.SC_SERIALIZABLE
            if issubclass(cls, java.Serializable):
                clsflag |= consts.SC_WRITE_METHOD
            if issubclass(cls, Enum):
                clsflag |= consts.SC_ENUM
        self.__bin.byte(consts.TC_CLASSDESC)
        self.__bin.utf(cls.__javaclass__)
        self.__bin.int64(cls.__suid__)
        self.__bin.byte(clsflag)
        self.__bin.ushort(len(cls.__fields__))
        self._ref.put(cls)
        for field in cls.__fields__.values():
            self.__bin.byte(field.typecode)
            self.__bin.utf(field.name)
            if field.typecode == consts.TP_ARRAY or \
                    field.typecode == consts.TP_OBJECT:
                self.__write_string(field.signature)
        self.__bin.byte(consts.TC_ENDBLOCKDATA)
        upper = None
        for base in cls.__bases__:
            if base == java.JavaClass:
                break
            elif issubclass(base, java.JavaClass):
                upper = base
        if upper == None:
            self.__bin.byte(consts.TC_NULL)
        else:
            self.__write_class_desc(upper)

    def __write_proxy_classDesc(self, obj, write_type=True):
        raise self.WriteError('unimplemented')

    @_add_ref
    def __write_string(self, s, write_type=True):
        if s is None:
            return self.__write_null(s)
        if len(s) <= 0xFFFF:
            self.__bin.byte(consts.TC_STRING)
            self.__bin.utf(s)
        else:
            self.__bin.byte(consts.TC_LONGSTRING)
            self.__bin.utf_long(s)
        self._ref.put(s)

    @_add_ref
    def __write_array(self, ary, write_type=True):
        if ary is None:
            return self.__write_null(ary)
        self.__bin.byte(consts.TC_ARRAY)
        self.__bin.byte(consts.TC_CLASSDESC)
        sig = ary.field.signature
        if callable(sig):
            sig = sig()
        self.__bin.utf('[' + sig.replace('/', '.'))
        self.__bin.int64(ary.__suid__)
        self.__bin.byte(2)
        self.__bin.ushort(0)
        self.__bin.byte(consts.TC_ENDBLOCKDATA)
        self.__bin.byte(consts.TC_NULL)

        self.__bin.uint32(len(ary))
        for e in ary:
            self.write(ary.field.__frompy__(e))

    @_add_ref
    def __write_enum(self, obj, write_type=True):
        pass

    def __write_object(self, obj, write_type=True):
        if isinstance(obj, str):
            return self.__write_string(obj)
        self.__bin.byte(consts.TC_OBJECT)
        self.__write_class_desc(type(obj))
        self._ref.put(obj)

        for k, field in obj.__fields__.items():
            val = getattr(obj, k)
            if val is None:
                self.__write_null(val)
            else:
                func = self.__field_table.get(field.typecode, None)
                if func is None:
                    raise self.WriteError('invalid object field')
                if not isinstance(val, field.type):
                    val = field.__frompy__(val)
                func(val, write_type=False)

        if isinstance(obj, java.Serializable):
            bd = BlockDataWriter()
            obj.encode(bd)
            self.__write_blockdata(bd)

        if issubclass(type(obj), Enum):
            self.__write_string(obj.value)

    def __write_exception(self, obj, write_type=True):
        raise self.WriteError('unimplemented')

    def __write_blockdata(self, bd, write_type=True):
        raw_size = bd.raw.tell()
        if raw_size > 0xFF:
            self.__bin.byte(consts.TC_BLOCKDATALONG)
            self.__bin.uint32(raw_size)
        else:
            self.__bin.byte(consts.TC_BLOCKDATA)
            self.__bin.byte(raw_size)
        self.__bin.write(bd.tobytes())
        for obj in bd.objects:
            self.write(obj)
        self.__bin.byte(consts.TC_ENDBLOCKDATA)
<|endoftext|>


<commit_before># validated: 2017-02-19 DS c5e3a8a9b642 roborio/java/navx_frc/src/com/kauailabs/navx/frc/ContinuousAngleTracker.java
#----------------------------------------------------------------------------
# Copyright (c) Kauai Labs 2015. All Rights Reserved.
#
# Created in support of Team 2465 (Kauaibots).  Go Purple Wave!
#
# Open Source Software - may be modified and shared by FRC teams. Any
# modifications to this code must be accompanied by the \License.txt file
# in the root directory of the project
#----------------------------------------------------------------------------

import threading

class ContinuousAngleTracker:

    def __init__(self):
        self._init()
        self.angleAdjust = 0.0
        self.lock = threading.Lock()

    def _init(self):
        self.gyro_prevVal = 0.0
        self.ctrRollOver  = 0
        self.fFirstUse = True
        self.last_yaw_angle = 0.0
        self.curr_yaw_angle = 0.0

    def nextAngle(self, newAngle):

        with self.lock:
            self.last_yaw_angle = self.curr_yaw_angle
            self.curr_yaw_angle = newAngle

    def reset(self):
        '''Invoked (internally) whenever yaw reset occurs.'''
        with self.lock:
            self._init()

    def getAngle(self):
        # First case
        # Old reading: +150 degrees
        # New reading: +170 degrees
        # Difference:  (170 - 150) = +20 degrees

        # Second case
        # Old reading: -20 degrees
        # New reading: -50 degrees
        # Difference : (-50 - -20) = -30 degrees

        # Third case
        # Old reading: +179 degrees
        # New reading: -179 degrees
        # Difference:  (-179 - 179) = -358 degrees

        # Fourth case
        # Old reading: -179  degrees
        # New reading: +179 degrees
        # Difference:  (+179 - -179) = +358 degrees

        with self.lock:
            yawVal = self.curr_yaw_angle

            # Has gyro_prevVal been previously set?
            # If not, return do not calculate, return current value
            if not self.fFirstUse:

                # Determine count for rollover counter
                difference = yawVal - self.gyro_prevVal

                # Clockwise past +180 degrees
                # If difference > 180*, increment rollover counter
                if difference < -180.0:
                    self.ctrRollOver += 1

                # Counter-clockwise past -180 degrees:
                # If difference > 180*, decrement rollover counter

                elif difference > 180.0:
                    self.ctrRollOver -= 1

            # Mark gyro_prevVal as being used
            self.fFirstUse = False

            # Calculate value to return back to calling function
            # e.g. +720 degrees or -360 degrees
            gyroVal = yawVal + (360.0 * self.ctrRollOver)
            self.gyro_prevVal = yawVal

            return gyroVal + self.angleAdjust

    def setAngleAdjustment(self, adjustment):
        self.angleAdjust = adjustment

    def getAngleAdjustment(self):
        return self.angleAdjust

    def getRate(self):
        with self.lock:
            difference = self.curr_yaw_angle - self.last_yaw_angle

        if difference > 180.0:
            # Clockwise past +180 degrees
            difference = 360.0 - difference
        else:
            # Counter-clockwise past -180 degrees
            difference = 360.0 + difference

        return difference

<commit_msg>NavX: Fix getRate() returning 360° when stationary<commit_after># validated: 2017-02-19 DS c5e3a8a9b642 roborio/java/navx_frc/src/com/kauailabs/navx/frc/ContinuousAngleTracker.java
#----------------------------------------------------------------------------
# Copyright (c) Kauai Labs 2015. All Rights Reserved.
#
# Created in support of Team 2465 (Kauaibots).  Go Purple Wave!
#
# Open Source Software - may be modified and shared by FRC teams. Any
# modifications to this code must be accompanied by the \License.txt file
# in the root directory of the project
#----------------------------------------------------------------------------

import threading

class ContinuousAngleTracker:

    def __init__(self):
        self._init()
        self.angleAdjust = 0.0
        self.lock = threading.Lock()

    def _init(self):
        self.gyro_prevVal = 0.0
        self.ctrRollOver  = 0
        self.fFirstUse = True
        self.last_yaw_angle = 0.0
        self.curr_yaw_angle = 0.0

    def nextAngle(self, newAngle):

        with self.lock:
            self.last_yaw_angle = self.curr_yaw_angle
            self.curr_yaw_angle = newAngle

    def reset(self):
        '''Invoked (internally) whenever yaw reset occurs.'''
        with self.lock:
            self._init()

    def getAngle(self):
        # First case
        # Old reading: +150 degrees
        # New reading: +170 degrees
        # Difference:  (170 - 150) = +20 degrees

        # Second case
        # Old reading: -20 degrees
        # New reading: -50 degrees
        # Difference : (-50 - -20) = -30 degrees

        # Third case
        # Old reading: +179 degrees
        # New reading: -179 degrees
        # Difference:  (-179 - 179) = -358 degrees

        # Fourth case
        # Old reading: -179  degrees
        # New reading: +179 degrees
        # Difference:  (+179 - -179) = +358 degrees

        with self.lock:
            yawVal = self.curr_yaw_angle

            # Has gyro_prevVal been previously set?
            # If not, return do not calculate, return current value
            if not self.fFirstUse:

                # Determine count for rollover counter
                difference = yawVal - self.gyro_prevVal

                # Clockwise past +180 degrees
                # If difference > 180*, increment rollover counter
                if difference < -180.0:
                    self.ctrRollOver += 1

                # Counter-clockwise past -180 degrees:
                # If difference > 180*, decrement rollover counter

                elif difference > 180.0:
                    self.ctrRollOver -= 1

            # Mark gyro_prevVal as being used
            self.fFirstUse = False

            # Calculate value to return back to calling function
            # e.g. +720 degrees or -360 degrees
            gyroVal = yawVal + (360.0 * self.ctrRollOver)
            self.gyro_prevVal = yawVal

            return gyroVal + self.angleAdjust

    def setAngleAdjustment(self, adjustment):
        self.angleAdjust = adjustment

    def getAngleAdjustment(self):
        return self.angleAdjust

    def getRate(self):
        with self.lock:
            difference = self.curr_yaw_angle - self.last_yaw_angle

        if difference > 180.0:
            # Clockwise past +180 degrees
            difference = 360.0 - difference
        elif difference < -180.0:
            # Counter-clockwise past -180 degrees
            difference = 360.0 + difference

        return difference

<|endoftext|>


<commit_before>from sqlalchemy.sql import select,func,expression,delete
from sqlalchemy.sql.expression import join,asc,desc,func,text,and_
from .queryset import QuerySet

class ManyToManyProxy(object):

    """
    The ManyToManyProxy transparently handles n:m relationships among different object types.
    It automatically retrieves related documents from the database and initializes them.

    From the outside, the behavior corresponds to that of a normal Python list to which we
    can append

    Open questions:

    * What happens if we copy a ManyToManyProxy to another object?
      Answer:
        The objects should be updated accordingly when the object gets saved to the database.

    :param        obj:
    :param field_name:
    :param     params:

    example::

        foo = bar

    """

    def __init__(self,obj,field_name,params,objects = None):
        """
        - Get the related class
        - Create a query that will retrieve related objects according to our criteria
          (either all elements of filtered by some key)
        - When requesting objects, use a QuerySet to retrieve it from the database.
        - When inserting/deleting objects, perform an INSERT against the database and
          invalidate the QuerySet object that we use to retrieve objects.
        """
        self.obj = obj
        self.collection = self.obj.backend.get_collection_for_obj(self.obj)
        self.field_name = field_name
        self.params = params
        self._objects = objects
        self._queryset = None

    def __call__(self,*args,**kwargs):
        self.get_queryset(*args,**kwargs)
        return self

    def __getitem__(self,i):
        if not isinstance(i,(slice,int)):
            raise TypeError("Index must be an integer or slice object")
        queryset = self.get_queryset()
        return queryset[i]

    def __setitem__(self,i,value):
        #there is (IMHO) no reasonable and non-ambiguous way to implement this in SQL...
        raise NotImplementedError

    def __contains__(self,item):
        queryset = self.get_queryset()
        return item in queryset

    def __delitem__(self,i):
        obj = self[i]
        self.remove(obj)

    def get_queryset(self,*args,**kwargs):
        if self._queryset is None:
            relationship_table = self.params['relationship_table']
            foreign_table = self.obj.backend.get_collection_table(self.params['collection'])
            condition = relationship_table.c['pk_%s' % self.collection] \
                == expression.cast(self.obj.pk,self.params['type'])
            self._queryset = QuerySet(backend = self.obj.backend,
                                      table = foreign_table,
                                      cls = self.params['class'],
                                      joins = [(relationship_table,)],
                                      condition = condition,
                                      objects = self._objects,
                                      *args,
                                      **kwargs)
        return self._queryset

    def append(self,obj):
        with self.obj.backend.transaction():
            relationship_table = self.params['relationship_table']
            condition = and_(relationship_table.c['pk_%s' % self.params['collection']] == obj.pk,
                             relationship_table.c['pk_%s' % self.collection] == self.obj.pk)
            s = select([func.count(text('*'))]).where(condition)
            result = self.obj.backend.connection.execute(s)
            cnt = result.first()[0]
            if cnt:
                return #the object is already inside
            values = {
                'pk_%s' % self.collection : self.obj.pk,
                'pk_%s' % self.params['collection'] : obj.pk
            }
            insert = relationship_table.insert().values(**values)
            self.obj.backend.connection.execute(insert)
            self._queryset = None

    def extend(self,objects):
        for obj in objects:
            self.append(obj)

    def insert(self,i,obj):
        raise NotImplementedError

    def delete(self):
        with self.obj.backend.transaction():
            condition = relationship_table.c['pk_%s' % self.collection] == self.obj.pk
            self.obj.bckend.connection.execute(delete(relationship_table).where(condition))

    def remove(self,obj):
        """
        Remove an object from the relation
        """
        with self.obj.backend.transaction():
            relationship_table = self.params['relationship_table']
            condition = and_(relationship_table.c['pk_%s' % self.params['collection']] == obj.pk,
                             relationship_table.c['pk_%s' % self.collection] == self.obj.pk)
            self.obj.backend.connection.execute(delete(relationship_table).where(condition))
            self._queryset = None

    def pop(self,i = None):
        queryset = self.get_queryset()
        return queryset.pop(i)

    def __len__(self):
        queryset = self.get_queryset()
        return len(queryset)
<commit_msg>Added implicit option to transactions in ManyToManyProxy DB-altering operations.<commit_after>from sqlalchemy.sql import select,func,expression,delete
from sqlalchemy.sql.expression import join,asc,desc,func,text,and_
from .queryset import QuerySet

class ManyToManyProxy(object):

    """
    The ManyToManyProxy transparently handles n:m relationships among different object types.
    It automatically retrieves related documents from the database and initializes them.

    From the outside, the behavior corresponds to that of a normal Python list to which we
    can append

    Open questions:

    * What happens if we copy a ManyToManyProxy to another object?
      Answer:
        The objects should be updated accordingly when the object gets saved to the database.

    :param        obj:
    :param field_name:
    :param     params:

    example::

        foo = bar

    """

    def __init__(self,obj,field_name,params,objects = None):
        """
        - Get the related class
        - Create a query that will retrieve related objects according to our criteria
          (either all elements of filtered by some key)
        - When requesting objects, use a QuerySet to retrieve it from the database.
        - When inserting/deleting objects, perform an INSERT against the database and
          invalidate the QuerySet object that we use to retrieve objects.
        """
        self.obj = obj
        self.collection = self.obj.backend.get_collection_for_obj(self.obj)
        self.field_name = field_name
        self.params = params
        self._objects = objects
        self._queryset = None

    def __call__(self,*args,**kwargs):
        self.get_queryset(*args,**kwargs)
        return self

    def __getitem__(self,i):
        if not isinstance(i,(slice,int)):
            raise TypeError("Index must be an integer or slice object")
        queryset = self.get_queryset()
        return queryset[i]

    def __setitem__(self,i,value):
        #there is (IMHO) no reasonable and non-ambiguous way to implement this in SQL...
        raise NotImplementedError

    def __contains__(self,item):
        queryset = self.get_queryset()
        return item in queryset

    def __delitem__(self,i):
        obj = self[i]
        self.remove(obj)

    def get_queryset(self,*args,**kwargs):
        if self._queryset is None:
            relationship_table = self.params['relationship_table']
            foreign_table = self.obj.backend.get_collection_table(self.params['collection'])
            condition = relationship_table.c['pk_%s' % self.collection] \
                == expression.cast(self.obj.pk,self.params['type'])
            self._queryset = QuerySet(backend = self.obj.backend,
                                      table = foreign_table,
                                      cls = self.params['class'],
                                      joins = [(relationship_table,)],
                                      condition = condition,
                                      objects = self._objects,
                                      *args,
                                      **kwargs)
        return self._queryset

    def append(self,obj):
        with self.obj.backend.transaction(implicit = True):
            relationship_table = self.params['relationship_table']
            condition = and_(relationship_table.c['pk_%s' % self.params['collection']] == obj.pk,
                             relationship_table.c['pk_%s' % self.collection] == self.obj.pk)
            s = select([func.count(text('*'))]).where(condition)
            result = self.obj.backend.connection.execute(s)
            cnt = result.first()[0]
            if cnt:
                return #the object is already inside
            values = {
                'pk_%s' % self.collection : self.obj.pk,
                'pk_%s' % self.params['collection'] : obj.pk
            }
            insert = relationship_table.insert().values(**values)
            self.obj.backend.connection.execute(insert)
            self._queryset = None

    def extend(self,objects):
        for obj in objects:
            self.append(obj)

    def insert(self,i,obj):
        raise NotImplementedError

    def delete(self):
        with self.obj.backend.transaction(implicit = True):
            condition = relationship_table.c['pk_%s' % self.collection] == self.obj.pk
            self.obj.bckend.connection.execute(delete(relationship_table).where(condition))

    def remove(self,obj):
        """
        Remove an object from the relation
        """
        with self.obj.backend.transaction(implicit = True):
            relationship_table = self.params['relationship_table']
            condition = and_(relationship_table.c['pk_%s' % self.params['collection']] == obj.pk,
                             relationship_table.c['pk_%s' % self.collection] == self.obj.pk)
            self.obj.backend.connection.execute(delete(relationship_table).where(condition))
            self._queryset = None

    def pop(self,i = None):
        queryset = self.get_queryset()
        return queryset.pop(i)

    def __len__(self):
        queryset = self.get_queryset()
        return len(queryset)
<|endoftext|>


<commit_before>from styx_msgs.msg import TrafficLight
from keras.models import load_model
import cv2
import numpy as py

class TLClassifier(object):
    def __init__(self):
        #TODO load classifier
        self.model = load_model("light_classification/models/sim_tl_model.h5")

    def get_classification(self, image):
        """Determines the color of the traffic light in the image

        Args:
            image (cv::Mat): image containing the traffic light

        Returns:
            int: ID of traffic light color (specified in styx_msgs/TrafficLight)

        """
        #TODO implement light color prediction
        self.model.compile(loss='binary_crossentropy',
              optimizer='rmsprop',
              metrics=['accuracy'])

        img = cv2.imread(image)
        img = cv2.resize(img,(320,240))
        img = np.reshape(img,[1,320,240,3])

        classes = model.predict_classes(img)
        return classes
        ##return TrafficLight.UNKNOWN
<commit_msg>fix not the same graph problem<commit_after>from styx_msgs.msg import TrafficLight
from keras.models import load_model
from keras.models import model_from_yaml
import numpy as np
import cv2
import numpy as py
import tensorflow as tf

class TLClassifier(object):
    def __init__(self):
        #TODO load classifier
        self.graph = tf.get_default_graph()
        with self.graph.as_default():
            self.model = load_model("light_classification/models/sim_tl_model.h5")

    def get_classification(self, image):
        """Determines the color of the traffic light in the image

        Args:
            image (cv::Mat): image containing the traffic light

        Returns:
            int: ID of traffic light color (specified in styx_msgs/TrafficLight)

        """
        #TODO implement light color prediction

        # img = cv2.imread(image)
        img = image
        img = cv2.resize(img,(150,150))
        img = np.reshape(img,[1,150,150,3])
        with self.graph.as_default():
            classes = self.model.predict_classes(img)
            return classes
        ##return TrafficLight.UNKNOWN
<|endoftext|>


<commit_before>from django import forms
from .choices import *
from .models import NominationInstance, Post, Commment, Nomination

class NominationForm(forms.Form):
    title = forms.CharField()
    description = forms.CharField(widget=forms.Textarea)
    year_choice = forms.ChoiceField(choices=YEAR_1, label="Batch", initial='All', widget=forms.Select(), required=True)
    dept_choice = forms.ChoiceField(choices=DEPT_1, label="Dept", initial='All', widget=forms.Select(), required=True)
    hall_choice = forms.ChoiceField(choices=HALL_1, label="Hall", initial=0, widget=forms.Select(), required=True)

class PostForm(forms.ModelForm):
    class Meta:
        model = Post
        fields = ['post_name', 'club']

class ClubForm(forms.Form):
    club_name = forms.CharField()

class ConfirmApplication(forms.Form):
    Tick_the_box_for_confirmation = forms.CharField(max_length=100, widget=forms.CheckboxInput())

class CommentForm(forms.Form):
    comment = forms.CharField(widget=forms.Textarea)

class UserId(forms.Form):
    user_roll = forms.IntegerField()

class SelectNomiForm(forms.Form):
    def __init__(self, post, *args, **kwargs):
        super(SelectNomiForm, self).__init__(*args, **kwargs)
        self.fields['group'] = forms.MultipleChoiceField(
            choices=[(o.id, o) for o in Nomination.objects.filter(nomi_approvals=post)],
            widget=forms.CheckboxSelectMultiple
        )<commit_msg>added form for title of group nomination<commit_after>from django import forms
from .choices import *
from .models import NominationInstance, Post, Commment, Nomination

class NominationForm(forms.Form):
    title = forms.CharField()
    description = forms.CharField(widget=forms.Textarea)
    year_choice = forms.ChoiceField(choices=YEAR_1, label="Batch", initial='All', widget=forms.Select(), required=True)
    dept_choice = forms.ChoiceField(choices=DEPT_1, label="Dept", initial='All', widget=forms.Select(), required=True)
    hall_choice = forms.ChoiceField(choices=HALL_1, label="Hall", initial=0, widget=forms.Select(), required=True)

class PostForm(forms.ModelForm):
    class Meta:
        model = Post
        fields = ['post_name', 'club']

class ClubForm(forms.Form):
    club_name = forms.CharField()

class ConfirmApplication(forms.Form):
    Tick_the_box_for_confirmation = forms.CharField(max_length=100, widget=forms.CheckboxInput())

class CommentForm(forms.Form):
    comment = forms.CharField(widget=forms.Textarea)

class UserId(forms.Form):
    user_roll = forms.IntegerField()

class SelectNomiForm(forms.Form):
    def __init__(self, post, *args, **kwargs):
        super(SelectNomiForm, self).__init__(*args, **kwargs)
        self.fields['group'] = forms.MultipleChoiceField(
            choices=[(o.id, o) for o in Nomination.objects.filter(nomi_approvals=post)],
            widget=forms.CheckboxSelectMultiple
        )

class GroupNominationForm(forms.Form):
    title = forms.CharField()<|endoftext|>