<commit_before># SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2015, ARM Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import logging
from system import System

class Screen(object):
    """
    Set of utility functions to control an Android Screen
    """

    @staticmethod
    def set_orientation(target, auto=True, portrait=None):
        """
        Set screen orientation mode
        """
        log = logging.getLogger('Screen')
        acc_mode = 1 if auto else 0
        # Force manual orientation of portrait specified
        if portrait is not None:
            acc_mode = 0
            log.info('Force manual orientation')
        if acc_mode == 0:
            usr_mode = 0 if portrait else 1
            usr_mode_str = 'PORTRAIT' if portrait else 'LANDSCAPE'
            log.info('Set orientation: %s', usr_mode_str)
        else:
            usr_mode = 0
            log.info('Set orientation: AUTO')

        if acc_mode == 0:
            target.execute('content insert '\
                           '--uri content://settings/system '\
                           '--bind name:s:accelerometer_rotation '\
                           '--bind value:i:{}'.format(acc_mode))
            target.execute('content insert '\
                           '--uri content://settings/system '\
                           '--bind name:s:user_rotation '\
                           '--bind value:i:{}'.format(usr_mode))
        else:
            # Force PORTRAIT mode when activation AUTO rotation
            target.execute('content insert '\
                           '--uri content://settings/system '\
                           '--bind name:s:user_rotation '\
                           '--bind value:i:{}'.format(usr_mode))
            target.execute('content insert '\
                           '--uri content://settings/system '\
                           '--bind name:s:accelerometer_rotation '\
                           '--bind value:i:{}'.format(acc_mode))

    @staticmethod
    def set_brightness(target, auto=True, percent=None):
        """
        Set screen brightness percentage
        """
        log = logging.getLogger('Screen')
        bri_mode = 1 if auto else 0
        # Force manual brightness if a percent specified
        if percent:
            bri_mode = 0
        target.execute('content insert '\
                       '--uri content://settings/system '\
                       '--bind name:s:screen_brightness_mode '\
                       '--bind value:i:{}'.format(bri_mode))
        if bri_mode == 0:
            if percent<0 or percent>100:
                msg = "Screen brightness {} out of range (0,100)"\
                      .format(percent)
                raise ValueError(msg)
            value = 255 * percent / 100
            target.execute('content insert '\
                           '--uri content://settings/system '\
                           '--bind name:s:screen_brightness '\
                           '--bind value:i:{}'.format(value))
            log.info('Set brightness: %d%%', percent)
        else:
            log.info('Set brightness: AUTO')

    @staticmethod
    def set_dim(target, auto=True):
        """
        Set screen dimming mode
        """
        log = logging.getLogger('Screen')
        dim_mode = 1 if auto else 0
        dim_mode_str = 'ON' if auto else 'OFF'
        target.execute('content insert '\
                       '--uri content://settings/system '\
                       '--bind name:s:dim_screen '\
                       '--bind value:i:{}'.format(dim_mode))
        log.info('Dim screen mode: %s', dim_mode_str)

    @staticmethod
    def set_timeout(target, seconds=30):
        """
        Set screen off timeout in seconds
        """
        log = logging.getLogger('Screen')
        if seconds<0:
            msg = "Screen timeout {}: cannot be negative".format(seconds)
            raise ValueError(msg)
        value = seconds * 1000
        target.execute('content insert '\
                       '--uri content://settings/system '\
                       '--bind name:s:screen_off_timeout '\
                       '--bind value:i:{}'.format(value))
        log.info('Screen timeout: %d [s]', seconds)

    @staticmethod
    def set_defaults(target):
        """
        Reset screen settings to a reasonable default
        """
        Screen.set_orientation(target)
        Screen.set_brightness(target)
        Screen.set_dim(target)
        Screen.set_timeout(target)

    @staticmethod
    def get_screen_density(target):
        """
        Get screen density of the device.
        """
        return target.execute('getprop ro.sf.lcd_density')

    @staticmethod
    def set_screen(target, on=True):
        log = logging.getLogger('Screen')
        if not on:
            log.info('Setting screen OFF')
            System.sleep(target)
            return
        log.info('Setting screen ON')
        System.wakeup(target)

    @staticmethod
    def unlock(target):
       Screen.set_screen(target, on=True)
       System.menu(target)
       System.home(target)

# vim :set tabstop=4 shiftwidth=4 expandtab
<commit_msg>android/screen: Give time for screen on<commit_after># SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2015, ARM Limited and contributors.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import logging
from system import System
from time import sleep

class Screen(object):
    """
    Set of utility functions to control an Android Screen
    """

    @staticmethod
    def set_orientation(target, auto=True, portrait=None):
        """
        Set screen orientation mode
        """
        log = logging.getLogger('Screen')
        acc_mode = 1 if auto else 0
        # Force manual orientation of portrait specified
        if portrait is not None:
            acc_mode = 0
            log.info('Force manual orientation')
        if acc_mode == 0:
            usr_mode = 0 if portrait else 1
            usr_mode_str = 'PORTRAIT' if portrait else 'LANDSCAPE'
            log.info('Set orientation: %s', usr_mode_str)
        else:
            usr_mode = 0
            log.info('Set orientation: AUTO')

        if acc_mode == 0:
            target.execute('content insert '\
                           '--uri content://settings/system '\
                           '--bind name:s:accelerometer_rotation '\
                           '--bind value:i:{}'.format(acc_mode))
            target.execute('content insert '\
                           '--uri content://settings/system '\
                           '--bind name:s:user_rotation '\
                           '--bind value:i:{}'.format(usr_mode))
        else:
            # Force PORTRAIT mode when activation AUTO rotation
            target.execute('content insert '\
                           '--uri content://settings/system '\
                           '--bind name:s:user_rotation '\
                           '--bind value:i:{}'.format(usr_mode))
            target.execute('content insert '\
                           '--uri content://settings/system '\
                           '--bind name:s:accelerometer_rotation '\
                           '--bind value:i:{}'.format(acc_mode))

    @staticmethod
    def set_brightness(target, auto=True, percent=None):
        """
        Set screen brightness percentage
        """
        log = logging.getLogger('Screen')
        bri_mode = 1 if auto else 0
        # Force manual brightness if a percent specified
        if percent:
            bri_mode = 0
        target.execute('content insert '\
                       '--uri content://settings/system '\
                       '--bind name:s:screen_brightness_mode '\
                       '--bind value:i:{}'.format(bri_mode))
        if bri_mode == 0:
            if percent<0 or percent>100:
                msg = "Screen brightness {} out of range (0,100)"\
                      .format(percent)
                raise ValueError(msg)
            value = 255 * percent / 100
            target.execute('content insert '\
                           '--uri content://settings/system '\
                           '--bind name:s:screen_brightness '\
                           '--bind value:i:{}'.format(value))
            log.info('Set brightness: %d%%', percent)
        else:
            log.info('Set brightness: AUTO')

    @staticmethod
    def set_dim(target, auto=True):
        """
        Set screen dimming mode
        """
        log = logging.getLogger('Screen')
        dim_mode = 1 if auto else 0
        dim_mode_str = 'ON' if auto else 'OFF'
        target.execute('content insert '\
                       '--uri content://settings/system '\
                       '--bind name:s:dim_screen '\
                       '--bind value:i:{}'.format(dim_mode))
        log.info('Dim screen mode: %s', dim_mode_str)

    @staticmethod
    def set_timeout(target, seconds=30):
        """
        Set screen off timeout in seconds
        """
        log = logging.getLogger('Screen')
        if seconds<0:
            msg = "Screen timeout {}: cannot be negative".format(seconds)
            raise ValueError(msg)
        value = seconds * 1000
        target.execute('content insert '\
                       '--uri content://settings/system '\
                       '--bind name:s:screen_off_timeout '\
                       '--bind value:i:{}'.format(value))
        log.info('Screen timeout: %d [s]', seconds)

    @staticmethod
    def set_defaults(target):
        """
        Reset screen settings to a reasonable default
        """
        Screen.set_orientation(target)
        Screen.set_brightness(target)
        Screen.set_dim(target)
        Screen.set_timeout(target)

    @staticmethod
    def get_screen_density(target):
        """
        Get screen density of the device.
        """
        return target.execute('getprop ro.sf.lcd_density')

    @staticmethod
    def set_screen(target, on=True):
        log = logging.getLogger('Screen')
        if not on:
            log.info('Setting screen OFF')
            System.sleep(target)
            return
        log.info('Setting screen ON')
        System.wakeup(target)

    @staticmethod
    def unlock(target):
       Screen.set_screen(target, on=True)
       sleep(1)
       System.menu(target)
       System.home(target)

# vim :set tabstop=4 shiftwidth=4 expandtab
<|endoftext|>


<commit_before>from django.db import connection
from django.template import Template, Context
from django.conf import settings

#
# Log all SQL statements direct to the console (when running in DEBUG)
# Intended for use with the django development server.
#

class SQLLogToConsoleMiddleware:
    def process_response(self, request, response):
        if settings.DEBUG and connection.queries:
            time = sum([float(q['time']) for q in connection.queries])
            t = Template("{{count}} quer{{count|pluralize:\"y,ies\"}} in {{time}} seconds:\n\n{% for sql in sqllog %}[{{forloop.counter}}] {{sql.time}}s: {{sql.sql|safe}}{% if not forloop.last %}\n\n{% endif %}{% endfor %}")
            print t.render(Context({'sqllog':connection.queries,'count':len(connection.queries),'time':time}))
        return response<commit_msg>added instructions for using the new middleware<commit_after>from django.db import connection
from django.template import Template, Context
from django.conf import settings

#
# Log all SQL statements direct to the console (when running in DEBUG)
# Intended for use with the django development server.
#

class SQLLogToConsoleMiddleware:
    """
    Add to settings_local like so to log sql commands:

    from lingcod.common.default_settings import MIDDLEWARE_CLASSES

    MIDDLEWARE_CLASSES += (
        'lingcod.common.middleware.SQLLogToConsoleMiddleware',
    )
    """
    def process_response(self, request, response):
        if settings.DEBUG and connection.queries:
            time = sum([float(q['time']) for q in connection.queries])
            t = Template("{{count}} quer{{count|pluralize:\"y,ies\"}} in {{time}} seconds:\n\n{% for sql in sqllog %}[{{forloop.counter}}] {{sql.time}}s: {{sql.sql|safe}}{% if not forloop.last %}\n\n{% endif %}{% endfor %}")
            print t.render(Context({'sqllog':connection.queries,'count':len(connection.queries),'time':time}))
        return response<|endoftext|>


<commit_before>from django.contrib.gis.db import models
from django.contrib.auth.models import User, Group, Permission
from django.conf import settings
from lingcod.features import registered_models

class ShareableGeoManager(models.GeoManager):
    def shared_with_user(self, user, filter_groups=None):
        """
        Returns a queryset containing any objects that have been
        shared with a group the user belongs to.

        Assumes that the model has been setup according to the instructions
        for implementing a shared model.
        """
        app_name = self.model._meta.app_label
        model_name = self.model.__name__.lower()
        perm = Permission.objects.get(codename='can_share_features')

        if user.is_anonymous() or not user.is_authenticated():
            # public users get special treatment -
            # ONLY get to see anything shared with a public group
            groups = Group.objects.filter(name__in=settings.SHARING_TO_PUBLIC_GROUPS)
        else:
            if user.is_staff:
                # Staff users get their groups,plus 'shared_to_staff_groups',  plus public groups
                groups = Group.objects.filter(
                            models.Q(
                                pk__in=[x.pk for x in user.groups.all()]
                            ) |
                            models.Q(
                                name__in=settings.SHARING_TO_PUBLIC_GROUPS
                            ) |
                            models.Q(
                                name__in=settings.SHARING_TO_STAFF_GROUPS
                            )
                        ).distinct()
            else:
                # Non-staff authenticated users get their groups plus public groups, MINUS shared_to_staff groups
                groups = Group.objects.filter(
                            models.Q(
                                pk__in=[x.pk for x in user.groups.all()]
                            ) |
                            models.Q(
                                name__in=settings.SHARING_TO_PUBLIC_GROUPS
                            )
                        ).distinct().exclude(name__in=settings.SHARING_TO_STAFF_GROUPS)

        if filter_groups and len(filter_groups)>0:
            groups = groups.filter(pk__in=[x.pk for x in filter_groups])
        else:
            filter_groups = None

        # Check for a Container
        potential_parents = self.model.get_options().get_potential_parents()
        if potential_parents:
            contained_ids = []
            for collection_model in potential_parents:
                # Avoid infinite recursion
                if collection_model == self.model:
                    continue
                # Get container objects shared with user
                if filter_groups:
                    shared_containers = collection_model.objects.shared_with_user(user,filter_groups=filter_groups)
                else:
                    shared_containers = collection_model.objects.shared_with_user(user)

                # Create list of contained object ids
                for sc in shared_containers:
                    #contained = sc.__getattribute__(shared_content_type.container_set_property)
                    contained = sc.feature_set(recurse=True,feature_classes=[self.model])
                    contained_ids.extend([x.id for x in contained])

            return self.filter(
                models.Q(
                    sharing_groups__permissions=perm,
                    sharing_groups__in=groups
                ) |
                models.Q(
                    pk__in=contained_ids
                )
            ).distinct()
        else:
            # No containers, just a straight 'is it shared' query
            return self.filter(
                models.Q(
                    sharing_groups__permissions=perm,
                    sharing_groups__in=groups
                )
            ).distinct()
<commit_msg>Fixed a bug in recursive sharing logic; collections just need to avoid including themselves on recursive calls but not on the first pass.<commit_after>from django.contrib.gis.db import models
from django.contrib.auth.models import User, Group, Permission
from django.conf import settings
from lingcod.features import registered_models

class ShareableGeoManager(models.GeoManager):
    def shared_with_user(self, user, filter_groups=None, exclude_models=None):
        """
        Returns a queryset containing any objects that have been
        shared with a group the user belongs to.

        Assumes that the model has been setup according to the instructions
        for implementing a shared model.
        """
        app_name = self.model._meta.app_label
        model_name = self.model.__name__.lower()
        perm = Permission.objects.get(codename='can_share_features')

        if user.is_anonymous() or not user.is_authenticated():
            # public users get special treatment -
            # ONLY get to see anything shared with a public group
            groups = Group.objects.filter(name__in=settings.SHARING_TO_PUBLIC_GROUPS)
        else:
            if user.is_staff:
                # Staff users get their groups,plus 'shared_to_staff_groups',  plus public groups
                groups = Group.objects.filter(
                            models.Q(
                                pk__in=[x.pk for x in user.groups.all()]
                            ) |
                            models.Q(
                                name__in=settings.SHARING_TO_PUBLIC_GROUPS
                            ) |
                            models.Q(
                                name__in=settings.SHARING_TO_STAFF_GROUPS
                            )
                        ).distinct()
            else:
                # Non-staff authenticated users get their groups plus public groups, MINUS shared_to_staff groups
                groups = Group.objects.filter(
                            models.Q(
                                pk__in=[x.pk for x in user.groups.all()]
                            ) |
                            models.Q(
                                name__in=settings.SHARING_TO_PUBLIC_GROUPS
                            )
                        ).distinct().exclude(name__in=settings.SHARING_TO_STAFF_GROUPS)

        if filter_groups and len(filter_groups)>0:
            groups = groups.filter(pk__in=[x.pk for x in filter_groups])
        else:
            filter_groups = None

        # Check for a Container
        potential_parents = self.model.get_options().get_potential_parents()
        if potential_parents:
            contained_ids = []
            for collection_model in potential_parents:
                # Avoid infinite recursion
                if exclude_models and collection_model in exclude_models:
                    continue

                exclude_models = None
                if collection_model == self.model:
                    # On the next recurisive go-round, avoid infinite recursion
                    exclude_models = [self.model]

                # Get container objects shared with user
                if filter_groups:
                    shared_containers = collection_model.objects.shared_with_user(user,filter_groups=filter_groups,exclude_models=exclude_models)
                else:
                    shared_containers = collection_model.objects.shared_with_user(user,exclude_models=exclude_models)

                # Create list of contained object ids
                for sc in shared_containers:
                    #contained = sc.__getattribute__(shared_content_type.container_set_property)
                    contained = sc.feature_set(recurse=True,feature_classes=[self.model])
                    contained_ids.extend([x.id for x in contained])

            return self.filter(
                models.Q(
                    sharing_groups__permissions=perm,
                    sharing_groups__in=groups
                ) |
                models.Q(
                    pk__in=contained_ids
                )
            ).distinct()
        else:
            # No containers, just a straight 'is it shared' query
            return self.filter(
                models.Q(
                    sharing_groups__permissions=perm,
                    sharing_groups__in=groups
                )
            ).distinct()
<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Helper for models_test."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf

from lingvo.core import base_model
from lingvo.core import base_input_generator

class BaseModelsTest(tf.test.TestCase):
  """Base model test class which does not define any test methods of its own."""

  def _testOneModelParams(self, registry, name):
    cls = registry.GetClass(name)
    p = cls.Model()
    self.assertTrue(issubclass(p.cls, base_model.BaseModel))
    self.assertTrue(p.model is not None)
    for dataset in ('Train', 'Dev', 'Test'):
      input_p = cls.GetDatasetParams(dataset)
      if issubclass(p.cls, base_model.SingleTaskModel):
        self.assertTrue(
            issubclass(input_p.cls, base_input_generator.BaseInputGenerator),
            'Error in %s' % dataset)
        if (dataset != 'Train') and issubclass(
            input_p.cls, base_input_generator.BaseSequenceInputGenerator) and (
                input_p.num_samples != 0):
          self.assertEquals(
              input_p.num_batcher_threads, 1,
              'num_batcher_threads too large in %s. Decoder '
              'or eval runs over this set might not span '
              'exactly one epoch.' % dataset)
      else:
        self.assertTrue(issubclass(p.cls, base_model.MultiTaskModel))
        for _, v in input_p.IterParams():
          self.assertTrue(
              issubclass(v.cls, base_input_generator.BaseInputGenerator),
              'Error in %s' % dataset)

  @classmethod
  def CreateTestMethodsForAllRegisteredModels(cls, registry):
    """Programmatically defines test methods for each registered model."""
    model_names = registry.GetAllRegisteredClasses().keys()
    for model_name in sorted(model_names):

      def test(self, name=model_name):
        self._testOneModelParams(registry, name)  # pylint: disable=protected-access

      setattr(cls, 'testModelParams_%s' % model_name, test)
<commit_msg>Add task_prefix_filter arg for CreateTestMethodsForAllRegisteredModels.<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Helper for models_test."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf

from lingvo.core import base_model
from lingvo.core import base_input_generator

class BaseModelsTest(tf.test.TestCase):
  """Base model test class which does not define any test methods of its own."""

  def _testOneModelParams(self, registry, name):
    cls = registry.GetClass(name)
    p = cls.Model()
    self.assertTrue(issubclass(p.cls, base_model.BaseModel))
    self.assertTrue(p.model is not None)
    for dataset in ('Train', 'Dev', 'Test'):
      input_p = cls.GetDatasetParams(dataset)
      if issubclass(p.cls, base_model.SingleTaskModel):
        self.assertTrue(
            issubclass(input_p.cls, base_input_generator.BaseInputGenerator),
            'Error in %s' % dataset)
        if (dataset != 'Train') and issubclass(
            input_p.cls, base_input_generator.BaseSequenceInputGenerator) and (
                input_p.num_samples != 0):
          self.assertEquals(
              input_p.num_batcher_threads, 1,
              'num_batcher_threads too large in %s. Decoder '
              'or eval runs over this set might not span '
              'exactly one epoch.' % dataset)
      else:
        self.assertTrue(issubclass(p.cls, base_model.MultiTaskModel))
        for _, v in input_p.IterParams():
          self.assertTrue(
              issubclass(v.cls, base_input_generator.BaseInputGenerator),
              'Error in %s' % dataset)

  @classmethod
  def CreateTestMethodsForAllRegisteredModels(cls,
                                              registry,
                                              task_prefix_filter=''):
    """Programmatically defines test methods for each registered model."""
    model_names = registry.GetAllRegisteredClasses().keys()
    for model_name in sorted(model_names):
      if task_prefix_filter and not model_name.startswith(task_prefix_filter):
        tf.logging.info('Skipping tests for registered model: %s', model_name)
        continue

      def test(self, name=model_name):
        self._testOneModelParams(registry, name)  # pylint: disable=protected-access

      setattr(cls, 'testModelParams_%s' % model_name, test)
<|endoftext|>


<commit_before>import copy
from collections import defaultdict
import datetime
from glob   import glob
import networkx as nx
import json
import os
from pprint import pprint
import re
import subprocess
import time
import uuid

#Lobbyists
#"lobbyists"

#afflilatedOrgs
#"affiliatedOrgs"

#foreignEntities
#"foreignEntities"

#Flags
#"selfSelect"

#Unknown
#"prefix","affiliatedUrl"

#Time
#"effectiveDate","regType","reportType","reportYear","signedDate"

def replaceWhitespace(s):
    return re.sub('  +', ' ', s)

#Questions
#alliance of/ alliance to?
#how to handle formely?
#center for/to'
#city of
#d/b/a doing business as
# if there is a 3-5 letter nonsense verb at the end of a word, remove it?
#ad hoc informal coalitions

def preProcess(s):
    s = s.encode("ascii","replace")
    s = re.sub('\n', ' ', s)
    s = s.strip().strip('"').strip("'").lower().strip()
    return replaceWhitespace(s)

#processClientName(preProcess("assn of J.H.Christ & The-All-Mighty l c llc lp"))
#'asociation of j h christ and the all mighty'
# preprocess ("aia") out
def processClientName(org):
    s = org
    s = re.sub('\?','\'',s)       #replace ? with '
    s = re.sub('[,\.]',' ',s)    #replace ,. with space
    s = re.sub('\\bu s a\\b','usa',s) #replace u.s.a. with usa
    s = re.sub('\\bu s\\b','us',s) #replace u.s.a. with usa
    s = re.sub('\\bassn\\b','asociation',s) # replace "assn" with "association"
    s = re.sub('&',' and ',s)#replace "&" with " and "
    if "city" not in s:
        s = re.sub("\bco\b"," ",s)

    useless =  ["l l c","llc",
                "l c","lc",
                "l l p","llp",
                "l p","lp",
                "ltd","l t d","company","corporation","corp","companies","incorporated","inc"]
    #remove various stopwords
    for sub in useless:
        s=re.sub("\\b"+sub+"\\b"," ",s) #TODO: look into "co" company vs. colorado

        # reverse chicago, city of
    #on behalf of cassidy & associates
    #on behalf of akin gump
    #"on behalf of akin gump strauss hauer & feld"
    breakers = ["on behalf of", "obo","o/b/o", "on behalf",
                "public policy partners (",
                "the livingston group - (client",
                "the livingston group (client:",
                "the livingston group (for ",
                "the livingston group ( for ",
                "the livingston group (",
                "the livingston group(",
                "the livingston group/",
                "akin gump strauss hauer and feld ("
                "(the livingston group)",
                "the livingston group-",
                "van scoyoc associates (",
                "the implementation group (",
                "jefferson consulting group (",
                "alcalde and fay (",
                "govbiz advantage (for",
                "govbiz advantage ("
    ]
    for b in breakers:
        if b in s:
            s = s.split(b)[-1]
    while ")" == s[0]:
        s = s[1:]
    while "(" == s[-1]:
        s = s[:-1]
    while ")" == s[-1] and "(" not in s:
        s = s[:-1]
    while "(" == s[-1] and ")" not in s:
        s = s[:-1]

    #ImmaLetYouFinish = [
    #                     "capitol strategies ",  "alcalde and fay \(","white house consulting",
    #                     "the ickes and enright group","corporation",
    #                     "dla piper us for", "obo" ]
    #remove "alcalde & fay (" "alenxander strategy group (" "apco worldwide ("
    # for s in ImmaLetYouFinish:
    #     if s in name:
    #         a = name.split(s)[-1]
    #         if len(a) > 2:
    #             name = replaceWhitespace(a)

    #remove on behalf of
    return preProcess(s)

def loadFile(f):
    jOb = {}
    corruption = "{http://www.PureEdge.com/XFDL/Custom}"
    try:
        jOb = json.loads(open(f).read())[u'LOBBYINGDISCLOSURE1']
    except KeyError:
        try:
            corruptedjOb = json.loads(open(f).read())[corruption+u'LOBBYINGDISCLOSURE1']
            jOb = {}
            for (k,v) in corruptedjOb.iteritems():
                jOb[k.split(corruption)[-1]]=v
        except KeyError:
            return None

    #Client node
    #"clientAddress","clientCity","clientCountry","clientGeneralDescription","clientGovtEntity",
    #"clientName","clientState","clientZip","clientZipExt","prinClientCity","prinClientCountry",
    #"prinClientState","prinClientZip","prinClientZipExt"
    client = {
        #Display data
        "label": processClientName(preProcess(jOb["clientName"])),
        "fillcolor": "darkolivegreen1",
        "style": "filled",

        #Type
        "type":"client",

        #Acutal data
        "name":        preProcess(jOb["clientName"]),
        "address":     preProcess(jOb["clientAddress"]),
        "city":        preProcess(jOb["clientCity"]),
        "country":     preProcess(jOb["clientCountry"]),
        "state":       preProcess(jOb["clientState"]),
        "zip":         preProcess(jOb["clientZip"]),

        "filename": f,
        "description":     preProcess(jOb["clientGeneralDescription"]),
        "specific_issues": preProcess(jOb["specific_issues"]),
    }

    #Firm node
    #"address1","address2","city","firstName","lastName","state","zip","zipext","country",
    #"principal_city","principal_country","principal_state","principal_zip","principal_zipext",
    #"organizationName","printedName","registrantGeneralDescription",

    firm = {
        #Display code
        "label":     preProcess(jOb["organizationName"]),
        "fillcolor": "deepskyblue",
        "style": "filled",
        #Type
        "type":"firm",

        #Actual data
        "firstname":   preProcess(jOb["firstName"]),
        "lastname":    preProcess(jOb["lastName"]),
        "orgname":     preProcess(jOb["organizationName"]),
        "printedname": preProcess(jOb["printedName"]),
        "description": preProcess(jOb["registrantGeneralDescription"]),

        "address1":    preProcess(jOb["address1"]),
        "address2":    preProcess(jOb["address2"]),
        "city":        preProcess(jOb["city"]),
        "country":     preProcess(jOb["country"]),
        "state":       preProcess(jOb["state"]),
        "zip":         preProcess(jOb["zip"]),

        "p_city":        preProcess(jOb["principal_city"]),
        "p_country":     preProcess(jOb["principal_country"]),
        "p_state":       preProcess(jOb["principal_state"]),
        "P_zip":         preProcess(jOb["principal_zip"]),
        "filename": f
    }

    #Relationship
    #"houseID", "senateID", "specific_issues","alis"
    employs = {
        #display code
        "label": "employs",
        #actual data
        "relation": "employs",
        "houseID":     preProcess(jOb["houseID"]),
        "senate":      preProcess(jOb["senateID"]),
        "alis":        frozenset(filter(lambda x: x != u"",jOb["alis"])),
    }
    return (client, firm, employs)

being  = {"label": "Being", "shape":"rectangle", "type": "Being"}
beingr = {"label":"represents", "relation":"represents"}

def loadData():
    print 'Reading into clients ...'
    universe = nx.Graph()
    for col in map(loadFile,glob(os.environ["HOUSEXML"]+"/LD1/*/*/*.json")):
        if col == None:
            continue
        (client,firm,employs) = col
        cnode = str(uuid.uuid1())
        fnode = str(uuid.uuid1())
        cbeing = str(uuid.uuid1())
        fbeing = str(uuid.uuid1())

        universe.add_node(cbeing,copy.copy(being))
        universe.add_node(cnode,client)
        universe.add_edge(cnode,cbeing,copy.copy(beingr))

        universe.add_node(fbeing,copy.copy(being))
        universe.add_node(fnode,firm)
        universe.add_edge(fnode,fbeing,copy.copy(beingr))

        universe.add_edge(fnode,cnode,employs)
    return universe

def mergeTheirBeings(universe,al,bl):
    a = findBeing(universe,al)
    b = findBeing(universe,bl)
    if a != b:
        for v in nx.neighbors(universe,b):
            universe.add_edge(v,a,copy.copy(beingr))
            universe.remove_edge(v,b)
        av = universe.node[a]
        bv = universe.node[b]
        if "names" in av and "names" in bv:
            av["names"] = av["names"].union(bv["names"])
        elif "names" not in av and "names" in bv:
            av["names"] = bv["names"]
    return al

def findBeing(universe,l):
    beings = []
    for lb,d in universe[l].iteritems():
        if d["relation"] == "represents":
            beings.append(lb)

    if len(beings) == 0:
        raise Exception("Cannot find a being for \"{}\"".format(l))
    elif len(beings) == 1:
        return beings[0]
    else:
        raise Exception("Found {} beings for {}: {}".format(len(beings),l,",".join(beings)))

def formerSplitter(name):
    splitters = ["\"fka\"","fka:","fka","f/k/a/",
                 "formerly known as",
                 "formerly know as",
                 "formerly filed as",
                 "formerly reported as",
                 "formerly",
                 "formally known as",
                 "formally",
                 "former",
                 "d/b/a",
                 "dba ",
    ]
    for s in splitters:
        if s in name:
            return name.split("\b"+s+"\b")
    return [name]

def mergeExactMatches(universe):
    nodes = universe.nodes(data=True)
    l = len(nodes)

    firmsOrg = defaultdict(list)
    firmsPrinted = defaultdict(list)
    clients = defaultdict(list)

    for k,v in nodes:
        if v["type"] == "client":
            if v["name"] != "":
                for split in map(processClientName,formerSplitter(v["name"])):
                    clients[split].append(k)
        if v["type"] == "firm":
            if v["orgname"] != "":
                firmsOrg[v["orgname"]].append(k)
            if v["printedname"] != "":
                firmsPrinted[v["printedname"]].append(k)

    for grouping in [firmsOrg,firmsPrinted,clients]:
        for k,v in grouping.iteritems():
            merged = reduce(lambda x,y: mergeTheirBeings(universe,x,y),v)
            found = findBeing(universe,merged)
            if "names" in universe.node[found]:
                universe.node[found]["names"].add(k)
            else:
                universe.node[found]["names"] = set([k])

    for (k,v) in universe.nodes(data=True):
        if k in universe and v["type"] == "Being" and len(nx.neighbors(universe,k)) == 0:
            universe.remove_node(k)

def save(universe):
    for k1,k2,v in universe.edges(data=True):
        if "alis" in v:
            v["alis"] = ", ".join(list(v["alis"]))

    for k,v in universe.nodes(data=True):
        if "names" in v:
            v["names"] = ", ".join(sorted(list(v["names"])))
    stamp = re.sub("[ :\.]","-",str(datetime.datetime.now()))[:-7]
    f = "output-{}.graphml".format(stamp)
    nx.write_graphml(universe,f)
    print("Saved in {}".format(f))

def main():
    print("Loading data")
    universe = loadData()
    print("Matching")
    mergeExactMatches(universe)
    print("Saving")
    save(universe)

if __name__ == "__main__":
    main()

<commit_msg>Bug fixes and small updates<commit_after>import copy
from collections import defaultdict
import datetime
from glob   import glob
import networkx as nx
import json
import os
from pprint import pprint
import re
import subprocess
import time
import uuid

#Lobbyists
#"lobbyists"

#afflilatedOrgs
#"affiliatedOrgs"

#foreignEntities
#"foreignEntities"

#Flags
#"selfSelect"

#Unknown
#"prefix","affiliatedUrl"

#Time
#"effectiveDate","regType","reportType","reportYear","signedDate"

def replaceWhitespace(s):
    return re.sub('  +', ' ', s)

#Questions
#alliance of/ alliance to?
#how to handle formely?
#center for/to'
#city of
#d/b/a doing business as
# if there is a 3-5 letter nonsense verb at the end of a word, remove it?
#ad hoc informal coalitions

def preProcess(s):
    s = s.encode("ascii","replace")
    s = re.sub('\n', ' ', s)
    s = s.strip().strip('"').strip("'").lower().strip()
    return replaceWhitespace(s)

#processClientName(preProcess("assn of J.H.Christ & The-All-Mighty l c llc lp"))
#'asociation of j h christ and the all mighty'
# preprocess ("aia") out
def processClientName(org):
    s = org
    if "city" not in s:
        s = re.sub("\\bco\\b"," ",s)
    s = re.sub('\?','\'',s)       #replace ? with '
    s = re.sub('[,\.]',' ',s)    #replace ,. with space
    s = re.sub('\\bu s a\\b','usa',s) #replace u.s.a. with usa
    s = re.sub('\\bu s\\b','us',s) #replace u.s.a. with usa
    s = re.sub('\\bassn\\b','asociation',s) # replace "assn" with "association"
    s = re.sub('&',' and ',s)#replace "&" with " and "

    useless =  ["l l c","llc",
                "l c","lc",
                "l l p","llp",
                "l p","lp",
                "ltd","l t d","company","corporation","corp","companies","incorporated","inc"]
    #remove various stopwords
    for sub in useless:
        s=re.sub("\\b"+sub+"\\b"," ",s) #TODO: look into "co" company vs. colorado

        # reverse chicago, city of
    #on behalf of cassidy & associates
    #on behalf of akin gump
    #"on behalf of akin gump strauss hauer & feld"
    breakers = ["on behalf of", "obo","o/b/o", "on behalf",
                "public policy partners (",
                "the livingston group - (client",
                "the livingston group (client:",
                "the livingston group (",
                "the livingston group(",
                "the livingston group/",
                "akin gump strauss hauer and feld ("
                "(the livingston group)",
                "the livingston group-",
                "van scoyoc associates (",
                "the implementation group (",
                "jefferson consulting group (",
                "alcalde and fay (",
                "govbiz advantage (",
    ]
    for b in breakers:
        if b in s:
            s = s.split(b)[-1]
    for c in [")",":","/","for"]:
        while c == s[0:len(c)+1]:
            s = s[1:]
    for c in ["(",":","/"]:
        while c == s[-1]:
            s = s[:-1]

    while ")" == s[-1] and "(" not in s:
        s = s[:-1]
    while "(" == s[-1] and ")" not in s:
        s = s[:-1]

    #ImmaLetYouFinish = [
    #                     "capitol strategies ",  "alcalde and fay \(","white house consulting",
    #                     "the ickes and enright group","corporation",
    #                     "dla piper us for", "obo" ]
    #remove "alcalde & fay (" "alenxander strategy group (" "apco worldwide ("
    # for s in ImmaLetYouFinish:
    #     if s in name:
    #         a = name.split(s)[-1]
    #         if len(a) > 2:
    #             name = replaceWhitespace(a)

    #remove on behalf of
    return preProcess(s)

def loadFile(f):
    jOb = {}
    corruption = "{http://www.PureEdge.com/XFDL/Custom}"
    try:
        jOb = json.loads(open(f).read())[u'LOBBYINGDISCLOSURE1']
    except KeyError:
        try:
            corruptedjOb = json.loads(open(f).read())[corruption+u'LOBBYINGDISCLOSURE1']
            jOb = {}
            for (k,v) in corruptedjOb.iteritems():
                jOb[k.split(corruption)[-1]]=v
        except KeyError:
            return None

    #Client node
    #"clientAddress","clientCity","clientCountry","clientGeneralDescription","clientGovtEntity",
    #"clientName","clientState","clientZip","clientZipExt","prinClientCity","prinClientCountry",
    #"prinClientState","prinClientZip","prinClientZipExt"
    client = {
        #Display data
        "label": processClientName(preProcess(jOb["clientName"])),
        "fillcolor": "darkolivegreen1",
        "style": "filled",

        #Type
        "type":"client",

        #Acutal data
        "name":        preProcess(jOb["clientName"]),
        "address":     preProcess(jOb["clientAddress"]),
        "city":        preProcess(jOb["clientCity"]),
        "country":     preProcess(jOb["clientCountry"]),
        "state":       preProcess(jOb["clientState"]),
        "zip":         preProcess(jOb["clientZip"]),

        "filename": f,
        "description":     preProcess(jOb["clientGeneralDescription"]),
        "specific_issues": preProcess(jOb["specific_issues"]),
    }

    #Firm node
    #"address1","address2","city","firstName","lastName","state","zip","zipext","country",
    #"principal_city","principal_country","principal_state","principal_zip","principal_zipext",
    #"organizationName","printedName","registrantGeneralDescription",

    firm = {
        #Display code
        "label":     preProcess(jOb["organizationName"]),
        "fillcolor": "deepskyblue",
        "style": "filled",
        #Type
        "type":"firm",

        #Actual data
        "firstname":   preProcess(jOb["firstName"]),
        "lastname":    preProcess(jOb["lastName"]),
        "orgname":     preProcess(jOb["organizationName"]),
        "printedname": preProcess(jOb["printedName"]),
        "description": preProcess(jOb["registrantGeneralDescription"]),

        "address1":    preProcess(jOb["address1"]),
        "address2":    preProcess(jOb["address2"]),
        "city":        preProcess(jOb["city"]),
        "country":     preProcess(jOb["country"]),
        "state":       preProcess(jOb["state"]),
        "zip":         preProcess(jOb["zip"]),

        "p_city":        preProcess(jOb["principal_city"]),
        "p_country":     preProcess(jOb["principal_country"]),
        "p_state":       preProcess(jOb["principal_state"]),
        "P_zip":         preProcess(jOb["principal_zip"]),
        "filename": f
    }

    #Relationship
    #"houseID", "senateID", "specific_issues","alis"
    employs = {
        #display code
        "label": "employs",
        #actual data
        "relation": "employs",
        "houseID":     preProcess(jOb["houseID"]),
        "senate":      preProcess(jOb["senateID"]),
        "alis":        frozenset(filter(lambda x: x != u"",jOb["alis"])),
    }
    return (client, firm, employs)

being  = {"label": "Being", "shape":"rectangle", "type": "Being"}
beingr = {"label":"represents", "relation":"represents"}

def loadData():
    print 'Reading into clients ...'
    universe = nx.Graph()
    for col in map(loadFile,glob(os.environ["HOUSEXML"]+"/LD1/*/*/*.json")):
        if col == None:
            continue
        (client,firm,employs) = col
        cnode = str(uuid.uuid1())
        fnode = str(uuid.uuid1())
        cbeing = str(uuid.uuid1())
        fbeing = str(uuid.uuid1())

        universe.add_node(cbeing,copy.copy(being))
        universe.add_node(cnode,client)
        universe.add_edge(cnode,cbeing,copy.copy(beingr))

        universe.add_node(fbeing,copy.copy(being))
        universe.add_node(fnode,firm)
        universe.add_edge(fnode,fbeing,copy.copy(beingr))

        universe.add_edge(fnode,cnode,employs)
    return universe

def mergeTheirBeings(universe,al,bl):
    a = findBeing(universe,al)
    b = findBeing(universe,bl)
    if a != b:
        for v in nx.neighbors(universe,b):
            universe.add_edge(v,a,copy.copy(beingr))
            universe.remove_edge(v,b)
        av = universe.node[a]
        bv = universe.node[b]
        if "names" in av and "names" in bv:
            av["names"] = av["names"].union(bv["names"])
        elif "names" not in av and "names" in bv:
            av["names"] = bv["names"]
    return al

def findBeing(universe,l):
    beings = []
    for lb,d in universe[l].iteritems():
        if d["relation"] == "represents":
            beings.append(lb)

    if len(beings) == 0:
        raise Exception("Cannot find a being for \"{}\"".format(l))
    elif len(beings) == 1:
        return beings[0]
    else:
        raise Exception("Found {} beings for {}: {}".format(len(beings),l,",".join(beings)))

def formerSplitter(name):
    splitters = ["\"fka\"","fka:","fka","f/k/a/",
                 "formerly known as",
                 "formerly know as",
                 "formerly filed as",
                 "formerly reported as",
                 "formerly",
                 "formally known as",
                 "formally",
                 "former", #united natural products alliance (former utah natural products alliance)?
                 "d/b/a",
                 "dba",
    ]#todo: compile regex ahead of times
    for s in splitters:
        if s in name and name != "dba international":
            return re.split("\\b"+s+"\\b",name)
    return [name]

def mergeExactMatches(universe):
    nodes = universe.nodes(data=True)
    l = len(nodes)

    firmsOrg = defaultdict(list)
    firmsPrinted = defaultdict(list)
    clients = defaultdict(list)

    for k,v in nodes:
        if v["type"] == "client":
            if v["name"] != "":
                for split in map(processClientName,formerSplitter(v["name"])):
                    clients[split].append(k)
        if v["type"] == "firm":
            if v["orgname"] != "":
                firmsOrg[v["orgname"]].append(k)
            if v["printedname"] != "":
                firmsPrinted[v["printedname"]].append(k)

    for grouping in [firmsOrg,firmsPrinted,clients]:
        for k,v in grouping.iteritems():
            merged = reduce(lambda x,y: mergeTheirBeings(universe,x,y),v)
            found = findBeing(universe,merged)
            if "names" in universe.node[found]:
                universe.node[found]["names"].add(k)
            else:
                universe.node[found]["names"] = set([k])

    for (k,v) in universe.nodes(data=True):
        if k in universe and v["type"] == "Being" and len(nx.neighbors(universe,k)) == 0:
            universe.remove_node(k)

def save(universe):
    for k1,k2,v in universe.edges(data=True):
        if "alis" in v:
            v["alis"] = ", ".join(list(v["alis"]))

    for k,v in universe.nodes(data=True):
        if "names" in v:
            v["names"] = ", ".join(sorted(list(v["names"])))
    stamp = re.sub("[ :\.]","-",str(datetime.datetime.now()))[:-7]
    f = "output-{}.graphml".format(stamp)
    nx.write_graphml(universe,f)
    nx.write_graphml(universe,"fresh-output.graphml")
    print("Saved in {}".format(f))

def main():
    print("Loading data")
    universe = loadData()
    print("Matching")
    mergeExactMatches(universe)
    print("Saving")
    save(universe)

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from lizard_auth_client.conf import settings

@python_2_unicode_compatible
class Role(models.Model):
    unique_id = models.CharField(max_length=32, unique=True)
    code = models.CharField(max_length=255, null=False, blank=False)
    name = models.CharField(max_length=255, null=False, blank=False)
    external_description = models.TextField()
    internal_description = models.TextField()

    BILLING_ROLE_CODE = 'billing'

    def __str__(self):
        return self.name

    @classmethod
    def create_from_dict(cls, dict_from_server):
        """Sync Role locally from the result of the server's
        Role.to_dict(). Return created role."""
        unique_id = dict_from_server['unique_id']

        # If server is on a later version than we are, the dict may contain
        # extra fields. Use only these.
        expected_fields = (
            'unique_id', 'code', 'name',
            'external_description', 'internal_description')

        d = {field: dict_from_server[field] for field in expected_fields}

        role, created = cls.objects.get_or_create(
            unique_id=unique_id, defaults=d)

        if not created:
            # Check for changes.
            changed = False
            for field in expected_fields:
                if getattr(role, field) != d[field]:
                    setattr(role, field, d[field])
                    changed = True
            if changed:
                role.save()

        return role

class OrganisationManager(models.Manager):
    def get_by_natural_key(self, unique_id):
        return self.get(unique_id=unique_id)

@python_2_unicode_compatible
class Organisation(models.Model):
    objects = OrganisationManager()

    # Don't make Organisation name unique because data is only
    # synchronized when someone in an organisation logs in -- we can't
    # guarantee that names will stay unique that way.
    name = models.CharField(max_length=255, null=False, blank=False)
    unique_id = models.CharField(max_length=32, unique=True)

    def __str__(self):
        return self.name

    def natural_key(self):
        return (self.unique_id, )

    @classmethod
    def create_from_dict(cls, dict_from_server):
        """Sync Organisation locally from the result of the server's
        Organisation.to_dict(). Return organisation"""
        unique_id = dict_from_server['unique_id']
        name = dict_from_server['name']

        org, created = cls.objects.get_or_create(
            unique_id=unique_id, defaults=dict(name=name))

        if not created and org.name != name:
            # It changed
            org.name = name
            org.save()

        return org

@python_2_unicode_compatible
class UserOrganisationRole(models.Model):
    """Stores which roles in which organisations a user has."""
    user_model = getattr(settings, 'AUTH_USER_MODEL', 'auth.User')
    user = models.ForeignKey(
        user_model,
        related_name='user_organisation_roles')
    organisation = models.ForeignKey(
        Organisation,
        related_name='user_organisation_roles')
    role = models.ForeignKey(
        Role,
        related_name='user_organisation_roles')

    def __str__(self):
        return '%s %s %s' % (
            str(self.user), str(self.organisation), str(self.role))

    @classmethod
    def create_from_list_of_dicts(cls, user, organisation_roles):
        """Sync from a user and a list of {organisation: ..., role: ...}
        dicts as returned from the server."""
        orgs_synced = dict()
        roles_synced = dict()

        # Remove old ones, then add the new ones.
        cls.objects.filter(user=user).delete()

        for organisation_role in organisation_roles:
            # Sync organisations and roles from their dicts, keep track
            # of which ones we have synced already.
            organisation_dict = organisation_role['organisation']
            role_dict = organisation_role['role']

            organisation_id = organisation_dict['unique_id']
            role_id = role_dict['unique_id']

            if organisation_id not in orgs_synced:
                orgs_synced[organisation_id] = Organisation.create_from_dict(
                    organisation_dict)
            organisation = orgs_synced[organisation_id]

            if role_id not in roles_synced:
                roles_synced[role_id] = Role.create_from_dict(role_dict)
            role = roles_synced[role_id]

            # Actually create the UserOrganisationRole instance.
            cls.objects.get_or_create(
                user=user, organisation=organisation, role=role)

def get_organisations_with_role(user, rolecode):
    """Return a queryset of organisations in which user has this role."""
    return Organisation.objects.filter(
        user_organisation_roles__user=user,
        user_organisation_roles__role__code=rolecode)

def get_organisation_with_role(user, rolecode):
    """Use this version of the function if there must be only a single
    organisation in which the user has this role.

    Raises Organisation.DoesNotExist if user has no such organisation, or
    Organisation.MultipleObjectsReturned if he has more than one.
    """
    return get_organisations_with_role(user, rolecode).get()

def get_user_org_role_dict(user):
    """
    :returns a dict for a given user containing infomration about the user
    itself, the organisations he/she belongs to, and the specific permissions
    (or roles) he has within these organisations::

        {u'email': u'<str>',
         u'is_superuser': <bool>,
         u'username': u'<str>',
         u'organisations': [{u'id': u'<str>',
                             u'name': u'<str>',
                             u'permissions': [u'<str>', u'<str>']},
                            {u'id': u'<str>',
                             u'name': u'<str>.',
                             u'permissions': [u'<str>']}],
         }
    """
    # add top level (user-) information
    payload = {
        'username': user.username,
        'email': user.email,
        'is_superuser': user.is_superuser,
        'organisations': []
    }
    # get all organisations the given user is attached to
    organisations = Organisation.objects.distinct().filter(
        user_organisation_roles__in=user.user_organisation_roles.all()
    )

    # add orgnisation details and user specific roles (read permissions)
    for orga in organisations:
        orga_dict = {'name': orga.name, 'id': orga.unique_id}
        permissions = set(
            orga.user_organisation_roles.filter(
                user=user).values_list('role__code', flat=True)
        )
        permissions.discard('is_connected')
        orga_dict['permissions'] = list(permissions)
        payload['organisations'].append(orga_dict)
    return payload
<commit_msg>typos<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from lizard_auth_client.conf import settings

@python_2_unicode_compatible
class Role(models.Model):
    unique_id = models.CharField(max_length=32, unique=True)
    code = models.CharField(max_length=255, null=False, blank=False)
    name = models.CharField(max_length=255, null=False, blank=False)
    external_description = models.TextField()
    internal_description = models.TextField()

    BILLING_ROLE_CODE = 'billing'

    def __str__(self):
        return self.name

    @classmethod
    def create_from_dict(cls, dict_from_server):
        """Sync Role locally from the result of the server's
        Role.to_dict(). Return created role."""
        unique_id = dict_from_server['unique_id']

        # If server is on a later version than we are, the dict may contain
        # extra fields. Use only these.
        expected_fields = (
            'unique_id', 'code', 'name',
            'external_description', 'internal_description')

        d = {field: dict_from_server[field] for field in expected_fields}

        role, created = cls.objects.get_or_create(
            unique_id=unique_id, defaults=d)

        if not created:
            # Check for changes.
            changed = False
            for field in expected_fields:
                if getattr(role, field) != d[field]:
                    setattr(role, field, d[field])
                    changed = True
            if changed:
                role.save()

        return role

class OrganisationManager(models.Manager):
    def get_by_natural_key(self, unique_id):
        return self.get(unique_id=unique_id)

@python_2_unicode_compatible
class Organisation(models.Model):
    objects = OrganisationManager()

    # Don't make Organisation name unique because data is only
    # synchronized when someone in an organisation logs in -- we can't
    # guarantee that names will stay unique that way.
    name = models.CharField(max_length=255, null=False, blank=False)
    unique_id = models.CharField(max_length=32, unique=True)

    def __str__(self):
        return self.name

    def natural_key(self):
        return (self.unique_id, )

    @classmethod
    def create_from_dict(cls, dict_from_server):
        """Sync Organisation locally from the result of the server's
        Organisation.to_dict(). Return organisation"""
        unique_id = dict_from_server['unique_id']
        name = dict_from_server['name']

        org, created = cls.objects.get_or_create(
            unique_id=unique_id, defaults=dict(name=name))

        if not created and org.name != name:
            # It changed
            org.name = name
            org.save()

        return org

@python_2_unicode_compatible
class UserOrganisationRole(models.Model):
    """Stores which roles in which organisations a user has."""
    user_model = getattr(settings, 'AUTH_USER_MODEL', 'auth.User')
    user = models.ForeignKey(
        user_model,
        related_name='user_organisation_roles')
    organisation = models.ForeignKey(
        Organisation,
        related_name='user_organisation_roles')
    role = models.ForeignKey(
        Role,
        related_name='user_organisation_roles')

    def __str__(self):
        return '%s %s %s' % (
            str(self.user), str(self.organisation), str(self.role))

    @classmethod
    def create_from_list_of_dicts(cls, user, organisation_roles):
        """Sync from a user and a list of {organisation: ..., role: ...}
        dicts as returned from the server."""
        orgs_synced = dict()
        roles_synced = dict()

        # Remove old ones, then add the new ones.
        cls.objects.filter(user=user).delete()

        for organisation_role in organisation_roles:
            # Sync organisations and roles from their dicts, keep track
            # of which ones we have synced already.
            organisation_dict = organisation_role['organisation']
            role_dict = organisation_role['role']

            organisation_id = organisation_dict['unique_id']
            role_id = role_dict['unique_id']

            if organisation_id not in orgs_synced:
                orgs_synced[organisation_id] = Organisation.create_from_dict(
                    organisation_dict)
            organisation = orgs_synced[organisation_id]

            if role_id not in roles_synced:
                roles_synced[role_id] = Role.create_from_dict(role_dict)
            role = roles_synced[role_id]

            # Actually create the UserOrganisationRole instance.
            cls.objects.get_or_create(
                user=user, organisation=organisation, role=role)

def get_organisations_with_role(user, rolecode):
    """Return a queryset of organisations in which user has this role."""
    return Organisation.objects.filter(
        user_organisation_roles__user=user,
        user_organisation_roles__role__code=rolecode)

def get_organisation_with_role(user, rolecode):
    """Use this version of the function if there must be only a single
    organisation in which the user has this role.

    Raises Organisation.DoesNotExist if user has no such organisation, or
    Organisation.MultipleObjectsReturned if he has more than one.
    """
    return get_organisations_with_role(user, rolecode).get()

def get_user_org_role_dict(user):
    """
    :returns a dict for a given user containing information about the user
    itself, the organisations he/she belongs to, and the specific permissions
    (or roles) he/she has within these organisations::

        {u'email': u'<str>',
         u'is_superuser': <bool>,
         u'username': u'<str>',
         u'organisations': [{u'id': u'<str>',
                             u'name': u'<str>',
                             u'permissions': [u'<str>', u'<str>']},
                            {u'id': u'<str>',
                             u'name': u'<str>.',
                             u'permissions': [u'<str>']}],
         }
    """
    # add top level (user-) information
    payload = {
        'username': user.username,
        'email': user.email,
        'is_superuser': user.is_superuser,
        'organisations': []
    }
    # get all organisations the given user is attached to
    organisations = Organisation.objects.distinct().filter(
        user_organisation_roles__in=user.user_organisation_roles.all()
    )

    # add orgnisation details and user specific roles (read permissions)
    for orga in organisations:
        orga_dict = {'name': orga.name, 'id': orga.unique_id}
        permissions = set(
            orga.user_organisation_roles.filter(
                user=user).values_list('role__code', flat=True)
        )
        permissions.discard('is_connected')
        orga_dict['permissions'] = list(permissions)
        payload['organisations'].append(orga_dict)
    return payload
<|endoftext|>


<commit_before># Copyright 2017 Max Planck Society
# Distributed under the BSD-3 Software license,
# (See accompanying file ./LICENSE.txt or copy at
# https://opensource.org/licenses/BSD-3-Clause)
"""Training AdaGAN on various datasets.

Refer to the arXiv paper 'AdaGAN: Boosting Generative Models'
Coded by Ilya Tolstikhin, Carl-Johann Simon-Gabriel
"""

import os
import argparse
import logging
import tensorflow as tf
import numpy as np
from datahandler import DataHandler
from adagan import AdaGan
from metrics import Metrics
import utils

flags = tf.app.flags
flags.DEFINE_float("g_learning_rate", 0.001,
                   "Learning rate for Generator optimizers [16e-4]")
flags.DEFINE_float("d_learning_rate", 0.0005,
                   "Learning rate for Discriminator optimizers [4e-4]")
flags.DEFINE_float("learning_rate", 0.003,
                   "Learning rate for other optimizers [8e-4]")
flags.DEFINE_float("adam_beta1", 0.5, "Beta1 parameter for Adam optimizer [0.5]")
flags.DEFINE_integer("zdim", 8, "Dimensionality of the latent space [100]")
flags.DEFINE_float("init_std", 0.0099999, "Initial variance for weights [0.02]")
flags.DEFINE_string("workdir", 'results_mnist', "Working directory ['results']")
flags.DEFINE_bool("unrolled", False, "Use unrolled GAN training [True]")
flags.DEFINE_bool("vae", False, "Use VAE instead of GAN")
flags.DEFINE_bool("pot", False, "Use POT instead of GAN")
flags.DEFINE_bool("is_bagging", False, "Do we want to use bagging instead of adagan? [False]")
FLAGS = flags.FLAGS

def main():
    opts = {}
    # Utility
    opts['random_seed'] = 66
    opts['dataset'] = 'mnist' # gmm, circle_gmm,  mnist, mnist3 ...
    opts['data_dir'] = 'mnist'
    opts['trained_model_path'] = None #'models'
    opts['mnist_trained_model_file'] = None #'mnist_trainSteps_19999_yhat' # 'mnist_trainSteps_20000'
    opts['work_dir'] = FLAGS.workdir
    opts['ckpt_dir'] = 'checkpoints'
    opts["verbose"] = 1
    opts['tf_run_batch_size'] = 128
    opts["early_stop"] = -1 # set -1 to run normally
    opts["plot_every"] = 50
    opts["save_every_epoch"] = 10
    opts['gmm_max_val'] = 15.

    # Datasets
    opts['toy_dataset_size'] = 10000
    opts['toy_dataset_dim'] = 2
    opts['mnist3_dataset_size'] = 2 * 64 # 64 * 2500
    opts['mnist3_to_channels'] = False # Hide 3 digits of MNIST to channels
    opts['input_normalize_sym'] = False # Normalize data to [-1, 1]
    opts['gmm_modes_num'] = 5

    # AdaGAN parameters
    opts['adagan_steps_total'] = 10
    opts['samples_per_component'] = 1000
    opts['is_bagging'] = FLAGS.is_bagging
    opts['beta_heur'] = 'uniform' # uniform, constant
    opts['weights_heur'] = 'theory_star' # theory_star, theory_dagger, topk
    opts['beta_constant'] = 0.5
    opts['topk_constant'] = 0.5
    opts["mixture_c_epoch_num"] = 5
    opts["eval_points_num"] = 25600
    opts['digit_classification_threshold'] = 0.999
    opts['inverse_metric'] = False # Use metric from the Unrolled GAN paper?
    opts['inverse_num'] = 100 # Number of real points to inverse.
    opts['objective'] = None

    # Generative model parameters
    opts["init_std"] = FLAGS.init_std
    opts["init_bias"] = 0.0
    opts['latent_space_distr'] = 'normal' # uniform, normal
    opts['latent_space_dim'] = FLAGS.zdim
    opts["gan_epoch_num"] = 100
    opts['convolutions'] = True # If False then encoder is MLP of 3 layers
    opts['d_num_filters'] = 512
    opts['d_num_layers'] = 4
    opts['g_num_filters'] = 1024
    opts['g_num_layers'] = 3
    opts['e_is_random'] = False
    opts['e_pretrain'] = False
    opts['e_add_noise'] = False
    opts['e_pretrain_bsize'] = 1000
    opts['e_num_filters'] = 1024
    opts['e_num_layers'] = 4
    opts['g_arch'] = 'dcgan_mod'
    opts['g_stride1_deconv'] = False
    opts['g_3x3_conv'] = 0
    opts['e_arch'] = 'dcgan'
    opts['e_3x3_conv'] = 0
    opts['conv_filters_dim'] = 4
    # --GAN specific:
    opts['conditional'] = False
    opts['unrolled'] = FLAGS.unrolled # Use Unrolled GAN? (only for images)
    opts['unrolling_steps'] = 5 # Used only if unrolled = True
    # --VAE specific
    opts['vae'] = FLAGS.vae
    opts['vae_sigma'] = 0.01
    # --POT specific
    opts['pot'] = FLAGS.pot
    opts['pot_pz_std'] = 2.
    opts['pot_lambda'] = FLAGS.pot_lambda
    opts['adv_c_loss'] = 'none'
    opts['vgg_layer'] = 'pool2'
    opts['adv_c_patches_size'] = 5
    opts['adv_c_num_units'] = 32
    opts['adv_c_loss_w'] = 1.0
    opts['cross_p_w'] = 0.0
    opts['diag_p_w'] = 0.0
    opts['emb_c_loss_w'] = 1.0
    opts['reconstr_w'] = 1.0
    opts['z_test'] = 'gan'
    opts['gan_p_trick'] = False
    opts['pz_transform'] = False
    opts['z_test_corr_w'] = 1.0
    opts['z_test_proj_dim'] = 10

    # Optimizer parameters
    opts['optimizer'] = 'adam' # sgd, adam
    opts["batch_size"] = 100
    opts["d_steps"] = 1
    opts['d_new_minibatch'] = False
    opts["g_steps"] = 2
    opts['batch_norm'] = True
    opts['dropout'] = False
    opts['dropout_keep_prob'] = 0.5
    opts['recon_loss'] = 'l2sq'
    # "manual" or number (float or int) giving the number of epochs to divide
    # the learning rate by 10 (converted into an exp decay per epoch).
    opts['decay_schedule'] = 'manual'
    opts['opt_learning_rate'] = FLAGS.learning_rate
    opts['opt_d_learning_rate'] = FLAGS.d_learning_rate
    opts['opt_g_learning_rate'] = FLAGS.g_learning_rate
    opts["opt_beta1"] = FLAGS.adam_beta1
    opts['batch_norm_eps'] = 1e-05
    opts['batch_norm_decay'] = 0.9

    if opts['e_is_random']:
        assert opts['latent_space_distr'] == 'normal',\
            'Random encoders currently work only with Gaussian Pz'
    # Data augmentation
    opts['data_augm'] = False

    if opts['verbose']:
        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s')
    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')

    utils.create_dir(opts['work_dir'])
    utils.create_dir(os.path.join(opts['work_dir'], opts['ckpt_dir']))

    with utils.o_gfile((opts['work_dir'], 'params.txt'), 'w') as text:
        text.write('Parameters:\n')
        for key in opts:
            text.write('%s : %s\n' % (key, opts[key]))

    data = DataHandler(opts)
    assert data.num_points >= opts['batch_size'], 'Training set too small'
    adagan = AdaGan(opts, data)
    metrics = Metrics()

    train_size = data.num_points
    random_idx = np.random.choice(train_size, 4*320, replace=False)
    metrics.make_plots(opts, 0, data.data,
            data.data[random_idx], adagan._data_weights, prefix='dataset_')

    for step in range(opts["adagan_steps_total"]):
        logging.info('Running step {} of AdaGAN'.format(step + 1))
        adagan.make_step(opts, data)
        num_fake = opts['eval_points_num']
        logging.debug('Sampling fake points')
        fake_points = adagan.sample_mixture(num_fake)
        logging.debug('Sampling more fake points')
        more_fake_points = adagan.sample_mixture(500)
        logging.debug('Plotting results')
        if opts['dataset'] == 'gmm':
            metrics.make_plots(opts, step, data.data[:500],
                    fake_points[0:100], adagan._data_weights[:500])
            logging.debug('Evaluating results')
            (likelihood, C) = metrics.evaluate(
                opts, step, data.data[:500],
                fake_points, more_fake_points, prefix='')
        else:
            metrics.make_plots(opts, step, data.data,
                    fake_points[:320], adagan._data_weights)
            if opts['inverse_metric']:
                logging.debug('Evaluating results')
                l2 = np.min(adagan._invert_losses[:step + 1], axis=0)
                logging.debug('MSE=%.5f, STD=%.5f' % (np.mean(l2), np.std(l2)))
            res = metrics.evaluate(
                opts, step, data.data[:500],
                fake_points, more_fake_points, prefix='')
    logging.debug("AdaGan finished working!")

if __name__ == '__main__':
    main()
<commit_msg>Missing flag<commit_after># Copyright 2017 Max Planck Society
# Distributed under the BSD-3 Software license,
# (See accompanying file ./LICENSE.txt or copy at
# https://opensource.org/licenses/BSD-3-Clause)
"""Training AdaGAN on various datasets.

Refer to the arXiv paper 'AdaGAN: Boosting Generative Models'
Coded by Ilya Tolstikhin, Carl-Johann Simon-Gabriel
"""

import os
import argparse
import logging
import tensorflow as tf
import numpy as np
from datahandler import DataHandler
from adagan import AdaGan
from metrics import Metrics
import utils

flags = tf.app.flags
flags.DEFINE_float("g_learning_rate", 0.001,
                   "Learning rate for Generator optimizers [16e-4]")
flags.DEFINE_float("d_learning_rate", 0.0005,
                   "Learning rate for Discriminator optimizers [4e-4]")
flags.DEFINE_float("learning_rate", 0.003,
                   "Learning rate for other optimizers [8e-4]")
flags.DEFINE_float("adam_beta1", 0.5, "Beta1 parameter for Adam optimizer [0.5]")
flags.DEFINE_integer("zdim", 8, "Dimensionality of the latent space [100]")
flags.DEFINE_float("init_std", 0.0099999, "Initial variance for weights [0.02]")
flags.DEFINE_string("workdir", 'results_mnist', "Working directory ['results']")
flags.DEFINE_bool("unrolled", False, "Use unrolled GAN training [True]")
flags.DEFINE_bool("vae", False, "Use VAE instead of GAN")
flags.DEFINE_bool("pot", False, "Use POT instead of GAN")
flags.DEFINE_bool("is_bagging", False, "Do we want to use bagging instead of adagan? [False]")
FLAGS = flags.FLAGS

def main():
    opts = {}
    # Utility
    opts['random_seed'] = 66
    opts['dataset'] = 'mnist' # gmm, circle_gmm,  mnist, mnist3 ...
    opts['data_dir'] = 'mnist'
    opts['trained_model_path'] = None #'models'
    opts['mnist_trained_model_file'] = None #'mnist_trainSteps_19999_yhat' # 'mnist_trainSteps_20000'
    opts['work_dir'] = FLAGS.workdir
    opts['ckpt_dir'] = 'checkpoints'
    opts["verbose"] = 1
    opts['tf_run_batch_size'] = 128
    opts["early_stop"] = -1 # set -1 to run normally
    opts["plot_every"] = 50
    opts["save_every_epoch"] = 10
    opts['gmm_max_val'] = 15.

    # Datasets
    opts['toy_dataset_size'] = 10000
    opts['toy_dataset_dim'] = 2
    opts['mnist3_dataset_size'] = 2 * 64 # 64 * 2500
    opts['mnist3_to_channels'] = False # Hide 3 digits of MNIST to channels
    opts['input_normalize_sym'] = False # Normalize data to [-1, 1]
    opts['gmm_modes_num'] = 5

    # AdaGAN parameters
    opts['adagan_steps_total'] = 10
    opts['samples_per_component'] = 1000
    opts['is_bagging'] = FLAGS.is_bagging
    opts['beta_heur'] = 'uniform' # uniform, constant
    opts['weights_heur'] = 'theory_star' # theory_star, theory_dagger, topk
    opts['beta_constant'] = 0.5
    opts['topk_constant'] = 0.5
    opts["mixture_c_epoch_num"] = 5
    opts["eval_points_num"] = 25600
    opts['digit_classification_threshold'] = 0.999
    opts['inverse_metric'] = False # Use metric from the Unrolled GAN paper?
    opts['inverse_num'] = 100 # Number of real points to inverse.
    opts['objective'] = None

    # Generative model parameters
    opts["init_std"] = FLAGS.init_std
    opts["init_bias"] = 0.0
    opts['latent_space_distr'] = 'normal' # uniform, normal
    opts['latent_space_dim'] = FLAGS.zdim
    opts["gan_epoch_num"] = 100
    opts['convolutions'] = True # If False then encoder is MLP of 3 layers
    opts['d_num_filters'] = 512
    opts['d_num_layers'] = 4
    opts['g_num_filters'] = 1024
    opts['g_num_layers'] = 3
    opts['e_is_random'] = False
    opts['e_pretrain'] = False
    opts['e_add_noise'] = False
    opts['e_pretrain_bsize'] = 1000
    opts['e_num_filters'] = 1024
    opts['e_num_layers'] = 4
    opts['g_arch'] = 'dcgan_mod'
    opts['g_stride1_deconv'] = False
    opts['g_3x3_conv'] = 0
    opts['e_arch'] = 'dcgan'
    opts['e_3x3_conv'] = 0
    opts['conv_filters_dim'] = 4
    # --GAN specific:
    opts['conditional'] = False
    opts['unrolled'] = FLAGS.unrolled # Use Unrolled GAN? (only for images)
    opts['unrolling_steps'] = 5 # Used only if unrolled = True
    # --VAE specific
    opts['vae'] = FLAGS.vae
    opts['vae_sigma'] = 0.01
    # --POT specific
    opts['pot'] = FLAGS.pot
    opts['pot_pz_std'] = 2.
    opts['adv_c_loss'] = 'none'
    opts['vgg_layer'] = 'pool2'
    opts['adv_c_patches_size'] = 5
    opts['adv_c_num_units'] = 32
    opts['adv_c_loss_w'] = 1.0
    opts['cross_p_w'] = 0.0
    opts['diag_p_w'] = 0.0
    opts['emb_c_loss_w'] = 1.0
    opts['reconstr_w'] = 1.0
    opts['z_test'] = 'gan'
    opts['gan_p_trick'] = False
    opts['pz_transform'] = False
    opts['z_test_corr_w'] = 1.0
    opts['z_test_proj_dim'] = 10

    # Optimizer parameters
    opts['optimizer'] = 'adam' # sgd, adam
    opts["batch_size"] = 100
    opts["d_steps"] = 1
    opts['d_new_minibatch'] = False
    opts["g_steps"] = 2
    opts['batch_norm'] = True
    opts['dropout'] = False
    opts['dropout_keep_prob'] = 0.5
    opts['recon_loss'] = 'l2sq'
    # "manual" or number (float or int) giving the number of epochs to divide
    # the learning rate by 10 (converted into an exp decay per epoch).
    opts['decay_schedule'] = 'manual'
    opts['opt_learning_rate'] = FLAGS.learning_rate
    opts['opt_d_learning_rate'] = FLAGS.d_learning_rate
    opts['opt_g_learning_rate'] = FLAGS.g_learning_rate
    opts["opt_beta1"] = FLAGS.adam_beta1
    opts['batch_norm_eps'] = 1e-05
    opts['batch_norm_decay'] = 0.9

    if opts['e_is_random']:
        assert opts['latent_space_distr'] == 'normal',\
            'Random encoders currently work only with Gaussian Pz'
    # Data augmentation
    opts['data_augm'] = False

    if opts['verbose']:
        logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(message)s')
    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')

    utils.create_dir(opts['work_dir'])
    utils.create_dir(os.path.join(opts['work_dir'], opts['ckpt_dir']))

    with utils.o_gfile((opts['work_dir'], 'params.txt'), 'w') as text:
        text.write('Parameters:\n')
        for key in opts:
            text.write('%s : %s\n' % (key, opts[key]))

    data = DataHandler(opts)
    assert data.num_points >= opts['batch_size'], 'Training set too small'
    adagan = AdaGan(opts, data)
    metrics = Metrics()

    train_size = data.num_points
    random_idx = np.random.choice(train_size, 4*320, replace=False)
    metrics.make_plots(opts, 0, data.data,
            data.data[random_idx], adagan._data_weights, prefix='dataset_')

    for step in range(opts["adagan_steps_total"]):
        logging.info('Running step {} of AdaGAN'.format(step + 1))
        adagan.make_step(opts, data)
        num_fake = opts['eval_points_num']
        logging.debug('Sampling fake points')
        fake_points = adagan.sample_mixture(num_fake)
        logging.debug('Sampling more fake points')
        more_fake_points = adagan.sample_mixture(500)
        logging.debug('Plotting results')
        if opts['dataset'] == 'gmm':
            metrics.make_plots(opts, step, data.data[:500],
                    fake_points[0:100], adagan._data_weights[:500])
            logging.debug('Evaluating results')
            (likelihood, C) = metrics.evaluate(
                opts, step, data.data[:500],
                fake_points, more_fake_points, prefix='')
        else:
            metrics.make_plots(opts, step, data.data,
                    fake_points[:320], adagan._data_weights)
            if opts['inverse_metric']:
                logging.debug('Evaluating results')
                l2 = np.min(adagan._invert_losses[:step + 1], axis=0)
                logging.debug('MSE=%.5f, STD=%.5f' % (np.mean(l2), np.std(l2)))
            res = metrics.evaluate(
                opts, step, data.data[:500],
                fake_points, more_fake_points, prefix='')
    logging.debug("AdaGan finished working!")

if __name__ == '__main__':
    main()
<|endoftext|>