<commit_before>import itertools as it
from collections import namedtuple

import numpy as np
from hfetch import HNumpyStore

from . import config, log
from .IStorage import IStorage
from .tools import extract_ks_tab, get_istorage_attrs

class StorageNumpy(IStorage, np.ndarray):
    class np_meta(object):
        def __init__(self, shape, dtype, block_id):
            self.dims = shape
            self.type = dtype
            self.block_id = block_id

    _build_args = None
    _prepared_store_meta = config.session.prepare('INSERT INTO hecuba.istorage'
                                                  '(storage_id, class_name, name, numpy_meta)'
                                                  'VALUES (?,?,?,?)')

    args_names = ["storage_id", "class_name", "name", "shape", "dtype", "block_id", "built_remotely"]
    args = namedtuple('StorageNumpyArgs', args_names)

    def __new__(cls, input_array=None, storage_id=None, name=None, built_remotely=False, **kwargs):
        if name:
            name = name + '_numpies'
        elif storage_id:
            metas = get_istorage_attrs(storage_id)
            name = metas[0].name
        if input_array is None and name and storage_id is not None:
            result = cls.reserve_numpy_array(storage_id, name)
            input_array = result[0]
            obj = np.asarray(input_array).view(cls)
            (obj._ksp, obj._table) = extract_ks_tab(name)
            obj._row_elem = result[1]
            obj._hcache = result[2]
            obj._numpy_full_loaded = False
            obj._loaded_coordinates = []
        elif not name and storage_id is not None:
            raise RuntimeError("hnumpy received storage id but not a name")
        else:
            obj = np.asarray(input_array).view(cls)
        # Finally, we must return the newly created object:
        obj._block_id = -1
        obj._built_remotely = built_remotely
        obj._class_name = '%s.%s' % (cls.__module__, cls.__name__)
        return obj

    def __init__(self, input_array=None, storage_id=None, name=None, **kwargs):
        IStorage.__init__(self, storage_id=storage_id, name=name, **kwargs)
        if name or storage_id:
            if input_array is not None:
                self.make_persistent(name)
            self._is_persistent = True

    # used as copy constructor
    def __array_finalize__(self, obj):
        if obj is None:
            return
        self.storage_id = getattr(obj, 'storage_id', None)
        self._hcache = getattr(obj, '_hcache', None)
        self._row_elem = getattr(obj, '_row_elem', None)
        self._loaded_coordinates = getattr(obj, '_loaded_coordinates', None)
        self._numpy_full_loaded = getattr(obj, '_numpy_full_loaded', None)
        self._is_persistent = getattr(obj, '_is_persistent', None)

    @staticmethod
    def _create_tables(name):
        (ksp, table) = extract_ks_tab(name)
        query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (ksp, config.replication)
        config.session.execute(query_keyspace)

        config.session.execute(
            'CREATE TABLE IF NOT EXISTS ' + ksp + '.' + table + '(storage_id uuid , '
                                                                'cluster_id int, '
                                                                'block_id int, '
                                                                'payload blob, '
                                                                'PRIMARY KEY((storage_id,cluster_id),block_id))')

    @staticmethod
    def _create_hcache(name):
        (ksp, table) = extract_ks_tab(name)
        hcache_params = (ksp, table,
                         {'cache_size': config.max_cache_size,
                          'writer_par': config.write_callbacks_number,
                          'write_buffer': config.write_buffer_size,
                          'timestamped_writes': False})

        return HNumpyStore(*hcache_params)

    @staticmethod
    def _store_meta(storage_args):
        """
            Saves the information of the object in the istorage table.
            Args:.
                storage_args (object): contains all data needed to restore the object from the workers
        """
        log.debug("StorageObj: storing media %s", storage_args)
        try:
            config.session.execute(StorageNumpy._prepared_store_meta,
                                   [storage_args.storage_id, storage_args.class_name,
                                    storage_args.name, StorageNumpy.np_meta(storage_args.shape, storage_args.dtype,
                                                                            storage_args.block_id)])

        except Exception as ex:
            log.warn("Error creating the StorageNumpy metadata with args: %s" % str(storage_args))
            raise ex

    @staticmethod
    def reserve_numpy_array(storage_id, name):
        '''Provides a numpy array with the number of elements obtained through storage_id'''

        hcache = StorageNumpy._create_hcache(name)
        result = hcache.allocate_numpy(storage_id)
        if len(result) == 2:
            return [result[0], result[1], hcache]
        else:
            raise KeyError

    def generate_coordinates(self, coordinates):
        if coordinates is None:
            return []
        # coords divided by number of elem in a row
        coord = [coordinates[:, coord] // self._row_elem for coord in range(len(coordinates[0]))]
        ranges = (range(*range_tuple) for range_tuple in zip(coord[0], coord[1] + 1))
        keys = list(it.product(*ranges))
        return keys

    def format_coords(self, coord):
        if not isinstance(coord, list):
            if isinstance(coord, set) or isinstance(coord, tuple):
                coord = list(coord)
            elif isinstance(coord, slice):
                coord = [coord]
            else:
                return Exception('Slice formats supported: list, set, tuple of slices and slice class')

        np_list = []
        count_none = 0
        for dim, coo in enumerate(coord):
            if coo.stop is None and coo.start is None:
                np_list.append([0, self.shape[dim]])
                count_none = count_none + 1
            elif coo.start is None:
                np_list.append([0, coo.stop])
            elif coo.stop is None:
                np_list.append([coo.start, self.shape[dim]])
            else:
                np_list.append([coo.start, coo.stop])

        if count_none == len(coord): return None
        coordinates = np.array(np_list)
        return coordinates

    def slices_match_numpy_shape(self, sliced_coord):
        if sliced_coord is None:
            return True
        elif len(self.shape) != len(sliced_coord):
            return False
        else:
            for i, queried_slice in enumerate(sliced_coord):
                return queried_slice[1] <= self.shape[i]

    def get_coords_match_numpy_shape(self, coo):
        new_coords = self.generate_coordinates(coo)
        shape_slices = [slice(0, coord) for coord in self.shape]
        formated_shape_coords = self.format_coords(shape_slices)
        numpy_coords = self.generate_coordinates(formated_shape_coords)
        coordinates = []
        for coord in new_coords:
            if coord in numpy_coords and coord not in coordinates:
                coordinates.append(coord)
        return coordinates

    def __getitem__(self, sliced_coord):
        log.info("RETRIEVING NUMPY")
        if self._is_persistent:
            # formats sliced coords
            new_coords = self.format_coords(sliced_coord)

            # checks if some coord in sliced_coords are inside the numpy
            if not self.slices_match_numpy_shape(new_coords):  # some coordinates match
                new_coords = self.get_coords_match_numpy_shape(new_coords)  # generates the coordinates
                if not new_coords:
                    self._hcache.load_numpy_slices([self.storage_id], [self.view(np.ndarray)],
                                                   None)  # any coordinates generated match
                    return super(StorageNumpy, self).__getitem__(sliced_coord)
            else:  # coordinates match
                new_coords = self.generate_coordinates(new_coords)

            # coordinates is the union between the loaded coordiantes and the new ones
            coordinates = list(set(it.chain.from_iterable((self._loaded_coordinates, new_coords))))

            # checks if we already loaded the coordinates
            if ((len(coordinates) != len(self._loaded_coordinates)) and not self._numpy_full_loaded) or (
                    not self._numpy_full_loaded and not coordinates):
                if not coordinates:
                    self._numpy_full_loaded = True
                    new_coords = None
                self._hcache.load_numpy_slices([self.storage_id], [self.view(np.ndarray)], new_coords)
                self._loaded_coordinates = coordinates
        return super(StorageNumpy, self).__getitem__(sliced_coord)

    def __setitem__(self, sliced_coord, values):
        log.info("WRITING NUMPY")
        if self._is_persistent:
            coo = self.format_coords(sliced_coord)
            coordinates = list(set(it.chain.from_iterable(
                (self._loaded_coordinates, self.generate_coordinates(coo)))))
            self._hcache.store_numpy_slices([self.storage_id], [self.view(np.ndarray)], coordinates)
        return super(StorageNumpy, self).__setitem__(sliced_coord, values)

    def make_persistent(self, name):
        if not name.endswith("_numpies"):
            name = name + '_numpies'

        super().make_persistent(name)

        self._build_args = self.args(self.storage_id, self._class_name, self._ksp + '.' + self._table,
                                     self.shape, self.dtype.num, self._block_id, self._built_remotely)

        if not self._built_remotely:
            self._create_tables(name)

        if not getattr(self, '_hcache', None):
            self._hcache = self._create_hcache(name)

        if len(self.shape) != 0:
            self._hcache.store_numpy_slices([self.storage_id], [self.view(np.ndarray)], None)
        StorageNumpy._store_meta(self._build_args)

    def stop_persistent(self):
        super().stop_persistent()

        self.storage_id = None

    def delete_persistent(self):
        """
            Deletes the Cassandra table where the persistent StorageObj stores data
        """
        super().delete_persistent()
        query = "DELETE FROM {}.{} WHERE storage_id = {} AND cluster_id=-1;".format(self._ksp, self._table + '_numpies',
                                                                                self.storage_id)
        query2 = "DELETE FROM hecuba.istorage WHERE storage_id = %s;" % self.storage_id
        log.debug("DELETE PERSISTENT: %s", query)
        config.session.execute(query)
        config.session.execute(query2)

    def __iter__(self):
        return iter(self.view(np.ndarray))

    def __contains__(self, item):
        return item in self.view(np.ndarray)

    def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
        args = []
        for input_ in inputs:
            if isinstance(input_, StorageNumpy):
                args.append(input_.view(np.ndarray))
            else:
                args.append(input_)

        outputs = kwargs.pop('out', None)
        if outputs:
            out_args = []
            for output in outputs:
                if isinstance(output, StorageNumpy):
                    out_args.append(output.view(np.ndarray))
                else:
                    out_args.append(output)
            kwargs['out'] = tuple(out_args)
        else:
            outputs = (None,) * ufunc.nout
        if self._is_persistent and len(self.shape):
            self._hcache.load_numpy_slices([self.storage_id], [self.view(np.ndarray)], None)

        results = super(StorageNumpy, self).__array_ufunc__(ufunc, method,
                                                            *args, **kwargs)
        if results is NotImplemented:
            return NotImplemented

        if method == 'at':
            return

        if self._is_persistent and len(self.shape):
            self._hcache.store_numpy_slices([self.storage_id], [self.view(np.ndarray)], None)

        if ufunc.nout == 1:
            results = (results,)

        results = tuple((result
                         if output is None else output)
                        for result, output in zip(results, outputs))

        return results[0] if len(results) == 1 else results
<commit_msg>fixing tests<commit_after>import itertools as it
from collections import namedtuple

import numpy as np
from hfetch import HNumpyStore

from . import config, log
from .IStorage import IStorage
from .tools import extract_ks_tab, get_istorage_attrs

class StorageNumpy(IStorage, np.ndarray):
    class np_meta(object):
        def __init__(self, shape, dtype, block_id):
            self.dims = shape
            self.type = dtype
            self.block_id = block_id

    _build_args = None
    _prepared_store_meta = config.session.prepare('INSERT INTO hecuba.istorage'
                                                  '(storage_id, class_name, name, numpy_meta)'
                                                  'VALUES (?,?,?,?)')

    args_names = ["storage_id", "class_name", "name", "shape", "dtype", "block_id", "built_remotely"]
    args = namedtuple('StorageNumpyArgs', args_names)

    def __new__(cls, input_array=None, storage_id=None, name=None, built_remotely=False, **kwargs):
        if name:
            name = name + '_numpies'
        elif storage_id:
            metas = get_istorage_attrs(storage_id)
            name = metas[0].name
        if input_array is None and name and storage_id is not None:
            result = cls.reserve_numpy_array(storage_id, name)
            input_array = result[0]
            obj = np.asarray(input_array).view(cls)
            (obj._ksp, obj._table) = extract_ks_tab(name)
            obj._row_elem = result[1]
            obj._hcache = result[2]
            obj._numpy_full_loaded = False
            obj._loaded_coordinates = []
        elif not name and storage_id is not None:
            raise RuntimeError("hnumpy received storage id but not a name")
        else:
            obj = np.asarray(input_array).view(cls)
        # Finally, we must return the newly created object:
        obj._block_id = -1
        obj._built_remotely = built_remotely
        obj._class_name = '%s.%s' % (cls.__module__, cls.__name__)
        return obj

    def __init__(self, input_array=None, storage_id=None, name=None, **kwargs):
        IStorage.__init__(self, storage_id=storage_id, name=name, **kwargs)
        if name or storage_id:
            if input_array is not None:
                self.make_persistent(name)
            self._is_persistent = True

    # used as copy constructor
    def __array_finalize__(self, obj):
        if obj is None:
            return
        self.storage_id = getattr(obj, 'storage_id', None)
        self._hcache = getattr(obj, '_hcache', None)
        self._row_elem = getattr(obj, '_row_elem', None)
        self._loaded_coordinates = getattr(obj, '_loaded_coordinates', None)
        self._numpy_full_loaded = getattr(obj, '_numpy_full_loaded', None)
        self._is_persistent = getattr(obj, '_is_persistent', None)

    @staticmethod
    def _create_tables(name):
        (ksp, table) = extract_ks_tab(name)
        query_keyspace = "CREATE KEYSPACE IF NOT EXISTS %s WITH replication = %s" % (ksp, config.replication)
        config.session.execute(query_keyspace)

        config.session.execute(
            'CREATE TABLE IF NOT EXISTS ' + ksp + '.' + table + '(storage_id uuid , '
                                                                'cluster_id int, '
                                                                'block_id int, '
                                                                'payload blob, '
                                                                'PRIMARY KEY((storage_id,cluster_id),block_id))')

    @staticmethod
    def _create_hcache(name):
        (ksp, table) = extract_ks_tab(name)
        hcache_params = (ksp, table,
                         {'cache_size': config.max_cache_size,
                          'writer_par': config.write_callbacks_number,
                          'write_buffer': config.write_buffer_size,
                          'timestamped_writes': False})

        return HNumpyStore(*hcache_params)

    @staticmethod
    def _store_meta(storage_args):
        """
            Saves the information of the object in the istorage table.
            Args:.
                storage_args (object): contains all data needed to restore the object from the workers
        """
        log.debug("StorageObj: storing media %s", storage_args)
        try:
            config.session.execute(StorageNumpy._prepared_store_meta,
                                   [storage_args.storage_id, storage_args.class_name,
                                    storage_args.name, StorageNumpy.np_meta(storage_args.shape, storage_args.dtype,
                                                                            storage_args.block_id)])

        except Exception as ex:
            log.warn("Error creating the StorageNumpy metadata with args: %s" % str(storage_args))
            raise ex

    @staticmethod
    def reserve_numpy_array(storage_id, name):
        '''Provides a numpy array with the number of elements obtained through storage_id'''

        hcache = StorageNumpy._create_hcache(name)
        result = hcache.allocate_numpy(storage_id)
        if len(result) == 2:
            return [result[0], result[1], hcache]
        else:
            raise KeyError

    def generate_coordinates(self, coordinates):
        if coordinates is None:
            return []
        # coords divided by number of elem in a row
        coord = [coordinates[:, coord] // self._row_elem for coord in range(len(coordinates[0]))]
        ranges = (range(*range_tuple) for range_tuple in zip(coord[0], coord[1] + 1))
        keys = list(it.product(*ranges))
        return keys

    def format_coords(self, coord):
        if not isinstance(coord, list):
            if isinstance(coord, set) or isinstance(coord, tuple):
                coord = list(coord)
            elif isinstance(coord, slice):
                coord = [coord]
            else:
                return Exception('Slice formats supported: list, set, tuple of slices and slice class')

        if not all(isinstance(c, slice) for c in coord): return Exception('All slices should be of class slice')

        np_list = []
        count_none = 0
        for dim, coo in enumerate(coord):
            if coo.stop is None and coo.start is None:
                np_list.append([0, self.shape[dim]])
                count_none = count_none + 1
            elif coo.start is None:
                np_list.append([0, coo.stop])
            elif coo.stop is None:
                np_list.append([coo.start, self.shape[dim]])
            else:
                np_list.append([coo.start, coo.stop])

        if count_none == len(coord): return None
        coordinates = np.array(np_list)
        return coordinates

    def slices_match_numpy_shape(self, sliced_coord):
        if sliced_coord is None:
            return True
        elif len(self.shape) != len(sliced_coord):
            return False
        else:
            for i, queried_slice in enumerate(sliced_coord):
                return queried_slice[1] <= self.shape[i]

    def get_coords_match_numpy_shape(self, coo):
        new_coords = self.generate_coordinates(coo)
        shape_slices = [slice(0, coord) for coord in self.shape]
        formated_shape_coords = self.format_coords(shape_slices)
        numpy_coords = self.generate_coordinates(formated_shape_coords)
        coordinates = []
        for coord in new_coords:
            if coord in numpy_coords and coord not in coordinates:
                coordinates.append(coord)
        return coordinates

    def __getitem__(self, sliced_coord):
        log.info("RETRIEVING NUMPY")
        if self._is_persistent:
            # formats sliced coords
            new_coords = self.format_coords(sliced_coord)

            # checks if some coord in sliced_coords are inside the numpy
            if not self.slices_match_numpy_shape(new_coords):  # some coordinates match
                new_coords = self.get_coords_match_numpy_shape(new_coords)  # generates the coordinates
                if not new_coords:
                    self._hcache.load_numpy_slices([self.storage_id], [self.view(np.ndarray)],
                                                   None)  # any coordinates generated match
                    return super(StorageNumpy, self).__getitem__(sliced_coord)
            else:  # coordinates match
                new_coords = self.generate_coordinates(new_coords)

            # coordinates is the union between the loaded coordiantes and the new ones
            coordinates = list(set(it.chain.from_iterable((self._loaded_coordinates, new_coords))))

            # checks if we already loaded the coordinates
            if ((len(coordinates) != len(self._loaded_coordinates)) and not self._numpy_full_loaded) or (
                    not self._numpy_full_loaded and not coordinates):
                if not coordinates:
                    self._numpy_full_loaded = True
                    new_coords = None
                self._hcache.load_numpy_slices([self.storage_id], [self.view(np.ndarray)], new_coords)
                self._loaded_coordinates = coordinates
        return super(StorageNumpy, self).__getitem__(sliced_coord)

    def __setitem__(self, sliced_coord, values):
        log.info("WRITING NUMPY")
        if self._is_persistent:
            coo = self.format_coords(sliced_coord)
            coordinates = list(set(it.chain.from_iterable(
                (self._loaded_coordinates, self.generate_coordinates(coo)))))
            self._hcache.store_numpy_slices([self.storage_id], [self.view(np.ndarray)], coordinates)
        return super(StorageNumpy, self).__setitem__(sliced_coord, values)

    def make_persistent(self, name):
        if not name.endswith("_numpies"):
            name = name + '_numpies'

        super().make_persistent(name)

        self._build_args = self.args(self.storage_id, self._class_name, self._ksp + '.' + self._table,
                                     self.shape, self.dtype.num, self._block_id, self._built_remotely)

        if not self._built_remotely:
            self._create_tables(name)

        if not getattr(self, '_hcache', None):
            self._hcache = self._create_hcache(name)

        if len(self.shape) != 0:
            self._hcache.store_numpy_slices([self.storage_id], [self.view(np.ndarray)], None)
        StorageNumpy._store_meta(self._build_args)

    def stop_persistent(self):
        super().stop_persistent()

        self.storage_id = None

    def delete_persistent(self):
        """
            Deletes the Cassandra table where the persistent StorageObj stores data
        """
        super().delete_persistent()
        query = "DELETE FROM {}.{} WHERE storage_id = {} AND cluster_id=-1;".format(self._ksp, self._table + '_numpies',
                                                                                self.storage_id)
        query2 = "DELETE FROM hecuba.istorage WHERE storage_id = %s;" % self.storage_id
        log.debug("DELETE PERSISTENT: %s", query)
        config.session.execute(query)
        config.session.execute(query2)

    def __iter__(self):
        return iter(self.view(np.ndarray))

    def __contains__(self, item):
        return item in self.view(np.ndarray)

    def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
        args = []
        for input_ in inputs:
            if isinstance(input_, StorageNumpy):
                args.append(input_.view(np.ndarray))
            else:
                args.append(input_)

        outputs = kwargs.pop('out', None)
        if outputs:
            out_args = []
            for output in outputs:
                if isinstance(output, StorageNumpy):
                    out_args.append(output.view(np.ndarray))
                else:
                    out_args.append(output)
            kwargs['out'] = tuple(out_args)
        else:
            outputs = (None,) * ufunc.nout
        if self._is_persistent and len(self.shape):
            self._hcache.load_numpy_slices([self.storage_id], [self.view(np.ndarray)], None)

        results = super(StorageNumpy, self).__array_ufunc__(ufunc, method,
                                                            *args, **kwargs)
        if results is NotImplemented:
            return NotImplemented

        if method == 'at':
            return

        if self._is_persistent and len(self.shape):
            self._hcache.store_numpy_slices([self.storage_id], [self.view(np.ndarray)], None)

        if ufunc.nout == 1:
            results = (results,)

        results = tuple((result
                         if output is None else output)
                        for result, output in zip(results, outputs))

        return results[0] if len(results) == 1 else results
<|endoftext|>


<commit_before>from django.contrib import admin
from django.utils.translation import ugettext_lazy as _
from django.utils.datetime_safe import date
from django.db.models import Q
from open_municipio.people.models import *
from open_municipio.votations.admin import VotationsInline
from open_municipio.acts.models import Speech
from open_municipio.widgets import SortWidget

from sorl.thumbnail.admin import AdminImageMixin

from django.contrib.admin.util import unquote
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404
from django.utils.functional import update_wrapper
from django.utils.html import strip_spaces_between_tags as short
from django.forms import ModelForm

from open_municipio.people.forms import SittingItemFormSet, SpeechInlineForm
from open_municipio.om.admin import LinkedTabularInline

class PersonResourceInline(admin.TabularInline):
    model = PersonResource
    extra = 0

class PersonAdminWithResources(AdminImageMixin, admin.ModelAdmin):
    list_display = ('id', '__unicode__', 'has_current_charges', 'birth_date', 'birth_location' )
    list_display_links = ('__unicode__',)
    search_fields = ['^first_name', '^last_name']
    prepopulated_fields = {"slug": ("first_name","last_name","birth_date", "birth_location",)}
    inlines = [PersonResourceInline, ]

class ChargeAdmin(admin.ModelAdmin):
    raw_id_fields = ('person', )

class GroupResourceInline(admin.TabularInline):
    model = GroupResource
    extra = 0

class GroupChargeInline(admin.TabularInline):
    model = GroupCharge
    raw_id_fields = ('charge', )
    extra = 1

class GroupIsMajorityInline(admin.TabularInline):
    model = GroupIsMajority
    extra = 0

class InCouncilNow(admin.SimpleListFilter):

    title = _("In council now")

    parameter_name = "in_council_now"

    def lookups(self, request, model_admin):

        return(
            ('1', _('Yes')),
            ('0', _('No')),
        )

    def queryset(self, request, queryset):

        val = self.value()

        today = date.today()

        # note: groups with no related item (groupismajority) will be considered
        # as not in council

        if val == '1':
            queryset = queryset.exclude(groupismajority__isnull=True).filter(Q(groupismajority__end_date__gt=today) | Q(groupismajority__end_date__isnull=True))

        elif val == '0':
            # the check for groups NOT in majority is more complex because
            # we have to check that ALL related objects (groupismajority)
            # have an end_date previous the current date
            groups_in_council = Group.objects.exclude(groupismajority__isnull=True).filter(Q(groupismajority__end_date__gt=today) | Q(groupismajority__end_date__isnull=True))

            queryset = queryset.exclude(pk__in=groups_in_council)

        return queryset

class GroupAdminWithCharges(AdminImageMixin, admin.ModelAdmin):
    prepopulated_fields = {"slug": ("name",)}
    list_display = ('name', 'acronym', 'is_majority_now', 'in_council_now')
    inlines = [GroupResourceInline, GroupIsMajorityInline, GroupChargeInline]

    list_filter = [ InCouncilNow, 'groupismajority__is_majority' ]

    ordering = [ 'name', 'acronym' ]

    def is_majority_now(self, obj):
        return obj.is_majority_now
    is_majority_now.short_description = _("Is majority now")

    def in_council_now(self, obj):
        return obj.in_council_now
    in_council_now.short_description = _("In council now")

class ChargeInline(admin.StackedInline):
    raw_id_fields = ('person', )
    fieldsets = (
        (None, {
            'fields': (('person', 'start_date', 'end_date'), )
        }),
        (_('Advanced options'), {
            'classes': ('collapse',),
            'fields': ('description', 'end_reason')
        })
    )
    extra = 1

class CompanyChargeInline(ChargeInline):
    model = CompanyCharge

class AdministrationChargeInline(ChargeInline):
    model = AdministrationCharge

class InstitutionResourceInline(admin.TabularInline):
    model = InstitutionResource
    extra = 0

class InstitutionChargeInline(ChargeInline):
    model = InstitutionCharge
    raw_id_fields = ('person', 'substitutes', 'substituted_by')
    fieldsets = (
        (None, {
            'fields': (('person', 'op_charge_id', 'start_date', 'end_date'), )
        }),
        (_('Advanced options'), {
            'classes': ('collapse',),
            'fields': ('description', 'end_reason', ('substitutes', 'substituted_by'))
        })
    )

class ResponsabilityInline(admin.TabularInline):
    raw_id_fields = ('charge',)
    extra = 0

class InstitutionResponsabilityInline(ResponsabilityInline):
    model = InstitutionResponsability
    fields = ('charge', 'charge_type', 'start_date', 'end_date', 'description')

class GroupResponsabilityInline(admin.TabularInline):
    model = GroupResponsability
    raw_id_fields = ('charge',)
    extra = 0
    fields = ('charge', 'charge_type', 'start_date', 'end_date', 'description')

class ChargeAdmin(admin.ModelAdmin):
    pass

class CompanyChargeAdmin(ChargeAdmin):
    model = CompanyCharge
    raw_id_fields = ('person', 'company')
    fieldsets = (
        (None, {
            'fields': (('person', 'company'),
                ('start_date', 'end_date', 'end_reason'),
                 'description')
        }),
    )

class AdministrationChargeAdmin(ChargeAdmin):
    model = AdministrationCharge
    raw_id_fields = ('person', 'office')
    fieldsets = (
        (None, {
            'fields': (('person', 'office'),
                 ('start_date', 'end_date', 'end_reason'),
                 'description')
        }),
    )

class InstitutionChargeAdmin(ChargeAdmin):
    model = InstitutionCharge
    raw_id_fields = ('person', 'substitutes', 'substituted_by', 'original_charge')
    search_fields = ['^person__first_name', '^person__last_name']

    fieldsets = (
        (None, {
            'fields': (('person', 'op_charge_id', 'institution', 'original_charge'),
                 ('start_date', 'end_date', 'end_reason'),
                  'description',
                 ('substitutes', 'substituted_by'))
        }),
        (_("Presences"), {
            'fields': (('n_present_votations', 'n_absent_votations'), ('n_present_attendances', 'n_absent_attendances'))
        }),
    )
    list_display = ('__unicode__', 'institution', 'start_date', 'end_date')
    list_select_related = True
    list_filter = ['institution__name']
    inlines = [InstitutionResponsabilityInline]

class GroupChargeAdmin(admin.ModelAdmin):
    raw_id_fields = ('charge', )
    list_display = ('__unicode__', 'start_date', 'end_date')
    list_select_related = True
    list_filter = ['group']
    inlines = [GroupResponsabilityInline]

    search_fields = [ 'charge__person__first_name', 'charge__person__last_name', 'group__name', 'group__acronym', ]

class BodyAdmin(admin.ModelAdmin):
    prepopulated_fields = {"slug": ("name",)}

class CompanyAdmin(BodyAdmin):
    inlines = [CompanyChargeInline]

class OfficeAdmin(BodyAdmin):
    inlines = [AdministrationChargeInline]

class InstitutionAdmin(BodyAdmin):

    list_filter = ("institution_type", )

    def get_urls(self):
        from django.conf.urls.defaults import patterns, url

        def wrap(view):
            def wrapper(*args, **kwargs):
                return self.admin_site.admin_view(view)(*args, **kwargs)
            return update_wrapper(wrapper, view)
        info = self.model._meta.app_label, self.model._meta.module_name
        return patterns('',
                        url(r'^(.+)/move-(up)/$',
                            wrap(self.move_view),
                            name='%s_%s_move_up' % info),
                        url(r'^(.+)/move-(down)/$',
                            wrap(self.move_view),
                            name='%s_%s_move_down' % info),
                        ) + super(InstitutionAdmin, self).get_urls()

    def move_view(self, request, object_id, direction):
        obj = get_object_or_404(self.model, pk=unquote(object_id))
        if direction == 'up':
            obj.move_up()
        else:
            obj.move_down()
        return HttpResponseRedirect('../../')

    link_html = short("""
        UP |
        DOWN (%(position)s)
        """)

    def move_up_down_links(self, obj):
        return self.link_html % {
            'app_label': self.model._meta.app_label,
            'module_name': self.model._meta.module_name,
            'object_id': obj.id,
            'position': obj.position,
        }
    move_up_down_links.allow_tags = True
    move_up_down_links.short_description = _(u'Move')

    inlines = [InstitutionResourceInline, InstitutionChargeInline]
    list_display = ('name', 'institution_type', 'move_up_down_links',)

class SpeechInline(LinkedTabularInline):
    model = Speech
    fields = ("author", "author_name_when_external", "title", \
              "seq_order", "admin_link", )
    raw_id_fields = ["author", ]
    extra = 0
    form = SpeechInlineForm

class SittingItemInline(LinkedTabularInline):
    model = SittingItem
    fields = ('title', 'related_act_set', 'item_type', 'seq_order','admin_link',)
    raw_id_fields = ('related_act_set',)
    extra = 0

    form = SittingItemFormSet

class SittingItemAdmin(admin.ModelAdmin):
    list_display = ( 'title','sitting', 'seq_order','item_type','num_related_acts')
    ordering = ('-sitting__date','seq_order')
    search_fields = ['^title', ]
    list_filter = ['sitting__institution','item_type',]

    raw_id_fields = ( 'sitting', 'related_act_set', )

    inlines = [SpeechInline, ]

class SittingAdmin(admin.ModelAdmin):
    inlines = [SittingItemInline, VotationsInline]

admin.site.register(SittingItem, SittingItemAdmin)
admin.site.register(Sitting, SittingAdmin)
admin.site.register(Person, PersonAdminWithResources)
admin.site.register(Group, GroupAdminWithCharges)
admin.site.register(GroupCharge, GroupChargeAdmin)
admin.site.register(InstitutionCharge, InstitutionChargeAdmin)
admin.site.register(CompanyCharge, CompanyChargeAdmin)
admin.site.register(AdministrationCharge, AdministrationChargeAdmin)
admin.site.register(Institution, InstitutionAdmin)
admin.site.register(Company, CompanyAdmin)
admin.site.register(Office, OfficeAdmin)
<commit_msg>add filters for group names and member names in group admin<commit_after>from django.contrib import admin
from django.utils.translation import ugettext_lazy as _
from django.utils.datetime_safe import date
from django.db.models import Q
from open_municipio.people.models import *
from open_municipio.votations.admin import VotationsInline
from open_municipio.acts.models import Speech
from open_municipio.widgets import SortWidget

from sorl.thumbnail.admin import AdminImageMixin

from django.contrib.admin.util import unquote
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404
from django.utils.functional import update_wrapper
from django.utils.html import strip_spaces_between_tags as short
from django.forms import ModelForm

from open_municipio.people.forms import SittingItemFormSet, SpeechInlineForm
from open_municipio.om.admin import LinkedTabularInline

class PersonResourceInline(admin.TabularInline):
    model = PersonResource
    extra = 0

class PersonAdminWithResources(AdminImageMixin, admin.ModelAdmin):
    list_display = ('id', '__unicode__', 'has_current_charges', 'birth_date', 'birth_location' )
    list_display_links = ('__unicode__',)
    search_fields = ['^first_name', '^last_name']
    prepopulated_fields = {"slug": ("first_name","last_name","birth_date", "birth_location",)}
    inlines = [PersonResourceInline, ]

class ChargeAdmin(admin.ModelAdmin):
    raw_id_fields = ('person', )

class GroupResourceInline(admin.TabularInline):
    model = GroupResource
    extra = 0

class GroupChargeInline(admin.TabularInline):
    model = GroupCharge
    raw_id_fields = ('charge', )
    extra = 1

class GroupIsMajorityInline(admin.TabularInline):
    model = GroupIsMajority
    extra = 0

class InCouncilNow(admin.SimpleListFilter):

    title = _("In council now")

    parameter_name = "in_council_now"

    def lookups(self, request, model_admin):

        return(
            ('1', _('Yes')),
            ('0', _('No')),
        )

    def queryset(self, request, queryset):

        val = self.value()

        today = date.today()

        # note: groups with no related item (groupismajority) will be considered
        # as not in council

        if val == '1':
            queryset = queryset.exclude(groupismajority__isnull=True).filter(Q(groupismajority__end_date__gt=today) | Q(groupismajority__end_date__isnull=True))

        elif val == '0':
            # the check for groups NOT in majority is more complex because
            # we have to check that ALL related objects (groupismajority)
            # have an end_date previous the current date
            groups_in_council = Group.objects.exclude(groupismajority__isnull=True).filter(Q(groupismajority__end_date__gt=today) | Q(groupismajority__end_date__isnull=True))

            queryset = queryset.exclude(pk__in=groups_in_council)

        return queryset

class GroupAdminWithCharges(AdminImageMixin, admin.ModelAdmin):
    prepopulated_fields = {"slug": ("name",)}
    list_display = ('name', 'acronym', 'is_majority_now', 'in_council_now')
    inlines = [GroupResourceInline, GroupIsMajorityInline, GroupChargeInline]

    search_fields = [ 'name', 'acronym', 'slug', 'charge_set__person__first_name', 'charge_set__person__last_name', ]

    list_filter = [ InCouncilNow, 'groupismajority__is_majority' ]

    ordering = [ 'name', 'acronym' ]

    def is_majority_now(self, obj):
        return obj.is_majority_now
    is_majority_now.short_description = _("Is majority now")

    def in_council_now(self, obj):
        return obj.in_council_now
    in_council_now.short_description = _("In council now")

class ChargeInline(admin.StackedInline):
    raw_id_fields = ('person', )
    fieldsets = (
        (None, {
            'fields': (('person', 'start_date', 'end_date'), )
        }),
        (_('Advanced options'), {
            'classes': ('collapse',),
            'fields': ('description', 'end_reason')
        })
    )
    extra = 1

class CompanyChargeInline(ChargeInline):
    model = CompanyCharge

class AdministrationChargeInline(ChargeInline):
    model = AdministrationCharge

class InstitutionResourceInline(admin.TabularInline):
    model = InstitutionResource
    extra = 0

class InstitutionChargeInline(ChargeInline):
    model = InstitutionCharge
    raw_id_fields = ('person', 'substitutes', 'substituted_by')
    fieldsets = (
        (None, {
            'fields': (('person', 'op_charge_id', 'start_date', 'end_date'), )
        }),
        (_('Advanced options'), {
            'classes': ('collapse',),
            'fields': ('description', 'end_reason', ('substitutes', 'substituted_by'))
        })
    )

class ResponsabilityInline(admin.TabularInline):
    raw_id_fields = ('charge',)
    extra = 0

class InstitutionResponsabilityInline(ResponsabilityInline):
    model = InstitutionResponsability
    fields = ('charge', 'charge_type', 'start_date', 'end_date', 'description')

class GroupResponsabilityInline(admin.TabularInline):
    model = GroupResponsability
    raw_id_fields = ('charge',)
    extra = 0
    fields = ('charge', 'charge_type', 'start_date', 'end_date', 'description')

class ChargeAdmin(admin.ModelAdmin):
    pass

class CompanyChargeAdmin(ChargeAdmin):
    model = CompanyCharge
    raw_id_fields = ('person', 'company')
    fieldsets = (
        (None, {
            'fields': (('person', 'company'),
                ('start_date', 'end_date', 'end_reason'),
                 'description')
        }),
    )

class AdministrationChargeAdmin(ChargeAdmin):
    model = AdministrationCharge
    raw_id_fields = ('person', 'office')
    fieldsets = (
        (None, {
            'fields': (('person', 'office'),
                 ('start_date', 'end_date', 'end_reason'),
                 'description')
        }),
    )

class InstitutionChargeAdmin(ChargeAdmin):
    model = InstitutionCharge
    raw_id_fields = ('person', 'substitutes', 'substituted_by', 'original_charge')
    search_fields = ['^person__first_name', '^person__last_name']

    fieldsets = (
        (None, {
            'fields': (('person', 'op_charge_id', 'institution', 'original_charge'),
                 ('start_date', 'end_date', 'end_reason'),
                  'description',
                 ('substitutes', 'substituted_by'))
        }),
        (_("Presences"), {
            'fields': (('n_present_votations', 'n_absent_votations'), ('n_present_attendances', 'n_absent_attendances'))
        }),
    )
    list_display = ('__unicode__', 'institution', 'start_date', 'end_date')
    list_select_related = True
    list_filter = ['institution__name']
    inlines = [InstitutionResponsabilityInline]

class GroupChargeAdmin(admin.ModelAdmin):
    raw_id_fields = ('charge', )
    list_display = ('__unicode__', 'start_date', 'end_date')
    list_select_related = True
    list_filter = ['group']
    inlines = [GroupResponsabilityInline]

    search_fields = [ 'charge__person__first_name', 'charge__person__last_name', 'group__name', 'group__acronym', ]

class BodyAdmin(admin.ModelAdmin):
    prepopulated_fields = {"slug": ("name",)}

class CompanyAdmin(BodyAdmin):
    inlines = [CompanyChargeInline]

class OfficeAdmin(BodyAdmin):
    inlines = [AdministrationChargeInline]

class InstitutionAdmin(BodyAdmin):

    list_filter = ("institution_type", )

    def get_urls(self):
        from django.conf.urls.defaults import patterns, url

        def wrap(view):
            def wrapper(*args, **kwargs):
                return self.admin_site.admin_view(view)(*args, **kwargs)
            return update_wrapper(wrapper, view)
        info = self.model._meta.app_label, self.model._meta.module_name
        return patterns('',
                        url(r'^(.+)/move-(up)/$',
                            wrap(self.move_view),
                            name='%s_%s_move_up' % info),
                        url(r'^(.+)/move-(down)/$',
                            wrap(self.move_view),
                            name='%s_%s_move_down' % info),
                        ) + super(InstitutionAdmin, self).get_urls()

    def move_view(self, request, object_id, direction):
        obj = get_object_or_404(self.model, pk=unquote(object_id))
        if direction == 'up':
            obj.move_up()
        else:
            obj.move_down()
        return HttpResponseRedirect('../../')

    link_html = short("""
        UP |
        DOWN (%(position)s)
        """)

    def move_up_down_links(self, obj):
        return self.link_html % {
            'app_label': self.model._meta.app_label,
            'module_name': self.model._meta.module_name,
            'object_id': obj.id,
            'position': obj.position,
        }
    move_up_down_links.allow_tags = True
    move_up_down_links.short_description = _(u'Move')

    inlines = [InstitutionResourceInline, InstitutionChargeInline]
    list_display = ('name', 'institution_type', 'move_up_down_links',)

class SpeechInline(LinkedTabularInline):
    model = Speech
    fields = ("author", "author_name_when_external", "title", \
              "seq_order", "admin_link", )
    raw_id_fields = ["author", ]
    extra = 0
    form = SpeechInlineForm

class SittingItemInline(LinkedTabularInline):
    model = SittingItem
    fields = ('title', 'related_act_set', 'item_type', 'seq_order','admin_link',)
    raw_id_fields = ('related_act_set',)
    extra = 0

    form = SittingItemFormSet

class SittingItemAdmin(admin.ModelAdmin):
    list_display = ( 'title','sitting', 'seq_order','item_type','num_related_acts')
    ordering = ('-sitting__date','seq_order')
    search_fields = ['^title', ]
    list_filter = ['sitting__institution','item_type',]

    raw_id_fields = ( 'sitting', 'related_act_set', )

    inlines = [SpeechInline, ]

class SittingAdmin(admin.ModelAdmin):
    inlines = [SittingItemInline, VotationsInline]

admin.site.register(SittingItem, SittingItemAdmin)
admin.site.register(Sitting, SittingAdmin)
admin.site.register(Person, PersonAdminWithResources)
admin.site.register(Group, GroupAdminWithCharges)
admin.site.register(GroupCharge, GroupChargeAdmin)
admin.site.register(InstitutionCharge, InstitutionChargeAdmin)
admin.site.register(CompanyCharge, CompanyChargeAdmin)
admin.site.register(AdministrationCharge, AdministrationChargeAdmin)
admin.site.register(Institution, InstitutionAdmin)
admin.site.register(Company, CompanyAdmin)
admin.site.register(Office, OfficeAdmin)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#                                                                             #
#   sale_payment_method for OpenERP                                            #
#   Copyright (C) 2011 Akretion Sébastien BEAU <sebastien.beau@akretion.com>  #
#                                                                             #
#   This program is free software: you can redistribute it and/or modify      #
#   it under the terms of the GNU Affero General Public License as            #
#   published by the Free Software Foundation, either version 3 of the        #
#   License, or (at your option) any later version.                           #
#                                                                             #
#   This program is distributed in the hope that it will be useful,           #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU Affero General Public License for more details.                       #
#                                                                             #
#   You should have received a copy of the GNU Affero General Public License  #
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.     #
#                                                                             #
###############################################################################

{
    'name': 'Sale Payment Method',
    'version': '0.2',
    'category': 'Generic Modules/Others',
    'license': 'AGPL-3',
    'description': """
Sale Payment Method
===================

This module adds low-levels features used for instance by modules:

- Sale Automatic Workflow
- Sale Quick Payment

It adds a payment method on the sales orders and allow to register
payments entries on sales orders.
""",
    'author': 'Akretion',
    'website': 'http://www.akretion.com/',
    'depends': [],
    'data': ['sale_view.xml',
             'payment_method_view.xml',
             'security/ir.model.access.csv',
             ],
    'demo': [],
    'installable': True,
}
<commit_msg>[FIX] missing dependency on sale<commit_after># -*- coding: utf-8 -*-
###############################################################################
#                                                                             #
#   sale_payment_method for OpenERP                                            #
#   Copyright (C) 2011 Akretion Sébastien BEAU <sebastien.beau@akretion.com>  #
#                                                                             #
#   This program is free software: you can redistribute it and/or modify      #
#   it under the terms of the GNU Affero General Public License as            #
#   published by the Free Software Foundation, either version 3 of the        #
#   License, or (at your option) any later version.                           #
#                                                                             #
#   This program is distributed in the hope that it will be useful,           #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU Affero General Public License for more details.                       #
#                                                                             #
#   You should have received a copy of the GNU Affero General Public License  #
#   along with this program.  If not, see <http://www.gnu.org/licenses/>.     #
#                                                                             #
###############################################################################

{
    'name': 'Sale Payment Method',
    'version': '0.2',
    'category': 'Generic Modules/Others',
    'license': 'AGPL-3',
    'description': """
Sale Payment Method
===================

This module adds low-levels features used for instance by modules:

- Sale Automatic Workflow
- Sale Quick Payment

It adds a payment method on the sales orders and allow to register
payments entries on sales orders.
""",
    'author': 'Akretion',
    'website': 'http://www.akretion.com/',
    'depends': ['sale',
                ],
    'data': ['sale_view.xml',
             'payment_method_view.xml',
             'security/ir.model.access.csv',
             ],
    'demo': [],
    'installable': True,
}
<|endoftext|>


<commit_before>import math
import numpy as np
from bokeh.charts import Bar
from bokeh.models import DataRange1d, GridPlot, Plot, PreviewSaveTool, Range1d
from bokeh.models.glyphs import AnnularWedge, Rect, Text
from bokeh.plotting import ColumnDataSource, output_file, show

def pie_chart(values, categories, colors, title, plot_width, legend_width, pie_slice_label=None):
    r"""Generate a pie chart.

    Generate a pie chart using the Bokeh library.

    The number of values, categories and colors must match. The labels shown on the pie slices are generated from
    `pie_slice_label`, which is formatted with Python's str.format method. You may refer to the percentage by
    '{0}' in the string. For example, if the percentage is 30.58 and `pie_slice_label` is '{0|.1f} %', then the string
    '30.6 %' is used as the label.

    No value must be negative, and at least one value must be non-zero.

    Parameters
    ----------
    values : array_like
        Values to plot on the pie chart. Values that are zero will be ignored.
    categories: array_like
        Categories corresponding to the values. These will be used in the legend.
    colors: array_like
        Colors corresponding to the categories.
    title: str
        Plot title.
    plot_width: int
        Width of the plot, in pixels.
    legend_width: int
        Width of the legend, in pixels.
    pie_slice_label: str, optional
        Label for a pie slice. Will be used with the str.format method.

    Returns
    -------
    Plot
        Bokeh plot containing the pie chart and its legend.

    Raises
    ------
    ValueError
        If any value is negative or a non-number, or if all values are zero, or if the number of values, categories and
        colors don't match.
    """

    # layout choices
    key_color_width = 30
    key_color_height = 16
    text_font = 'times'
    text_font_size = '13pt'
    plot_border = 10
    pie_legend_gap = 20
    legend_bottom_margin = 20

    # check that arguments are fine
    if len(values) != len(colors) or len(colors) != len(categories):
        raise ValueError('the number of values, colors and categories don\'t match')
    for v in values:
        if v < 0:
            raise ValueError('values must be non-negative: {0}'.format(v))

# convert values to angles and percentages
    angles = normalize(values, 2 * math.pi)
    percentages = normalize(values, 100.)

    # remove all zero values
    data = zip(angles, categories, colors, percentages)
    data = [item for item in data if item[0] != 0]
    colors = [item[2] for item in data]

    # collect start and end angles
    start_angles = []
    end_angles = []
    for i, item in enumerate(data):
        start_angles.append(0 if i == 0 else end_angles[i - 1])
        end_angles.append(start_angles[i] + item[0])
    start_angles = np.array(start_angles)
    end_angles = np.array(end_angles)
    mid_angles = 0.5 * (start_angles + end_angles)

    # plot dimensions
    pie_radius = (plot_width - 2 * plot_border - legend_width - pie_legend_gap) / 2
    plot_height = 20 + 2 * pie_radius
    plot_width = 20 + 2 * pie_radius + legend_width

    # create plot
    xdr = DataRange1d(start=0, end=1)
    ydr = DataRange1d(start=0, end=1)
    plot = Plot(
        x_range=xdr,
        y_range=ydr,
        title=None,
        background_fill="white",
        border_fill='white',
        outline_line_color='white',
        min_border=2,
        plot_width=plot_width,
        plot_height=plot_height)
    plot.add_tools(PreviewSaveTool())

    # create pie slices
    inner_radius = 0
    outer_radius = pie_radius
    pie_center = (plot_border + outer_radius, plot_border + outer_radius)
    source = ColumnDataSource(
        data=dict(
            x=pie_center[0] * np.ones(len(data)),
            y=pie_center[1] * np.ones(len(data)),
            inner_radius=inner_radius * np.ones(len(data)),
            outer_radius=outer_radius * np.ones(len(data)),
            start_angle=start_angles,
            end_angle=end_angles,
            fill_color=colors,
            category=[item[1] for item in data]
        )
    )
    a = AnnularWedge(x=dict(field='x', units='screen'),
                     y=dict(field='y', units='screen'),
                     inner_radius=dict(field='inner_radius', units='screen'),
                     outer_radius=dict(field='outer_radius', units='screen'),
                     start_angle=dict(field='start_angle'),
                     end_angle=dict(field='end_angle'),
                     fill_color=dict(field='fill_color'),
                     line_color='white')
    plot.add_glyph(source, a)

    # create the pie slice labels
    if pie_slice_label is not None:
        mid_radius = 0.5 * (inner_radius + outer_radius)
        xs_label = pie_center[0] + mid_radius * np.cos(mid_angles)
        ys_label = pie_center[1] + mid_radius * np.sin(mid_angles)
        label_source = ColumnDataSource(
            data=dict(
                x=xs_label,
                y=ys_label,
                text=[pie_slice_label.format(item[3]) for item in data]
            )
        )
        t = Text(x=dict(field='x', units='screen'),
                 y=dict(field='y', units='screen'),
                 text=dict(field='text', units='screen'),
                 text_font=text_font,
                 text_font_size=text_font_size,
                 text_color='white',
                 text_align='center',
                 text_baseline='middle')
        plot.add_glyph(label_source, t)

    # create the legend colors
    xs_legends = np.array([pie_center[0] + pie_radius + pie_legend_gap + key_color_width / 2 for _ in range(len(data))])
    ys_legends = np.array([pie_center[1] - pie_radius + legend_bottom_margin + 30 * i + key_color_height / 2 for i in range(len(data))])
    legend_color_width = 30
    legend_color_height = 15
    key_color_source = ColumnDataSource(
        data=dict(
            x=xs_legends,
            y=ys_legends,
            width=legend_color_width * np.ones(len(data)),
            height=legend_color_height * np.ones(len(data)),
            fill_color=[color for color in reversed(colors)]
        )
    )
    r = Rect(x=dict(field='x', units='screen'),
             y=dict(field='y', units='screen'),
             width=dict(field='width', units='screen'),
             height=dict(field='height', units='screen'),
             line_color=dict(field='fill_color'),
             fill_color=dict(field='fill_color'))
    plot.add_glyph(key_color_source, r)

    # create the legend labels
    key_label_source = ColumnDataSource(
        data=dict(
            x=legend_color_width + 10 + xs_legends,
            y=ys_legends,
            text=[item[1] for item in reversed(data)]
        )
    )
    t = Text(x=dict(field='x', units='screen'),
             y=dict(field='y', units='screen'),
             text=dict(field='text', units='screen'),
             text_font=text_font,
             text_font_size=text_font_size,
             text_color='black',
             text_align='left',
             text_baseline='middle')
    plot.add_glyph(key_label_source, t)

    # title plot
    t_plot = title_plot(title, plot_width)

    # create a grid plot
    # we do this as grid plots don't feature the Bokeh logo and display the tool icons on the side
    grid = GridPlot(children=[[t_plot], [plot]], title=None)
    return grid

def stacked_bar_chart(values, categories, colors, x_label, y_label, title, plot_width, plot_height, legend_height):
    r"""Generate a stacked bar chart.

    Generate a stacked bar chart with the Bokeh library.

    The values must be a dictionary of lists. For example,

    values = dict(engineering=[1.2, 7.8, 3.8],
                  science=[10., 2.5])

    The number of keys for this dictionary must equal the number of colors.

    The categories will be used as the labels for the x axis.

    The legend height is an estimate of the vertical space required for the legend, in pixels. It is used to ensure that
    the legend isn't plotted on top of a bar.

    Parameters
    ----------
    values : array_like
        Values to plot on the pie chart. Values that are zero will be ignored.
    categories: array_like
        Categories corresponding to the values. These will be used in the legend.
    colors: array_like
        Colors corresponding to the categories.
    x_label: str
        Label for the x axis.
    y_label: str
        Label for the y axis.
    title: str
        Plot title.
    plot_width: int
        Width of the plot, in pixels.
    legend_height: int
        Height of the legend, in pixels.

    Returns
    -------
    Plot
        Bokeh plot.

    Raises
    ------
    ValueError
        If the number of colors and keys in `values` don't match, or if any value is negative, or if the legend height
        isn't smaller than the plot height.
    """

    # check that arguments are fine
    if len(values.keys()) != len(colors):
        raise ValueError('the number of value keys and colors don\'t match')
    for key in values:
        for v in values[key]:
            if v < 0:
                raise ValueError('all values must be non-negative')
    if legend_height >= plot_height:
        raise ValueError('the legend height must be smaller than the plot height')

    # ensure there is space for the legend
    max_value = max([sum([values[key][i] for key in values.keys()]) for i in range(len(values.keys()[0]))])
    max_y = max_value * plot_height / (plot_height - legend_height)
    ydr = Range1d(start=0, end=max_y)

    # bar plot
    plot = Bar(values=values,
               cat=categories,
               palette=colors,
               stacked=True,
               legend=True,
               xlabel=x_label,
               ylabel=y_label,
               title=None,
               width=plot_width,
               height=plot_height,
               continuous_range=ydr,
               tools='previewsave')

    # title plot
    t_plot = title_plot(title, plot_width)

    # create a grid plot
    # we do this as grid plots don't feature the Bokeh logo and display the tool icons on the side
    grid = GridPlot(children=[[t_plot], [plot]], title=None)
    return grid

def title_plot(title, width):
    r"""Generate a plot just containing a title.

    The generated plot just contains a title. The title may contain newline characters.

    Parameters
    ----------
    title : str
        Title.
    width: int
        Plot width, in pixels

    Returns
    -------
    Plot
        Bokeh plot containing the title.
    describe : type
        Explanation of return value named `describe`.
    out : type
        Explanation of `out`.
    """

    # layout choices
    plot_border = 5
    line_height = 30
    text_font = 'times'
    text_font_size = '14pt'

    # figure out the text position and plot height
    lines = [line for line in reversed(title.split('\n'))]
    line_xs = [width / 2 for _ in range(len(lines))]
    line_ys = [plot_border + i * line_height for i, _ in enumerate(lines)]
    height = 2 * plot_border + len(lines) * line_height

    # create the plot
    xdr = DataRange1d(start=0, end=1)
    ydr = DataRange1d(start=0, end=1)
    plot = Plot(x_range=xdr,
                y_range=ydr,
                title=None,
                background_fill="white",
                border_fill='white',
                outline_line_color='white',
                min_border=2,
                plot_width=width,
                plot_height=height)
    plot.add_tools(PreviewSaveTool())

    # add the title
    source = ColumnDataSource(
        data=dict(
            x=line_xs,
            y=line_ys,
            text=lines
        )
    )
    t = Text(x=dict(field='x', units='screen'),
             y=dict(field='y', units='screen'),
             text=dict(field='text', screen='units'),
             text_font=text_font,
             text_font_size=text_font_size,
             text_color='black',
             text_align='center',
             text_baseline='bottom')
    plot.add_glyph(source, t)

    return plot

def normalize(values, normalized_total):
    r"""Normalize a list of values.

    The values are scaled so that their sum is equal to `normalized_total`, and a list of the scaled values is returned.

    Parameters
    ----------
    values : array_like
        Numerical values to normalize.
    normalized_total: number
        Sum of the scaled values.

    Returns
    -------
    list
        Normalized values.

    Raises
    ------
    ValueError
        If any of the values or the total is a non-number, or if all values are zero.

    Examples
    --------
    >>> normalize([1, 2], 2 * math.pi)
    [2.0943951023931953, 4.1887902047863905]

    >>> normalize([1, 0, 1], 100)
    [50.0, 0.0, 50.0]

    >>> normalize([0, 0], 1)
    Traceback (most recent call last):
    ...
    ValueError: at least one value must be non-zero
    """

    # make sure we have numbers
    values = [float(v) for v in values]
    normalized_total = float(normalized_total)

    # check for negative values
    for v in values:
        if v < 0:
            raise ValueError('negative values are not allowed: {0}'.format(v))

    # there must be non-zero values
    total = sum(values)
    if total == 0:
        raise ValueError('at least one value must be non-zero')

    # normalize
    return [normalized_total * v / total for v in values]
<commit_msg>Let user choose label colour in pie charts<commit_after>import math
import numpy as np
from bokeh.charts import Bar
from bokeh.models import DataRange1d, GridPlot, Plot, PreviewSaveTool, Range1d
from bokeh.models.glyphs import AnnularWedge, Rect, Text
from bokeh.plotting import ColumnDataSource, output_file, show

def pie_chart(values, categories, colors, title, plot_width, legend_width, text_color, pie_slice_label=None):
    r"""Generate a pie chart.

    Generate a pie chart using the Bokeh library.

    The number of values, categories and colors must match. The labels shown on the pie slices are generated from
    `pie_slice_label`, which is formatted with Python's str.format method. You may refer to the percentage by
    '{0}' in the string. For example, if the percentage is 30.58 and `pie_slice_label` is '{0|.1f} %', then the string
    '30.6 %' is used as the label.

    No value must be negative, and at least one value must be non-zero.

    Parameters
    ----------
    values : array_like
        Values to plot on the pie chart. Values that are zero will be ignored.
    categories: array_like
        Categories corresponding to the values. These will be used in the legend.
    colors: array_like
        Colors corresponding to the categories.
    title: str
        Plot title.
    plot_width: int
        Width of the plot, in pixels.
    legend_width: int
        Width of the legend, in pixels.
    pie_slice_label: str, optional
        Label for a pie slice. Will be used with the str.format method.

    Returns
    -------
    Plot
        Bokeh plot containing the pie chart and its legend.

    Raises
    ------
    ValueError
        If any value is negative or a non-number, or if all values are zero, or if the number of values, categories and
        colors don't match.
    """

    # layout choices
    key_color_width = 30
    key_color_height = 16
    text_font = 'times'
    text_font_size = '13pt'
    plot_border = 10
    pie_legend_gap = 20
    legend_bottom_margin = 20

    # check that arguments are fine
    if len(values) != len(colors) or len(colors) != len(categories):
        raise ValueError('the number of values, colors and categories don\'t match')
    for v in values:
        if v < 0:
            raise ValueError('values must be non-negative: {0}'.format(v))

    # convert values to angles and percentages
    angles = normalize(values, 2 * math.pi)
    percentages = normalize(values, 100.)

    # remove all zero values
    data = zip(angles, categories, colors, percentages)
    data = [item for item in data if item[0] != 0]
    colors = [item[2] for item in data]

    # collect start and end angles
    start_angles = []
    end_angles = []
    for i, item in enumerate(data):
        start_angles.append(0 if i == 0 else end_angles[i - 1])
        end_angles.append(start_angles[i] + item[0])
    start_angles = np.array(start_angles)
    end_angles = np.array(end_angles)
    mid_angles = 0.5 * (start_angles + end_angles)

    # plot dimensions
    pie_radius = (plot_width - 2 * plot_border - legend_width - pie_legend_gap) / 2
    plot_height = 20 + 2 * pie_radius
    plot_width = 20 + 2 * pie_radius + legend_width

    # create plot
    xdr = DataRange1d(start=0, end=1)
    ydr = DataRange1d(start=0, end=1)
    plot = Plot(
        x_range=xdr,
        y_range=ydr,
        title=None,
        background_fill="white",
        border_fill='white',
        outline_line_color='white',
        min_border=2,
        plot_width=plot_width,
        plot_height=plot_height)
    plot.add_tools(PreviewSaveTool())

    # create pie slices
    inner_radius = 0
    outer_radius = pie_radius
    pie_center = (plot_border + outer_radius, plot_border + outer_radius)
    source = ColumnDataSource(
        data=dict(
            x=pie_center[0] * np.ones(len(data)),
            y=pie_center[1] * np.ones(len(data)),
            inner_radius=inner_radius * np.ones(len(data)),
            outer_radius=outer_radius * np.ones(len(data)),
            start_angle=start_angles,
            end_angle=end_angles,
            fill_color=colors,
            category=[item[1] for item in data]
        )
    )
    a = AnnularWedge(x=dict(field='x', units='screen'),
                     y=dict(field='y', units='screen'),
                     inner_radius=dict(field='inner_radius', units='screen'),
                     outer_radius=dict(field='outer_radius', units='screen'),
                     start_angle=dict(field='start_angle'),
                     end_angle=dict(field='end_angle'),
                     fill_color=dict(field='fill_color'),
                     line_color='white')
    plot.add_glyph(source, a)

    # create the pie slice labels
    if pie_slice_label is not None:
        mid_radius = 0.5 * (inner_radius + outer_radius)
        xs_label = pie_center[0] + mid_radius * np.cos(mid_angles)
        ys_label = pie_center[1] + mid_radius * np.sin(mid_angles)
        label_source = ColumnDataSource(
            data=dict(
                x=xs_label,
                y=ys_label,
                text=[pie_slice_label.format(item[3]) for item in data]
            )
        )
        t = Text(x=dict(field='x', units='screen'),
                 y=dict(field='y', units='screen'),
                 text=dict(field='text', units='screen'),
                 text_font=text_font,
                 text_font_size=text_font_size,
                 text_color=text_color,
                 text_align='center',
                 text_baseline='middle')
        plot.add_glyph(label_source, t)

    # create the legend colors
    xs_legends = np.array([pie_center[0] + pie_radius + pie_legend_gap + key_color_width / 2 for _ in range(len(data))])
    ys_legends = np.array([pie_center[1] - pie_radius + legend_bottom_margin + 30 * i + key_color_height / 2 for i in range(len(data))])
    legend_color_width = 30
    legend_color_height = 15
    key_color_source = ColumnDataSource(
        data=dict(
            x=xs_legends,
            y=ys_legends,
            width=legend_color_width * np.ones(len(data)),
            height=legend_color_height * np.ones(len(data)),
            fill_color=[color for color in reversed(colors)]
        )
    )
    r = Rect(x=dict(field='x', units='screen'),
             y=dict(field='y', units='screen'),
             width=dict(field='width', units='screen'),
             height=dict(field='height', units='screen'),
             line_color=dict(field='fill_color'),
             fill_color=dict(field='fill_color'))
    plot.add_glyph(key_color_source, r)

    # create the legend labels
    key_label_source = ColumnDataSource(
        data=dict(
            x=legend_color_width + 10 + xs_legends,
            y=ys_legends,
            text=[item[1] for item in reversed(data)]
        )
    )
    t = Text(x=dict(field='x', units='screen'),
             y=dict(field='y', units='screen'),
             text=dict(field='text', units='screen'),
             text_font=text_font,
             text_font_size=text_font_size,
             text_color='black',
             text_align='left',
             text_baseline='middle')
    plot.add_glyph(key_label_source, t)

    # title plot
    t_plot = title_plot(title, plot_width)

    # create a grid plot
    # we do this as grid plots don't feature the Bokeh logo and display the tool icons on the side
    grid = GridPlot(children=[[t_plot], [plot]], title=None)
    return grid

def stacked_bar_chart(values, categories, colors, x_label, y_label, title, plot_width, plot_height, legend_height):
    r"""Generate a stacked bar chart.

    Generate a stacked bar chart with the Bokeh library.

    The values must be a dictionary of lists. For example,

    values = dict(engineering=[1.2, 7.8, 3.8],
                  science=[10., 2.5])

    The number of keys for this dictionary must equal the number of colors.

    The categories will be used as the labels for the x axis.

    The legend height is an estimate of the vertical space required for the legend, in pixels. It is used to ensure that
    the legend isn't plotted on top of a bar.

    Parameters
    ----------
    values : array_like
        Values to plot on the pie chart. Values that are zero will be ignored.
    categories: array_like
        Categories corresponding to the values. These will be used in the legend.
    colors: array_like
        Colors corresponding to the categories.
    x_label: str
        Label for the x axis.
    y_label: str
        Label for the y axis.
    title: str
        Plot title.
    plot_width: int
        Width of the plot, in pixels.
    legend_height: int
        Height of the legend, in pixels.

    Returns
    -------
    Plot
        Bokeh plot.

    Raises
    ------
    ValueError
        If the number of colors and keys in `values` don't match, or if any value is negative, or if the legend height
        isn't smaller than the plot height.
    """

    # check that arguments are fine
    if len(values.keys()) != len(colors):
        raise ValueError('the number of value keys and colors don\'t match')
    for key in values:
        for v in values[key]:
            if v < 0:
                raise ValueError('all values must be non-negative')
    if legend_height >= plot_height:
        raise ValueError('the legend height must be smaller than the plot height')

    # ensure there is space for the legend
    max_value = max([sum([values[key][i] for key in values.keys()]) for i in range(len(values.keys()[0]))])
    max_y = max_value * plot_height / (plot_height - legend_height)
    ydr = Range1d(start=0, end=max_y)

    # bar plot
    plot = Bar(values=values,
               cat=categories,
               palette=colors,
               stacked=True,
               legend=True,
               xlabel=x_label,
               ylabel=y_label,
               title=None,
               width=plot_width,
               height=plot_height,
               continuous_range=ydr,
               tools='previewsave')

    # title plot
    t_plot = title_plot(title, plot_width)

    # create a grid plot
    # we do this as grid plots don't feature the Bokeh logo and display the tool icons on the side
    grid = GridPlot(children=[[t_plot], [plot]], title=None)
    return grid

def title_plot(title, width):
    r"""Generate a plot just containing a title.

    The generated plot just contains a title. The title may contain newline characters.

    Parameters
    ----------
    title : str
        Title.
    width: int
        Plot width, in pixels

    Returns
    -------
    Plot
        Bokeh plot containing the title.
    describe : type
        Explanation of return value named `describe`.
    out : type
        Explanation of `out`.
    """

    # layout choices
    plot_border = 5
    line_height = 30
    text_font = 'times'
    text_font_size = '14pt'

    # figure out the text position and plot height
    lines = [line for line in reversed(title.split('\n'))]
    line_xs = [width / 2 for _ in range(len(lines))]
    line_ys = [plot_border + i * line_height for i, _ in enumerate(lines)]
    height = 2 * plot_border + len(lines) * line_height

    # create the plot
    xdr = DataRange1d(start=0, end=1)
    ydr = DataRange1d(start=0, end=1)
    plot = Plot(x_range=xdr,
                y_range=ydr,
                title=None,
                background_fill="white",
                border_fill='white',
                outline_line_color='white',
                min_border=2,
                plot_width=width,
                plot_height=height)
    plot.add_tools(PreviewSaveTool())

    # add the title
    source = ColumnDataSource(
        data=dict(
            x=line_xs,
            y=line_ys,
            text=lines
        )
    )
    t = Text(x=dict(field='x', units='screen'),
             y=dict(field='y', units='screen'),
             text=dict(field='text', screen='units'),
             text_font=text_font,
             text_font_size=text_font_size,
             text_color='black',
             text_align='center',
             text_baseline='bottom')
    plot.add_glyph(source, t)

    return plot

def normalize(values, normalized_total):
    r"""Normalize a list of values.

    The values are scaled so that their sum is equal to `normalized_total`, and a list of the scaled values is returned.

    Parameters
    ----------
    values : array_like
        Numerical values to normalize.
    normalized_total: number
        Sum of the scaled values.

    Returns
    -------
    list
        Normalized values.

    Raises
    ------
    ValueError
        If any of the values or the total is a non-number, or if all values are zero.

    Examples
    --------
    >>> normalize([1, 2], 2 * math.pi)
    [2.0943951023931953, 4.1887902047863905]

    >>> normalize([1, 0, 1], 100)
    [50.0, 0.0, 50.0]

    >>> normalize([0, 0], 1)
    Traceback (most recent call last):
    ...
    ValueError: at least one value must be non-zero
    """

    # make sure we have numbers
    values = [float(v) for v in values]
    normalized_total = float(normalized_total)

    # check for negative values
    for v in values:
        if v < 0:
            raise ValueError('negative values are not allowed: {0}'.format(v))

    # there must be non-zero values
    total = sum(values)
    if total == 0:
        raise ValueError('at least one value must be non-zero')

    # normalize
    return [normalized_total * v / total for v in values]
<|endoftext|>


<commit_before>import hera_librarian
import os
import psutil
import sys

def get_size(file):
    if not os.path.exists(file):
        raise OSError('file: ' + file + ' does not exist')

    if os.path.isdir(file):
        file_size = 0
        for dirpath, dirnames, filenames in os.walk(file):
            for f in filenames:
                fp = os.path.join(dirpath, f)
                file_size += os.path.getsize(fp)
    else:
        statinfo = os.stat(file)
        file_size = statinfo.st_size

    return file_size

def get_recommendation(site, file_size):
    client = hera_librarian.LibrarianClient(site)
    store_dict = client.recommended_store(file_size)
    path = store_dict['store']['path_prefix']
    store = store_dict['store']['name']
    ssh_prefix = store_dict['store']['ssh_prefix']

    return store, ssh_prefix, path

def bash_command(command):
    """
    Issue a command to the bash shell, wait until command finishes before
        returning, if there's an error then raise and exception with the
        command that was called.

    Parameters
    ----------
    command: list
        all the elements of the command as seperate items in a list
            (ie the command will be the list with spaces inserted between
            the elements)
    """
    p = psutil.Popen(command)
    if p.wait():
        raise Exception('bash command failed, command was: ',
                        ' '.join(command))

def uploader(site, files, paths):
    """
    Upload a list of file to the librarian

    Parameters
    ----------
    site: string
        name of librarian site
    files: list
        list of local files to upload
    paths: list
        list of paths for files (paths + files should be full paths to files)
    """
    if len(paths) != 1 and len(paths) != len(files):
        raise ValueError('paths must be a list of length 1 or the same ' +
                         'length as files')

    for i, file in enumerate(files):
        if len(paths) == 1:
            full_filepath = paths[0] + file
        else:
            full_filepath = paths[i] + file

        file_size = get_size(full_filepath)
        store, ssh_prefix, path = get_recommendation(site, file_size)

        dir_part, filename = os.path.split(file)

        mkdir_cmd = ['ssh', ssh_prefix, 'mkdir', '-p', path + '/' + dir_part]

        scp_cmd = ['scp', '-r', '-c', 'arcfour256', '-o',
                   'UserKnownHostsFile=/dev/null', '-o',
                   'StrictHostKeyChecking=no', full_filepath,
                   ssh_prefix + ':' + path + '/' + file]
        add_obs_cmd = ['ssh', ssh_prefix, 'add_obs_librarian.py', '--site ',
                       site, '--store ', store, '--store_path ', path + '/',
                       file]
        print ' '.join(add_obs_cmd)

        bash_command(mkdir_cmd)
        bash_command(scp_cmd)
        bash_command(add_obs_cmd)

def main():
    arg_list = sys.argv
    site = arg_list[1]
    files = arg_list[2:]

    uploader(site, files)

if __name__ == '__main__':
    main()
<commit_msg>change variable name for clarity<commit_after>import hera_librarian
import os
import psutil
import sys

def get_size(file):
    if not os.path.exists(file):
        raise OSError('file: ' + file + ' does not exist')

    if os.path.isdir(file):
        file_size = 0
        for dirpath, dirnames, filenames in os.walk(file):
            for f in filenames:
                fp = os.path.join(dirpath, f)
                file_size += os.path.getsize(fp)
    else:
        statinfo = os.stat(file)
        file_size = statinfo.st_size

    return file_size

def get_recommendation(site, file_size):
    client = hera_librarian.LibrarianClient(site)
    store_dict = client.recommended_store(file_size)
    path = store_dict['store']['path_prefix']
    store = store_dict['store']['name']
    ssh_prefix = store_dict['store']['ssh_prefix']

    return store, ssh_prefix, path

def bash_command(command):
    """
    Issue a command to the bash shell, wait until command finishes before
        returning, if there's an error then raise and exception with the
        command that was called.

    Parameters
    ----------
    command: list
        all the elements of the command as seperate items in a list
            (ie the command will be the list with spaces inserted between
            the elements)
    """
    p = psutil.Popen(command)
    if p.wait():
        raise Exception('bash command failed, command was: ',
                        ' '.join(command))

def uploader(site, files, root_paths):
    """
    Upload a list of file to the librarian

    Parameters
    ----------
    site: string
        name of librarian site
    files: list
        list of local files to upload
    root_paths: list
        list of root_paths for files (root_paths + files should give full paths
            to files)
    """
    if len(root_paths) != 1 and len(root_paths) != len(files):
        raise ValueError('root_paths must be a list of length 1 or the same ' +
                         'length as files')

    for i, file in enumerate(files):
        if len(root_paths) == 1:
            full_filepath = root_paths[0] + file
        else:
            full_filepath = root_paths[i] + file

        file_size = get_size(full_filepath)
        store, ssh_prefix, path = get_recommendation(site, file_size)

        dir_part, filename = os.path.split(file)

        mkdir_cmd = ['ssh', ssh_prefix, 'mkdir', '-p', path + '/' + dir_part]

        scp_cmd = ['scp', '-r', '-c', 'arcfour256', '-o',
                   'UserKnownHostsFile=/dev/null', '-o',
                   'StrictHostKeyChecking=no', full_filepath,
                   ssh_prefix + ':' + path + '/' + file]
        add_obs_cmd = ['ssh', ssh_prefix, 'add_obs_librarian.py', '--site ',
                       site, '--store ', store, '--store_path ', path + '/',
                       file]
        print ' '.join(add_obs_cmd)

        bash_command(mkdir_cmd)
        bash_command(scp_cmd)
        bash_command(add_obs_cmd)

def main():
    arg_list = sys.argv
    site = arg_list[1]
    files = arg_list[2:]

    uploader(site, files)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python3.4.2
'''
Created on Apr 6, 2015

Name something that might be worn on the foot.

Change one letter in it without changing the
order of the other letters. The result
will name something one might wear on the
upper part of the body. What is it?

Here's a hint: The thing on the upper part
of the body is a two-word phrase.

@author: john.obrien, leiran.biton
'''

import sys
from string import ascii_lowercase as alphabet
from nltk.corpus import words, wordnet

'''
search
parameters:
1. a list of keywords which we are trying to find
in the definitions of the words

returns:
a list of words for which any of the keywords searched for
were found in the definitions
'''

def search(kws, stype=all, word_type="noun"):
    """
kws   - list of keywords to be searched for
stype - either all or any (default = all)
word_type - part of speech (default = 'noun')
"""
    if stype not in (all, any):
        return TypeError("Invalid stype argument supplied. stype must be 'any' or 'all' functions.")
    matches = []
    # check kws against all defs in wordnets list of words
    for word in words.words():
        synset = wordnet.synsets(word)
        for syn in synset:
            if stype([kw.lower() in syn.definition().lower() for kw in kws]) and \
               syn.lexname().startswith(word_type):
                matches.append(word.lower())
    return matches

def build_candidates(footwear, upperwear):
    candidates = []
    for word in footwear:
        for offset, letter in enumerate(word):
            for new_letter in list(set(alphabet).difference(letter)):
                new_word = word[:offset] + new_letter + word[offset+1:]
                if new_word in upperwear:
                    candidates.append((word, new_word))
    return candidates

if __name__ == "__main__":
    print("Test Run...")
    footwear   = ["boot", "shoe"]
    upperwear  = ["coot", "tshirt"]
    candidates = build_candidates(footwear, upperwear)
    for word, new_word in candidates:
        print("    Footwear: {0}-> Upperwear: {1}".format(word, new_word))

    print("Real run...")
    footwear   = search(["foot", "worn"], stype=all)
    footwear.extend(search(["feet", "worn"], stype=all))
    footwear.extend(search(["foot", "wear"], stype=all))
    footwear.extend(search(["feet", "wear"], stype=all))
    footwear = list(set(footwear)) # remove duplicates
    print("Footwear: {0}".format(footwear))

    upperwear      = search(["upper", "wear"], stype=all)
    upperwear.extend(search(["arm", "worn"], stype=all))
    upperwear.extend(search(["torso", "worn"], stype=all))
    upperwear.extend(search(["shoulder", "worn"], stype=all))
    upperwear.extend(search(["chest", "worn"], stype=all))
    upperwear.extend(search(["back", "worn"], stype=all))
    upperwear.extend(search(["upper", "wear"], stype=all))
    upperwear.extend(search(["arm", "wear"], stype=all))
    upperwear.extend(search(["torso", "wear"], stype=all))
    upperwear.extend(search(["shoulder", "wear"], stype=all))
    upperwear.extend(search(["chest", "wear"], stype=all))
    upperwear.extend(search(["back", "wear"], stype=all))
    upperwear = list(set(upperwear)) # remove duplicates
    print("Upperwear: {0}".format(upperwear))
    candidates = build_candidates(footwear, upperwear)
    if len(candidates) is 0:
        print("No candidates found.")
    else:
        for word, new_word in candidates:
            print("    Footwear: {0}-> Upperwear: {1}".format(word, new_word))
<commit_msg>Additional improvements<commit_after>#!/usr/bin/python3.4.2
'''
Created on Apr 6, 2015

Name something that might be worn on the foot.

Change one letter in it without changing the
order of the other letters. The result
will name something one might wear on the
upper part of the body. What is it?

Here's a hint: The thing on the upper part
of the body is a two-word phrase.

@author: john.obrien, leiran.biton
'''

import sys
from string import ascii_lowercase as alphabet
from nltk.corpus import words, wordnet

'''
search
parameters:
1. a list of keywords which we are trying to find
in the definitions of the words

returns:
a list of words for which any of the keywords searched for
were found in the definitions
'''

def search(kws, stype=all, word_type="noun"):
    """
kws   - list of keywords to be searched for
stype - either all or any (default = all)
word_type - part of speech (default = 'noun')
"""
    if stype not in (all, any):
        return TypeError("Invalid stype argument supplied. stype must be 'any' or 'all' functions.")
    matches = []
    # check kws against all defs in wordnets list of words
    for word in words.words():
        synset = wordnet.synsets(word)
        for syn in synset:
            if stype([kw.lower() in syn.definition().lower() for kw in kws]) and \
               syn.lexname().startswith(word_type):
                matches.append(word.lower())
    return matches

def build_candidates(footwear, upperwear):
    candidates = []
    for word in footwear:
        for offset, letter in enumerate(word):
            for new_letter in list(set(alphabet).difference(letter)):
                new_word = word[:offset] + new_letter + word[offset+1:]
                if new_word in upperwear:
                    candidates.append((word, new_word))
    return candidates

if __name__ == "__main__":
    print("Test Run...")
    footwear   = ["boot", "shoe"]
    upperwear  = ["coot", "tshirt"]
    candidates = build_candidates(footwear, upperwear)
    for word, new_word in candidates:
        print("    Footwear: {0}-> Upperwear: {1}".format(word, new_word))

    print("Real run...")
    footwear   = search(["foot", "worn"], stype=all)
    footwear.extend(search(["feet", "worn"], stype=all))
    footwear.extend(search(["foot", "wear"], stype=all))
    footwear.extend(search(["feet", "wear"], stype=all))
    footwear.extend(search(["shoe", "sock", "boot"], stype=any))
    footwear = list(set(footwear)) # remove duplicates
    print("Footwear: {0}".format(footwear))

    upperwear      = search(["upper", "wear"], stype=all)
    upperwear.extend(search(["arm", "worn"], stype=all))
    upperwear.extend(search(["torso", "worn"], stype=all))
    upperwear.extend(search(["shoulder", "worn"], stype=all))
    upperwear.extend(search(["chest", "worn"], stype=all))
    upperwear.extend(search(["back", "worn"], stype=all))
    upperwear.extend(search(["top", "worn"], stype=all))
    upperwear.extend(search(["upper", "wear"], stype=all))
    upperwear.extend(search(["arm", "wear"], stype=all))
    upperwear.extend(search(["torso", "wear"], stype=all))
    upperwear.extend(search(["shoulder", "wear"], stype=all))
    upperwear.extend(search(["chest", "wear"], stype=all))
    upperwear.extend(search(["back", "wear"], stype=all))
    upperwear.extend(search(["top", "wear"], stype=all))
    upperwear.extend(search(["coat", "shirt", "jacket", "necklace", "sweater"], stype=any))
    upperwear = list(set(upperwear)) # remove duplicates
    print("Upperwear: {0}".format(upperwear))
    candidates = build_candidates(footwear, upperwear)
    if len(candidates) is 0:
        print("No candidates found.")
    else:
        for word, new_word in candidates:
            print("    Footwear: {0}-> Upperwear: {1}".format(word, new_word))
<|endoftext|>