<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
decorators.py

Created by Maximillian Dornseif on 2007-05-10.
Copyright (c) 2007, 2015 HUDORA GmbH. All rights reserved.
"""
import cPickle as pickle
import functools
import hashlib
import threading
import time

from _decorator import decorator
from collections import namedtuple
from functools import update_wrapper

# TODO: können wir _decorator durch "from functools import wraps" ersetzen?

def _getattr_(obj, name, default_thunk):
    "Similar to .setdefault in dictionaries."
    try:
        return getattr(obj, name)
    except AttributeError:
        default = default_thunk()
        setattr(obj, name, default)
        return default

# from https://github.com/carlsverre/wraptor/blob/master/wraptor/decorators/memoize.py

class memoize(object):
    """ Memoize the results of a function.  Supports an optional timeout
        for automatic cache expiration.
        If the optional manual_flush argument is True, a function called
        "flush_cache" will be added to the wrapped function.  When
        called, it will remove all the timed out values from the cache.
        If you use this decorator as a class method, you must specify
        instance_method=True otherwise you will have a single shared
        cache for every instance of your class.
        This decorator is thread safe.
    """
    def __init__(self, timeout=None, manual_flush=False, instance_method=False):
        self.timeout = timeout
        self.manual_flush = manual_flush
        self.instance_method = instance_method
        self.cache = {}
        self.cache_lock = threading.RLock()

    def __call__(self, fn):
        if self.instance_method:
            @functools.wraps(fn)
            def rewrite_instance_method(instance, *args, **kwargs):
                # the first time we are called we overwrite the method
                # on the class instance with a new memoize instance
                if hasattr(instance, fn.__name__):
                    bound_fn = fn.__get__(instance, instance.__class__)
                    new_memoizer = memoize(self.timeout, self.manual_flush)(bound_fn)
                    setattr(instance, fn.__name__, new_memoizer)
                    return getattr(instance, fn.__name__)(*args, **kwargs)

            return rewrite_instance_method

        def flush_cache():
            with self.cache_lock:
                for key in self.cache.keys():
                    if (time.time() - self.cache[key][1]) > self.timeout:
                        del(self.cache[key])

        @functools.wraps(fn)
        def wrapped(*args, **kwargs):
            kw = kwargs.items()
            kw.sort()
            key_str = repr((args, kw))
            key = hashlib.md5(key_str).hexdigest()

            with self.cache_lock:
                try:
                    result, cache_time = self.cache[key]
                    if self.timeout is not None and (time.time() - cache_time) > self.timeout:
                        raise KeyError
                except KeyError:
                    result, _ = self.cache[key] = (fn(*args, **kwargs), time.time())

            if not self.manual_flush and self.timeout is not None:
                flush_cache()

            return result

        if self.manual_flush:
            wrapped.flush_cache = flush_cache

        return wrapped

# from somewhere on the internet

def cache_function(length):
    """
    A variant of the snippet posted by Jeff Wheeler at
    http://www.djangosnippets.org/snippets/109/

    Caches a function, using the function and its arguments as the key, and the return
    value as the value saved. It passes all arguments on to the function, as
    it should.

    The decorator itself takes a length argument, which is the number of
    seconds the cache will keep the result around.
    """

    def decorator(func):

        def inner_func(*args, **kwargs):
            from django.core.cache import cache

            raw = [func.__name__, func.__module__, args, kwargs]
            pickled = pickle.dumps(raw, protocol=pickle.HIGHEST_PROTOCOL)
            key = hashlib.md5(pickled).hexdigest()
            value = cache.get(key)
            if value:
                return value
            else:
                result = func(*args, **kwargs)
                cache.set(key, result, length)
                return result
        return inner_func
    return decorator

# from http://code.activestate.com/recipes/578078-py26-and-py30-backport-of-python-33s-lru-cache/

_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])

class _HashedSeq(list):
    __slots__ = 'hashvalue'

    def __init__(self, tup, hash=hash):
        self[:] = tup
        self.hashvalue = hash(tup)

    def __hash__(self):
        return self.hashvalue

def _make_key(args, kwds, typed,
              kwd_mark=(object(),),
              fasttypes={int, str, frozenset, type(None)},
              sorted=sorted, tuple=tuple, type=type, len=len):
    """Make a cache key from optionally typed positional and keyword arguments"""
    key = args
    if kwds:
        sorted_items = sorted(kwds.items())
        key += kwd_mark
        for item in sorted_items:
            key += item
    if typed:
        key += tuple(type(v) for v in args)
        if kwds:
            key += tuple(type(v) for k, v in sorted_items)
    elif len(key) == 1 and type(key[0]) in fasttypes:
        return key[0]
    return _HashedSeq(key)

def lru_cache(maxsize=64, typed=False, ttl=None):
    """Least-recently-used cache decorator.

    If *maxsize* is set to None, the LRU features are disabled and the cache
    can grow without bound.

    If *typed* is True, arguments of different types will be cached separately.
    For example, f(3.0) and f(3) will be treated as distinct calls with
    distinct results.

    if *ttl* is set, cache entries are only served for `ttl` seconds.

    Arguments to the cached function must be hashable.

    View the cache statistics named tuple (hits, misses, maxsize, currsize) with
    f.cache_info().  Clear the cache and statistics with f.cache_clear().
    Access the underlying function with f.__wrapped__.

    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used

    """

    # Users should only access the lru_cache through its public API:
    #       cache_info, cache_clear, and f.__wrapped__
    # The internals of the lru_cache are encapsulated for thread safety and
    # to allow the implementation to change (including a possible C version).

    def decorating_function(user_function):

        cache = dict()
        maxage = dict()                 # stores the timestamp after wich result should be regeneratd
        stats = [0, 0]                  # make statistics updateable non-locally
        HITS, MISSES = 0, 1             # names for the stats fields
        make_key = _make_key
        cache_get = cache.get           # bound method to lookup key or return None
        maxage_get = maxage.get
        _len = len                      # localize the global len() function
        lock = threading.RLock()                  # because linkedlist updates aren't threadsafe
        root = []                       # root of the circular doubly linked list
        root[:] = [root, root, None, None]      # initialize by pointing to self
        nonlocal_root = [root]                  # make updateable non-locally
        PREV, NEXT, KEY, RESULT = 0, 1, 2, 3    # names for the link fields

        if maxsize == 0:

            def wrapper(*args, **kwds):
                # no caching, just do a statistics update after a successful call
                result = user_function(*args, **kwds)
                stats[MISSES] += 1
                return result

        elif maxsize is None:

            def wrapper(*args, **kwds):
                # simple caching without ordering or size limit
                key = make_key(args, kwds, typed)
                result = cache_get(key, root)   # root used here as a unique not-found sentinel
                if result is not root:
                    if maxage_get(key, 0) < time.time():
                        stats[HITS] += 1
                        return result
                result = user_function(*args, **kwds)
                cache[key] = result
                if ttl:
                    maxage[key] = int(time.time() + ttl)
                stats[MISSES] += 1
                return result

        else:

            def wrapper(*args, **kwds):
                # size limited caching that tracks accesses by recency
                key = make_key(args, kwds, typed) if kwds or typed else args
                with lock:
                    link = cache_get(key)
                    if link is not None:
                        if maxage_get(key, None) < time.time():
                            # record recent use of the key by moving it to the front of the list
                            root, = nonlocal_root
                            link_prev, link_next, key, result = link
                            link_prev[NEXT] = link_next
                            link_next[PREV] = link_prev
                            last = root[PREV]
                            last[NEXT] = root[PREV] = link
                            link[PREV] = last
                            link[NEXT] = root
                            stats[HITS] += 1
                            return result
                result = user_function(*args, **kwds)
                if ttl:
                    maxage[key] = int(time.time() + ttl)
                with lock:
                    root, = nonlocal_root
                    if key in cache:
                        # getting here means that this same key was added to the
                        # cache while the lock was released.  since the link
                        # update is already done, we need only return the
                        # computed result and update the count of misses.
                        pass
                    elif _len(cache) >= maxsize:
                        # use the old root to store the new key and result
                        oldroot = root
                        oldroot[KEY] = key
                        oldroot[RESULT] = result
                        # empty the oldest link and make it the new root
                        root = nonlocal_root[0] = oldroot[NEXT]
                        oldkey = root[KEY]
                        oldvalue = root[RESULT]
                        root[KEY] = root[RESULT] = None
                        # now update the cache dictionary for the new links
                        del cache[oldkey]
                        cache[key] = oldroot
                    else:
                        # put result in a new link at the front of the list
                        last = root[PREV]
                        link = [last, root, key, result]
                        last[NEXT] = root[PREV] = cache[key] = link
                    stats[MISSES] += 1
                return result

        def cache_info():
            """Report cache statistics"""
            with lock:
                return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache))

        def cache_clear():
            """Clear the cache and cache statistics"""
            with lock:
                cache.clear()
                root = nonlocal_root[0]
                root[:] = [root, root, None, None]
                stats[:] = [0, 0]

        wrapper.__wrapped__ = user_function
        wrapper.cache_info = cache_info
        wrapper.cache_clear = cache_clear
        return update_wrapper(wrapper, user_function)

    return decorating_function

# from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/425445
def func_once(func):
    "A decorator that runs a function only once."

    def decorated(*args, **kwargs):
        try:
            return decorated._once_result
        except AttributeError:
            decorated._once_result = func(*args, **kwargs)
            return decorated._once_result
    return decorated

def method_once(method):
    "A decorator that runs a method only once."
    attrname = "_%s_once_result" % id(method)

    def decorated(self, *args, **kwargs):
        try:
            return getattr(self, attrname)
        except AttributeError:
            setattr(self, attrname, method(self, *args, **kwargs))
            return getattr(self, attrname)
    return decorated

def none_on_exception(func):
    """A decorator that returns the return value of a function or None if an exception is raised."""

    @functools.wraps(func)
    def _decorator(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception:
            return None
    return _decorator
<commit_msg>Bugfix für TTL von lru_cache<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
decorators.py

Created by Maximillian Dornseif on 2007-05-10.
Copyright (c) 2007, 2015 HUDORA GmbH. All rights reserved.
"""
import cPickle as pickle
import functools
import hashlib
import threading
import time

from _decorator import decorator
from collections import namedtuple
from functools import update_wrapper

# TODO: können wir _decorator durch "from functools import wraps" ersetzen?

def _getattr_(obj, name, default_thunk):
    "Similar to .setdefault in dictionaries."
    try:
        return getattr(obj, name)
    except AttributeError:
        default = default_thunk()
        setattr(obj, name, default)
        return default

# from https://github.com/carlsverre/wraptor/blob/master/wraptor/decorators/memoize.py

class memoize(object):
    """ Memoize the results of a function.  Supports an optional timeout
        for automatic cache expiration.
        If the optional manual_flush argument is True, a function called
        "flush_cache" will be added to the wrapped function.  When
        called, it will remove all the timed out values from the cache.
        If you use this decorator as a class method, you must specify
        instance_method=True otherwise you will have a single shared
        cache for every instance of your class.
        This decorator is thread safe.
    """
    def __init__(self, timeout=None, manual_flush=False, instance_method=False):
        self.timeout = timeout
        self.manual_flush = manual_flush
        self.instance_method = instance_method
        self.cache = {}
        self.cache_lock = threading.RLock()

    def __call__(self, fn):
        if self.instance_method:
            @functools.wraps(fn)
            def rewrite_instance_method(instance, *args, **kwargs):
                # the first time we are called we overwrite the method
                # on the class instance with a new memoize instance
                if hasattr(instance, fn.__name__):
                    bound_fn = fn.__get__(instance, instance.__class__)
                    new_memoizer = memoize(self.timeout, self.manual_flush)(bound_fn)
                    setattr(instance, fn.__name__, new_memoizer)
                    return getattr(instance, fn.__name__)(*args, **kwargs)

            return rewrite_instance_method

        def flush_cache():
            with self.cache_lock:
                for key in self.cache.keys():
                    if (time.time() - self.cache[key][1]) > self.timeout:
                        del(self.cache[key])

        @functools.wraps(fn)
        def wrapped(*args, **kwargs):
            kw = kwargs.items()
            kw.sort()
            key_str = repr((args, kw))
            key = hashlib.md5(key_str).hexdigest()

            with self.cache_lock:
                try:
                    result, cache_time = self.cache[key]
                    if self.timeout is not None and (time.time() - cache_time) > self.timeout:
                        raise KeyError
                except KeyError:
                    result, _ = self.cache[key] = (fn(*args, **kwargs), time.time())

            if not self.manual_flush and self.timeout is not None:
                flush_cache()

            return result

        if self.manual_flush:
            wrapped.flush_cache = flush_cache

        return wrapped

# from somewhere on the internet

def cache_function(length):
    """
    A variant of the snippet posted by Jeff Wheeler at
    http://www.djangosnippets.org/snippets/109/

    Caches a function, using the function and its arguments as the key, and the return
    value as the value saved. It passes all arguments on to the function, as
    it should.

    The decorator itself takes a length argument, which is the number of
    seconds the cache will keep the result around.
    """

    def decorator(func):

        def inner_func(*args, **kwargs):
            from django.core.cache import cache

            raw = [func.__name__, func.__module__, args, kwargs]
            pickled = pickle.dumps(raw, protocol=pickle.HIGHEST_PROTOCOL)
            key = hashlib.md5(pickled).hexdigest()
            value = cache.get(key)
            if value:
                return value
            else:
                result = func(*args, **kwargs)
                cache.set(key, result, length)
                return result
        return inner_func
    return decorator

# from http://code.activestate.com/recipes/578078-py26-and-py30-backport-of-python-33s-lru-cache/

_CacheInfo = namedtuple("CacheInfo", ["hits", "misses", "maxsize", "currsize"])

class _HashedSeq(list):
    __slots__ = 'hashvalue'

    def __init__(self, tup, hash=hash):
        self[:] = tup
        self.hashvalue = hash(tup)

    def __hash__(self):
        return self.hashvalue

def _make_key(args, kwds, typed,
              kwd_mark=(object(),),
              fasttypes={int, str, frozenset, type(None)},
              sorted=sorted, tuple=tuple, type=type, len=len):
    """Make a cache key from optionally typed positional and keyword arguments"""
    key = args
    if kwds:
        sorted_items = sorted(kwds.items())
        key += kwd_mark
        for item in sorted_items:
            key += item
    if typed:
        key += tuple(type(v) for v in args)
        if kwds:
            key += tuple(type(v) for k, v in sorted_items)
    elif len(key) == 1 and type(key[0]) in fasttypes:
        return key[0]
    return _HashedSeq(key)

def lru_cache(maxsize=64, typed=False, ttl=None):
    """Least-recently-used cache decorator.

    If *maxsize* is set to None, the LRU features are disabled and the cache
    can grow without bound.

    If *typed* is True, arguments of different types will be cached separately.
    For example, f(3.0) and f(3) will be treated as distinct calls with
    distinct results.

    if *ttl* is set, cache entries are only served for `ttl` seconds.

    Arguments to the cached function must be hashable.

    View the cache statistics named tuple (hits, misses, maxsize, currsize) with
    f.cache_info().  Clear the cache and statistics with f.cache_clear().
    Access the underlying function with f.__wrapped__.

    See:  http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used

    """

    # Users should only access the lru_cache through its public API:
    #       cache_info, cache_clear, and f.__wrapped__
    # The internals of the lru_cache are encapsulated for thread safety and
    # to allow the implementation to change (including a possible C version).

    def decorating_function(user_function):

        cache = dict()
        maxage = dict()                 # stores the timestamp after wich result should be regeneratd
        stats = [0, 0]                  # make statistics updateable non-locally
        HITS, MISSES = 0, 1             # names for the stats fields
        make_key = _make_key
        cache_get = cache.get           # bound method to lookup key or return None
        maxage_get = maxage.get
        _len = len                      # localize the global len() function
        lock = threading.RLock()                  # because linkedlist updates aren't threadsafe
        root = []                       # root of the circular doubly linked list
        root[:] = [root, root, None, None]      # initialize by pointing to self
        nonlocal_root = [root]                  # make updateable non-locally
        PREV, NEXT, KEY, RESULT = 0, 1, 2, 3    # names for the link fields

        if maxsize == 0:

            def wrapper(*args, **kwds):
                # no caching, just do a statistics update after a successful call
                result = user_function(*args, **kwds)
                stats[MISSES] += 1
                return result

        elif maxsize is None:

            def wrapper(*args, **kwds):
                # simple caching without ordering or size limit
                key = make_key(args, kwds, typed)
                result = cache_get(key, root)   # root used here as a unique not-found sentinel
                if result is not root:
                    if time.time() <= maxage_get(key, 0):
                        stats[HITS] += 1
                        return result
                result = user_function(*args, **kwds)
                cache[key] = result
                if ttl:
                    maxage[key] = int(time.time() + ttl)
                stats[MISSES] += 1
                return result

        else:

            def wrapper(*args, **kwds):
                # size limited caching that tracks accesses by recency
                key = make_key(args, kwds, typed) if kwds or typed else args
                with lock:
                    link = cache_get(key)
                    if link is not None:
                        if time.time() <= maxage_get(key, None):
                            # record recent use of the key by moving it to the front of the list
                            root, = nonlocal_root
                            link_prev, link_next, key, result = link
                            link_prev[NEXT] = link_next
                            link_next[PREV] = link_prev
                            last = root[PREV]
                            last[NEXT] = root[PREV] = link
                            link[PREV] = last
                            link[NEXT] = root
                            stats[HITS] += 1
                            return result
                result = user_function(*args, **kwds)
                if ttl:
                    maxage[key] = int(time.time() + ttl)
                with lock:
                    root, = nonlocal_root
                    if key in cache:
                        # getting here means that this same key was added to the
                        # cache while the lock was released.  since the link
                        # update is already done, we need only return the
                        # computed result and update the count of misses.
                        pass
                    elif _len(cache) >= maxsize:
                        # use the old root to store the new key and result
                        oldroot = root
                        oldroot[KEY] = key
                        oldroot[RESULT] = result
                        # empty the oldest link and make it the new root
                        root = nonlocal_root[0] = oldroot[NEXT]
                        oldkey = root[KEY]
                        oldvalue = root[RESULT]
                        root[KEY] = root[RESULT] = None
                        # now update the cache dictionary for the new links
                        del cache[oldkey]
                        cache[key] = oldroot
                    else:
                        # put result in a new link at the front of the list
                        last = root[PREV]
                        link = [last, root, key, result]
                        last[NEXT] = root[PREV] = cache[key] = link
                    stats[MISSES] += 1
                return result

        def cache_info():
            """Report cache statistics"""
            with lock:
                return _CacheInfo(stats[HITS], stats[MISSES], maxsize, len(cache))

        def cache_clear():
            """Clear the cache and cache statistics"""
            with lock:
                cache.clear()
                root = nonlocal_root[0]
                root[:] = [root, root, None, None]
                stats[:] = [0, 0]

        wrapper.__wrapped__ = user_function
        wrapper.cache_info = cache_info
        wrapper.cache_clear = cache_clear
        return update_wrapper(wrapper, user_function)

    return decorating_function

# from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/425445
def func_once(func):
    "A decorator that runs a function only once."

    def decorated(*args, **kwargs):
        try:
            return decorated._once_result
        except AttributeError:
            decorated._once_result = func(*args, **kwargs)
            return decorated._once_result
    return decorated

def method_once(method):
    "A decorator that runs a method only once."
    attrname = "_%s_once_result" % id(method)

    def decorated(self, *args, **kwargs):
        try:
            return getattr(self, attrname)
        except AttributeError:
            setattr(self, attrname, method(self, *args, **kwargs))
            return getattr(self, attrname)
    return decorated

def none_on_exception(func):
    """A decorator that returns the return value of a function or None if an exception is raised."""

    @functools.wraps(func)
    def _decorator(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except Exception:
            return None
    return _decorator
<|endoftext|>


<commit_before># Copyright (c) 2011-2013 Kunal Mehta. All rights reserved.
# Use of this source code is governed by a BSD License found in README.md.

from django.db import models

class Conference(models.Model):
    session         = models.PositiveSmallIntegerField(default=0)
    start_date      = models.DateField()
    end_date        = models.DateField()
    reg_open        = models.DateField()
    early_reg_close = models.DateField()
    reg_close       = models.DateField()
    min_attendance  = models.PositiveSmallIntegerField(default=0)
    max_attendance  = models.PositiveSmallIntegerField(default=0)

    @staticmethod
    def auto_country_assign(school):
        '''Automatically assign a school country and committee assignments
        based on preference, and then by default order.'''
        spots_left = school.max_delegation_size
        if spots_left:
            spots_left = Conference.auto_assign(Country.objects.filter(special=True),
                                                school.get_committee_preferences(),
                                                school,
                                                spots_left,
                                                (school.max_delegation_size/10)+1)
        if spots_left:
            spots_left = Conference.auto_assign(Country.objects.filter(special=False),
                                                school.get_committee_preferences(),
                                                school,
                                                spots_left,
                                                (school.max_delegation_size/5)+1)
        if spots_left:
            spots_left = Conference.auto_assign(school.get_country_preferences(),
                                                Committee.objects.filter(special=False),
                                                school,
                                                spots_left)
        if spots_left:
            spots_left = Conference.auto_assign(Country.objects.filter(special=False),
                                                Committee.objects.filter(special=False),
                                                school,
                                                spots_left)

    @staticmethod
    def auto_assign(countries, committees, school, spots_left, max_spots=100):
        '''Assign schools to unassigned Assignment objects based on a set of
        available countries and committees.'''
        for country in countries:
            for committee in committees:
                try:
                    assignment = Assignment.objects.get(committee=committee,
                                                        country=country)
                    if assignment.school is None:
                        assignment.school = school
                        spots_left -= committee.delegation_size
                        max_spots -= committee.delegation_size:
                        assignment.save()
                        if spots_left < 3:
                            return 0
                        if max_spots < 0:
                            break
                except Assignment.DoesNotExist:
                    pass

        return spots_left

    def __unicode__(self):
        return 'BMUN %d' % self.session

    class Meta:
        db_table = u'conference'
        get_latest_by = 'start_date'

class Country(models.Model):
    name    = models.CharField(max_length=128)
    special = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    class Meta:
        db_table = u'country'

class Committee(models.Model):
    name            = models.CharField(max_length=8)
    full_name       = models.CharField(max_length=128)
    countries       = models.ManyToManyField(Country, through='Assignment')
    delegation_size = models.PositiveSmallIntegerField(default=2)
    special         = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    class Meta:
        db_table = u'committee'

class School(models.Model):
    TYPE_CLUB = 1
    TYPE_CLASS = 2
    PROGRAM_TYPE_OPTIONS = (
        (TYPE_CLUB, 'Club'),
        (TYPE_CLASS, 'Class'),
    )

    LOCATION_USA = 'location/usa'
    LOCATION_INTERNATIONAL = 'location/international'
    LOCATION_OPTIONS = (
        (LOCATION_USA, 'United States of America'),
        (LOCATION_INTERNATIONAL, 'International'),
    )

    registered          = models.DateTimeField(auto_now_add=True)
    name                = models.CharField(max_length=128)
    address             = models.CharField(max_length=128)
    city                = models.CharField(max_length=128)
    state               = models.CharField(max_length=16)
    zip_code            = models.CharField(max_length=16)
    country             = models.CharField(max_length=64)
    primary_name        = models.CharField(max_length=128)
    primary_email       = models.EmailField()
    primary_phone       = models.CharField(max_length=32)
    secondary_name      = models.CharField(max_length=128, blank=True)
    secondary_email     = models.EmailField(blank=True)
    secondary_phone     = models.CharField(max_length=32, blank=True)
    program_type        = models.PositiveSmallIntegerField(choices=PROGRAM_TYPE_OPTIONS)
    times_attended      = models.PositiveSmallIntegerField(default=0)
    min_delegation_size = models.PositiveSmallIntegerField(default=0)
    max_delegation_size = models.PositiveSmallIntegerField(default=0)
    international       = models.BooleanField(default=False)

    countrypreferences   = models.ManyToManyField(Country, through='CountryPreference')
    committeepreferences = models.ManyToManyField(Committee, limit_choices_to={'special':True})

    registration_fee         = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    registration_fee_paid    = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    registration_fee_balance = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    delegation_fee           = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    delegation_fee_paid      = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    delegation_fee_balance   = models.DecimalField(max_digits=6, decimal_places=2, default=0)

    def update_country_preferences(self, country_ids, shuffled=False):
        """ Refreshes a school's country preferences, given a list of
            country IDs. If shuffled is True, then the country_ids are
            ordered [1, 6, 2, 7, 3, 8, ...] due to double columning in
            the template. """
        if shuffled:
            country_ids = CountryPreference.unshuffle(country_ids)
        self.countrypreferences.clear()
        seen = set()
        for rank, country_id in enumerate(country_ids, start=1):
            if country_id and country_id not in seen:
                seen.add(country_id)
                CountryPreference.objects.create(school=self,
                                                 country_id=country_id,
                                                 rank=rank)

    def update_committee_preferences(self, committee_ids):
        """ Refreshes a school's committee preferences. """
        self.committeepreferences.clear()
        self.committeepreferences = committee_ids
        self.save()

    def update_delegate_slots(self, slot_data):
        """ Adds, deletes, or updates delegates attached to this school's
            delegate slots. """
        for slot_id, delegate_data in slot_data:
            slot = DelegateSlot.objects.get(id=slot_id)
            if 'name' in delegate_data and 'email' in delegate_data:
                slot.update_or_create_delegate(delegate_data)
            else:
                slot.delete_delegate_if_exists()

    def get_country_preferences(self):
        """ Returns a list of this school's country preferences,
            ordered by rank. Note that these are Country objects,
            not CountryPreference objects."""
        return list(self.countrypreferences.all()
                        .order_by('countrypreference__rank'))

    def get_committee_preferences(self):
        return self.committeepreferences.all()

    def get_delegate_slots(self):
        """ Returns a list of this school's delegate slots,
            ordered by committee name. """
        return list(DelegateSlot.objects.filter(assignment__school=self)
                                 .order_by('assignment__committee__name'))

    def __unicode__(self):
        return self.name

    class Meta:
        db_table = u'school'

class Assignment(models.Model):
    committee = models.ForeignKey(Committee)
    country   = models.ForeignKey(Country)
    school    = models.ForeignKey(School, null=True, blank=True, default=None)

    def __unicode__(self):
        return self.committee.name + " : " + self.country.name + " : " + (self.school or "Unassigned")

    class Meta:
        db_table = u'assignment'

class CountryPreference(models.Model):
    school  = models.ForeignKey(School)
    country = models.ForeignKey(Country, limit_choices_to={'special':False})
    rank    = models.PositiveSmallIntegerField()

    @staticmethod
    def unshuffle(countries):
        """ Given a list of 2-tuples, Returns a list of countries (or IDs) in correct,
            unshuffled order. """
        countries = [list(t) for t in zip(*countries)]
        return filter(None, countries[0] + countries[1])

    @staticmethod
    def shuffle(countries):
        """ Returns a list of countries (or IDs) in shuffled order
            for double columning, i.e. [1, 6, 2, 7, 3, 8, ...]. Before
            shuffling, the list is padded to length 10. """
        countries += [None]*(10 - len(countries))
        c1, c2 = countries[:5], countries[5:]
        return zip(c1, c2)

    def __unicode__(self):
        return "%s : %s (%d)" % self.school.name, self.country.name, self.rank

    class Meta:
        db_table = u'country_preference'
        ordering = ['rank']

class DelegateSlot(models.Model):
    assignment        = models.ForeignKey(Assignment)
    attended_session1 = models.BooleanField(default=False)
    attended_session2 = models.BooleanField(default=False)
    attended_session3 = models.BooleanField(default=False)
    attended_session4 = models.BooleanField(default=False)

    def update_or_create_delegate(self, delegate_data):
        """ Updates this slot's delegate object, or creates one if
            the slot has no delegate. """
        try:
            delegate = self.delegate
            for attr, value in delegate_data.items():
                setattr(delegate, attr, value)
            delegate.save()
        except Delegate.DoesNotExist:
            Delegate.objects.create(delegate_slot=self, **delegate_data)

    def delete_delegate_if_exists(self):
        """ Deletes this slot's delegate or fails silently. """
        try:
            self.delegate.delete()
        except Delegate.DoesNotExist:
            pass

    def update_delegate_attendance(self, slot_data):
        """ Updates this slot's attendance information. """
        self.attended_session1 = slot_data['session1']
        self.attended_session2 = slot_data['session2']
        self.attended_session3 = slot_data['session3']
        self.attended_session4 = slot_data['session4']
        self.save()

    @property
    def country(self):
        return self.assignment.country

    @property
    def committee(self):
        return self.assignment.committee

    @property
    def school(self):
        return self.assignment.school

    def __unicode__(self):
        return str(self.assignment)

    class Meta:
        db_table = u'delegate_slot'

class Delegate(models.Model):
    name          = models.CharField(max_length=64, blank=True)
    email         = models.EmailField(blank=True)
    delegate_slot = models.OneToOneField(DelegateSlot, related_name='delegate', null=True, default=None)
    created_at    = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        return self.name

    @property
    def country(self):
        return self.delegate_slot.country

    @property
    def committee(self):
        return self.delegate_slot.committee

    @property
    def school(self):
        return self.delegate_slot.school

    class Meta:
        db_table = u'delegate'

class HelpCategory(models.Model):
    name = models.CharField(max_length=128, unique=True)

    def __unicode__(self):
        return self.name

    class Meta:
        db_table = u'help_category'

class HelpQuestion(models.Model):
    category = models.ForeignKey(HelpCategory)
    question = models.CharField(max_length=255)
    answer   = models.TextField()

    def __unicode__(self):
        return self.question

    class Meta:
        db_table = u'help_question'

<commit_msg>updates algorithm for randomness<commit_after># Copyright (c) 2011-2013 Kunal Mehta. All rights reserved.
# Use of this source code is governed by a BSD License found in README.md.

from django.db import models

class Conference(models.Model):
    session         = models.PositiveSmallIntegerField(default=0)
    start_date      = models.DateField()
    end_date        = models.DateField()
    reg_open        = models.DateField()
    early_reg_close = models.DateField()
    reg_close       = models.DateField()
    min_attendance  = models.PositiveSmallIntegerField(default=0)
    max_attendance  = models.PositiveSmallIntegerField(default=0)

    @staticmethod
    def auto_country_assign(school):
        '''Automatically assign a school country and committee assignments
        based on preference, and then by default order.'''
        spots_left = school.max_delegation_size
        if spots_left:
            spots_left = Conference.auto_assign(Country.objects.filter(special=True).order_by('?'),
                                                school.get_committee_preferences().,
                                                school,
                                                spots_left,
                                                (school.max_delegation_size*.15)+1)
        if spots_left:
            spots_left = Conference.auto_assign(Country.objects.filter(special=False).order_by('?'),
                                                school.get_committee_preferences(),
                                                school,
                                                spots_left,
                                                (school.max_delegation_size*.20)+1)
        if spots_left:
            spots_left = Conference.auto_assign(school.get_country_preferences(),
                                                Committee.objects.filter(special=False).order_by('?'),
                                                school,
                                                spots_left)
        if spots_left:
            spots_left = Conference.auto_assign(Country.objects.filter(special=False).order_by('?'),
                                                Committee.objects.filter(special=False).order_by('?'),
                                                school,
                                                spots_left)

    @staticmethod
    def auto_assign(countries, committees, school, spots_left, max_spots=100):
        '''Assign schools to unassigned Assignment objects based on a set of
        available countries and committees.'''
        for country in countries:
            for committee in committees:
                try:
                    assignment = Assignment.objects.get(committee=committee,
                                                        country=country)
                    if assignment.school is None:
                        assignment.school = school
                        spots_left -= committee.delegation_size
                        max_spots -= committee.delegation_size:
                        assignment.save()
                        if spots_left < 3:
                            return 0
                        if max_spots < 0:
                            break
                except Assignment.DoesNotExist:
                    pass

        return spots_left

    def __unicode__(self):
        return 'BMUN %d' % self.session

    class Meta:
        db_table = u'conference'
        get_latest_by = 'start_date'

class Country(models.Model):
    name    = models.CharField(max_length=128)
    special = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    class Meta:
        db_table = u'country'

class Committee(models.Model):
    name            = models.CharField(max_length=8)
    full_name       = models.CharField(max_length=128)
    countries       = models.ManyToManyField(Country, through='Assignment')
    delegation_size = models.PositiveSmallIntegerField(default=2)
    special         = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    class Meta:
        db_table = u'committee'

class School(models.Model):
    TYPE_CLUB = 1
    TYPE_CLASS = 2
    PROGRAM_TYPE_OPTIONS = (
        (TYPE_CLUB, 'Club'),
        (TYPE_CLASS, 'Class'),
    )

    LOCATION_USA = 'location/usa'
    LOCATION_INTERNATIONAL = 'location/international'
    LOCATION_OPTIONS = (
        (LOCATION_USA, 'United States of America'),
        (LOCATION_INTERNATIONAL, 'International'),
    )

    registered          = models.DateTimeField(auto_now_add=True)
    name                = models.CharField(max_length=128)
    address             = models.CharField(max_length=128)
    city                = models.CharField(max_length=128)
    state               = models.CharField(max_length=16)
    zip_code            = models.CharField(max_length=16)
    country             = models.CharField(max_length=64)
    primary_name        = models.CharField(max_length=128)
    primary_email       = models.EmailField()
    primary_phone       = models.CharField(max_length=32)
    secondary_name      = models.CharField(max_length=128, blank=True)
    secondary_email     = models.EmailField(blank=True)
    secondary_phone     = models.CharField(max_length=32, blank=True)
    program_type        = models.PositiveSmallIntegerField(choices=PROGRAM_TYPE_OPTIONS)
    times_attended      = models.PositiveSmallIntegerField(default=0)
    min_delegation_size = models.PositiveSmallIntegerField(default=0)
    max_delegation_size = models.PositiveSmallIntegerField(default=0)
    international       = models.BooleanField(default=False)

    countrypreferences   = models.ManyToManyField(Country, through='CountryPreference')
    committeepreferences = models.ManyToManyField(Committee, limit_choices_to={'special':True})

    registration_fee         = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    registration_fee_paid    = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    registration_fee_balance = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    delegation_fee           = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    delegation_fee_paid      = models.DecimalField(max_digits=6, decimal_places=2, default=0)
    delegation_fee_balance   = models.DecimalField(max_digits=6, decimal_places=2, default=0)

    def update_country_preferences(self, country_ids, shuffled=False):
        """ Refreshes a school's country preferences, given a list of
            country IDs. If shuffled is True, then the country_ids are
            ordered [1, 6, 2, 7, 3, 8, ...] due to double columning in
            the template. """
        if shuffled:
            country_ids = CountryPreference.unshuffle(country_ids)
        self.countrypreferences.clear()
        seen = set()
        for rank, country_id in enumerate(country_ids, start=1):
            if country_id and country_id not in seen:
                seen.add(country_id)
                CountryPreference.objects.create(school=self,
                                                 country_id=country_id,
                                                 rank=rank)

    def update_committee_preferences(self, committee_ids):
        """ Refreshes a school's committee preferences. """
        self.committeepreferences.clear()
        self.committeepreferences = committee_ids
        self.save()

    def update_delegate_slots(self, slot_data):
        """ Adds, deletes, or updates delegates attached to this school's
            delegate slots. """
        for slot_id, delegate_data in slot_data:
            slot = DelegateSlot.objects.get(id=slot_id)
            if 'name' in delegate_data and 'email' in delegate_data:
                slot.update_or_create_delegate(delegate_data)
            else:
                slot.delete_delegate_if_exists()

    def get_country_preferences(self):
        """ Returns a list of this school's country preferences,
            ordered by rank. Note that these are Country objects,
            not CountryPreference objects."""
        return list(self.countrypreferences.all()
                        .order_by('countrypreference__rank'))

    def get_committee_preferences(self):
        return self.committeepreferences.all()

    def get_delegate_slots(self):
        """ Returns a list of this school's delegate slots,
            ordered by committee name. """
        return list(DelegateSlot.objects.filter(assignment__school=self)
                                 .order_by('assignment__committee__name'))

    def __unicode__(self):
        return self.name

    class Meta:
        db_table = u'school'

class Assignment(models.Model):
    committee = models.ForeignKey(Committee)
    country   = models.ForeignKey(Country)
    school    = models.ForeignKey(School, null=True, blank=True, default=None)

    def __unicode__(self):
        return self.committee.name + " : " + self.country.name + " : " + (self.school or "Unassigned")

    class Meta:
        db_table = u'assignment'

class CountryPreference(models.Model):
    school  = models.ForeignKey(School)
    country = models.ForeignKey(Country, limit_choices_to={'special':False})
    rank    = models.PositiveSmallIntegerField()

    @staticmethod
    def unshuffle(countries):
        """ Given a list of 2-tuples, Returns a list of countries (or IDs) in correct,
            unshuffled order. """
        countries = [list(t) for t in zip(*countries)]
        return filter(None, countries[0] + countries[1])

    @staticmethod
    def shuffle(countries):
        """ Returns a list of countries (or IDs) in shuffled order
            for double columning, i.e. [1, 6, 2, 7, 3, 8, ...]. Before
            shuffling, the list is padded to length 10. """
        countries += [None]*(10 - len(countries))
        c1, c2 = countries[:5], countries[5:]
        return zip(c1, c2)

    def __unicode__(self):
        return "%s : %s (%d)" % self.school.name, self.country.name, self.rank

    class Meta:
        db_table = u'country_preference'
        ordering = ['rank']

class DelegateSlot(models.Model):
    assignment        = models.ForeignKey(Assignment)
    attended_session1 = models.BooleanField(default=False)
    attended_session2 = models.BooleanField(default=False)
    attended_session3 = models.BooleanField(default=False)
    attended_session4 = models.BooleanField(default=False)

    def update_or_create_delegate(self, delegate_data):
        """ Updates this slot's delegate object, or creates one if
            the slot has no delegate. """
        try:
            delegate = self.delegate
            for attr, value in delegate_data.items():
                setattr(delegate, attr, value)
            delegate.save()
        except Delegate.DoesNotExist:
            Delegate.objects.create(delegate_slot=self, **delegate_data)

    def delete_delegate_if_exists(self):
        """ Deletes this slot's delegate or fails silently. """
        try:
            self.delegate.delete()
        except Delegate.DoesNotExist:
            pass

    def update_delegate_attendance(self, slot_data):
        """ Updates this slot's attendance information. """
        self.attended_session1 = slot_data['session1']
        self.attended_session2 = slot_data['session2']
        self.attended_session3 = slot_data['session3']
        self.attended_session4 = slot_data['session4']
        self.save()

    @property
    def country(self):
        return self.assignment.country

    @property
    def committee(self):
        return self.assignment.committee

    @property
    def school(self):
        return self.assignment.school

    def __unicode__(self):
        return str(self.assignment)

    class Meta:
        db_table = u'delegate_slot'

class Delegate(models.Model):
    name          = models.CharField(max_length=64, blank=True)
    email         = models.EmailField(blank=True)
    delegate_slot = models.OneToOneField(DelegateSlot, related_name='delegate', null=True, default=None)
    created_at    = models.DateTimeField(auto_now_add=True)

    def __unicode__(self):
        return self.name

    @property
    def country(self):
        return self.delegate_slot.country

    @property
    def committee(self):
        return self.delegate_slot.committee

    @property
    def school(self):
        return self.delegate_slot.school

    class Meta:
        db_table = u'delegate'

class HelpCategory(models.Model):
    name = models.CharField(max_length=128, unique=True)

    def __unicode__(self):
        return self.name

    class Meta:
        db_table = u'help_category'

class HelpQuestion(models.Model):
    category = models.ForeignKey(HelpCategory)
    question = models.CharField(max_length=255)
    answer   = models.TextField()

    def __unicode__(self):
        return self.question

    class Meta:
        db_table = u'help_question'

<|endoftext|>


<commit_before>"""
feature extraction
"""

# from standard library
import sys
import os
import glob

# from dependencies
import numpy as np
from sklearn.externals import joblib

# from hvc
from .parseconfig import parse_config
from . import features
from .utils import timestamp

SELECT_TEMPLATE = """select:
  num_replicates: None
  num_train_samples:
    start : None
    stop : None
    step : None
  num_test_samples: None

    models:"""

MODELS_TEMPLATE = """
    -
      model: {0}
      feature_list_indices: {1}
      hyperparameters:
        {2}
"""

SVM_HYPERPARAMS = """C : None
        gamma : None
"""

KNN_HYPERPARAMS = """k : None
"""

TODO_TEMPLATE = """  todo_list:
    -
      feature_file : {0}
      output_dir: {1}"""

def write_select_config(summary_ftr_file_dict,
                       summary_filename,
                       output_dir):
    """writes summary output dict from extract to a config file for select
    Only called when feature_group is a list, to save user from figuring
    out manually which features belong to which group.

    Parameters
    ----------
    summary_ftr_file_dict : dictionary
        as defined in featureextract.extract
    summary_filename : string
        name of summary feature file
    output_dir : string
        name of output directory -- assumes it will be the same as it was for extract.yml

    Returns
    -------
    None

    Doesn't return anything, just saves .yml file
    """

    select_config_filename = 'select.config.from_' + summary_filename + '.yml'
    with open(select_config_filename, 'w') as yml_outfile:
        yml_outfile.write(SELECT_TEMPLATE)
        for model_name, model_ID in summary_ftr_file_dict['feature_group_ID_dict'].items():
            inds = [ftr_list_ind for ftr_list_ind, grp_ID in
                    enumerate(summary_ftr_file_dict['feature_list_group_ID'])
                    if grp_ID == model_ID]
            if model_name == 'svm':
                hyperparams = SVM_HYPERPARAMS
            elif model_name == 'knn':
                hyperparams = KNN_HYPERPARAMS
            yml_outfile.write(MODELS_TEMPLATE.format(model_name,
                                                     inds,
                                                     hyperparams))
        yml_outfile.write(TODO_TEMPLATE.format(summary_filename,
                                               output_dir))

def _extract(extract_params):
    """helper function that loops through data dirs,
    extracts features, saves feature files, and then
    makes summary feature file.
    Put into a separate helper function so that it can
    also be used by labelpredict.predict

    Parameters
    ----------
    extract_params : dict with following key, value pairs
        data_dirs : list
            list of directories with data files
        home_dir : str
        output_dir :
        file_format : str
            {'evtaf', 'koumura'}
        feature_list : list
            list of features generated by hvc.parse.extract
        spect_params : dict
            as defined for hvc.audiofileIO.Spectrogram class
        labelset : str
            labels for syllables as a string, e.g., 'iabcdefg'
        segment_params : dict
            as defined for hvc.audiofileIO.Song and hvc.audiofileIO.segment_song

    Returns
    -------
    None
    Saves output files in output dir

    """

    for data_dir in extract_params['data_dirs']:
        print('Changing to data directory: {}'.format(data_dir))
        os.chdir(extract_params['home_dir'])
        os.chdir(data_dir)

        if 'features_from_all_files' in locals():
            # from last time through loop
            # (need to re-initialize for each directory)
            del features_from_all_files

        if 'neuralnet_inputs_dict' in locals():
            del neuralnet_inputs_dict

        if extract_params['file_format'] == 'evtaf':
            songfiles_list = glob.glob('*.cbin')
        elif extract_params['file_format'] == 'koumura':
            songfiles_list = glob.glob('*.wav')

        num_songfiles = len(songfiles_list)
        all_labels = []
        song_IDs = []
        song_ID_counter = 0
        for file_num, songfile in enumerate(songfiles_list):
            print('Processing audio file {} of {}.'.format(file_num + 1, num_songfiles))
            # segment_params defined for todo_list item takes precedence over any default
            # defined for `extract` config
            extract_dict = features.extract.from_file(songfile,
                                                      extract_params['file_format'],
                                                      extract_params['feature_list'],
                                                      extract_params['spect_params'],
                                                      extract_params['labelset'],
                                                      extract_params['segment_params'])

            if extract_dict is None:
                # because no labels from labels_to_use were found in songfile
                continue

            all_labels.extend(extract_dict['labels'])
            song_IDs.extend([song_ID_counter] * len(extract_dict['labels']))
            song_ID_counter += 1

            if 'features_arr' in extract_dict:
                if 'features_from_all_files' in locals():
                    features_from_all_files = np.concatenate((features_from_all_files,
                                                              extract_dict['features_arr']),
                                                             axis=0)
                else:
                    features_from_all_files = extract_dict['features_arr']

            if 'neuralnet_inputs_dict' in extract_dict:
                if 'neuralnet_inputs_all_files' in locals():
                    for key, val in neuralnet_inputs_all_files.items():
                        new_val = np.concatenate((neuralnet_inputs_all_files[key],
                                                  extract_dict['neuralnet_inputs_dict'][key]))
                        neuralnet_inputs_all_files[key] = new_val
                else:
                    neuralnet_inputs_all_files = extract_dict['neuralnet_inputs_dict']

        # get dir name without the rest of path so it doesn't have separators in the name
        # because those can't be in filename
        just_dir_name = os.getcwd().split(os.path.sep)[-1]
        feature_file = os.path.join(extract_params['output_dir'],
                                    'features_from_' + just_dir_name + '_created_' + timestamp())
        feature_file_dict = {
            'labels': all_labels,
            'feature_list': extract_params['feature_list'],
            'spect_params': extract_params['spect_params'],
            'segment_params': extract_params['segment_params'],
            'labelset': extract_params['labelset'],
            'file_format': extract_params['file_format'],
            'bird_ID': extract_params['bird_ID'],
            'song_IDs': song_IDs
        }

        if 'features_from_all_files' in locals():
            feature_file_dict['features'] = features_from_all_files
            feature_file_dict['features_arr_column_IDs'] = extract_dict['feature_inds']

            if 'feature_list_group_ID' in extract_params:
                feature_file_dict['feature_list_group_ID'] = extract_params['feature_list_group_ID']
                feature_file_dict['feature_group_ID_dict'] = extract_params['feature_group_ID_dict']

        if 'neuralnet_inputs_all_files' in locals():
            feature_file_dict['neuralnet_inputs'] = neuralnet_inputs_all_files

        joblib.dump(feature_file_dict,
                    feature_file,
                    compress=3)

    ##########################################################
    # after looping through all data_dirs for this todo_item #
    ##########################################################
    print('making summary file')
    os.chdir(extract_params['output_dir'])
    summary_filename = 'summary_feature_file_created_' + timestamp()
    summary_filename_with_path = os.path.join(extract_params['output_dir'],
                                              summary_filename)
    ftr_output_files = glob.glob('*features_from_*')
    if len(ftr_output_files) > 1:
        # make a 'summary' data file
        list_of_output_dicts = []
        summary_ftr_file_dict = {}
        for feature_file in ftr_output_files:
            feature_file_dict = joblib.load(feature_file)

            if 'labels' not in summary_ftr_file_dict:
                summary_ftr_file_dict['labels'] = feature_file_dict['labels']
            else:
                summary_ftr_file_dict['labels'] = summary_ftr_file_dict['labels'] + feature_file_dict['labels']

            if 'spect_params' not in summary_ftr_file_dict:
                summary_ftr_file_dict['spect_params'] = feature_file_dict['spect_params']
            else:
                if feature_file_dict['spect_params'] != summary_ftr_file_dict['spect_params']:
                    raise ValueError('mismatch between spect_params in {} '
                                     'and other feature files'.format(feature_file))

            if 'segment_params' not in summary_ftr_file_dict:
                summary_ftr_file_dict['segment_params'] = feature_file_dict['segment_params']
            else:
                if feature_file_dict['segment_params'] != summary_ftr_file_dict['segment_params']:
                    raise ValueError('mismatch between segment_params in {} '
                                     'and other feature files'.format(feature_file))

            if 'labelset' not in summary_ftr_file_dict:
                summary_ftr_file_dict['labelset'] = feature_file_dict['labelset']
            else:
                if feature_file_dict['labelset'] != summary_ftr_file_dict['labelset']:
                    raise ValueError('mismatch between labelset in {} '
                                     'and other feature files'.format(feature_file))

            if 'file_format' not in summary_ftr_file_dict:
                summary_ftr_file_dict['file_format'] = feature_file_dict['file_format']
            else:
                if feature_file_dict['file_format'] != summary_ftr_file_dict['file_format']:
                    raise ValueError('mismatch between file_format in {} '
                                     'and other feature files'.format(feature_file))

            if 'bird_ID' not in summary_ftr_file_dict:
                summary_ftr_file_dict['bird_ID'] = feature_file_dict['bird_ID']
            else:
                if feature_file_dict['bird_ID'] != summary_ftr_file_dict['bird_ID']:
                    raise ValueError('mismatch between bird_ID in {} '
                                     'and other feature files'.format(feature_file))

            if 'song_IDs' not in summary_ftr_file_dict:
                summary_ftr_file_dict['song_IDs'] = feature_file_dict['song_IDs']
            else:
                curr_last_ID = summary_ftr_file_dict['song_IDs'][-1]
                tmp_song_IDs = [el + curr_last_ID + 1 for el in feature_file_dict['song_IDs']]
                summary_ftr_file_dict['song_IDs'].extend(tmp_song_IDs)

            if 'feature_list' not in summary_ftr_file_dict:
                summary_ftr_file_dict['feature_list'] = feature_file_dict['feature_list']
            else:
                if feature_file_dict['feature_list'] != summary_ftr_file_dict['feature_list']:
                    raise ValueError('mismatch between feature_list in {} '
                                     'and other feature files'.format(feature_file))

            # only if not-neuralnet features were used
            if 'features' in feature_file_dict:
                if 'features' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['features'] = feature_file_dict['features']
                else:
                    summary_ftr_file_dict['features'] = np.concatenate((summary_ftr_file_dict['features'],
                                                                        feature_file_dict['features']))

                if 'features_arr_column_IDs' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['features_arr_column_IDs'] = feature_file_dict['features_arr_column_IDs']
                else:
                    if any(feature_file_dict['features_arr_column_IDs'] !=
                                   summary_ftr_file_dict['features_arr_column_IDs']):
                        raise ValueError('mismatch between features_arr_column_IDs in {} '
                                         'and other feature files'.format(feature_file))

                if 'feature_list_group_ID' in extract_params:
                    if 'feature_list_group_ID' not in summary_ftr_file_dict:
                        summary_ftr_file_dict['feature_list_group_ID'] = feature_file_dict['feature_list_group_ID']
                    else:
                        if feature_file_dict['feature_list_group_ID'] != \
                                summary_ftr_file_dict['feature_list_group_ID']:
                            raise ValueError('mismatch between feature_list_group_ID in {} '
                                             'and other feature files'.format(feature_file))

                    if 'feature_group_ID_dict' not in summary_ftr_file_dict:
                        summary_ftr_file_dict['feature_group_ID_dict'] = \
                            feature_file_dict['feature_group_ID_dict']
                    else:
                        if feature_file_dict['feature_group_ID_dict'] != \
                                summary_ftr_file_dict['feature_group_ID_dict']:
                            raise ValueError('mismatch between feature_group_ID_dict in {} '
                                             'and other feature files'.format(feature_file))

            # if extracting inputs for neuralnets
            if 'neuralnet_inputs' in feature_file_dict:
                if 'neuralnet_inputs' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['neuralnet_inputs'] = \
                        feature_file_dict['neuralnet_inputs']
                else:
                    for key, val in summary_ftr_file_dict['neuralnet_inputs'].items():
                        newval = np.concatenate((summary_ftr_file_dict['neuralnet_inputs'][key],
                                                 feature_file_dict['neuralnet_inputs'][key]))
                        summary_ftr_file_dict['neuralnet_inputs'][key] = newval

        joblib.dump(summary_ftr_file_dict,
                    summary_filename)

    else:  # if only one feature_file
        os.rename(ftr_output_files[0],
                  summary_filename)
        summary_ftr_file_dict = joblib.load(summary_filename)

    if 'feature_list_group_ID' in summary_ftr_file_dict:
        write_select_config(summary_ftr_file_dict,
                            summary_filename,
                            extract_params['output_dir'])

def extract(config_file):
    """main function that runs feature extraction.
    Does not return anything, just runs through directories specified in config_file
    and extracts features.

    Parameters
    ----------
    config_file : string
        filename of YAML file that configures feature extraction
    """

    extract_config = parse_config(config_file, 'extract')
    print('Parsed extract config.')

    home_dir = os.getcwd()

    todo_list = extract_config['todo_list']
    for ind, todo in enumerate(todo_list):

        print('Completing item {} of {} in to-do list'.format(ind+1, len(todo_list)))
        file_format = todo['file_format']
        if file_format == 'evtaf':
            if 'evfuncs' not in sys.modules:
                from . import evfuncs
        elif file_format == 'koumura':
            if 'koumura' not in sys.modules:
                from . import koumura

        output_dir = 'extract_output_' + timestamp()
        output_dir_with_path = os.path.join(todo['output_dir'], output_dir)
        if not os.path.isdir(output_dir_with_path):
            os.mkdir(output_dir_with_path)

        extract_params = {
            'bird_ID': todo['bird_ID'],
            'feature_list': todo['feature_list'],
            'output_dir': output_dir_with_path,
            'home_dir': home_dir,
            'data_dirs': todo['data_dirs'],
            'labelset': todo['labelset'],
            'file_format': todo['file_format']
        }

        # segment_params defined for todo_list item takes precedence over any default
        # defined for `extract` config
        if 'segment_params' in todo:
            extract_params['segment_params'] = todo['segment_params']
        else:
            extract_params['segment_params'] = extract_config['segment_params']

        if 'spect_params' in todo:
            extract_params['spect_params'] = todo['spect_params']
        else:
            extract_params['spect_params'] = extract_config['spect_params']

        if 'feature_list_group_ID' in todo:
            extract_params['feature_list_group_ID'] = todo['feature_list_group_ID']
            extract_params['feature_group_ID_dict'] = todo['feature_group_ID_dict']

        _extract(extract_params)

        os.chdir(home_dir)
<commit_msg>add 'make_sumamry_file' argument to _extract'<commit_after>"""
feature extraction
"""

# from standard library
import sys
import os
import glob

# from dependencies
import numpy as np
from sklearn.externals import joblib

# from hvc
from .parseconfig import parse_config
from . import features
from .utils import timestamp

SELECT_TEMPLATE = """select:
  num_replicates: None
  num_train_samples:
    start : None
    stop : None
    step : None
  num_test_samples: None

    models:"""

MODELS_TEMPLATE = """
    -
      model: {0}
      feature_list_indices: {1}
      hyperparameters:
        {2}
"""

SVM_HYPERPARAMS = """C : None
        gamma : None
"""

KNN_HYPERPARAMS = """k : None
"""

TODO_TEMPLATE = """  todo_list:
    -
      feature_file : {0}
      output_dir: {1}"""

def write_select_config(summary_ftr_file_dict,
                       summary_filename,
                       output_dir):
    """writes summary output dict from extract to a config file for select
    Only called when feature_group is a list, to save user from figuring
    out manually which features belong to which group.

    Parameters
    ----------
    summary_ftr_file_dict : dictionary
        as defined in featureextract.extract
    summary_filename : string
        name of summary feature file
    output_dir : string
        name of output directory -- assumes it will be the same as it was for extract.yml

    Returns
    -------
    None

    Doesn't return anything, just saves .yml file
    """

    select_config_filename = 'select.config.from_' + summary_filename + '.yml'
    with open(select_config_filename, 'w') as yml_outfile:
        yml_outfile.write(SELECT_TEMPLATE)
        for model_name, model_ID in summary_ftr_file_dict['feature_group_ID_dict'].items():
            inds = [ftr_list_ind for ftr_list_ind, grp_ID in
                    enumerate(summary_ftr_file_dict['feature_list_group_ID'])
                    if grp_ID == model_ID]
            if model_name == 'svm':
                hyperparams = SVM_HYPERPARAMS
            elif model_name == 'knn':
                hyperparams = KNN_HYPERPARAMS
            yml_outfile.write(MODELS_TEMPLATE.format(model_name,
                                                     inds,
                                                     hyperparams))
        yml_outfile.write(TODO_TEMPLATE.format(summary_filename,
                                               output_dir))

def _extract(extract_params, make_summary_file=True):
    """helper function that loops through data dirs,
    extracts features, saves feature files, and then
    makes summary feature file.
    Put into a separate helper function so that it can
    also be used by labelpredict.predict

    Parameters
    ----------
    extract_params : dict with following key, value pairs
        data_dirs : list
            list of directories with data files
        home_dir : str
        output_dir :
        file_format : str
            {'evtaf', 'koumura'}
        feature_list : list
            list of features generated by hvc.parse.extract
        spect_params : dict
            as defined for hvc.audiofileIO.Spectrogram class
        labelset : str
            labels for syllables as a string, e.g., 'iabcdefg'
        segment_params : dict
            as defined for hvc.audiofileIO.Song and hvc.audiofileIO.segment_song

    Returns
    -------
    None
    Saves output files in output dir

    """

    for data_dir in extract_params['data_dirs']:
        print('Changing to data directory: {}'.format(data_dir))
        os.chdir(extract_params['home_dir'])
        os.chdir(data_dir)

        if 'features_from_all_files' in locals():
            # from last time through loop
            # (need to re-initialize for each directory)
            del features_from_all_files

        if 'neuralnet_inputs_dict' in locals():
            del neuralnet_inputs_dict

        if extract_params['file_format'] == 'evtaf':
            songfiles_list = glob.glob('*.cbin')
        elif extract_params['file_format'] == 'koumura':
            songfiles_list = glob.glob('*.wav')

        num_songfiles = len(songfiles_list)
        all_labels = []
        song_IDs = []
        song_ID_counter = 0
        for file_num, songfile in enumerate(songfiles_list):
            print('Processing audio file {} of {}.'.format(file_num + 1, num_songfiles))
            # segment_params defined for todo_list item takes precedence over any default
            # defined for `extract` config
            extract_dict = features.extract.from_file(songfile,
                                                      extract_params['file_format'],
                                                      extract_params['feature_list'],
                                                      extract_params['spect_params'],
                                                      extract_params['labelset'],
                                                      extract_params['segment_params'])

            if extract_dict is None:
                # because no labels from labels_to_use were found in songfile
                continue

            all_labels.extend(extract_dict['labels'])
            song_IDs.extend([song_ID_counter] * len(extract_dict['labels']))
            song_ID_counter += 1

            if 'features_arr' in extract_dict:
                if 'features_from_all_files' in locals():
                    features_from_all_files = np.concatenate((features_from_all_files,
                                                              extract_dict['features_arr']),
                                                             axis=0)
                else:
                    features_from_all_files = extract_dict['features_arr']

            if 'neuralnet_inputs_dict' in extract_dict:
                if 'neuralnet_inputs_all_files' in locals():
                    for key, val in neuralnet_inputs_all_files.items():
                        new_val = np.concatenate((neuralnet_inputs_all_files[key],
                                                  extract_dict['neuralnet_inputs_dict'][key]))
                        neuralnet_inputs_all_files[key] = new_val
                else:
                    neuralnet_inputs_all_files = extract_dict['neuralnet_inputs_dict']

        # get dir name without the rest of path so it doesn't have separators in the name
        # because those can't be in filename
        just_dir_name = os.getcwd().split(os.path.sep)[-1]
        feature_file = os.path.join(extract_params['output_dir'],
                                    'features_from_' + just_dir_name + '_created_' + timestamp())
        feature_file_dict = {
            'labels': all_labels,
            'feature_list': extract_params['feature_list'],
            'spect_params': extract_params['spect_params'],
            'segment_params': extract_params['segment_params'],
            'labelset': extract_params['labelset'],
            'file_format': extract_params['file_format'],
            'bird_ID': extract_params['bird_ID'],
            'song_IDs': song_IDs
        }

        if 'features_from_all_files' in locals():
            feature_file_dict['features'] = features_from_all_files
            feature_file_dict['features_arr_column_IDs'] = extract_dict['feature_inds']

            if 'feature_list_group_ID' in extract_params:
                feature_file_dict['feature_list_group_ID'] = extract_params['feature_list_group_ID']
                feature_file_dict['feature_group_ID_dict'] = extract_params['feature_group_ID_dict']

        if 'neuralnet_inputs_all_files' in locals():
            feature_file_dict['neuralnet_inputs'] = neuralnet_inputs_all_files

        joblib.dump(feature_file_dict,
                    feature_file,
                    compress=3)

    ##########################################################
    # after looping through all data_dirs for this todo_item #
    ##########################################################
    if make_summary_file:
        print('making summary file')
        os.chdir(extract_params['output_dir'])
        summary_filename = 'summary_feature_file_created_' + timestamp()
        summary_filename_with_path = os.path.join(extract_params['output_dir'],
                                                  summary_filename)
        ftr_output_files = glob.glob('*features_from_*')
        if len(ftr_output_files) > 1:
            # make a 'summary' data file
            list_of_output_dicts = []
            summary_ftr_file_dict = {}
            for feature_file in ftr_output_files:
                feature_file_dict = joblib.load(feature_file)

                if 'labels' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['labels'] = feature_file_dict['labels']
                else:
                    summary_ftr_file_dict['labels'] = \
                        summary_ftr_file_dict['labels'] + feature_file_dict['labels']

                if 'spect_params' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['spect_params'] = feature_file_dict['spect_params']
                else:
                    if feature_file_dict['spect_params'] != summary_ftr_file_dict['spect_params']:
                        raise ValueError('mismatch between spect_params in {} '
                                         'and other feature files'.format(feature_file))

                if 'segment_params' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['segment_params'] = feature_file_dict['segment_params']
                else:
                    if feature_file_dict['segment_params'] != summary_ftr_file_dict['segment_params']:
                        raise ValueError('mismatch between segment_params in {} '
                                         'and other feature files'.format(feature_file))

                if 'labelset' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['labelset'] = feature_file_dict['labelset']
                else:
                    if feature_file_dict['labelset'] != summary_ftr_file_dict['labelset']:
                        raise ValueError('mismatch between labelset in {} '
                                         'and other feature files'.format(feature_file))

                if 'file_format' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['file_format'] = feature_file_dict['file_format']
                else:
                    if feature_file_dict['file_format'] != summary_ftr_file_dict['file_format']:
                        raise ValueError('mismatch between file_format in {} '
                                         'and other feature files'.format(feature_file))

                if 'bird_ID' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['bird_ID'] = feature_file_dict['bird_ID']
                else:
                    if feature_file_dict['bird_ID'] != summary_ftr_file_dict['bird_ID']:
                        raise ValueError('mismatch between bird_ID in {} '
                                         'and other feature files'.format(feature_file))

                if 'song_IDs' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['song_IDs'] = feature_file_dict['song_IDs']
                else:
                    curr_last_ID = summary_ftr_file_dict['song_IDs'][-1]
                    tmp_song_IDs = [el + curr_last_ID + 1 for el in feature_file_dict['song_IDs']]
                    summary_ftr_file_dict['song_IDs'].extend(tmp_song_IDs)

                if 'feature_list' not in summary_ftr_file_dict:
                    summary_ftr_file_dict['feature_list'] = feature_file_dict['feature_list']
                else:
                    if feature_file_dict['feature_list'] != summary_ftr_file_dict['feature_list']:
                        raise ValueError('mismatch between feature_list in {} '
                                         'and other feature files'.format(feature_file))

                # only if not-neuralnet features were used
                if 'features' in feature_file_dict:
                    if 'features' not in summary_ftr_file_dict:
                        summary_ftr_file_dict['features'] = feature_file_dict['features']
                    else:
                        summary_ftr_file_dict['features'] = np.concatenate((summary_ftr_file_dict['features'],
                                                                            feature_file_dict['features']))

                    if 'features_arr_column_IDs' not in summary_ftr_file_dict:
                        summary_ftr_file_dict['features_arr_column_IDs'] = feature_file_dict['features_arr_column_IDs']
                    else:
                        if any(feature_file_dict['features_arr_column_IDs'] !=
                                       summary_ftr_file_dict['features_arr_column_IDs']):
                            raise ValueError('mismatch between features_arr_column_IDs in {} '
                                             'and other feature files'.format(feature_file))

                    if 'feature_list_group_ID' in extract_params:
                        if 'feature_list_group_ID' not in summary_ftr_file_dict:
                            summary_ftr_file_dict['feature_list_group_ID'] = feature_file_dict['feature_list_group_ID']
                        else:
                            if feature_file_dict['feature_list_group_ID'] != \
                                    summary_ftr_file_dict['feature_list_group_ID']:
                                raise ValueError('mismatch between feature_list_group_ID in {} '
                                                 'and other feature files'.format(feature_file))

                        if 'feature_group_ID_dict' not in summary_ftr_file_dict:
                            summary_ftr_file_dict['feature_group_ID_dict'] = \
                                feature_file_dict['feature_group_ID_dict']
                        else:
                            if feature_file_dict['feature_group_ID_dict'] != \
                                    summary_ftr_file_dict['feature_group_ID_dict']:
                                raise ValueError('mismatch between feature_group_ID_dict in {} '
                                                 'and other feature files'.format(feature_file))

                # if extracting inputs for neuralnets
                if 'neuralnet_inputs' in feature_file_dict:
                    if 'neuralnet_inputs' not in summary_ftr_file_dict:
                        summary_ftr_file_dict['neuralnet_inputs'] = \
                            feature_file_dict['neuralnet_inputs']
                    else:
                        for key, val in summary_ftr_file_dict['neuralnet_inputs'].items():
                            newval = np.concatenate((summary_ftr_file_dict['neuralnet_inputs'][key],
                                                     feature_file_dict['neuralnet_inputs'][key]))
                            summary_ftr_file_dict['neuralnet_inputs'][key] = newval

            joblib.dump(summary_ftr_file_dict,
                        summary_filename)

        else:  # if only one feature_file
            os.rename(ftr_output_files[0],
                      summary_filename)
            summary_ftr_file_dict = joblib.load(summary_filename)

        if 'feature_list_group_ID' in summary_ftr_file_dict:
            write_select_config(summary_ftr_file_dict,
                                summary_filename,
                                extract_params['output_dir'])

def extract(config_file):
    """main function that runs feature extraction.
    Does not return anything, just runs through directories specified in config_file
    and extracts features.

    Parameters
    ----------
    config_file : string
        filename of YAML file that configures feature extraction
    """

    extract_config = parse_config(config_file, 'extract')
    print('Parsed extract config.')

    home_dir = os.getcwd()

    todo_list = extract_config['todo_list']
    for ind, todo in enumerate(todo_list):

        print('Completing item {} of {} in to-do list'.format(ind+1, len(todo_list)))
        file_format = todo['file_format']
        if file_format == 'evtaf':
            if 'evfuncs' not in sys.modules:
                from . import evfuncs
        elif file_format == 'koumura':
            if 'koumura' not in sys.modules:
                from . import koumura

        output_dir = 'extract_output_' + timestamp()
        output_dir_with_path = os.path.join(todo['output_dir'], output_dir)
        if not os.path.isdir(output_dir_with_path):
            os.mkdir(output_dir_with_path)

        extract_params = {
            'bird_ID': todo['bird_ID'],
            'feature_list': todo['feature_list'],
            'output_dir': output_dir_with_path,
            'home_dir': home_dir,
            'data_dirs': todo['data_dirs'],
            'labelset': todo['labelset'],
            'file_format': todo['file_format']
        }

        # segment_params defined for todo_list item takes precedence over any default
        # defined for `extract` config
        if 'segment_params' in todo:
            extract_params['segment_params'] = todo['segment_params']
        else:
            extract_params['segment_params'] = extract_config['segment_params']

        if 'spect_params' in todo:
            extract_params['spect_params'] = todo['spect_params']
        else:
            extract_params['spect_params'] = extract_config['spect_params']

        if 'feature_list_group_ID' in todo:
            extract_params['feature_list_group_ID'] = todo['feature_list_group_ID']
            extract_params['feature_group_ID_dict'] = todo['feature_group_ID_dict']

        _extract(extract_params)

        os.chdir(home_dir)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import datetime

#hideous code to create the html report file with the prefix in the format of year month day hour minute
report = open(str(datetime.datetime.today()).replace(':','').replace('.','').replace(' ','').replace('-','')[0:14]+'report.html','w')

#this method expects a two-dimensional list containing the data as lst, and a one-dimensional list containing titles as titlelst
def generate(lst,titlelst):
    report.write('
')
    report.write('
\n')
    #writes titles to first row
    for title in titlelst:
        report.write(''+title+'	\n')
    report.write('
\n')
    #writes data
    for i in range(0,len(lst)):
        report.write('
\n')
        for j in range(0,len(lst[i])):
            report.write(''+str(j)+'	\n')
        report.write('
\n')
    report.write('
')

generate([[0,1],[2,3]],['cats','dogs'])

<commit_msg>Fixed bug<commit_after>#!/usr/bin/env python
import datetime

#hideous code to create the html report file with the prefix in the format of year month day hour minute
report = open(str(datetime.datetime.today()).replace(':','').replace('.','').replace(' ','').replace('-','')[0:14]+'report.html','w')

#this method expects a two-dimensional list containing the data as lst, and a one-dimensional list containing titles as titlelst
def generate(lst,titlelst):
    #make the document pretty
    #report.write("""
    #
    #             """)
    report.write('
')
    report.write('
\n')
    #writes titles to first row
    for title in titlelst:
        report.write(''+title+'	\n')
    report.write('
\n')
    #writes data
    for i in range(0,len(lst)):
        report.write('
\n')
        for j in range(0,len(lst[i])):
            report.write(''+str(lst[i][j])+'	\n')

        report.write('
\n')
    report.write('
')

generate([[0,1],[2,3]],['cats','dogs'])

<|endoftext|>