<commit_before>from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import (
    compat_chr,
    compat_ord,
    compat_urllib_parse_unquote,
)
from ..utils import (
    int_or_none,
    parse_iso8601,
    urljoin,
)

class BeegIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
    _TEST = {
        'url': 'http://beeg.com/5416503',
        'md5': 'a1a1b1a8bc70a89e49ccfd113aed0820',
        'info_dict': {
            'id': '5416503',
            'ext': 'mp4',
            'title': 'Sultry Striptease',
            'description': 'md5:d22219c09da287c14bed3d6c37ce4bc2',
            'timestamp': 1391813355,
            'upload_date': '20140207',
            'duration': 383,
            'tags': list,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        cpl_url = self._search_regex(
            r'<script[^>]+src=(["\'])(?P<url>(?:/static|(?:https?:)?//static\.beeg\.com)/cpl/\d+\.js.*?)\1',
            webpage, 'cpl', default=None, group='url')

        cpl_url = urljoin(url, cpl_url)

        beeg_version, beeg_salt = [None] * 2

        if cpl_url:
            cpl = self._download_webpage(
                self._proto_relative_url(cpl_url), video_id,
                'Downloading cpl JS', fatal=False)
            if cpl:
                beeg_version = int_or_none(self._search_regex(
                    r'beeg_version\s*=\s*([^\b]+)', cpl,
                    'beeg version', default=None)) or self._search_regex(
                    r'/(\d+)\.js', cpl_url, 'beeg version', default=None)
                beeg_salt = self._search_regex(
                    r'beeg_salt\s*=\s*(["\'])(?P<beeg_salt>.+?)\1', cpl, 'beeg salt',
                    default=None, group='beeg_salt')

        beeg_version = beeg_version or '2185'
        beeg_salt = beeg_salt or 'pmweAkq8lAYKdfWcFCUj0yoVgoPlinamH5UE1CB3H'

        for api_path in ('', 'api.'):
            video = self._download_json(
                'https://%sbeeg.com/api/v6/%s/video/%s'
                % (api_path, beeg_version, video_id), video_id,
                fatal=api_path == 'api.')
            if video:
                break

        def split(o, e):
            def cut(s, x):
                n.append(s[:x])
                return s[x:]
            n = []
            r = len(o) % e
            if r > 0:
                o = cut(o, r)
            while len(o) > e:
                o = cut(o, e)
            n.append(o)
            return n

        def decrypt_key(key):
            # Reverse engineered from http://static.beeg.com/cpl/1738.js
            a = beeg_salt
            e = compat_urllib_parse_unquote(key)
            o = ''.join([
                compat_chr(compat_ord(e[n]) - compat_ord(a[n % len(a)]) % 21)
                for n in range(len(e))])
            return ''.join(split(o, 3)[::-1])

        def decrypt_url(encrypted_url):
            encrypted_url = self._proto_relative_url(
                encrypted_url.replace('{DATA_MARKERS}', ''), 'https:')
            key = self._search_regex(
                r'/key=(.*?)%2Cend=', encrypted_url, 'key', default=None)
            if not key:
                return encrypted_url
            return encrypted_url.replace(key, decrypt_key(key))

        formats = []
        for format_id, video_url in video.items():
            if not video_url:
                continue
            height = self._search_regex(
                r'^(\d+)[pP]$', format_id, 'height', default=None)
            if not height:
                continue
            formats.append({
                'url': decrypt_url(video_url),
                'format_id': format_id,
                'height': int(height),
            })
        self._sort_formats(formats)

        title = video['title']
        video_id = video.get('id') or video_id
        display_id = video.get('code')
        description = video.get('desc')

        timestamp = parse_iso8601(video.get('date'), ' ')
        duration = int_or_none(video.get('duration'))

        tags = [tag.strip() for tag in video['tags'].split(',')] if video.get('tags') else None

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'timestamp': timestamp,
            'duration': duration,
            'tags': tags,
            'formats': formats,
            'age_limit': self._rta_search(webpage),
        }
<commit_msg>[beeg] Fix extraction (closes #18610, closes #18626)<commit_after>from __future__ import unicode_literals

from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
    int_or_none,
    unified_timestamp,
)

class BeegIE(InfoExtractor):
    _VALID_URL = r'https?://(?:www\.)?beeg\.com/(?P<id>\d+)'
    _TEST = {
        'url': 'http://beeg.com/5416503',
        'md5': 'a1a1b1a8bc70a89e49ccfd113aed0820',
        'info_dict': {
            'id': '5416503',
            'ext': 'mp4',
            'title': 'Sultry Striptease',
            'description': 'md5:d22219c09da287c14bed3d6c37ce4bc2',
            'timestamp': 1391813355,
            'upload_date': '20140207',
            'duration': 383,
            'tags': list,
            'age_limit': 18,
        }
    }

    def _real_extract(self, url):
        video_id = self._match_id(url)

        webpage = self._download_webpage(url, video_id)

        beeg_version = self._search_regex(
            r'beeg_version\s*=\s*([\da-zA-Z_-]+)', webpage, 'beeg version',
            default='1546225636701')

        for api_path in ('', 'api.'):
            video = self._download_json(
                'https://%sbeeg.com/api/v6/%s/video/%s'
                % (api_path, beeg_version, video_id), video_id,
                fatal=api_path == 'api.')
            if video:
                break

        formats = []
        for format_id, video_url in video.items():
            if not video_url:
                continue
            height = self._search_regex(
                r'^(\d+)[pP]$', format_id, 'height', default=None)
            if not height:
                continue
            formats.append({
                'url': self._proto_relative_url(
                    video_url.replace('{DATA_MARKERS}', 'data=pc_XX__%s_0' % beeg_version), 'https:'),
                'format_id': format_id,
                'height': int(height),
            })
        self._sort_formats(formats)

        title = video['title']
        video_id = compat_str(video.get('id') or video_id)
        display_id = video.get('code')
        description = video.get('desc')
        series = video.get('ps_name')

        timestamp = unified_timestamp(video.get('date'))
        duration = int_or_none(video.get('duration'))

        tags = [tag.strip() for tag in video['tags'].split(',')] if video.get('tags') else None

        return {
            'id': video_id,
            'display_id': display_id,
            'title': title,
            'description': description,
            'series': series,
            'timestamp': timestamp,
            'duration': duration,
            'tags': tags,
            'formats': formats,
            'age_limit': self._rta_search(webpage),
        }
<|endoftext|>


<commit_before>#!/usr/bin/python

##
## Copyright 2009 Adriana Lukas & Alec Muffett
##
## Licensed under the Apache License, Version 2.0 (the "License"); you
## may not use this file except in compliance with the License. You
## may obtain a copy of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
## implied. See the License for the specific language governing
## permissions and limitations under the License.
##

import os
from UserDict import UserDict

class Mine:
    """the master container-object for a per-username Mine"""

    def __init__(self, username):
	"""initialise a mine for user 'username'"""

        # who am i?
	self.username = username

        # where shall we find the Things?
	self.path = os.path.join("database", username)

        # hack for Config to become a Thing without an aggregator
        self.mine = self

        # primary importance: configuration
        self.config = Config(self)

        # secondary importance: cache (so the rest can reference it)
	self.cache = Cache(self)

        # the remainder; items come last, and have comments beneath them
	self.tags = Tags(self)
	self.relations = Relations(self)
	self.items = Items(self)

    def Import(self, file):
	"""returns nothing, imports into a new mine the zipfile referenced by 'file'"""
	pass

    def Export(self, file, srchctx):
	"""returns a filehandle on a zipfile representing the mine with items represented by srchctx, or all if None"""
	pass

##################################################################

class Cache:
    def __init__(self, mine):
	"""setup"""
        self.mine = mine

    def get(self, wotsit):
        pass

    def put(self, wotsit):
        pass

##################################################################

class Things:

    """
    aggregate manager for Thing objects, and parent for classes of
    similar functionality below.
    """

    subpath = "things" # probably does not exist

    def __init__(self, mine):
	"""setup"""

	self.mine = mine
        self.path = os.path.join(mine.path, self.subpath)

    def New(self):
	"""
        Returns a new Thing, properly initialised under this Things
        aggregator, rather than using a direct constructor.

        This Thing /may/ not be visible to List() (etc) until Commit()
        is performed
        """
	pass

    def ListIds(self):
	"""returns a list of all Thing.id (ie: list of int) known to this Thing aggregator"""
	pass

    def List(self):
	"""returns a list of all Thing (ie: list of Thing objects) known to this Thing aggregator"""
	pass

    def Exists(self, id):
	"""returns a boolean, does the Thing numbered 'id' exist?"""
	pass

    def Named(self, name):
	"""returns a list of Thing, of name 'name'; returns empty list if no match"""
	pass

    def NamedSingle(self, name):
	"""returns a single Thing of name 'name'; raises exception if multiple identical names are permitted or found"""
	pass

    def Select(self, srchw):
	"""returns a list of Thing matching SearchWotsit 'srchw'"""
	pass

    def Freeze(self):
	"""locks this Things object against addition/deletion"""
	pass

    def Thaw(self):
	"""unlocks this Things object, see 'Freeze'"""
	pass

##################################################################

class Thing(UserDict):

    """base class for all Mine database objects, and parent for classes of similar functionality below."""

    keyPrefix = 'thing'
    keyRegexp = '^thing[A-Z]'
    keyNamesUnique = True

    # keySettings is critical - syntax:
    # keysuffix : ( isReadOnly, isRequired, isLine, isVirtual, enumeration )

    keySettings = {
	'Id' : ( True, True, True, True, None ),
	'Name' : ( False, True, True, False, None ),
	}

    keyId = keyPrefix + "Id"
    keyName = keyPrefix + "Name"
    dictValidKeys = {}
    dictReadOnly = {}
    dictRequired = {}
    dictLine = {}
    dictVirtual = {}
    dictEnumeration = {}

    for suffix, ( isReadOnly, isRequired, isLine, isVirtual, enumtuple ) in keySettings.items():
        key = keyPrefix + suffix
        dictValidKeys[key] = True
        if (isReadOnly): dictReadOnly[key] = True
        if (isRequired): dictRequired[key] = True
        if (isLine): dictLine[key] = True
        if (isVirtual): dictVirtual[key] = True
        if (enumtuple): dictEnumeration = enumtuple

    def __init__(self, parent, id):

	"""set up the thing object-tables"""

        # we are a UserDict
        UserDict.__init__(self)

	# memorise my aggregator/parent
	self.parent = parent

	# memorise my mine
	self.mine = parent.mine

        # and my id
        self.id = id

    # ----

    def MapOutbound(self, key, value):
	"""
        Takes string 'value' and maps/processes it, returning the data
        in on-disk format for key 'key'

        In this, the superclass version which MUST be called, it
        rewrites any 'isLine' keys as a single line of text without
        newline and with single whitespaces.

        Blobs are left verbatim.
        """
        if (key in self.dictLine): value = " ".join(value.split())
	return value

    def Set(self, key, value): # __set_item__
	"""stores value (string) of key 'key, passing it through MapOutbound() before storage'"""
	pass

    # ----

    def MapInbound(self, key, value):
	"""takes string 'value' and maps/processes it, returning the data in userland format for key 'key'"""
	return value

    def Get(self, key): # __get_item__
	"""retreives value (string) of key 'key' and returns it after passing through MapInbound()"""

        if (key in self.dictVirtual):
            if (key == self.keyId): return str(self.id)

    # ----

    def Id(self):
	"""returns this Thing's id (integer > 0)"""
	return self.Get(keyId)

    def Name(self):
	"""returns this Thing's name (string)"""
	return self.Get(keyName)

    def Keys(self):
	"""returns a list of keys (string) for this Thing"""
	pass

    def Commit(self):
	"""
	Ensures the state of this Thing is written to disk; the mode
	of update is implementation dependent (params may be written
	to disk synchronously without Commit) but use of Commit is
	mandatory to ensure updates.

	Raises an exception on failure.

	Note: implementations should strive to implement atomic Set()
	"""
	pass

    def Delete(self):
	"""deletes this Thing or raises an exception"""
	pass

    def Has(self, key):
	"""returns boolean, does this Thing have a key named 'key'"""
	return True

    def Compare(self, thing): # __cmp__
	"""compares one Thing with another, implements __cmp__ semantics"""
	pass

    def Printable(self): # -> __repr__
	"""supplies a Printable representation of a Thing, implements __repr_ semantics"""
	pass

    def Meta(self):
	"""returns a ThingMeta object for this Thing, providing useful metainformation"""
	pass

    def Lock(self):
	"""locks this Thing against amendment"""
	pass

    def Unlock(self):
	"""unlocks this Thing (see Lock)"""
	pass

##################################################################
##################################################################
##################################################################

class Tags(Things):

    """..."""

    subpath = "tags"

    def __init__(self, mine):
	"""..."""
	Things.__init__(self, mine)
	pass

##################################################################

class Tag(Thing):

    """..."""

    def __init__(self, parent):
	"""..."""
	Thing.__init__(self, parent)
	pass

##################################################################
##################################################################
##################################################################

class Relations(Things):

    """..."""

    subpath = "relations"

    def __init__(self, mine):
	"""..."""
	Things.__init__(self, mine)
	pass

##################################################################

class Relation(Thing):

    """..."""

    def __init__(self, parent):
	"""..."""
	Thing.__init__(self, parent)
	pass

##################################################################
##################################################################
##################################################################

class Items(Things):

    """..."""

    subpath = "items"

    def __init__(self, mine):
	"""..."""
	Things.__init__(self, mine)
	pass

##################################################################

class Item(Thing):

    """..."""

    def __init__(self, parent):
	"""..."""
	Thing.__init__(self, parent)
	pass

##################################################################
##################################################################
##################################################################

class Comments(Things):

    """..."""

    subpath = ".../comments" # fix this

    def __init__(self, mine):
	"""..."""
	Things.__init__(self, mine)
	pass

##################################################################

class Comment(Thing):

    """..."""

    def __init__(self, parent):
	"""..."""
	Thing.__init__(self, parent)
	pass

##################################################################
##################################################################
##################################################################

class Config(Thing):

    """..."""

    def __init__(self, mine):
	"""..."""
	Thing.__init__(self, mine, 0)
	pass

##################################################################
##################################################################
##################################################################

class MineKey():

    """..."""

    def __init__(self):
	"""..."""
	pass

##################################################################

class ThingMeta:

    """Container object that provides platform meta-information about a Thing."""

    lastmodified = None
    created = None
    size = None
    path = None
    name = None
    type = None

    def __init__(self):
	"""..."""
	pass

##################################################################

class RequestWotsit:
    """..."""
    def __init__(self):
	"""..."""
	pass

##################################################################

class SearchWotsit:
    """..."""
    def __init__(self):
	"""..."""
	pass

##################################################################

class CryptoEngine:
    """..."""
    def __init__(self):
	"""..."""
	pass

##################################################################

if __name__ == '__main__':
    foo = Mine("alecm")

<commit_msg>more experimentation<commit_after>#!/usr/bin/python

##
## Copyright 2009 Adriana Lukas & Alec Muffett
##
## Licensed under the Apache License, Version 2.0 (the "License"); you
## may not use this file except in compliance with the License. You
## may obtain a copy of the License at
##
## http://www.apache.org/licenses/LICENSE-2.0
##
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
## implied. See the License for the specific language governing
## permissions and limitations under the License.
##

import os
from UserDict import UserDict

class Mine:
    """the master container-object for a per-username Mine"""

    def __init__(self, username):
	"""initialise a mine for user 'username'"""

        # who am i?
	self.username = username

        # where shall we find the Things?
	self.path = os.path.join("database", username)

        # hack for Config to become a Thing without an aggregator
        self.mine = self

        # primary importance: configuration
        self.config = Config(self)

        # secondary importance: cache (so the rest can reference it)
	self.cache = Cache(self)

        # the remainder; items come last, and have comments beneath them
	self.tags = Tags(self)
	self.relations = Relations(self)
	self.items = Items(self)

    def Import(self, file):
	"""returns nothing, imports into a new mine the zipfile referenced by 'file'"""
	pass

    def Export(self, file, srchctx):
	"""returns a filehandle on a zipfile representing the mine with items represented by srchctx, or all if None"""
	pass

##################################################################

class Cache:
    def __init__(self, mine):
	"""setup"""
        self.mine = mine

    def get(self, wotsit):
        pass

    def put(self, wotsit):
        pass

##################################################################

class Things:

    """
    aggregate manager for Thing objects, and parent for classes of
    similar functionality below.
    """

    subpath = "things" # probably does not exist

    def __init__(self, mine):
	"""setup"""

	self.mine = mine
        self.path = os.path.join(mine.path, self.subpath)

    def New(self):
	"""
        Returns a new Thing, properly initialised under this Things
        aggregator, rather than using a direct constructor.

        This Thing /may/ not be visible to List() (etc) until Commit()
        is performed
        """
	pass

    def ListIds(self):
	"""returns a list of all Thing.id (ie: list of int) known to this Thing aggregator"""
	pass

    def List(self):
	"""returns a list of all Thing (ie: list of Thing objects) known to this Thing aggregator"""
	pass

    def Exists(self, id):
	"""returns a boolean, does the Thing numbered 'id' exist?"""
	pass

    def Named(self, name):
	"""returns a list of Thing, of name 'name'; returns empty list if no match"""
	pass

    def NamedSingle(self, name):
	"""returns a single Thing of name 'name'; raises exception if multiple identical names are permitted or found"""
	pass

    def Select(self, srchw):
	"""returns a list of Thing matching SearchWotsit 'srchw'"""
	pass

    def Freeze(self):
	"""locks this Things object against addition/deletion"""
	pass

    def Thaw(self):
	"""unlocks this Things object, see 'Freeze'"""
	pass

##################################################################

def populateThingCommon(prefix, settings):

    """subclasses cannot inherit class-attribute initialisation code, but
    most of it is common with Things and their subclasses and need
    only be done once at class-initiation time, so this routine
    factors that code out."""

    thingSettings = {}

    keyId = keyPrefix + "Id"
    keyName = keyPrefix + "Name"
    dictValidKeys = {}
    dictReadOnly = {}
    dictRequired = {}
    dictLine = {}
    dictVirtual = {}
    dictEnumeration = {}

    for suffix, ( isReadOnly, isRequired, isLine, isVirtual, enumtuple ) in keySettings.items():
        key = keyPrefix + suffix
        dictValidKeys[key] = True
        if (isReadOnly): dictReadOnly[key] = True
        if (isRequired): dictRequired[key] = True
        if (isLine): dictLine[key] = True
        if (isVirtual): dictVirtual[key] = True
        if (enumtuple): dictEnumeration = enumtuple

    thingSettings['keyId'] = keyId
    thingSettings['keyName'] = keyName
    thingSettings['dictValidKeys'] = dictValidKeys
    thingSettings['dictReadOnly'] = dictReadOnly
    thingSettings['dictRequired'] = dictRequired
    thingSettings['dictLine'] = dictLine
    thingSettings['dictVirtual'] = dictVirtual
    thingSettings['dictEnumeration'] = dictEnumeration

    return thingSettings

class Thing(UserDict):

    """base class for all Mine database objects, and parent for classes of similar functionality below."""

    # what is the name of this Thing / prefix for keys
    keyPrefix = 'thing'

    # regexp which matches keys
    keyRegexp = '^thing[A-Z]'

    # are names of Things meant to be unique?
    keyNamesUnique = True

    # what are the settings per-key
    # keysuffix : ( isReadOnly, isRequired, isLine, isVirtual, enumeration )
    keySettings = {
	'Id' : ( True, True, True, True, None ),
	'Name' : ( False, True, True, False, None ),
	}

    # do some magic once only, at compile time
    thingSettings = populateThingCommon(prefix, settings)
    keyId = thingSettings['keyId']
    keyName = thingSettings['keyName']
    dictValidKeys = thingSettings['dictValidKeys']
    dictReadOnly = thingSettings['dictReadOnly']
    dictRequired = thingSettings['dictRequired']
    dictLine = thingSettings['dictLine']
    dictVirtual = thingSettings['dictVirtual']
    dictEnumeration = thingSettings['dictEnumeration']

    # instance methods from here down

    def __init__(self, parent, id):

	"""set up the thing object-tables"""

        # we are a UserDict
        UserDict.__init__(self)

	# memorise my aggregator/parent
	self.parent = parent

	# memorise my mine
	self.mine = parent.mine

        # and my id
        self.id = id

    # ----

    def MapOutbound(self, key, value):
	"""
        Takes string 'value' and maps/processes it, returning the data
        in on-disk format for key 'key'

        In this, the superclass version which MUST be called, it
        rewrites any 'isLine' keys as a single line of text without
        newline and with single whitespaces.

        Blobs are left verbatim.
        """
        if (key in self.dictLine): value = " ".join(value.split())
	return value

    def Set(self, key, value): # __set_item__
	"""stores value (string) of key 'key, passing it through MapOutbound() before storage'"""
	pass

    # ----

    def MapInbound(self, key, value):
	"""takes string 'value' and maps/processes it, returning the data in userland format for key 'key'"""
	return value

    def Get(self, key): # __get_item__
	"""retreives value (string) of key 'key' and returns it after passing through MapInbound()"""

        if (key in self.dictVirtual):
            if (key == self.keyId): return str(self.id)

    # ----

    def Id(self):
	"""returns this Thing's id (integer > 0)"""
	return self.Get(keyId)

    def Name(self):
	"""returns this Thing's name (string)"""
	return self.Get(keyName)

    def Keys(self):
	"""returns a list of keys (string) for this Thing"""
	pass

    def Commit(self):
	"""
	Ensures the state of this Thing is written to disk; the mode
	of update is implementation dependent (params may be written
	to disk synchronously without Commit) but use of Commit is
	mandatory to ensure updates.

	Raises an exception on failure.

	Note: implementations should strive to implement atomic Set()
	"""
	pass

    def Delete(self):
	"""deletes this Thing or raises an exception"""
	pass

    def Has(self, key):
	"""returns boolean, does this Thing have a key named 'key'"""
	return True

    def Compare(self, thing): # __cmp__
	"""compares one Thing with another, implements __cmp__ semantics"""
	pass

    def Printable(self): # -> __repr__
	"""supplies a Printable representation of a Thing, implements __repr_ semantics"""
	pass

    def Meta(self):
	"""returns a ThingMeta object for this Thing, providing useful metainformation"""
	pass

    def Lock(self):
	"""locks this Thing against amendment"""
	pass

    def Unlock(self):
	"""unlocks this Thing (see Lock)"""
	pass

##################################################################
##################################################################
##################################################################

class Tags(Things):

    """..."""

    subpath = "tags"

    def __init__(self, mine):
	"""..."""
	Things.__init__(self, mine)
	pass

##################################################################

class Tag(Thing):

    """..."""

    def __init__(self, parent):
	"""..."""
	Thing.__init__(self, parent)
	pass

##################################################################
##################################################################
##################################################################

class Relations(Things):

    """..."""

    subpath = "relations"

    def __init__(self, mine):
	"""..."""
	Things.__init__(self, mine)
	pass

##################################################################

class Relation(Thing):

    """..."""

    def __init__(self, parent):
	"""..."""
	Thing.__init__(self, parent)
	pass

##################################################################
##################################################################
##################################################################

class Items(Things):

    """..."""

    subpath = "items"

    def __init__(self, mine):
	"""..."""
	Things.__init__(self, mine)
	pass

##################################################################

class Item(Thing):

    """..."""

    def __init__(self, parent):
	"""..."""
	Thing.__init__(self, parent)
	pass

##################################################################
##################################################################
##################################################################

class Comments(Things):

    """..."""

    subpath = ".../comments" # fix this

    def __init__(self, mine):
	"""..."""
	Things.__init__(self, mine)
	pass

##################################################################

class Comment(Thing):

    """..."""

    def __init__(self, parent):
	"""..."""
	Thing.__init__(self, parent)
	pass

##################################################################
##################################################################
##################################################################

class Config(Thing):

    """..."""

    def __init__(self, mine):
	"""..."""
	Thing.__init__(self, mine, 0)
	pass

##################################################################
##################################################################
##################################################################

class MineKey():

    """..."""

    def __init__(self):
	"""..."""
	pass

##################################################################

class ThingMeta:

    """Container object that provides platform meta-information about a Thing."""

    lastmodified = None
    created = None
    size = None
    path = None
    name = None
    type = None

    def __init__(self):
	"""..."""
	pass

##################################################################

class RequestWotsit:
    """..."""
    def __init__(self):
	"""..."""
	pass

##################################################################

class SearchWotsit:
    """..."""
    def __init__(self):
	"""..."""
	pass

##################################################################

class CryptoEngine:
    """..."""
    def __init__(self):
	"""..."""
	pass

##################################################################

if __name__ == '__main__':
    foo = Mine("alecm")

<|endoftext|>


<commit_before>import calendar
from collections import defaultdict, OrderedDict
from datetime import date, timedelta
import itertools

from django import forms
from django.contrib.admin.views.decorators import staff_member_required
from django.db.models import Max, Min, Q
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.utils.translation import ugettext_lazy

from towel.forms import SearchForm

from zivinetz.models import Assignment, Specification, WaitList

# Calendar week calculation according to ISO 8601

def _thursday(day):
    return day + timedelta(days=3 - day.weekday())

def _thursday_week1(year):
    # 4.1. is always in the first calendar week
    return _thursday(date(year, 1, 4))

def _has_53_weeks(year):
    first = _thursday_week1(year)

    if calendar.isleap(year):
        # Year begins on a wednesday
        return first.day == 2
    else:
        # Year begins on a thursday
        return first.day == 1

def calendar_week(day):
    delta = _thursday(day) - _thursday_week1(day.year)
    week = int(1.5 + delta.days / 7)

    if 0 < week < 53:
        return (day.year, week)

    if week == 0:
        if day < _thursday_week1(day.year):
            # Day belongs to last calendar week of the year before
            return (day.year - 1, _has_53_weeks(day.year - 1) and 53 or 52)

        return (day.year, _has_53_weeks(day.year) and 53 or 52)

    if week == 53:
        return (day.year, _has_53_weeks(day.year) and 53 or 1)

def daterange(start_date, end_date):
    """Iterate over a range of dates (includes the end_date)"""
    for nr in range((end_date - start_date).days + 1):
        yield start_date + timedelta(days=nr)

class Scheduler(object):
    def __init__(self, assignments, date_range):
        self.queryset = assignments
        self.date_range = date_range
        self.waitlist = None

        self.date_from = _monday(date_range[0])
        self.date_until = date_range[1]

        if self.date_from:  # Is None if no assignments in queryset
            self.week_count = (
                self.date_range[1] - self.date_range[0]).days // 7

            self.date_slice = slice(
                max(0, (self.date_range[0] - self.date_from).days // 7),
                (self.date_range[1] - self.date_from).days // 7 + 1)

        self.date_list = list(daterange(self.date_from, self.date_until))
        self.data_weeks = OrderedDict()
        for day in self.date_list:
            self.data_weeks.setdefault(
                calendar_week(day),
                (day, defaultdict(lambda: 0)))

    def add_waitlist(self, queryset):
        self.waitlist = queryset

        if not self.queryset.exists() and self.waitlist.exists():
            max_min = self.waitlist.aggregate(
                min=Min('assignment_date_from'),
                max=Max('assignment_date_until'))

            self.date_from = max_min['min']
            self.date_until = max_min['max']

            if self.date_from:  # Is None if no assignments in queryset
                self.week_count = (
                    self.date_until - self.date_from).days // 7 + 1

                self.date_slice = slice(
                    max(0, (self.date_range[0] - self.date_from).days // 7),
                    (self.date_range[1] - self.date_from).days // 7 + 2)

    def weeks(self):
        ret = []

        if self.date_from:
            monday = _monday(self.date_from)
            this_monday = _monday(date.today())

            while True:
                ret.append(
                    (monday,)
                    + calendar_week(monday)
                    + (monday == this_monday,))

                monday += timedelta(days=7)
                if monday > self.date_until:
                    break

            ret = ret[self.date_slice]

        return ret

    def _schedule_assignment(self, date_from, date_until):
        weeks = []
        cw = None
        inside = False

        for day in self.date_list:
            new_cw = calendar_week(day)
            if new_cw == cw:
                continue
            else:
                cw = new_cw

            if date_from <= day <= date_until:
                if inside:
                    weeks.append([1, ''])
                else:
                    inside = True
                    if calendar_week(day) == calendar_week(date_from):
                        weeks.append([1, date_from.day])
                    else:
                        # assignment has not started this week
                        weeks.append([1, ''])
            else:
                if inside:
                    inside = False
                    weeks[-1][1] = date_until.day
                weeks.append([0, ''])
        return weeks

    def assignments(self):
        assignments_dict = OrderedDict()

        assignments = self.queryset.select_related(
            'specification__scope_statement', 'drudge__user',
        ).order_by('date_from', 'date_until')

        for assignment in assignments:
            for day in daterange(
                    assignment.date_from,
                    assignment.determine_date_until(),
            ):
                self.data_weeks.setdefault(
                    calendar_week(day),
                    (day, defaultdict(int)),
                )[1][assignment.drudge_id] += 1

            if assignment.drudge not in assignments_dict:
                assignments_dict[assignment.drudge] = []

            assignments_dict[assignment.drudge].append((
                assignment,
                self._schedule_assignment(
                    assignment.date_from, assignment.determine_date_until()),
            ))

        # linearize assignments, but still give precedence to drudge
        assignments = list(itertools.chain(*assignments_dict.values()))

        waitlist = []
        if self.waitlist is not None:
            for entry in self.waitlist.select_related(
                'specification__scope_statement',
                'drudge__user',
            ).order_by('assignment_date_from'):
                entry.status = 'wl'  # special waitlist entry status
                item = (
                    entry,
                    self._schedule_assignment(
                        entry.assignment_date_from,
                        entry.assignment_date_until),
                )

                if entry.drudge in assignments_dict:
                    assignments_dict[entry.drudge].append(item)
                else:
                    waitlist.append(item)

            # linearize assignments with waitlist entries intermingled
            assignments = list(itertools.chain(*assignments_dict.values()))

        filtered_data_weeks = [
            (day, week)
            for day, week in self.data_weeks.values()
            if self.date_from <= _monday(day) <= self.date_until]

        # Weekly count is determined by the count of drudges which are
        # available at least 3 days in a week.
        sums = [(
            sum((1 for days in week.values() if days >= 3), 0),
            u'%d' % sum((1 for days in week.values() if days >= 3), 0),
        ) for day, week in filtered_data_weeks]

        try:
            self.average = sum(
                (sum(week.values(), 0.0) for day, week in filtered_data_weeks),
                0.0) / (self.date_until - self.date_from).days
        except ArithmeticError:
            self.average = 0

        return [[None, sums]] + waitlist + assignments

def _monday(day):
    return day - timedelta(days=day.weekday())

class SchedulingSearchForm(SearchForm):
    default = {
        'date_until__gte': lambda request: _monday(date.today()),
        'date_from__lte': lambda request: _monday(
            date.today()) + timedelta(days=35 * 7 + 4),
        'status': (
            Assignment.TENTATIVE, Assignment.ARRANGED, Assignment.MOBILIZED),
        'mode': 'both',
    }

    specification = forms.ModelMultipleChoiceField(
        Specification.objects.all(),
        label=ugettext_lazy('specification'), required=False)
    status = forms.MultipleChoiceField(
        choices=Assignment.STATUS_CHOICES,
        label=ugettext_lazy('status'), required=False)
    date_until__gte = forms.DateField(
        label=ugettext_lazy('Start date'),
        required=False, widget=forms.DateInput(attrs={'class': 'dateinput'}))
    date_from__lte = forms.DateField(
        label=ugettext_lazy('End date'),
        required=False, widget=forms.DateInput(attrs={'class': 'dateinput'}))

    drudge__motor_saw_course = forms.MultipleChoiceField(
        label=ugettext_lazy('motor saw course'), required=False, choices=(
            ('2-day', ugettext_lazy('2 day course')),
            ('5-day', ugettext_lazy('5 day course')),
        ))
    drudge__driving_license = forms.NullBooleanField(
        label=ugettext_lazy('driving license'), required=False)

    mode = forms.ChoiceField(label=ugettext_lazy('Mode'), choices=(
        ('both', ugettext_lazy('both')),
        ('assignments', ugettext_lazy('only assignments')),
        ('waitlist', ugettext_lazy('only waitlist entries')),
    ), required=False)

    def queryset(self):
        data = self.safe_cleaned_data
        queryset = self.apply_filters(
            Assignment.objects.search(data.get('query')),
            data, exclude=('mode', 'date_until__gte'))
        if data.get('date_until__gte'):
            queryset = queryset.filter(
                Q(
                    date_until_extension__isnull=True,
                    date_until__gte=data.get('date_until__gte'),
                ) | Q(
                    date_until_extension__isnull=False,
                    date_until_extension__gte=data.get('date_until__gte'),
                )
            )
        return queryset

    def waitlist_queryset(self):
        data = self.safe_cleaned_data
        return self.apply_filters(WaitList.objects.search(data.get('query')), {
            'assignment_date_until__gte': data.get('date_until__gte'),
            'assignment_date_from__lte': data.get('date_from__lte'),
        })

@staff_member_required
def scheduling(request):
    search_form = SchedulingSearchForm(request.GET, request=request)

    date_range = (
        search_form.safe_cleaned_data.get('date_until__gte'),
        search_form.safe_cleaned_data.get('date_from__lte'))
    if not all(date_range):
        return HttpResponseRedirect('?clear=1')

    mode = search_form.safe_cleaned_data.get('mode')

    if mode != 'waitlist':
        scheduler = Scheduler(search_form.queryset(), date_range)
    else:
        scheduler = Scheduler(search_form.queryset().none(), date_range)

    if mode != 'assignments':
        scheduler.add_waitlist(search_form.waitlist_queryset())

    return render(request, 'zivinetz/scheduling.html', {
        'scheduler': scheduler,
        'search_form': search_form,
    })
<commit_msg>Fix the calendar week calculation<commit_after>import calendar
from collections import defaultdict, OrderedDict
from datetime import date, timedelta
import itertools

from django import forms
from django.contrib.admin.views.decorators import staff_member_required
from django.db.models import Max, Min, Q
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.utils.translation import ugettext_lazy

from towel.forms import SearchForm

from zivinetz.models import Assignment, Specification, WaitList

# Calendar week calculation according to ISO 8601

def _thursday(day):
    return day + timedelta(days=3 - day.weekday())

def _thursday_week1(year):
    # 4.1. is always in the first calendar week
    return _thursday(date(year, 1, 4))

def _has_53_weeks(year):
    first = _thursday_week1(year)

    if calendar.isleap(year):
        # Year begins on a wednesday
        return first.day == 2
    else:
        # Year begins on a thursday
        return first.day == 1

def calendar_week(day):
    delta = _thursday(day) - _thursday_week1(day.year)
    week = int(1.5 + delta.days / 7)

    if 0 < week < 53:
        return (day.year, week)

    if week == 0:
        if day < _thursday_week1(day.year):
            # Day belongs to last calendar week of the year before
            return (day.year - 1, _has_53_weeks(day.year - 1) and 53 or 52)

        return (day.year, _has_53_weeks(day.year) and 53 or 52)

    if week == 53:
        if _has_53_weeks(day.year):
            return (day.year, 53)
        # Otherwise, it is already week one of the next year
        return (day.year + 1, 1)

def daterange(start_date, end_date):
    """Iterate over a range of dates (includes the end_date)"""
    for nr in range((end_date - start_date).days + 1):
        yield start_date + timedelta(days=nr)

class Scheduler(object):
    def __init__(self, assignments, date_range):
        self.queryset = assignments
        self.date_range = date_range
        self.waitlist = None

        self.date_from = _monday(date_range[0])
        self.date_until = date_range[1]

        if self.date_from:  # Is None if no assignments in queryset
            self.week_count = (
                self.date_range[1] - self.date_range[0]).days // 7

            self.date_slice = slice(
                max(0, (self.date_range[0] - self.date_from).days // 7),
                (self.date_range[1] - self.date_from).days // 7 + 1)

        self.date_list = list(daterange(self.date_from, self.date_until))
        self.data_weeks = OrderedDict()
        for day in self.date_list:
            self.data_weeks.setdefault(
                calendar_week(day),
                (day, defaultdict(lambda: 0)))

    def add_waitlist(self, queryset):
        self.waitlist = queryset

        if not self.queryset.exists() and self.waitlist.exists():
            max_min = self.waitlist.aggregate(
                min=Min('assignment_date_from'),
                max=Max('assignment_date_until'))

            self.date_from = max_min['min']
            self.date_until = max_min['max']

            if self.date_from:  # Is None if no assignments in queryset
                self.week_count = (
                    self.date_until - self.date_from).days // 7 + 1

                self.date_slice = slice(
                    max(0, (self.date_range[0] - self.date_from).days // 7),
                    (self.date_range[1] - self.date_from).days // 7 + 2)

    def weeks(self):
        ret = []

        if self.date_from:
            monday = _monday(self.date_from)
            this_monday = _monday(date.today())

            while True:
                ret.append(
                    (monday,)
                    + calendar_week(monday)
                    + (monday == this_monday,))

                monday += timedelta(days=7)
                if monday > self.date_until:
                    break

            ret = ret[self.date_slice]

        return ret

    def _schedule_assignment(self, date_from, date_until):
        weeks = []
        cw = None
        inside = False

        for day in self.date_list:
            new_cw = calendar_week(day)
            if new_cw == cw:
                continue
            else:
                cw = new_cw

            if date_from <= day <= date_until:
                if inside:
                    weeks.append([1, ''])
                else:
                    inside = True
                    if calendar_week(day) == calendar_week(date_from):
                        weeks.append([1, date_from.day])
                    else:
                        # assignment has not started this week
                        weeks.append([1, ''])
            else:
                if inside:
                    inside = False
                    weeks[-1][1] = date_until.day
                weeks.append([0, ''])
        return weeks

    def assignments(self):
        assignments_dict = OrderedDict()

        assignments = self.queryset.select_related(
            'specification__scope_statement', 'drudge__user',
        ).order_by('date_from', 'date_until')

        for assignment in assignments:
            for day in daterange(
                    assignment.date_from,
                    assignment.determine_date_until(),
            ):
                self.data_weeks.setdefault(
                    calendar_week(day),
                    (day, defaultdict(int)),
                )[1][assignment.drudge_id] += 1

            if assignment.drudge not in assignments_dict:
                assignments_dict[assignment.drudge] = []

            assignments_dict[assignment.drudge].append((
                assignment,
                self._schedule_assignment(
                    assignment.date_from, assignment.determine_date_until()),
            ))

        # linearize assignments, but still give precedence to drudge
        assignments = list(itertools.chain(*assignments_dict.values()))

        waitlist = []
        if self.waitlist is not None:
            for entry in self.waitlist.select_related(
                'specification__scope_statement',
                'drudge__user',
            ).order_by('assignment_date_from'):
                entry.status = 'wl'  # special waitlist entry status
                item = (
                    entry,
                    self._schedule_assignment(
                        entry.assignment_date_from,
                        entry.assignment_date_until),
                )

                if entry.drudge in assignments_dict:
                    assignments_dict[entry.drudge].append(item)
                else:
                    waitlist.append(item)

            # linearize assignments with waitlist entries intermingled
            assignments = list(itertools.chain(*assignments_dict.values()))

        filtered_data_weeks = [
            (day, week)
            for day, week in self.data_weeks.values()
            if self.date_from <= _monday(day) <= self.date_until]

        # Weekly count is determined by the count of drudges which are
        # available at least 3 days in a week.
        sums = [(
            sum((1 for days in week.values() if days >= 3), 0),
            u'%d' % sum((1 for days in week.values() if days >= 3), 0),
        ) for day, week in filtered_data_weeks]

        try:
            self.average = sum(
                (sum(week.values(), 0.0) for day, week in filtered_data_weeks),
                0.0) / (self.date_until - self.date_from).days
        except ArithmeticError:
            self.average = 0

        return [[None, sums]] + waitlist + assignments

def _monday(day):
    return day - timedelta(days=day.weekday())

class SchedulingSearchForm(SearchForm):
    default = {
        'date_until__gte': lambda request: _monday(date.today()),
        'date_from__lte': lambda request: _monday(
            date.today()) + timedelta(days=35 * 7 + 4),
        'status': (
            Assignment.TENTATIVE, Assignment.ARRANGED, Assignment.MOBILIZED),
        'mode': 'both',
    }

    specification = forms.ModelMultipleChoiceField(
        Specification.objects.all(),
        label=ugettext_lazy('specification'), required=False)
    status = forms.MultipleChoiceField(
        choices=Assignment.STATUS_CHOICES,
        label=ugettext_lazy('status'), required=False)
    date_until__gte = forms.DateField(
        label=ugettext_lazy('Start date'),
        required=False, widget=forms.DateInput(attrs={'class': 'dateinput'}))
    date_from__lte = forms.DateField(
        label=ugettext_lazy('End date'),
        required=False, widget=forms.DateInput(attrs={'class': 'dateinput'}))

    drudge__motor_saw_course = forms.MultipleChoiceField(
        label=ugettext_lazy('motor saw course'), required=False, choices=(
            ('2-day', ugettext_lazy('2 day course')),
            ('5-day', ugettext_lazy('5 day course')),
        ))
    drudge__driving_license = forms.NullBooleanField(
        label=ugettext_lazy('driving license'), required=False)

    mode = forms.ChoiceField(label=ugettext_lazy('Mode'), choices=(
        ('both', ugettext_lazy('both')),
        ('assignments', ugettext_lazy('only assignments')),
        ('waitlist', ugettext_lazy('only waitlist entries')),
    ), required=False)

    def queryset(self):
        data = self.safe_cleaned_data
        queryset = self.apply_filters(
            Assignment.objects.search(data.get('query')),
            data, exclude=('mode', 'date_until__gte'))
        if data.get('date_until__gte'):
            queryset = queryset.filter(
                Q(
                    date_until_extension__isnull=True,
                    date_until__gte=data.get('date_until__gte'),
                ) | Q(
                    date_until_extension__isnull=False,
                    date_until_extension__gte=data.get('date_until__gte'),
                )
            )
        return queryset

    def waitlist_queryset(self):
        data = self.safe_cleaned_data
        return self.apply_filters(WaitList.objects.search(data.get('query')), {
            'assignment_date_until__gte': data.get('date_until__gte'),
            'assignment_date_from__lte': data.get('date_from__lte'),
        })

@staff_member_required
def scheduling(request):
    search_form = SchedulingSearchForm(request.GET, request=request)

    date_range = (
        search_form.safe_cleaned_data.get('date_until__gte'),
        search_form.safe_cleaned_data.get('date_from__lte'))
    if not all(date_range):
        return HttpResponseRedirect('?clear=1')

    mode = search_form.safe_cleaned_data.get('mode')

    if mode != 'waitlist':
        scheduler = Scheduler(search_form.queryset(), date_range)
    else:
        scheduler = Scheduler(search_form.queryset().none(), date_range)

    if mode != 'assignments':
        scheduler.add_waitlist(search_form.waitlist_queryset())

    return render(request, 'zivinetz/scheduling.html', {
        'scheduler': scheduler,
        'search_form': search_form,
    })
<|endoftext|>


<commit_before>from DiffusionModel import DiffusionModel
import networkx as nx
import numpy as np

__author__ = "Alina Sirbu"
__email__ = "alina.sirbu@unipi.it"

class CognitiveOpDynModel(DiffusionModel):
    """
    Implements the cognitive model of opinion dynamics by Villone et al.
    One model parameter: 'I' external information value in [0,1].
    Node states are continuous values in [0,1].
    Additional node parameters encode risk sensitivity R in {0,-1,+1}, tendency to communicate B in [0,1], trust towards institutions T in [0,1].  These are stored in the 'cognitive' node parameter.
    The initial state is generated randomly uniformly from the domain.
    """

    def set_initial_status(self, configuration=None):
	"""
	Override behaviour of methods in class DiffusionModel.
	Overwrites initial status using random real values.
	Generates random node profiles.
	"""
	super(CognitiveOpDynModel, self).set_initial_status(configuration)

	#set node status
	for node in self.status:
		self.status[node]=np.random.random_sample()
	self.initial_status = self.status.copy()
	#set new node parameters
	RDomain=[-1,0,1]
	self.params['nodes']['cognitive']={}
	for node in self.graph.nodes():
		#R, B and T parameters in a tuple
		self.params['nodes']['cognitive'][node]=(RDomain[np.random.randint(0,3)],np.random.random_sample(),np.random.random_sample())

    def iteration(self):
        """
        One iteration changes the opinion of all agents using the following procedure:
        - first all agents communicate with institutional information I using a deffuant like rule
        - then random pairs of agents are selected to interact  (N pairs)
	- interaction depends on state of agents but also internal cognitive structure
        """

	#first interact with I
	I= self.params['I']
	for node in self.graph.nodes():
		T=self.params['nodes']['cognitive'][node][2]
		R=self.params['nodes']['cognitive'][node][0]
		self.status[node]=self.status[node]+T*(I-self.status[node])
		if R==1:
			self.status[node]=0.5*(1-self.status[node])
		if R==-1:
			self.status[node]=0.5* self.status[node]

	#then interact with peers
	for i in range(0,self.graph.number_of_nodes()):
       		# select a random node
        	n1 = self.graph.nodes()[np.random.randint(0, self.graph.number_of_nodes())]

        	# select all of the nodes neighbours (no digraph possible)
        	neighbours = self.graph.neighbors(n1)

        	# select second node - a random neighbour
        	n2 = neighbours[np.random.randint(0, len(neighbours))]

            	# update status of n1 and n2
		p1=pow(self.status[n1],1.0/self.params['nodes']['cognitive'][n1][1])
		p2=pow(self.status[n2],1.0/self.params['nodes']['cognitive'][n2][1])

		oldn1=self.status[n1]
		if np.random.random_sample()<p2:#if node 2 talks, node 1 gets changed
			T1=self.params['nodes']['cognitive'][n1][2]
			R1=self.params['nodes']['cognitive'][n1][0]
            		self.status[n1] = self.status[n1]+(1-T1)*(self.status[n2]-self.status[n1])
			if R1==1:
				self.status[n1]=0.5*(1-self.status[n1])
			if R1==-1:
				self.status[n1]=0.5* self.status[n1]

		if np.random.random_sample()<p1:#if node 1 talks, node 2 gets changed
			T2=self.params['nodes']['cognitive'][n2][2]
			R2=self.params['nodes']['cognitive'][n2][0]
            		self.status[n2] = self.status[n2]+(1-T2)*(oldn1-self.status[n2])
			if R2==1:
				self.status[n2]=0.5*(1-self.status[n2])
			if R2==-1:
				self.status[n2]=0.5* self.status[n2]

        self.actual_iteration += 1

        return self.actual_iteration, self.status

<commit_msg>PEP8 standardization<commit_after>from DiffusionModel import DiffusionModel
import numpy as np

__author__ = "Alina Sirbu"
__email__ = "alina.sirbu@unipi.it"

class CognitiveOpDynModel(DiffusionModel):
    """
    Implements the cognitive model of opinion dynamics by Villone et al.
    One model parameter: 'I' external information value in [0,1].
    Node states are continuous values in [0,1].
    Additional node parameters encode risk sensitivity R in {0,-1,+1}, tendency to communicate B in [0,1], trust towards institutions T in [0,1].  These are stored in the 'cognitive' node parameter.
    The initial state is generated randomly uniformly from the domain.
    """

    def set_initial_status(self, configuration=None):
        """
        Override behaviour of methods in class DiffusionModel.
        Overwrites initial status using random real values.
        Generates random node profiles.
        """
        super(CognitiveOpDynModel, self).set_initial_status(configuration)

        # set node status
        for node in self.status:
            self.status[node] = np.random.random_sample()
        self.initial_status = self.status.copy()
        # set new node parameters
        RDomain = [-1, 0, 1]
        self.params['nodes']['cognitive'] = {}
        for node in self.graph.nodes():
            # R, B and T parameters in a tuple
            self.params['nodes']['cognitive'][node] = (
            RDomain[np.random.randint(0, 3)], np.random.random_sample(), np.random.random_sample())

    def iteration(self):
        """
        One iteration changes the opinion of all agents using the following procedure:
        - first all agents communicate with institutional information I using a deffuant like rule
        - then random pairs of agents are selected to interact  (N pairs)
        - interaction depends on state of agents but also internal cognitive structure
        """

        # first interact with I
        I = self.params['I']
        for node in self.graph.nodes():
            T = self.params['nodes']['cognitive'][node][2]
            R = self.params['nodes']['cognitive'][node][0]
            self.status[node] = self.status[node] + T * (I - self.status[node])
            if R == 1:
                self.status[node] = 0.5 * (1 - self.status[node])
            if R == -1:
                self.status[node] *= 0.5

        # then interact with peers
        for i in range(0, self.graph.number_of_nodes()):
            # select a random node
            n1 = self.graph.nodes()[np.random.randint(0, self.graph.number_of_nodes())]

            # select all of the nodes neighbours (no digraph possible)
            neighbours = self.graph.neighbors(n1)

            # select second node - a random neighbour
            n2 = neighbours[np.random.randint(0, len(neighbours))]

            # update status of n1 and n2
            p1 = pow(self.status[n1], 1.0 / self.params['nodes']['cognitive'][n1][1])
            p2 = pow(self.status[n2], 1.0 / self.params['nodes']['cognitive'][n2][1])

            oldn1 = self.status[n1]
            if np.random.random_sample() < p2:  # if node 2 talks, node 1 gets changed
                T1 = self.params['nodes']['cognitive'][n1][2]
                R1 = self.params['nodes']['cognitive'][n1][0]
                self.status[n1] += (1 - T1) * (self.status[n2] - self.status[n1])
                if R1 == 1:
                    self.status[n1] = 0.5 * (1 - self.status[n1])
                if R1 == -1:
                    self.status[n1] *= 0.5

            if np.random.random_sample() < p1:  # if node 1 talks, node 2 gets changed
                T2 = self.params['nodes']['cognitive'][n2][2]
                R2 = self.params['nodes']['cognitive'][n2][0]
                self.status[n2] += (1 - T2) * (oldn1 - self.status[n2])
                if R2 == 1:
                    self.status[n2] = 0.5 * (1 - self.status[n2])
                if R2 == -1:
                    self.status[n2] *= 0.5

        self.actual_iteration += 1

        return self.actual_iteration, self.status
<|endoftext|>


<commit_before># ----------------------------------------------------------------------------
# Copyright 2015 Nervana Systems Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ----------------------------------------------------------------------------

from neon import NervanaObject
import numpy as np

def get_param_list(layer_list):
    '''
    returns a flattened list of params
    '''
    plist = []
    for l in layer_list:
        ptuple = l.get_params()
        plist.extend(ptuple) if isinstance(ptuple, list) else plist.append(ptuple)
    return plist

class Optimizer(NervanaObject):

    '''
    Optimizers will take a param, update, and state
    will be responsible for keeping track of a schedule
    '''

    def optimize(self, layer_list, epoch):
        raise NotImplementedError()

class Schedule(NervanaObject):
    """
    Learning rate schedule for constant or step learning rates.

    By default implements a constant learning rate.
    """
    def __init__(self, step_config=None, change=1.):
        """
        Arguments:
            step_config (int or list, optional): Configure the epoch step rate (int)
                or step times (list of epoch indices). Defaults to None (constant).

            change (float or list, optional): In step mode, learning rate is
                multiplied by ``change ** steps``, where ``steps`` is the number of
                steps in the step schedule that have passed. If ``change`` is a list,
                ``step_config`` must also be a list. Then at ``step[i]``, the
                learning rate is set to ``change[i]``.
        """

        if isinstance(step_config, list) and isinstance(change, list):
            assert len(step_config) == len(change), "change and step_config must have the same" \
                "length after step_config is deduplicated to do epoch-level LR assignment."

        self.step_config = step_config
        self.change = change
        self.steps = 0

    def get_learning_rate(self, learning_rate, epoch):
        """
        Get the current learning rate given the epoch and initial rate

        Arguments:
            learning_rate (float): the initial learning rate
            epoch (int): the current epoch, used to calculate the new effective learning rate.
        """

        if isinstance(self.step_config, list) and isinstance(self.change, list):
            if epoch in self.step_config:
                # steps will store the current lr
                self.steps = self.change[self.step_config.index(epoch)]
            if self.steps == 0:
                return learning_rate
            else:
                return self.steps

        elif isinstance(self.step_config, int):
            self.steps = np.floor((epoch + 1) / self.step_config)

        elif isinstance(self.step_config, list):
            self.steps = np.sum(epoch >= np.array(self.step_config))

        return float(learning_rate * self.change ** self.steps)

class ExpSchedule(Schedule):
    """
    Exponential learning rate schedule.

    Arguments:
        decay (float): how much exponential decay to apply to the learning rate
    """
    def __init__(self, decay):
        self.decay = decay

    def get_learning_rate(self, learning_rate, epoch):
        return learning_rate / (1. + self.decay * epoch)

class GradientDescentMomentum(Optimizer):

    """
    Stochastic gradient descent with momentum
    """

    def __init__(self, learning_rate, momentum_coef, stochastic_round=False,
                 wdecay=0.0, name="gdm", schedule=Schedule()):
        """
        Arguments:
            learning_rate (float): the multiplicative coefficient of updates
            momentum_coef (float): the coefficient of momentum
            stochastic_round (bool): Set this to True for stochastic rounding.
                                     If False rounding will be to nearest.
                                     If True will perform stochastic rounding using default width.
                                     Only affects the gpu backend
            wdecay (float): the weight decay
            name (str): the optimizer's layer's pretty-print name.
        """
        super(GradientDescentMomentum, self).__init__(name=name)
        self.learning_rate, self.momentum_coef = (learning_rate, momentum_coef)
        self.wdecay = wdecay
        self.schedule = schedule
        self.stochastic_round = stochastic_round

    def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            layer_list (list): a list of Layer objects to optimize.
            epoch (int): the current epoch, needed for the Schedule object.
        """
        lrate = self.schedule.get_learning_rate(self.learning_rate, epoch)
        param_list = get_param_list(layer_list)
        for (param, grad), states in param_list:
            param.rounding = self.stochastic_round
            if len(states) == 0:
                states.append(self.be.zeros_like(grad))
            grad = grad / self.be.bsz
            velocity = states[0]
            velocity[:] = velocity * self.momentum_coef - lrate * (grad + self.wdecay * param)
            param[:] = param + velocity

class RMSProp(Optimizer):

    """
    Root Mean Square propagation (leaving out schedule for now).
    """

    def __init__(self, stochastic_round=False, decay_rate=0.95, learning_rate=2e-3, epsilon=1e-6,
                 clip_gradients=False, gradient_limit=5, name="rmsprop"):
        """
        Arguments:
            stochastic_round (bool): Set this to True for stochastic rounding.
                                     If False rounding will be to nearest.
                                     If True will perform stochastic rounding using default width.
                                     Only affects the gpu backend.
            decay_rate (float): decay rate of states
            learning_rate (float): the multiplication coefficent of updates
            epsilon (float): smoothing epsilon to avoid divide by zeros
            clip_gradients (bool): whether to truncate the gradients.
            gradient_limit (float): positive value to clip gradients between.
        """
        self.state_list = None

        self.epsilon = epsilon
        self.decay_rate = decay_rate
        self.learning_rate = learning_rate
        self.clip_gradients = clip_gradients
        self.gradient_limit = gradient_limit
        self.stochastic_round = stochastic_round

    def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            layer_list (list): a list of Layer objects to optimize.
            epoch (int): the current epoch, needed for the Schedule object.
        """
        lrate, epsilon, decay = (self.learning_rate, self.epsilon, self.decay_rate)

        param_list = get_param_list(layer_list)

        for (param, grad), states in param_list:

            param.rounding = self.stochastic_round
            if len(states) == 0:
                states.append(self.be.zeros_like(grad))

            if self.clip_gradients:
                grad = self.be.clip(grad, -self.gradient_limit, self.gradient_limit)
            grad = grad / self.be.bsz
            # update state
            state = states[0]
            state[:] = decay * state + self.be.square(grad) * (1.0 - decay)

            param[:] = param - grad * lrate / (self.be.sqrt(state + epsilon) + epsilon)

class Adadelta(Optimizer):

    """
    Adadelta based learning rule updates.
    See Zeiler2012 for instance.
    """

    def __init__(self, stochastic_round=False, decay=0.95, epsilon=1e-6, name="ada"):
        """
        Args:
            stochastic_round (bool): Set this to True for stochastic rounding.
                                     If False rounding will be to nearest.
                                     If True will perform stochastic rounding using default width.
                                     Only affects the gpu backend.
            decay: decay parameter in Adadelta
            epsilon: epsilon parameter in Adadelta
        """
        super(Adadelta, self).__init__(name=name)
        self.decay = decay
        self.epsilon = epsilon
        self.stochastic_round = stochastic_round

    def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            param_list (list): a list of tuples of the form ((param, grad), state),
                               corresponding to parameters, grads,
                               and states of layers to be updated
            epoch (int): the current epoch, needed for the Schedule object.
        """
        epsilon, decay = (self.epsilon, self.decay)

        param_list = get_param_list(layer_list)

        for (param, grad), states in param_list:
            param.rounding = self.stochastic_round

            if len(states) == 0:
                # E[Grad^2], E[Delt^2], updates
                states.extend([self.be.zeros_like(grad) for i in range(3)])

            grad = grad / self.be.bsz
            states[0][:] = states[0] * decay + (1. - decay) * grad * grad
            states[2][:] = self.be.sqrt((states[1] + epsilon) / (states[0] + epsilon)) * grad
            states[1][:] = states[1] * decay + (1. - decay) * states[2] * states[2]

            param[:] = param - states[2]

class Adam(Optimizer):

    """
    Adam based learning rule updates. http://arxiv.org/pdf/1412.6980v8.pdf
    """

    def __init__(self, stochastic_round=False, learning_rate=0.001, beta_1=0.9, beta_2=0.999,
                 epsilon=1e-8, name="adam"):
        """
        Args:
            stochastic_round (bool): Set this to True for stochastic rounding.
                                     If False rounding will be to nearest.
                                     If True will perform stochastic rounding using default width.
                                     Only affects the gpu backend.
            learning_rate (float): the multiplicative coefficient of updates
            beta_1 (float): Adam parameter beta1
            beta_2 (float): Adam parameter beta2
            epsilon (float): numerical stability parameter
        """
        super(Adam, self).__init__(name=name)
        self.beta_1 = beta_1
        self.beta_2 = beta_2
        self.epsilon = epsilon
        self.learning_rate = learning_rate
        self.stochastic_round = stochastic_round

    def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            param_list (list): a list of tuples of the form ((param, grad), state),
                corresponding to parameters, grads, and states of layers to be updated
            epoch (int): the current epoch, needed for the Schedule object.
        """
        t = epoch + 1
        l = self.learning_rate * self.be.sqrt(1 - self.beta_2 ** t) / (1 - self.beta_1 ** t)

        param_list = get_param_list(layer_list)

        for (param, grad), states in param_list:
            param.rounding = self.stochastic_round
            if len(states) == 0:
                # running_1st_mom, running_2nd_mom
                states.extend([self.be.zeros_like(grad) for i in range(2)])

            grad = grad / self.be.bsz
            m, v = states
            m[:] = m * self.beta_1 + (1. - self.beta_1) * grad
            v[:] = v * self.beta_2 + (1. - self.beta_2) * grad * grad

            param[:] = param - l * m / (self.be.sqrt(v) + self.epsilon)

class MultiOptimizer(Optimizer):

    """
    A wrapper class for using multiple Optimizers within the same model.
    """

    def __init__(self, optimizer_mapping, name="multiopt"):
        """

        Args:
            optimizer_mapping (dict): dictionary specifying the mapping of layers to optimizers.
                Key: Layer class name or Layer `name` attribute. The latter takes
                precedence over the former for finer layer-to-layer control.
                Don't name your layers ``'default'``. Value: the optimizer object to use for those
                layers. For instance, ``{'default': optimizer1, 'Bias': optimizer2,
                'special_bias': optimizer3}`` will use ``optimizer3`` for the layer named
                ``special_bias``, ``optimizer2`` for all other Bias layers, and ``optimizer1``
                for all other layers.
        """
        super(MultiOptimizer, self).__init__(name=name)
        self.optimizer_mapping = optimizer_mapping
        assert 'default' in self.optimizer_mapping, "Must specify a default" \
            "optimizer in layer type to optimizer mapping"

        self.map_list = None

    def map_optimizers(self, layer_list):
        """
        maps the optimizers to their corresponding layers
        """
        map_list = dict()
        for layer in layer_list:
            classname = layer.__class__.__name__
            name = layer.name
            opt = None
            if name in self.optimizer_mapping:
                opt = self.optimizer_mapping[name]
            elif classname in self.optimizer_mapping:
                opt = self.optimizer_mapping[classname]
            else:
                opt = self.optimizer_mapping['default']

            if opt not in map_list:
                map_list[opt] = [layer]
            else:
                map_list[opt].append(layer)
        return map_list

    def reset_mapping(self, new_mapping):
        """
        Pass this optimizer a new mapping, and on subsequent optimize call, the
        mapping will be refreshed (since map_list will be recreated)
        """
        self.optimizer_mapping = new_mapping
        self.map_list = None

    def optimize(self, layer_list, epoch):
        """
        Determine which optimizer in the container should go with which layers,
        then apply their optimize functions to those layers.

        Notes:

        We can recalculate ``map_list`` in case ``optimizer_mapping`` changes
        during training.
        """

        if self.map_list is None:
            self.map_list = self.map_optimizers(layer_list)

        for opt in self.map_list:
            opt.optimize(self.map_list[opt], epoch)

    def get_description(self):
        desc = {'type': self.__class__.__name__}
        for key in self.optimizer_mapping:
            desc[key] = self.optimizer_mapping[key].get_description()
        return desc
<commit_msg>Scale grads by batch size before clipping.<commit_after># ----------------------------------------------------------------------------
# Copyright 2015 Nervana Systems Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ----------------------------------------------------------------------------

from neon import NervanaObject
import numpy as np

def get_param_list(layer_list):
    '''
    returns a flattened list of params
    '''
    plist = []
    for l in layer_list:
        ptuple = l.get_params()
        plist.extend(ptuple) if isinstance(ptuple, list) else plist.append(ptuple)
    return plist

class Optimizer(NervanaObject):

    '''
    Optimizers will take a param, update, and state
    will be responsible for keeping track of a schedule
    '''

    def optimize(self, layer_list, epoch):
        raise NotImplementedError()

class Schedule(NervanaObject):
    """
    Learning rate schedule for constant or step learning rates.

    By default implements a constant learning rate.
    """
    def __init__(self, step_config=None, change=1.):
        """
        Arguments:
            step_config (int or list, optional): Configure the epoch step rate (int)
                or step times (list of epoch indices). Defaults to None (constant).

            change (float or list, optional): In step mode, learning rate is
                multiplied by ``change ** steps``, where ``steps`` is the number of
                steps in the step schedule that have passed. If ``change`` is a list,
                ``step_config`` must also be a list. Then at ``step[i]``, the
                learning rate is set to ``change[i]``.
        """

        if isinstance(step_config, list) and isinstance(change, list):
            assert len(step_config) == len(change), "change and step_config must have the same" \
                "length after step_config is deduplicated to do epoch-level LR assignment."

        self.step_config = step_config
        self.change = change
        self.steps = 0

    def get_learning_rate(self, learning_rate, epoch):
        """
        Get the current learning rate given the epoch and initial rate

        Arguments:
            learning_rate (float): the initial learning rate
            epoch (int): the current epoch, used to calculate the new effective learning rate.
        """

        if isinstance(self.step_config, list) and isinstance(self.change, list):
            if epoch in self.step_config:
                # steps will store the current lr
                self.steps = self.change[self.step_config.index(epoch)]
            if self.steps == 0:
                return learning_rate
            else:
                return self.steps

        elif isinstance(self.step_config, int):
            self.steps = np.floor((epoch + 1) / self.step_config)

        elif isinstance(self.step_config, list):
            self.steps = np.sum(epoch >= np.array(self.step_config))

        return float(learning_rate * self.change ** self.steps)

class ExpSchedule(Schedule):
    """
    Exponential learning rate schedule.

    Arguments:
        decay (float): how much exponential decay to apply to the learning rate
    """
    def __init__(self, decay):
        self.decay = decay

    def get_learning_rate(self, learning_rate, epoch):
        return learning_rate / (1. + self.decay * epoch)

class GradientDescentMomentum(Optimizer):

    """
    Stochastic gradient descent with momentum
    """

    def __init__(self, learning_rate, momentum_coef, stochastic_round=False,
                 wdecay=0.0, name="gdm", schedule=Schedule()):
        """
        Arguments:
            learning_rate (float): the multiplicative coefficient of updates
            momentum_coef (float): the coefficient of momentum
            stochastic_round (bool): Set this to True for stochastic rounding.
                                     If False rounding will be to nearest.
                                     If True will perform stochastic rounding using default width.
                                     Only affects the gpu backend
            wdecay (float): the weight decay
            name (str): the optimizer's layer's pretty-print name.
        """
        super(GradientDescentMomentum, self).__init__(name=name)
        self.learning_rate, self.momentum_coef = (learning_rate, momentum_coef)
        self.wdecay = wdecay
        self.schedule = schedule
        self.stochastic_round = stochastic_round

    def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            layer_list (list): a list of Layer objects to optimize.
            epoch (int): the current epoch, needed for the Schedule object.
        """
        lrate = self.schedule.get_learning_rate(self.learning_rate, epoch)
        param_list = get_param_list(layer_list)
        for (param, grad), states in param_list:
            param.rounding = self.stochastic_round
            if len(states) == 0:
                states.append(self.be.zeros_like(grad))
            grad = grad / self.be.bsz
            velocity = states[0]
            velocity[:] = velocity * self.momentum_coef - lrate * (grad + self.wdecay * param)
            param[:] = param + velocity

class RMSProp(Optimizer):

    """
    Root Mean Square propagation (leaving out schedule for now).
    """

    def __init__(self, stochastic_round=False, decay_rate=0.95, learning_rate=2e-3, epsilon=1e-6,
                 clip_gradients=False, gradient_limit=5, name="rmsprop"):
        """
        Arguments:
            stochastic_round (bool): Set this to True for stochastic rounding.
                                     If False rounding will be to nearest.
                                     If True will perform stochastic rounding using default width.
                                     Only affects the gpu backend.
            decay_rate (float): decay rate of states
            learning_rate (float): the multiplication coefficent of updates
            epsilon (float): smoothing epsilon to avoid divide by zeros
            clip_gradients (bool): whether to truncate the gradients.
            gradient_limit (float): positive value to clip gradients between.
        """
        self.state_list = None

        self.epsilon = epsilon
        self.decay_rate = decay_rate
        self.learning_rate = learning_rate
        self.clip_gradients = clip_gradients
        self.gradient_limit = gradient_limit
        self.stochastic_round = stochastic_round

    def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            layer_list (list): a list of Layer objects to optimize.
            epoch (int): the current epoch, needed for the Schedule object.
        """
        lrate, epsilon, decay = (self.learning_rate, self.epsilon, self.decay_rate)

        param_list = get_param_list(layer_list)

        for (param, grad), states in param_list:

            param.rounding = self.stochastic_round
            if len(states) == 0:
                states.append(self.be.zeros_like(grad))

            grad = grad / self.be.bsz
            if self.clip_gradients:
                grad = self.be.clip(grad, -self.gradient_limit, self.gradient_limit)
            # update state
            state = states[0]
            state[:] = decay * state + self.be.square(grad) * (1.0 - decay)

            param[:] = param - grad * lrate / (self.be.sqrt(state + epsilon) + epsilon)

class Adadelta(Optimizer):

    """
    Adadelta based learning rule updates.
    See Zeiler2012 for instance.
    """

    def __init__(self, stochastic_round=False, decay=0.95, epsilon=1e-6, name="ada"):
        """
        Args:
            stochastic_round (bool): Set this to True for stochastic rounding.
                                     If False rounding will be to nearest.
                                     If True will perform stochastic rounding using default width.
                                     Only affects the gpu backend.
            decay: decay parameter in Adadelta
            epsilon: epsilon parameter in Adadelta
        """
        super(Adadelta, self).__init__(name=name)
        self.decay = decay
        self.epsilon = epsilon
        self.stochastic_round = stochastic_round

    def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            param_list (list): a list of tuples of the form ((param, grad), state),
                               corresponding to parameters, grads,
                               and states of layers to be updated
            epoch (int): the current epoch, needed for the Schedule object.
        """
        epsilon, decay = (self.epsilon, self.decay)

        param_list = get_param_list(layer_list)

        for (param, grad), states in param_list:
            param.rounding = self.stochastic_round

            if len(states) == 0:
                # E[Grad^2], E[Delt^2], updates
                states.extend([self.be.zeros_like(grad) for i in range(3)])

            grad = grad / self.be.bsz
            states[0][:] = states[0] * decay + (1. - decay) * grad * grad
            states[2][:] = self.be.sqrt((states[1] + epsilon) / (states[0] + epsilon)) * grad
            states[1][:] = states[1] * decay + (1. - decay) * states[2] * states[2]

            param[:] = param - states[2]

class Adam(Optimizer):

    """
    Adam based learning rule updates. http://arxiv.org/pdf/1412.6980v8.pdf
    """

    def __init__(self, stochastic_round=False, learning_rate=0.001, beta_1=0.9, beta_2=0.999,
                 epsilon=1e-8, name="adam"):
        """
        Args:
            stochastic_round (bool): Set this to True for stochastic rounding.
                                     If False rounding will be to nearest.
                                     If True will perform stochastic rounding using default width.
                                     Only affects the gpu backend.
            learning_rate (float): the multiplicative coefficient of updates
            beta_1 (float): Adam parameter beta1
            beta_2 (float): Adam parameter beta2
            epsilon (float): numerical stability parameter
        """
        super(Adam, self).__init__(name=name)
        self.beta_1 = beta_1
        self.beta_2 = beta_2
        self.epsilon = epsilon
        self.learning_rate = learning_rate
        self.stochastic_round = stochastic_round

    def optimize(self, layer_list, epoch):
        """
        Apply the learning rule to all the layers and update the states.

        Arguments:
            param_list (list): a list of tuples of the form ((param, grad), state),
                corresponding to parameters, grads, and states of layers to be updated
            epoch (int): the current epoch, needed for the Schedule object.
        """
        t = epoch + 1
        l = self.learning_rate * self.be.sqrt(1 - self.beta_2 ** t) / (1 - self.beta_1 ** t)

        param_list = get_param_list(layer_list)

        for (param, grad), states in param_list:
            param.rounding = self.stochastic_round
            if len(states) == 0:
                # running_1st_mom, running_2nd_mom
                states.extend([self.be.zeros_like(grad) for i in range(2)])

            grad = grad / self.be.bsz
            m, v = states
            m[:] = m * self.beta_1 + (1. - self.beta_1) * grad
            v[:] = v * self.beta_2 + (1. - self.beta_2) * grad * grad

            param[:] = param - l * m / (self.be.sqrt(v) + self.epsilon)

class MultiOptimizer(Optimizer):

    """
    A wrapper class for using multiple Optimizers within the same model.
    """

    def __init__(self, optimizer_mapping, name="multiopt"):
        """

        Args:
            optimizer_mapping (dict): dictionary specifying the mapping of layers to optimizers.
                Key: Layer class name or Layer `name` attribute. The latter takes
                precedence over the former for finer layer-to-layer control.
                Don't name your layers ``'default'``. Value: the optimizer object to use for those
                layers. For instance, ``{'default': optimizer1, 'Bias': optimizer2,
                'special_bias': optimizer3}`` will use ``optimizer3`` for the layer named
                ``special_bias``, ``optimizer2`` for all other Bias layers, and ``optimizer1``
                for all other layers.
        """
        super(MultiOptimizer, self).__init__(name=name)
        self.optimizer_mapping = optimizer_mapping
        assert 'default' in self.optimizer_mapping, "Must specify a default" \
            "optimizer in layer type to optimizer mapping"

        self.map_list = None

    def map_optimizers(self, layer_list):
        """
        maps the optimizers to their corresponding layers
        """
        map_list = dict()
        for layer in layer_list:
            classname = layer.__class__.__name__
            name = layer.name
            opt = None
            if name in self.optimizer_mapping:
                opt = self.optimizer_mapping[name]
            elif classname in self.optimizer_mapping:
                opt = self.optimizer_mapping[classname]
            else:
                opt = self.optimizer_mapping['default']

            if opt not in map_list:
                map_list[opt] = [layer]
            else:
                map_list[opt].append(layer)
        return map_list

    def reset_mapping(self, new_mapping):
        """
        Pass this optimizer a new mapping, and on subsequent optimize call, the
        mapping will be refreshed (since map_list will be recreated)
        """
        self.optimizer_mapping = new_mapping
        self.map_list = None

    def optimize(self, layer_list, epoch):
        """
        Determine which optimizer in the container should go with which layers,
        then apply their optimize functions to those layers.

        Notes:

        We can recalculate ``map_list`` in case ``optimizer_mapping`` changes
        during training.
        """

        if self.map_list is None:
            self.map_list = self.map_optimizers(layer_list)

        for opt in self.map_list:
            opt.optimize(self.map_list[opt], epoch)

    def get_description(self):
        desc = {'type': self.__class__.__name__}
        for key in self.optimizer_mapping:
            desc[key] = self.optimizer_mapping[key].get_description()
        return desc
<|endoftext|>