<commit_before># -*- coding: utf-8 -*-

###############################################################################
# Copyright 2014 OCLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################

"""Class that represents a Web Services Key (WSKey)

Stores the WSKey parameters and methods for HMAC Hashing and requesting access tokens.

"""

import copy
import base64
import hashlib
import hmac
import math
import random
import string
import time

from .authcode import AuthCode
from .accesstoken import AccessToken

import six

AUTHORIZATION_SERVER = 'https://authn.sd00.worldcat.org/oauth2'
SIGNATURE_URL = 'https://www.oclc.org/wskey'

class InvalidObject(Exception):
    """Custom exception - invalid object was passed to class"""

    def __init__(self, message):
        self.message = message

class InvalidParameter(Exception):
    """Custom exception - invalid parameter was passed to class"""

    def __init__(self, message):
        self.message = message

class Wskey(object):
    """Web Services Key object

    Class variables
    authorization_server string   url of the authorization server
    valid_options        dict     list of valid options that can be passed to this class
    key                  string   the clientID (public) portion of the WSKey
    secret               string   the secret (private) portion of the WSKey
    redirect_uri         string   the url of the web app, for example https://localhost:8000/auth/
    services             list     the web services associated with the WSKey, for example ['WorldCatMetadataAPI']
    debug_time_stamp     string   if not None, then overrides the calculated timestamp. Used for unit tests.
    debug_nonce          string   if not None, then overrides the calculated nonce. Used for unit tests.
    body_hash            string   set to None - current implementation of OCLC's OAuth2 does not use body hashing
    auth_params          dict     custom list of authentication parameters - used for some specialized APIs
    user                 object   a user object associated with the key. See user.py in the authliboclc library folder.
    """

    authorization_server = AUTHORIZATION_SERVER
    valid_options = ['redirect_uri', 'services']
    key = None
    secret = None
    redirect_uri = None
    services = None
    debug_time_stamp = None
    debug_nonce = None
    body_hash = None
    auth_params = None
    user = None

    def __init__(self, key, secret, options=None):
        """Constructor.

        Args:
          key: string, the clientID (public) portion of the WSKey
          secret: string, the secret (private) portion of the WSKey
          options: dict
                   - redirect_uri: string, the url that the client authenticates from
                                  ie, https://localhost:8000/auth/
                   - services: list, the services associated with the key
                               ie, ['WMS_ACQ','WorldCatMetadataAPI']
                               * note that including 'refresh_token' as a service causes access token requests
                                 to return a refresh token with the access token.
        """
        if key is None or secret is None:
            raise InvalidObject('A valid key and secret are required to construct a WSKey.')
        elif options == '':
            raise InvalidObject('Options must be sent as a dictionary object.')

        self.key = key
        self.secret = secret

        """If options are included, they must include a redirect_uri and one or more services."""
        if options:
            if 'redirect_uri' in options:
                if options['redirect_uri'] is None:
                    raise InvalidParameter('redirect_uri must contain a value.')
                else:
                    scheme = six.moves.urllib.parse.urlparse(options['redirect_uri']).scheme
                    if not scheme == 'http' and not scheme == 'https':
                        raise InvalidParameter('Invalid redirect_uri. Must begin with http:// or https://')

            if 'services' not in options:
                raise InvalidParameter('Missing service option.')
            elif not options['services']:
                raise InvalidParameter('A list containing at least one service is required.')

            for key, value in six.iteritems(options):
                if key in Wskey.valid_options:
                    setattr(self, key, value)

    def get_login_url(self, authenticating_institution_id=None, context_institution_id=None):
        """Creates a login url.

        Args:
            authenticating_institution_id: string, the institution which the user authenticates against
            context_institution_id: string, the institution which the user will make requests against

        Returns:
            string, the login URL to be used to authenticate the user
        """
        if authenticating_institution_id is None:
            raise InvalidParameter('You must pass an authenticating institution ID')
        if context_institution_id is None:
            raise InvalidParameter('You must pass a context institution ID')

        authCode = AuthCode(
            authorization_server=self.authorization_server,
            client_id=self.key,
            authenticating_institution_id=authenticating_institution_id,
            context_institution_id=context_institution_id,
            redirect_uri=self.redirect_uri,
            scopes=self.services
        )

        return authCode.get_login_url()

    def get_access_token_with_auth_code(self, code=None, authenticating_institution_id=None,
                                        context_institution_id=None):
        """Retrieves an Access Token using an Authentication Code

        Args:
            code: string, the authentication code returned after the user authenticates
            authenticating_institution_id: string, the institution the user authenticates against
            context_institution_id: string, the institution that the requests will be made against

        Returns:
            object, an access token
        """

        if not code:
            raise InvalidParameter('You must pass a code')
        if not authenticating_institution_id:
            raise InvalidParameter('You must pass an authenticating_institution_id')
        if not context_institution_id:
            raise InvalidParameter('You must pass a context_institution_id')

        accessToken = AccessToken(
            authorization_server=self.authorization_server,
            grant_type='authorization_code',
            options={
                'code': code,
                'authenticating_institution_id': authenticating_institution_id,
                'context_institution_id': context_institution_id,
                'redirect_uri': self.redirect_uri
            }
        )

        accessToken.create(wskey=self, user=None)

        return accessToken

    def get_access_token_with_client_credentials(self, authenticating_institution_id=None, context_institution_id=None):
        """Retrieves an Access Token using a Client Credentials Grant

        Args:
            authenticating_institution_id: string, the institution the user authenticates against
            context_institution_id: string, the institution that the requests will be made against

        Returns:
            object, an access token
        """

        if not authenticating_institution_id:
            raise InvalidParameter('You must pass an authenticating_institution_id')
        if not context_institution_id:
            raise InvalidParameter('You must pass a context_institution_id')
        if not self.services or self.services == ['']:
            raise InvalidParameter('You must set at least one service on the Wskey')

        accessToken = AccessToken(
            authorization_server=self.authorization_server,
            grant_type='client_credentials',
            options={
                'authenticating_institution_id': authenticating_institution_id,
                'context_institution_id': context_institution_id,
                'scope': self.services
            }
        )

        accessToken.create(wskey=self)

        return accessToken

    def get_hmac_signature(self, method=None, request_url=None, options=None):
        """Signs a url with an HMAC signature and builds an Authorization header

        Args:
            method: string, GET, POST, PUT, DELETE, etc.
            request_url: string, the url to be signed
            options: dict
                     - user: object, a user object
                     - auth_params: dict, various key value pairs to be added to the authorization header. For example,
                                   userid and password. Depends on the API and its specialized needs.

        Returns:
            authorization_header: string, the Authorization header to be added to the request.
        """

        if not self.secret:
            raise InvalidParameter('You must construct a WSKey with a secret to build an HMAC Signature.')
        if not method:
            raise InvalidParameter('You must pass an HTTP Method to build an HMAC Signature.')
        if not request_url:
            raise InvalidParameter('You must pass a valid request URL to build an HMAC Signature.')

        if options is not None:
            for key, value in six.iteritems(options):
                setattr(self, key, value)

        timestamp = self.debug_time_stamp
        if not timestamp:
            timestamp = str(int(time.time()))

        nonce = self.debug_nonce
        if not nonce:
            nonce = str(hex(int(math.floor(random.random() * 4026531839 + 268435456))))

        signature = self.sign_request(
            method=method,
            request_url=request_url,
            timestamp=timestamp,
            nonce=nonce
        )

        q = '"'
        qc = '",'

        authorization_header = ("http://www.worldcat.org/wskey/v2/hmac/v1 " +
                                "clientID=" + q + self.key + qc +
                                "timestamp=" + q + timestamp + qc +
                                "nonce=" + q + nonce + qc +
                                "signature=" + q + signature)

        if self.user is not None or self.auth_params is not None:
            authorization_header += (qc + self.add_auth_params(self.user, self.auth_params))
        else:
            authorization_header += q

        return authorization_header

    def sign_request(self, method, request_url, timestamp, nonce):
        """Requests a normalized request and hashes it

        Args:
            method: string, GET, POST, etc.
            request_url: string, the URL to be hashed
            timestamp: string, POSIX time
            nonce: string, a random 32 bit integer expressed in hexadecimal format

        Returns:
            A base 64 encoded SHA 256 HMAC hash
        """
        normalized_request = self.normalize_request(
            method=method,
            request_url=request_url,
            timestamp=timestamp,
            nonce=nonce
        )

        digest = hmac.new(self.secret.encode('utf-8'),
                          msg=normalized_request.encode('utf-8'),
                          digestmod=hashlib.sha256).digest()
        return str(base64.b64encode(digest).decode())

    def normalize_request(self, method, request_url, timestamp, nonce):
        """Prepares a normalized request for hashing

         Args:
            method: string, GET, POST, etc.
            request_url: string, the URL to be hashed
            timestamp: string, POSIX time
            nonce: string, a random 32 bit integer expressed in hexadecimal format

        Returns:
            normalized_request: string, the normalized request to be hashed
        """
        signature_url = SIGNATURE_URL
        parsed_signature_url = six.moves.urllib.parse.urlparse(six.moves.urllib.parse.unquote(signature_url))
        parsed_request_url = six.moves.urllib.parse.urlparse(six.moves.urllib.parse.unquote(request_url))

        host = str(parsed_signature_url.netloc)

        if parsed_signature_url.port is not None:
            port = str(parsed_signature_url.port)
        else:
            if str(parsed_signature_url.scheme) == 'http':
                port = '80'
            elif str(parsed_signature_url.scheme) == 'https':
                port = '443'

        path = str(parsed_signature_url.path)

        """ OCLC's OAuth implementation does not currently use body hashing, so this should always be ''."""
        body_hash = ''
        if self.body_hash is not None:
            body_hash = self.body_hash

        """The base normalized request."""
        normalized_request = (self.key + '\n' +
                              timestamp + '\n' +
                              nonce + '\n' +
                              body_hash + '\n' +
                              method + '\n' +
                              host + '\n' +
                              port + '\n' +
                              path + '\n')

        """Add the request parameters to the normalized request."""
        parameters = {}
        if parsed_request_url.query:
            for param in parsed_request_url.query.split('&'):
                key = (param.split('='))[0]
                value = (param.split('='))[1]
                parameters[key] = value

        """URL encode normalized request per OAuth 2 Official Specification."""
        for key in sorted(parameters):
            nameAndValue = six.moves.urllib.parse.urlencode({key: parameters[key]})
            nameAndValue = nameAndValue.replace('+', '%20')
            nameAndValue = nameAndValue.replace('*', '%2A')
            nameAndValue = nameAndValue.replace('%7E', '~')
            normalized_request += nameAndValue + '\n'

        return normalized_request

    def add_auth_params(self, user, auth_params):
        """Adds users custom authentication parameters, if any, to the Normalized request

        Args:
            user: object, a user object
            auth_params: dict, a list of parameters to add the custom parameters to

        Returns:
            authValuePairs: dict, the auth_params with any custom parameters added to them
        """
        authValuePairs = ''
        combinedParams = copy.copy(auth_params)

        if not combinedParams:
            combinedParams = {}

        if user is not None:
            combinedParams['principalID'] = user.principal_id
            combinedParams['principalIDNS'] = user.principal_idns

        counter = 0

        for key in sorted(combinedParams):

            authValuePairs += key + '=' + '"' + combinedParams[key]
            counter += 1
            if counter == len(combinedParams):
                authValuePairs += '"'
            else:
                authValuePairs += '",'

        return authValuePairs

    def __str__(self):

        return string.Template("""key:              $key
secret:           $secret
redirect_uri:     $redirect_uri
services:         $services
debug_time_stamp: $debug_time_stamp
debug_nonce:      $debug_nonce
body_hash:        $body_hash
auth_params:      $auth_params
user:
$user""").substitute({
            'key': self.key,
            'secret': self.secret,
            'redirect_uri': self.redirect_uri,
            'services': self.services,
            'debug_time_stamp': self.debug_time_stamp,
            'debug_nonce': self.debug_nonce,
            'body_hash': self.body_hash,
            'auth_params': self.auth_params,
            'user': self.user
        })<commit_msg>added user param to get_access_token_with_client_credentials method<commit_after># -*- coding: utf-8 -*-

###############################################################################
# Copyright 2014 OCLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################

"""Class that represents a Web Services Key (WSKey)

Stores the WSKey parameters and methods for HMAC Hashing and requesting access tokens.

"""

import copy
import base64
import hashlib
import hmac
import math
import random
import string
import time

from .authcode import AuthCode
from .accesstoken import AccessToken

import six

AUTHORIZATION_SERVER = 'https://authn.sd00.worldcat.org/oauth2'
SIGNATURE_URL = 'https://www.oclc.org/wskey'

class InvalidObject(Exception):
    """Custom exception - invalid object was passed to class"""

    def __init__(self, message):
        self.message = message

class InvalidParameter(Exception):
    """Custom exception - invalid parameter was passed to class"""

    def __init__(self, message):
        self.message = message

class Wskey(object):
    """Web Services Key object

    Class variables
    authorization_server string   url of the authorization server
    valid_options        dict     list of valid options that can be passed to this class
    key                  string   the clientID (public) portion of the WSKey
    secret               string   the secret (private) portion of the WSKey
    redirect_uri         string   the url of the web app, for example https://localhost:8000/auth/
    services             list     the web services associated with the WSKey, for example ['WorldCatMetadataAPI']
    debug_time_stamp     string   if not None, then overrides the calculated timestamp. Used for unit tests.
    debug_nonce          string   if not None, then overrides the calculated nonce. Used for unit tests.
    body_hash            string   set to None - current implementation of OCLC's OAuth2 does not use body hashing
    auth_params          dict     custom list of authentication parameters - used for some specialized APIs
    user                 object   a user object associated with the key. See user.py in the authliboclc library folder.
    """

    authorization_server = AUTHORIZATION_SERVER
    valid_options = ['redirect_uri', 'services']
    key = None
    secret = None
    redirect_uri = None
    services = None
    debug_time_stamp = None
    debug_nonce = None
    body_hash = None
    auth_params = None
    user = None

    def __init__(self, key, secret, options=None):
        """Constructor.

        Args:
          key: string, the clientID (public) portion of the WSKey
          secret: string, the secret (private) portion of the WSKey
          options: dict
                   - redirect_uri: string, the url that the client authenticates from
                                  ie, https://localhost:8000/auth/
                   - services: list, the services associated with the key
                               ie, ['WMS_ACQ','WorldCatMetadataAPI']
                               * note that including 'refresh_token' as a service causes access token requests
                                 to return a refresh token with the access token.
        """
        if key is None or secret is None:
            raise InvalidObject('A valid key and secret are required to construct a WSKey.')
        elif options == '':
            raise InvalidObject('Options must be sent as a dictionary object.')

        self.key = key
        self.secret = secret

        """If options are included, they must include a redirect_uri and one or more services."""
        if options:
            if 'redirect_uri' in options:
                if options['redirect_uri'] is None:
                    raise InvalidParameter('redirect_uri must contain a value.')
                else:
                    scheme = six.moves.urllib.parse.urlparse(options['redirect_uri']).scheme
                    if not scheme == 'http' and not scheme == 'https':
                        raise InvalidParameter('Invalid redirect_uri. Must begin with http:// or https://')

            if 'services' not in options:
                raise InvalidParameter('Missing service option.')
            elif not options['services']:
                raise InvalidParameter('A list containing at least one service is required.')

            for key, value in six.iteritems(options):
                if key in Wskey.valid_options:
                    setattr(self, key, value)

    def get_login_url(self, authenticating_institution_id=None, context_institution_id=None):
        """Creates a login url.

        Args:
            authenticating_institution_id: string, the institution which the user authenticates against
            context_institution_id: string, the institution which the user will make requests against

        Returns:
            string, the login URL to be used to authenticate the user
        """
        if authenticating_institution_id is None:
            raise InvalidParameter('You must pass an authenticating institution ID')
        if context_institution_id is None:
            raise InvalidParameter('You must pass a context institution ID')

        authCode = AuthCode(
            authorization_server=self.authorization_server,
            client_id=self.key,
            authenticating_institution_id=authenticating_institution_id,
            context_institution_id=context_institution_id,
            redirect_uri=self.redirect_uri,
            scopes=self.services
        )

        return authCode.get_login_url()

    def get_access_token_with_auth_code(self, code=None, authenticating_institution_id=None,
                                        context_institution_id=None):
        """Retrieves an Access Token using an Authentication Code

        Args:
            code: string, the authentication code returned after the user authenticates
            authenticating_institution_id: string, the institution the user authenticates against
            context_institution_id: string, the institution that the requests will be made against

        Returns:
            object, an access token
        """

        if not code:
            raise InvalidParameter('You must pass a code')
        if not authenticating_institution_id:
            raise InvalidParameter('You must pass an authenticating_institution_id')
        if not context_institution_id:
            raise InvalidParameter('You must pass a context_institution_id')

        accessToken = AccessToken(
            authorization_server=self.authorization_server,
            grant_type='authorization_code',
            options={
                'code': code,
                'authenticating_institution_id': authenticating_institution_id,
                'context_institution_id': context_institution_id,
                'redirect_uri': self.redirect_uri
            }
        )

        accessToken.create(wskey=self, user=None)

        return accessToken

    def get_access_token_with_client_credentials(self, authenticating_institution_id=None, context_institution_id=None, user=None):
        """Retrieves an Access Token using a Client Credentials Grant

        Args:
            authenticating_institution_id: string, the institution the user authenticates against
            context_institution_id: string, the institution that the requests will be made against
            user: object, a user object associated with the key. See user.py in the authliboclc library folder.

        Returns:
            object, an access token
        """

        if not authenticating_institution_id:
            raise InvalidParameter('You must pass an authenticating_institution_id')
        if not context_institution_id:
            raise InvalidParameter('You must pass a context_institution_id')
        if not self.services or self.services == ['']:
            raise InvalidParameter('You must set at least one service on the Wskey')

        accessToken = AccessToken(
            authorization_server=self.authorization_server,
            grant_type='client_credentials',
            options={
                'authenticating_institution_id': authenticating_institution_id,
                'context_institution_id': context_institution_id,
                'scope': self.services
            }
        )

        accessToken.create(wskey=self, user=user)

        return accessToken

    def get_hmac_signature(self, method=None, request_url=None, options=None):
        """Signs a url with an HMAC signature and builds an Authorization header

        Args:
            method: string, GET, POST, PUT, DELETE, etc.
            request_url: string, the url to be signed
            options: dict
                     - user: object, a user object
                     - auth_params: dict, various key value pairs to be added to the authorization header. For example,
                                   userid and password. Depends on the API and its specialized needs.

        Returns:
            authorization_header: string, the Authorization header to be added to the request.
        """

        if not self.secret:
            raise InvalidParameter('You must construct a WSKey with a secret to build an HMAC Signature.')
        if not method:
            raise InvalidParameter('You must pass an HTTP Method to build an HMAC Signature.')
        if not request_url:
            raise InvalidParameter('You must pass a valid request URL to build an HMAC Signature.')

        if options is not None:
            for key, value in six.iteritems(options):
                setattr(self, key, value)

        timestamp = self.debug_time_stamp
        if not timestamp:
            timestamp = str(int(time.time()))

        nonce = self.debug_nonce
        if not nonce:
            nonce = str(hex(int(math.floor(random.random() * 4026531839 + 268435456))))

        signature = self.sign_request(
            method=method,
            request_url=request_url,
            timestamp=timestamp,
            nonce=nonce
        )

        q = '"'
        qc = '",'

        authorization_header = ("http://www.worldcat.org/wskey/v2/hmac/v1 " +
                                "clientID=" + q + self.key + qc +
                                "timestamp=" + q + timestamp + qc +
                                "nonce=" + q + nonce + qc +
                                "signature=" + q + signature)

        if self.user is not None or self.auth_params is not None:
            authorization_header += (qc + self.add_auth_params(self.user, self.auth_params))
        else:
            authorization_header += q

        return authorization_header

    def sign_request(self, method, request_url, timestamp, nonce):
        """Requests a normalized request and hashes it

        Args:
            method: string, GET, POST, etc.
            request_url: string, the URL to be hashed
            timestamp: string, POSIX time
            nonce: string, a random 32 bit integer expressed in hexadecimal format

        Returns:
            A base 64 encoded SHA 256 HMAC hash
        """
        normalized_request = self.normalize_request(
            method=method,
            request_url=request_url,
            timestamp=timestamp,
            nonce=nonce
        )

        digest = hmac.new(self.secret.encode('utf-8'),
                          msg=normalized_request.encode('utf-8'),
                          digestmod=hashlib.sha256).digest()
        return str(base64.b64encode(digest).decode())

    def normalize_request(self, method, request_url, timestamp, nonce):
        """Prepares a normalized request for hashing

         Args:
            method: string, GET, POST, etc.
            request_url: string, the URL to be hashed
            timestamp: string, POSIX time
            nonce: string, a random 32 bit integer expressed in hexadecimal format

        Returns:
            normalized_request: string, the normalized request to be hashed
        """
        signature_url = SIGNATURE_URL
        parsed_signature_url = six.moves.urllib.parse.urlparse(six.moves.urllib.parse.unquote(signature_url))
        parsed_request_url = six.moves.urllib.parse.urlparse(six.moves.urllib.parse.unquote(request_url))

        host = str(parsed_signature_url.netloc)

        if parsed_signature_url.port is not None:
            port = str(parsed_signature_url.port)
        else:
            if str(parsed_signature_url.scheme) == 'http':
                port = '80'
            elif str(parsed_signature_url.scheme) == 'https':
                port = '443'

        path = str(parsed_signature_url.path)

        """ OCLC's OAuth implementation does not currently use body hashing, so this should always be ''."""
        body_hash = ''
        if self.body_hash is not None:
            body_hash = self.body_hash

        """The base normalized request."""
        normalized_request = (self.key + '\n' +
                              timestamp + '\n' +
                              nonce + '\n' +
                              body_hash + '\n' +
                              method + '\n' +
                              host + '\n' +
                              port + '\n' +
                              path + '\n')

        """Add the request parameters to the normalized request."""
        parameters = {}
        if parsed_request_url.query:
            for param in parsed_request_url.query.split('&'):
                key = (param.split('='))[0]
                value = (param.split('='))[1]
                parameters[key] = value

        """URL encode normalized request per OAuth 2 Official Specification."""
        for key in sorted(parameters):
            nameAndValue = six.moves.urllib.parse.urlencode({key: parameters[key]})
            nameAndValue = nameAndValue.replace('+', '%20')
            nameAndValue = nameAndValue.replace('*', '%2A')
            nameAndValue = nameAndValue.replace('%7E', '~')
            normalized_request += nameAndValue + '\n'

        return normalized_request

    def add_auth_params(self, user, auth_params):
        """Adds users custom authentication parameters, if any, to the Normalized request

        Args:
            user: object, a user object
            auth_params: dict, a list of parameters to add the custom parameters to

        Returns:
            authValuePairs: dict, the auth_params with any custom parameters added to them
        """
        authValuePairs = ''
        combinedParams = copy.copy(auth_params)

        if not combinedParams:
            combinedParams = {}

        if user is not None:
            combinedParams['principalID'] = user.principal_id
            combinedParams['principalIDNS'] = user.principal_idns

        counter = 0

        for key in sorted(combinedParams):

            authValuePairs += key + '=' + '"' + combinedParams[key]
            counter += 1
            if counter == len(combinedParams):
                authValuePairs += '"'
            else:
                authValuePairs += '",'

        return authValuePairs

    def __str__(self):

        return string.Template("""key:              $key
secret:           $secret
redirect_uri:     $redirect_uri
services:         $services
debug_time_stamp: $debug_time_stamp
debug_nonce:      $debug_nonce
body_hash:        $body_hash
auth_params:      $auth_params
user:
$user""").substitute({
            'key': self.key,
            'secret': self.secret,
            'redirect_uri': self.redirect_uri,
            'services': self.services,
            'debug_time_stamp': self.debug_time_stamp,
            'debug_nonce': self.debug_nonce,
            'body_hash': self.body_hash,
            'auth_params': self.auth_params,
            'user': self.user
        })<|endoftext|>


<commit_before>"""CONWAY'S GAME OF LIFE, EXTENDED"""

##################
# TYPES OF CELLS #
##################

"""
1. Live cell - a live member of the game
2. Active cell - a cell that was once a live cell or another type of cell but has died/vanished
3. Inactive cell - the default state of a cell, has not been touched by other cells
4. Fire cell - a cell that extinguishes life
5. Water cell - a cell that extinguishes fire, but may cause drowning
6. Virus cell - a cell that moves on its own, spreads, and slowly causes life to perish
7. Diseased cell - a live cell that has been infected, and will infect other cells
8. Hospital cell - a cell that cures and eliminates Viruses and the Diseased
9. Technology cell - a cell that creates money
10. Money cell - a cell that attracts live cells
"""
#########
# RULES #
#########

# Standard
"""
1. SOLITUDE: Any live cell with fewer than two live neighbours dies, as if caused by under-population.
2. STASIS: Any live cell with two or three live neighbours lives on to the next generation.
3. OVERPOPULATION: Any live cell with more than three live neighbours dies
4. REPRODUCTION: Any dead cell with exactly three live neighbours becomes a live cell
"""

# Extra
"""
- BURNOUT: Fire that is not directly adjacent to other fire cells will extinguish the next tick
- SPREADING FIRE: Active/inactive cells neighboring 4 or more fire cells will become a fire cell
- SCORCH: Any live cell directly adjacent (no corners) to fire will die the next tick
- FIREFIGHTER: Any fire cell neighboring two or more live cells but not directly adjacent will turn into a water cell
- ABLAZE: Hospital cells neighboring 4 or more fire cells will be burnt down (-> active)
- EXTINGUISH: Fire cells neighboring water cells will be extinguished (-> active)
- DROWNING: Live cells neighboring 6 or more water cells will drown
- EVAPORATION: Water that is not directly adjacent to other water cells will evaporate the next tick
"""
<commit_msg>Cells and worlds<commit_after>"""CONWAY'S GAME OF LIFE, EXTENDED"""

##################
# TYPES OF CELLS #
##################

"""
1. Live cell - a live member of the game
2. Active cell - a cell that was once a live cell or another type of cell but has died/vanished
3. Inactive cell - the default state of a cell, has not been touched by other cells
4. Fire cell - a cell that extinguishes life
5. Water cell - a cell that extinguishes fire, but may cause drowning
6. Virus cell - a cell that moves on its own, spreads, and slowly causes life to perish
7. Diseased cell - a live cell that has been infected, and will infect other cells
8. Hospital cell - a cell that cures and eliminates Viruses and the Diseased
9. Technology cell - a cell that creates money
10. Money cell - a cell that attracts live cells
"""
#########
# RULES #
#########

# Standard
"""
1. SOLITUDE: Any live cell with fewer than two live neighbours dies, as if caused by under-population.
2. STASIS: Any live cell with two or three live neighbours lives on to the next generation.
3. OVERPOPULATION: Any live cell with more than three live neighbours dies
4. REPRODUCTION: Any dead cell with exactly three live neighbours becomes a live cell
"""

# Extra
"""
- BURNOUT: Fire that is not directly adjacent to other fire cells will extinguish the next tick
- SPREADING FIRE: Active/inactive cells neighboring 4 or more fire cells will become a fire cell
- SCORCH: Any live cell directly adjacent (no corners) to fire will die the next tick
- FIREFIGHTER: Any fire cell neighboring two or more live cells but not directly adjacent will turn into a water cell
- ABLAZE: Hospital cells neighboring 4 or more fire cells will be burnt down (-> active)
- EXTINGUISH: Fire cells neighboring water cells will be extinguished (-> active)
- DROWNING: Live cells neighboring 6 or more water cells will drown
- EVAPORATION: Water that is not directly adjacent to other water cells will evaporate the next tick
"""
import pygame

class Cell(object):
    """A cell in the game."""
    def __init__(self, world, row, col, identity="inactive"):
        """Initializes a cell in a certain world object at position (row, col)"""
        self.world = world
        self.row = row
        self.col = col
        self.identity = identity

    def __repr__(self):
        return "Cell(%s, %s, %s, '%s')" %(repr(self.world), self.row, self.col, self.identity)

    def __str__(self):
        return self.identity

class World(object):
    """The board that represents the game world."""
    def __init__(self, name, rows, cols):
        self.name = name
        self.rows = rows
        self.cols = cols
        # self.cells = {(i, j) : Cell(self, i, j) for i in range(rows) for j in range(cols)}
        self.cells = [[Cell(self, i, j) for j in range(cols)] for i in range(rows)]

    def __repr__(self):
        return "World('%s', %s, %s)" %(self.name, self.rows, self.cols)

    def __str__(self):
        return "World %s [%s x %s]" %(self.name, self.rows, self.cols)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import tensorflow as tf

class OthelloModel(object):
    def __init__(self):
        self.x = tf.placeholder(tf.float32, shape=[None,64], name="board")
        self.y = tf.placeholder(tf.float32, shape=[None,1], name="y")

        w1 = tf.Variable(tf.truncated_normal([8,8], mean=0.0, stddev=0.2), name="weight1")
        b1 = tf.Variable(tf.zeros(1), name="bias1")
        w2 = tf.Variable(tf.truncated_normal([4,1], mean=0.0, stddev=0.2), name="weight2")
        b2 = tf.Variable(tf.zeros(1), name="bias2")
        self.params = [w1, b1, w2, b2]

        # model
        w11 = tf.reshape(w1, (64,1))
        w12 = tf.reshape(tf.transpose(w1), (64,1))
        reversed_w1 = tf.reverse(w1, [True, True])
        w13 = tf.reshape(reversed_w1, (64,1))
        w14 = tf.reshape(tf.transpose(reversed_w1), (64,1))
        w = tf.concat(1, [w11,w12,w13,w14])
        h = tf.nn.relu(tf.matmul(self.x, w) + b1)

        self.logits = tf.matmul(h, w2) + b2
        self.prediction = tf.nn.sigmoid(self.logits)

        # cost
        cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(self.logits, self.y))

        # optimizer
        optimizer = tf.train.AdagradOptimizer(learning_rate=0.2)
        # optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)
        self.opt_op = optimizer.minimize(cost)

        self.init_op = tf.initialize_all_variables()
        self.sess = tf.Session()
        self.saver = tf.train.Saver(self.params)

    def init_params(self):
        self.sess.run(self.init_op)

    @property
    def parameters(self):
        return self.params
    @property
    def session(self):
        return self.sess

    def save_params(self, file_path):
        self.saver.save(self.sess, file_path)

    def restore_params(self, file_path):
        self.saver.restore(self.sess, file_path)

    def predict(self, x):
        return self.sess.run(self.prediction, feed_dict={self.x: x})

    def update_param(self, x, y):
        self.sess.run([self.opt_op], feed_dict={self.x: x, self.y: y})

from othello import Board
from ai import Agent, ScoreEvaluator, AlphaBeta
import numpy as np

class TDLProcessor(object):
    def __init__(self, role, model_path, batch_size=50):
        self.role = role
        self.model = OthelloModel()
        self.model.init_params()
        self.model_path = model_path
        self.batch_size = batch_size
        self.game_processed = 0
        self.x = []
        self.y = []

    def __call__(self, player, i, j, result, board):
        if player == self.role:
            with board.flip2(i,j,player):
                target = 0.0
                if self.role == Board.BLACK and result > 0:
                    target = 1.0
                if self.role == Board.WHITE and result < 0:
                    target = 1.0
                self.x.append(board.board2)
                self.y.append(target)
            if len(self.x) >= self.batch_size:
                xx = np.vstack(self.x)
                yy = np.vstack(self.y)
                self.model.update_param(xx, yy)
                self.x = []
                self.y = []

    def after_one_game(self):
        self.game_processed += 1
        if self.game_processed % 10000 == 0:
            print "# game processed: ", self.game_processed

    def after_one_replay(self):
        self.save_model()

    def save_model(self, model_path=None):
        if model_path is None:
            model_path = self.model_path
        self.model.save_params(model_path)

class TDLAgent(Agent):
    def __init__(self, role, update=False, alpha=0.1, epsilon=0.1, model_file=None, depth=3):
        super(TDLAgent, self).__init__(role)
        self._model = OthelloModel()
        self._model.init_params()
        if model_file is not None:
            self._model.restore_params(model_file)
        self._alpha = alpha
        self._update = update
        self._epsilon = epsilon
        self._prev_state = None
        self._depth = depth
        score_evaluator = ScoreEvaluator(role)
        self._searcher = AlphaBeta(score_evaluator, depth)

    def _epsilon_greedy(self, pos, val):
        r = np.random.rand()
        if r < 1 - self._epsilon:
            idx = np.argmax(val)
        else:
            idx = np.random.randint(low=0, high=len(pos))
        return pos[idx], val[idx]

    def save_model(self, model_path):
        if self._update:
            self._model.save_params(model_path)

    def _is_first_step(self, board):
        return board.blanks >= 59

    def _update_param(self, v):
        prev_val = self._model.predict(self._prev_state)[0][0]
        target = (1.0-self._alpha) * prev_val + self._alpha * v
        self._model.update_param(self._prev_state, [[target]])

    def play(self, board):
        if self._is_first_step(board):
            self._prev_state = None

        if board.blanks <= self._depth:
            s, p = self._searcher.search(board, self.role)
            if s >=32:
                v = 1.0
            else:
                v = 0.0
        else:
            pos = board.feasible_pos(self.role)
            next_vals = []
            for i,j in pos:
                with board.flip2(i, j, self.role):
                    if board.is_terminal_state():
                        if board.wins(self.role):
                            next_vals.append(1.0)
                        else:
                            next_vals.append(0.0)
                    else:
                        next_vals.append(self._model.predict(board.board2)[0][0])
            p,v = self._epsilon_greedy(pos, next_vals)

        if self._update:
            if  self._prev_state is not None:
                self._update_param(v)
            with board.flip2(p[0], p[1], self.role):
                self._prev_state = board.board2.copy()

        return p

    def tell_result(self, board):
        if self._update and self._prev_state is not None:
            v = 0.0
            if board.wins(self.role):
                v = 1.0
            self._update_param(v)
<commit_msg>use cnn model<commit_after># -*- coding: utf-8 -*-

import tensorflow as tf

class OthelloModel(object):
    def __init__(self):
        self.x = tf.placeholder(tf.float32, shape=[None,8,8], name="board")
        self.y = tf.placeholder(tf.float32, shape=[None,1], name="y")

        self._params = []

        # model
        with tf.name_scope("conv1"):
            output_channels1 = 32
            kernel = tf.Variable(tf.truncated_normal([3,3,1,output_channels1], stddev=0.1))
            bias = tf.Variable(tf.zeros([32]))
            x = tf.reshape(self.x, [-1,8,8,1])
            h = tf.nn.relu(tf.nn.conv2d(x, kernel, strides=[1,1,1,1], padding="SAME") + bias)
            p1 = tf.nn.max_pool(h, ksize=[1,2,2,1], strides=[1,2,2,1], padding="SAME")
            self._params.extend([kernel, bias])
        with tf.name_scope("conv2"):
            output_channels2 = 64
            kernel = tf.Variable(tf.truncated_normal([3,3,output_channels1,output_channels2], stddev=0.1))
            bias = tf.Variable(tf.zeros([64]))
            h = tf.nn.relu(tf.nn.conv2d(p1, kernel, strides=[1,1,1,1], padding="SAME")+ bias)
            p2 = tf.nn.max_pool(h, ksize=[1,2,2,1], strides=[1,2,2,1], padding="SAME")
            self._params.extend([kernel, bias])
        with tf.name_scope("full_connect1"):
            full1_size = 8
            conv2_size = 8 // 4 * 8 // 4 * output_channels2
            x = tf.reshape(p2, [-1, conv2_size])
            f1_weights = tf.Variable(tf.truncated_normal([conv2_size, full1_size], stddev=0.1))
            f1_bias = tf.Variable(tf.truncated_normal([full1_size], stddev=0.1))
            f1 = tf.nn.relu(tf.matmul(x,f1_weights) + f1_bias)
            self._params.extend([f1_weights, f1_bias])
        with tf.name_scope("full_connect2"):
            f2_weights = tf.Variable(tf.truncated_normal([full1_size, 1], stddev=0.1))
            f2_bias = tf.Variable(tf.truncated_normal([1], stddev=0.1))
            logits = tf.matmul(f1,f2_weights) + f2_bias
            self._params.extend([f2_weights, f2_bias])

        # prediction
        self._prediction = tf.nn.sigmoid(logits)

        # cost
        regularizers = (tf.nn.l2_loss(f1_weights) + tf.nn.l2_loss(f1_bias) +
                  tf.nn.l2_loss(f2_weights) + tf.nn.l2_loss(f2_bias))
        self._cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits, self.y)) + 5e-4 * regularizers

        # optimizer
        optimizer = tf.train.AdagradOptimizer(learning_rate=0.05)
        self._train_step = optimizer.minimize(self._cost)

        # init params
        init_op = tf.initialize_all_variables()
        self._sess = tf.Session()
        self._sess.run(init_op)

        # saver
        self._saver = tf.train.Saver(self._params)

    @property
    def parameters(self):
        return self._params
    @property
    def session(self):
        return self._sess

    def save_params(self, file_path):
        self._saver.save(self._sess, file_path)

    def restore_params(self, file_path):
        self._saver.restore(self._sess, file_path)

    def predict(self, x):
        return self._sess.run(self._prediction, feed_dict={self.x: x})

    def update_param(self, x, y):
        self._sess.run([self._train_step], feed_dict={self.x: x, self.y: y})

from othello import Board
from ai import Agent, ScoreEvaluator, AlphaBeta
import numpy as np

class TDLProcessor(object):
    def __init__(self, role, model_path, batch_size=50):
        self.role = role
        self.model = OthelloModel()
        self.model_path = model_path
        self.batch_size = batch_size
        self.game_processed = 0
        self.x = []
        self.y = []

    def __call__(self, player, i, j, result, board):
        if player == self.role:
            with board.flip2(i,j,player):
                target = 0.0
                if self.role == Board.BLACK and result > 0:
                    target = 1.0
                if self.role == Board.WHITE and result < 0:
                    target = 1.0
                self.x.append(board.board.copy())
                self.y.append([target])
            if len(self.x) >= self.batch_size:
                xx = np.stack(self.x)
                yy = np.stack(self.y)
                self.model.update_param(xx, yy)
                self.x = []
                self.y = []

    def after_one_game(self):
        self.game_processed += 1
        if self.game_processed % 10000 == 0:
            print "# game processed: ", self.game_processed

    def after_one_replay(self):
        self.save_model()

    def save_model(self, model_path=None):
        if model_path is None:
            model_path = self.model_path
        self.model.save_params(model_path)

class TDLAgent(Agent):
    def __init__(self, role, update=False, alpha=0.1, epsilon=0.1, model_file=None, depth=3):
        super(TDLAgent, self).__init__(role)
        self._model = OthelloModel()
        if model_file is not None:
            self._model.restore_params(model_file)
        self._alpha = alpha
        self._update = update
        self._epsilon = epsilon
        self._prev_state = None
        self._depth = depth
        score_evaluator = ScoreEvaluator(role)
        self._searcher = AlphaBeta(score_evaluator, depth)

    def _epsilon_greedy(self, pos, val):
        r = np.random.rand()
        if r < 1 - self._epsilon:
            idx = np.argmax(val)
        else:
            idx = np.random.randint(low=0, high=len(pos))
        return pos[idx], val[idx]

    def save_model(self, model_path):
        if self._update:
            self._model.save_params(model_path)

    def _is_first_step(self, board):
        return board.blanks >= 59

    def _update_param(self, v):
        prev_val = self._model.predict(self._prev_state)[0][0]
        target = (1.0-self._alpha) * prev_val + self._alpha * v
        self._model.update_param(self._prev_state, [[target]])

    def play(self, board):
        if self._is_first_step(board):
            self._prev_state = None

        if board.blanks <= self._depth:
            s, p = self._searcher.search(board, self.role)
            if s >=32:
                v = 1.0
            else:
                v = 0.0
        else:
            pos = board.feasible_pos(self.role)
            next_vals = []
            for i,j in pos:
                with board.flip2(i, j, self.role):
                    if board.is_terminal_state():
                        if board.wins(self.role):
                            next_vals.append(1.0)
                        else:
                            next_vals.append(0.0)
                    else:
                        next_vals.append(self._model.predict(board.board)[0][0])
            p,v = self._epsilon_greedy(pos, next_vals)

        if self._update:
            if  self._prev_state is not None:
                self._update_param(v)
            with board.flip2(p[0], p[1], self.role):
                self._prev_state = board.board.copy()

        return p

    def tell_result(self, board):
        if self._update and self._prev_state is not None:
            v = 0.0
            if board.wins(self.role):
                v = 1.0
            self._update_param(v)
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2009 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Starts an interactive shell with statistic helpers.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
]

import cPickle
import datetime
import operator
import sys
import time

import interactive

def dateFetch(queryGen, last=None, batchSize=100):
  """Iterator that yields an entity in batches.

  Args:
    queryGen: should return a Query object
    last: used to .filter() for last_modified_on
    batchSize: how many entities to retrieve in one datastore call

  Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook).
  """

  from google.appengine.ext import db

   # AppEngine will not fetch more than 1000 results
  batchSize = min(batchSize,1000)

  query = None
  done = False
  count = 0

  while not done:
    print count
    query = queryGen()
    query.order('last_modified_on')
    if last:
      query.filter("last_modified_on > ", last)
    results = query.fetch(batchSize)
    for result in results:
      count += 1
      yield result
    if batchSize > len(results):
      done = True
    else:
      last = results[-1].last_modified_on

def addKey(target, fieldname):
  """Adds the key of the specified field.
  """

  result = target.copy()
  result['%s_key' % fieldname] = target[fieldname].key().name()
  return result

def getEntities(model):
  """Returns all users as dictionary.
  """

  def wrapped():
    gen = lambda: model.all()
    it = interactive.deepFetch(gen)

    entities = [(i.key().name(), i) for i in it]
    return dict(entities)

  return wrapped

def getProps(last=None):
  """Returns all proposals as a list of dictionaries.
  """

  key_order = [
      'link_id', 'scope_path', 'title', 'abstract', 'content',
      'additional_info', '_mentor', 'possible_mentors', 'score',
      'status', '_org', 'created_on', 'last_modified_on']

  from soc.models.student_proposal import StudentProposal

  gen = lambda: StudentProposal.all()

  it = dateFetch(gen, last)

  proposals = [(i.key().name(), i.toDict(key_order)) for i in it]
  if proposals:
    last = i.last_modified_on # last modified entity
  else:
    last = datetime.datetime.now()

  return dict(proposals), last

def orgStats(target, orgs):
  """Retrieves org stats.
  """

  from soc.logic import dicts

  orgs = [(v.key(), v) for k, v in orgs.iteritems()]
  orgs = dict(orgs)

  grouped = dicts.groupby(target.values(), '_org')

  grouped = [(orgs[k], v) for k, v in grouped.iteritems()]
  popularity = [(k.link_id, len(v)) for k, v in grouped]

  return dict(grouped), dict(popularity)

def countStudentsWithProposals():
  """Retrieves number of Students who have submitted at least one Student Proposal.
  """

  proposals = getStudentProposals()
  students = {}

  for proposal_key in proposals.keys():
    students[proposals[proposal_key].scope_path] = True

  return len(students)

def printPopularity(popularity):
  """Prints the popularity for the specified proposals.
  """

  g = operator.itemgetter(1)

  for item in sorted(popularity.iteritems(), key=g, reverse=True):
    print "%s: %d" % item

def saveValues(values, saver):
  """Saves the specified popularities.
  """

  import logging
  from google.appengine.ext import db

  from soc.models.organization import Organization

  def txn(key, value):
    org = Organization.get_by_key_name(key)
    saver(org, value)
    org.put()

  for key, value in sorted(values.iteritems()):
    print key
    db.run_in_transaction_custom_retries(10, txn, key, value)

  print "done"

def addFollower(follower, proposals, add_public=True, add_private=True):
  """Adds a user as follower to the specified proposals.

  Args:
    follower: the User to add as follower
    proposals: a list with the StudnetProposals that should be subscribed to
    add_public: whether the user is subscribed to public updates
    add_private: whether the user should be subscribed to private updates
  """

  from soc.models.review_follower import ReviewFollower

  result = []

  for i in proposals:
     properties = {
       'user': follower,
       'link_id': follower.link_id,
       'scope': i,
       'scope_path': i.key().name(),
       'key_name': '%s/%s' % (i.key().name(), follower.link_id),
       'subscribed_public': add_public,
       'subscribed_private': add_private,
     }

     entity = ReviewFollower(**properties)
     result.append(entity)

  return result

def convertProposals(org):
  """Convert all proposals for the specified organization.

  Args:
    org: the organization for which all proposals will be converted
  """

  from soc.logic.models.student_proposal import logic as proposal_logic
  from soc.logic.models.student_project import logic as project_logic

  proposals = proposal_logic.getProposalsToBeAcceptedForOrg(org)

  for proposal in proposals:
    fields = {
        'link_id': 't%i' % (int(time.time()*100)),
        'scope_path': proposal.org.key().id_or_name(),
        'scope': proposal.org,
        'program': proposal.program,
        'student': proposal.scope,
        'title': proposal.title,
        'abstract': proposal.abstract,
        'mentor': proposal.mentor,
        }

    project = project_logic.updateOrCreateFromFields(fields, silent=True)

    fields = {
        'status':'accepted',
        }

    proposal_logic.updateEntityProperties(proposal, fields, silent=True)

  fields = {
      'status': ['new', 'pending'],
      'org': org,
      }

  querygen = lambda: proposal_logic.getQueryForFields(fields)
  proposals = [i for i in interactive.deepFetch(querygen)]

  print "rejecting %d proposals" % len(proposals)

  fields = {
      'status': 'rejected',
      }

  for proposal in proposals:
    proposal_logic.updateEntityProperties(proposal, fields, silent=True)

def startSpam():
  from soc.logic.models.job import logic as job_logic
  from soc.logic.models.priority_group import logic as priority_logic
  from soc.logic.models.program import logic as program_logic

  program_entity = program_logic.getFromKeyName('google/gsoc2009')

  priority_group = priority_logic.getGroup(priority_logic.EMAIL)
  job_fields = {
      'priority_group': priority_group,
      'task_name': 'setupStudentProposalMailing',
      'key_data': [program_entity.key()]}

  job_logic.updateOrCreateFromFields(job_fields)

def deleteEntities(model, step_size=25):
  """Deletes all entities of the specified type
  """

  print "Deleting..."
  count = 0

  while True:
    entities = model.all().fetch(step_size)

    if not entities:
      break

    for entity in entities:
      entity.delete()

    count += step_size

    print "deleted %d entities" % count

  print "Done"

def loadPickle(name):
  """Loads a pickle.
  """

  f = open(name + '.dat')
  return cPickle.load(f)

def dumpPickle(target, name):
  """Dumps a pickle.
  """

  f = open("%s.dat" % name, 'w')
  cPickle.dump(target, f)

def main(args):
  """Main routine.
  """

  interactive.setup()

  from soc.models.organization import Organization
  from soc.models.user import User
  from soc.models.student import Student
  from soc.models.mentor import Mentor
  from soc.models.org_admin import OrgAdmin
  from soc.models.job import Job
  from soc.models.student_proposal import StudentProposal
  from soc.models.student_project import StudentProject

  def slotSaver(org, value):
    org.slots = value
  def popSaver(org, value):
    org.nr_applications = value
  def rawSaver(org, value):
    org.slots_calculated = value

  context = {
      'load': loadPickle,
      'dump': dumpPickle,
      'orgStats': orgStats,
      'printPopularity': printPopularity,
      'saveValues': saveValues,
      'getEntities': getEntities,
      'deleteEntities': deleteEntities,
      'getOrgs': getEntities(Organization),
      'getUsers': getEntities(User),
      'getStudents': getEntities(Student),
      'getMentors': getEntities(Mentor),
      'getOrgAdmins': getEntities(OrgAdmin),
      'getProps': getProps,
      'countStudentsWithProposals': countStudentsWithProposals,
      'convertProposals': convertProposals,
      'addFollower': addFollower,
      'Organization': Organization,
      'Job': Job,
      'User': User,
      'Student': Student,
      'Mentor': Mentor,
      'OrgAdmin': OrgAdmin,
      'StudentProject': StudentProject,
      'StudentProposal': StudentProposal,
      'slotSaver': slotSaver,
      'popSaver': popSaver,
      'rawSaver': rawSaver,
      'startSpam': startSpam,
  }

  interactive.remote(args, context)

if __name__ == '__main__':
  if len(sys.argv) < 2:
    print "Usage: %s app_id [host]" % (sys.argv[0],)
    sys.exit(1)

  main(sys.argv[1:])
<commit_msg>Set batch size to 10<commit_after>#!/usr/bin/python2.5
#
# Copyright 2009 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Starts an interactive shell with statistic helpers.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
]

import cPickle
import datetime
import operator
import sys
import time

import interactive

def dateFetch(queryGen, last=None, batchSize=100):
  """Iterator that yields an entity in batches.

  Args:
    queryGen: should return a Query object
    last: used to .filter() for last_modified_on
    batchSize: how many entities to retrieve in one datastore call

  Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook).
  """

  from google.appengine.ext import db

   # AppEngine will not fetch more than 1000 results
  batchSize = min(batchSize,1000)

  query = None
  done = False
  count = 0

  while not done:
    print count
    query = queryGen()
    query.order('last_modified_on')
    if last:
      query.filter("last_modified_on > ", last)
    results = query.fetch(batchSize)
    for result in results:
      count += 1
      yield result
    if batchSize > len(results):
      done = True
    else:
      last = results[-1].last_modified_on

def addKey(target, fieldname):
  """Adds the key of the specified field.
  """

  result = target.copy()
  result['%s_key' % fieldname] = target[fieldname].key().name()
  return result

def getEntities(model):
  """Returns all users as dictionary.
  """

  def wrapped():
    gen = lambda: model.all()
    it = interactive.deepFetch(gen)

    entities = [(i.key().name(), i) for i in it]
    return dict(entities)

  return wrapped

def getProps(last=None):
  """Returns all proposals as a list of dictionaries.
  """

  key_order = [
      'link_id', 'scope_path', 'title', 'abstract', 'content',
      'additional_info', '_mentor', 'possible_mentors', 'score',
      'status', '_org', 'created_on', 'last_modified_on']

  from soc.models.student_proposal import StudentProposal

  gen = lambda: StudentProposal.all()

  it = dateFetch(gen, last)

  proposals = [(i.key().name(), i.toDict(key_order)) for i in it]
  if proposals:
    last = i.last_modified_on # last modified entity
  else:
    last = datetime.datetime.now()

  return dict(proposals), last

def orgStats(target, orgs):
  """Retrieves org stats.
  """

  from soc.logic import dicts

  orgs = [(v.key(), v) for k, v in orgs.iteritems()]
  orgs = dict(orgs)

  grouped = dicts.groupby(target.values(), '_org')

  grouped = [(orgs[k], v) for k, v in grouped.iteritems()]
  popularity = [(k.link_id, len(v)) for k, v in grouped]

  return dict(grouped), dict(popularity)

def countStudentsWithProposals():
  """Retrieves number of Students who have submitted at least one Student Proposal.
  """

  proposals = getStudentProposals()
  students = {}

  for proposal_key in proposals.keys():
    students[proposals[proposal_key].scope_path] = True

  return len(students)

def printPopularity(popularity):
  """Prints the popularity for the specified proposals.
  """

  g = operator.itemgetter(1)

  for item in sorted(popularity.iteritems(), key=g, reverse=True):
    print "%s: %d" % item

def saveValues(values, saver):
  """Saves the specified popularities.
  """

  import logging
  from google.appengine.ext import db

  from soc.models.organization import Organization

  def txn(key, value):
    org = Organization.get_by_key_name(key)
    saver(org, value)
    org.put()

  for key, value in sorted(values.iteritems()):
    print key
    db.run_in_transaction_custom_retries(10, txn, key, value)

  print "done"

def addFollower(follower, proposals, add_public=True, add_private=True):
  """Adds a user as follower to the specified proposals.

  Args:
    follower: the User to add as follower
    proposals: a list with the StudnetProposals that should be subscribed to
    add_public: whether the user is subscribed to public updates
    add_private: whether the user should be subscribed to private updates
  """

  from soc.models.review_follower import ReviewFollower

  result = []

  for i in proposals:
     properties = {
       'user': follower,
       'link_id': follower.link_id,
       'scope': i,
       'scope_path': i.key().name(),
       'key_name': '%s/%s' % (i.key().name(), follower.link_id),
       'subscribed_public': add_public,
       'subscribed_private': add_private,
     }

     entity = ReviewFollower(**properties)
     result.append(entity)

  return result

def convertProposals(org):
  """Convert all proposals for the specified organization.

  Args:
    org: the organization for which all proposals will be converted
  """

  from soc.logic.models.student_proposal import logic as proposal_logic
  from soc.logic.models.student_project import logic as project_logic

  proposals = proposal_logic.getProposalsToBeAcceptedForOrg(org)

  print "accepting %d proposals, with %d slots" % (len(proposals), org.slots)

  for proposal in proposals:
    fields = {
        'link_id': 't%i' % (int(time.time()*100)),
        'scope_path': proposal.org.key().id_or_name(),
        'scope': proposal.org,
        'program': proposal.program,
        'student': proposal.scope,
        'title': proposal.title,
        'abstract': proposal.abstract,
        'mentor': proposal.mentor,
        }

    project = project_logic.updateOrCreateFromFields(fields, silent=True)

    fields = {
        'status':'accepted',
        }

    proposal_logic.updateEntityProperties(proposal, fields, silent=True)

  fields = {
      'status': ['new', 'pending'],
      'org': org,
      }

  querygen = lambda: proposal_logic.getQueryForFields(fields)
  proposals = [i for i in interactive.deepFetch(querygen, batchSize=10)]

  print "rejecting %d proposals" % len(proposals)

  fields = {
      'status': 'rejected',
      }

  for proposal in proposals:
    proposal_logic.updateEntityProperties(proposal, fields, silent=True)

def startSpam():
  from soc.logic.models.job import logic as job_logic
  from soc.logic.models.priority_group import logic as priority_logic
  from soc.logic.models.program import logic as program_logic

  program_entity = program_logic.getFromKeyName('google/gsoc2009')

  priority_group = priority_logic.getGroup(priority_logic.EMAIL)
  job_fields = {
      'priority_group': priority_group,
      'task_name': 'setupStudentProposalMailing',
      'key_data': [program_entity.key()]}

  job_logic.updateOrCreateFromFields(job_fields)

def deleteEntities(model, step_size=25):
  """Deletes all entities of the specified type
  """

  print "Deleting..."
  count = 0

  while True:
    entities = model.all().fetch(step_size)

    if not entities:
      break

    for entity in entities:
      entity.delete()

    count += step_size

    print "deleted %d entities" % count

  print "Done"

def loadPickle(name):
  """Loads a pickle.
  """

  f = open(name + '.dat')
  return cPickle.load(f)

def dumpPickle(target, name):
  """Dumps a pickle.
  """

  f = open("%s.dat" % name, 'w')
  cPickle.dump(target, f)

def main(args):
  """Main routine.
  """

  interactive.setup()

  from soc.models.organization import Organization
  from soc.models.user import User
  from soc.models.student import Student
  from soc.models.mentor import Mentor
  from soc.models.org_admin import OrgAdmin
  from soc.models.job import Job
  from soc.models.student_proposal import StudentProposal
  from soc.models.student_project import StudentProject

  def slotSaver(org, value):
    org.slots = value
  def popSaver(org, value):
    org.nr_applications = value
  def rawSaver(org, value):
    org.slots_calculated = value

  context = {
      'load': loadPickle,
      'dump': dumpPickle,
      'orgStats': orgStats,
      'printPopularity': printPopularity,
      'saveValues': saveValues,
      'getEntities': getEntities,
      'deleteEntities': deleteEntities,
      'getOrgs': getEntities(Organization),
      'getUsers': getEntities(User),
      'getStudents': getEntities(Student),
      'getMentors': getEntities(Mentor),
      'getOrgAdmins': getEntities(OrgAdmin),
      'getProps': getProps,
      'countStudentsWithProposals': countStudentsWithProposals,
      'convertProposals': convertProposals,
      'addFollower': addFollower,
      'Organization': Organization,
      'Job': Job,
      'User': User,
      'Student': Student,
      'Mentor': Mentor,
      'OrgAdmin': OrgAdmin,
      'StudentProject': StudentProject,
      'StudentProposal': StudentProposal,
      'slotSaver': slotSaver,
      'popSaver': popSaver,
      'rawSaver': rawSaver,
      'startSpam': startSpam,
  }

  interactive.remote(args, context)

if __name__ == '__main__':
  if len(sys.argv) < 2:
    print "Usage: %s app_id [host]" % (sys.argv[0],)
    sys.exit(1)

  main(sys.argv[1:])
<|endoftext|>


<commit_before>import copy
import numpy as np
import theano
from inspect import getargspec

from autodiff.context import Context
from autodiff.compat import OrderedDict
import autodiff.utils as utils

class Symbolic(object):
    def __init__(self, pyfn):
        # make deepcopy of pyfn because we might change its defaults
        self._pyfn = copy.deepcopy(pyfn)

        self.s_vars = OrderedDict()
        self.s_args = OrderedDict()
        self.s_results = OrderedDict()
        self._cache = dict()

        # replace integer defaults in pyfn to avoid problems
        if self._pyfn.func_defaults:
            new_defaults = tuple([np.int_(d)
                                  if type(d) is int and -5 <= d <= 256 else d
                                  for d in self._pyfn.func_defaults])
            self._pyfn.func_defaults = new_defaults

    @property
    def pyfn(self):
        return self._pyfn

    @property
    def cache(self):
        return self._cache

    def trace(self, args=None, kwargs=None):
        """
        Given args and kwargs, call the Python function and get its
        symbolic representation.

        Three ordered dictionaries are maintained:
            self.s_vars   : {id(obj) : sym_var}
                            Contains all symbolic variables traced during
                            function execution, indexed by the id of the
                            corresponding Python object.

            self.s_args   : {arg name : sym_var}
                            Contains all symbolic inputs to the function,
                            indexed by the argument name.

            self.s_results : {id(obj) : sym_var}
                            Contains the symbolic function results, indexed by
                            the id of the corresponding Python object.

            The dictionaries are cleared every time this method is run.

        """
        if args is None:
            args = ()
        elif not isinstance(args, tuple):
            raise TypeError('args must be a tuple')

        if kwargs is None:
            kwargs = dict()
        elif not isinstance(kwargs, dict):
            raise TypeError('args must be a dict')

        # check for small ints and collections
        def check(name, i):
            # Check argument i (with name 'name') for small ints or
            # collections.  If it is a small int, replace it with a numpy int.
            # If a collection, raise a helpful error.
            #
            # This is required because:
            #     1. PyAutoDiff can not shadow CPython ints because they are
            #     cached objects that reuse ids.
            #
            #     2. Theano functions can not accept arguments that are
            #     collections.
            if type(i) is int and -5 <= i <= 256:
                return np.int_(i)
            elif isinstance(i, (list, tuple, dict)):
                raise TypeError('Function arguments can not be '
                                'containers (received {0} for '
                                'argument \'{1}\').'.format(i, name))
            else:
                return i

        argspec = getargspec(self.pyfn)
        tmp_args = tuple(check(n, a) for n, a in zip(argspec.args, args))
        args = tmp_args + tuple(check(argspec.varargs, a)
                                for a in args[len(argspec.args):])
        kwargs = OrderedDict((k, check(k, v)) for k, v in kwargs.iteritems())

        # clear symbolic dictionaries
        self.s_vars.clear()
        self.s_args.clear()
        self.s_results.clear()

        # trace the function
        c = Context()
        results = c.call(self.pyfn, args, kwargs)

        # collect symbolic variables in s_vars
        self.s_vars.update(c.svars)

        # collect symbolic arguments in s_args
        callargs = utils.orderedcallargs(self.pyfn, *args, **kwargs)

        for name, arg in callargs.iteritems():

            # collect variable args
            if name == argspec.varargs:
                self.s_args[name] = ()
                for i, a in enumerate(arg):
                    try:
                        self.s_args[name] += (self.s_vars[id(a)],)
                        self.s_args[name][-1].name = '{0}_{1}'.format(name, i)
                    except KeyError:
                        raise KeyError('Unable to trace item {0} of variable '
                                       'argument \'{1}\'.'.format(i, name))
                    except:
                        raise

            # collect variable kwargs
            elif name == argspec.keywords:
                for n, a in arg.iteritems():
                    try:
                        self.s_args[n] = self.s_vars[id(a)]
                        self.s_args[n].name = n
                    except KeyError:
                        raise KeyError('Unable to trace argument '
                                       '\'{0}\'.'.format(n))
                    except:
                        raise

            # collect positional args
            else:
                try:
                    self.s_args[name] = self.s_vars[id(arg)]
                    self.s_args[name].name = name
                except KeyError:
                    raise KeyError('Unable to trace argument '
                                   '\'{0}\'.'.format(name))
                except:
                    raise

        # collect symbolic results in s_results
        if not isinstance(results, tuple):
            results = [results]
        for i, r in enumerate(results):
            try:
                self.s_results[id(r)] = self.s_vars[id(r)]
            except KeyError:
                raise KeyError('Unable to trace result #{0} '
                               '(indexed from 1).'.format(i + 1))
            except:
                raise

    def get_theano_vars(self, args, kwargs):
        """
        Returns a dict containing inputs, outputs and givens corresponding to
        the Theano version of the pyfn.

        """

        argspec = getargspec(self.pyfn)

        # trace the function
        self.trace(args, kwargs)

        defaults = dict()
        if argspec.defaults:
            defaults.update(zip(reversed(argspec.args),
                                reversed(argspec.defaults)))

        # ========== collect inputs, givens
        inputs, inputvars, givens = OrderedDict(), OrderedDict(), OrderedDict()
        for name, arg in self.s_args.iteritems():
            if name != argspec.varargs:
                givens[arg] = arg.type(name=arg.name)
                inputs[name] = theano.Param(givens[arg],
                                            default=defaults.get(name, None),
                                            name=name)
            else:
                for i, a in enumerate(arg):
                    givens[a] = a.type(name='{0}_{1}'.format(name, i))
                    inputs[i] = givens[a]

        # ========== collect outputs
        outputs = OrderedDict(enumerate(self.s_results.values()))

        theano_vars = {'inputs': inputs,
                       'outputs': outputs,
                       'givens': givens}

        return theano_vars

    def get_symbolic_arg(self, x):
        """
        Retrieve the symbolic version of x.

        x : python object or string

        If x is a string, it is matched to the names of the function arguments.
        If x is an object, it must have been traced by the Symbolic class.
        If x is a small int, raises an error.
        """
        if type(x) is int and -5 <= x <= 256:
            raise ValueError('Small integer arguments can not be traced '
                             'selectively. Either recast or redesign your '
                             'function.')
        elif isinstance(x, basestring):
            if x in self.s_args:
                return self.s_args[x]
            else:
                raise ValueError('Argument {0} has not been traced.'.format(x))
        else:
            if id(x) in self.s_vars:
                return self.s_vars[id(x)]
            else:
                raise ValueError('Object {0} has not been traced.'.format(x))

class Function(Symbolic):

    def __call__(self, *args, **kwargs):
        return self.call(*args, **kwargs)

    def compile_function(self, args, kwargs):
        theano_vars = self.get_theano_vars(args, kwargs)

        inputs = theano_vars['inputs'].values()
        outputs = theano_vars['outputs'].values()
        givens = theano_vars['givens']

        if len(outputs) == 1:
            outputs = outputs[0]

        # compile function
        fn = theano.function(inputs=inputs,
                             outputs=outputs,
                             givens=givens,
                             on_unused_input='ignore')

        # store in cache corresponding to the number of positional inputs
        argspec = getargspec(self.pyfn)
        callargs = utils.orderedcallargs(self.pyfn, *args, **kwargs)
        self.cache[len(callargs.get(argspec.varargs, ()))] = fn

        return fn

    def call(self, *args, **kwargs):
        argspec = getargspec(self.pyfn)
        callargs = utils.orderedcallargs(self.pyfn, *args, **kwargs)

        # try to retrieve function from cache; otherwise compile
        fn = self.cache.get(len(callargs.get(argspec.varargs, ())),
                            self.compile_function(args, kwargs))

        pos_args = [callargs[arg] for arg in argspec.args]
        pos_args.extend(callargs.get(argspec.varargs, ()))
        kw_args = callargs.get(argspec.keywords, {})

        return fn(*pos_args, **kw_args)
<commit_msg>add Symbolic call<commit_after>import copy
import numpy as np
import theano
from inspect import getargspec

from autodiff.context import Context
from autodiff.compat import OrderedDict
import autodiff.utils as utils

class Symbolic(object):
    def __init__(self, pyfn):
        # make deepcopy of pyfn because we might change its defaults
        self._pyfn = copy.deepcopy(pyfn)

        self.s_vars = OrderedDict()
        self.s_args = OrderedDict()
        self.s_results = OrderedDict()
        self._cache = dict()

        # replace integer defaults in pyfn to avoid problems
        if self._pyfn.func_defaults:
            new_defaults = tuple([np.int_(d)
                                  if type(d) is int and -5 <= d <= 256 else d
                                  for d in self._pyfn.func_defaults])

            self._pyfn.func_defaults = new_defaults

    def __call__(self, *args, **kwargs):
        return self.get_theano_vars(args, kwargs)

    @property
    def pyfn(self):
        return self._pyfn

    @property
    def cache(self):
        return self._cache

    def trace(self, args=None, kwargs=None):
        """
        Given args and kwargs, call the Python function and get its
        symbolic representation.

        Three ordered dictionaries are maintained:
            self.s_vars   : {id(obj) : sym_var}
                            Contains all symbolic variables traced during
                            function execution, indexed by the id of the
                            corresponding Python object.

            self.s_args   : {arg name : sym_var}
                            Contains all symbolic inputs to the function,
                            indexed by the argument name.

            self.s_results : {id(obj) : sym_var}
                            Contains the symbolic function results, indexed by
                            the id of the corresponding Python object.

            The dictionaries are cleared every time this method is run.

        """
        if args is None:
            args = ()
        elif not isinstance(args, tuple):
            raise TypeError('args must be a tuple')

        if kwargs is None:
            kwargs = dict()
        elif not isinstance(kwargs, dict):
            raise TypeError('args must be a dict')

        # check for small ints and collections
        def check(name, i):
            # Check argument i (with name 'name') for small ints or
            # collections.  If it is a small int, replace it with a numpy int.
            # If a collection, raise a helpful error.
            #
            # This is required because:
            #     1. PyAutoDiff can not shadow CPython ints because they are
            #     cached objects that reuse ids.
            #
            #     2. Theano functions can not accept arguments that are
            #     collections.
            if type(i) is int and -5 <= i <= 256:
                return np.int_(i)
            elif isinstance(i, (list, tuple, dict)):
                raise TypeError('Function arguments can not be '
                                'containers (received {0} for '
                                'argument \'{1}\').'.format(i, name))
            else:
                return i

        argspec = getargspec(self.pyfn)
        tmp_args = tuple(check(n, a) for n, a in zip(argspec.args, args))
        args = tmp_args + tuple(check(argspec.varargs, a)
                                for a in args[len(argspec.args):])
        kwargs = OrderedDict((k, check(k, v)) for k, v in kwargs.iteritems())

        # clear symbolic dictionaries
        self.s_vars.clear()
        self.s_args.clear()
        self.s_results.clear()

        # trace the function
        c = Context()
        results = c.call(self.pyfn, args, kwargs)

        # collect symbolic variables in s_vars
        self.s_vars.update(c.svars)

        # collect symbolic arguments in s_args
        callargs = utils.orderedcallargs(self.pyfn, *args, **kwargs)

        for name, arg in callargs.iteritems():

            # collect variable args
            if name == argspec.varargs:
                self.s_args[name] = ()
                for i, a in enumerate(arg):
                    try:
                        self.s_args[name] += (self.s_vars[id(a)],)
                        self.s_args[name][-1].name = '{0}_{1}'.format(name, i)
                    except KeyError:
                        raise KeyError('Unable to trace item {0} of variable '
                                       'argument \'{1}\'.'.format(i, name))
                    except:
                        raise

            # collect variable kwargs
            elif name == argspec.keywords:
                for n, a in arg.iteritems():
                    try:
                        self.s_args[n] = self.s_vars[id(a)]
                        self.s_args[n].name = n
                    except KeyError:
                        raise KeyError('Unable to trace argument '
                                       '\'{0}\'.'.format(n))
                    except:
                        raise

            # collect positional args
            else:
                try:
                    self.s_args[name] = self.s_vars[id(arg)]
                    self.s_args[name].name = name
                except KeyError:
                    raise KeyError('Unable to trace argument '
                                   '\'{0}\'.'.format(name))
                except:
                    raise

        # collect symbolic results in s_results
        if not isinstance(results, tuple):
            results = [results]
        for i, r in enumerate(results):
            try:
                self.s_results[id(r)] = self.s_vars[id(r)]
            except KeyError:
                raise KeyError('Unable to trace result #{0} '
                               '(indexed from 1).'.format(i + 1))
            except:
                raise

    def get_theano_vars(self, args, kwargs):
        """
        Returns a dict containing inputs, outputs and givens corresponding to
        the Theano version of the pyfn.

        """

        argspec = getargspec(self.pyfn)

        # trace the function
        self.trace(args, kwargs)

        defaults = dict()
        if argspec.defaults:
            defaults.update(zip(reversed(argspec.args),
                                reversed(argspec.defaults)))

        # ========== collect inputs, givens
        inputs, inputvars, givens = OrderedDict(), OrderedDict(), OrderedDict()
        for name, arg in self.s_args.iteritems():
            if name != argspec.varargs:
                givens[arg] = arg.type(name=arg.name)
                inputs[name] = theano.Param(givens[arg],
                                            default=defaults.get(name, None),
                                            name=name)
            else:
                for i, a in enumerate(arg):
                    givens[a] = a.type(name='{0}_{1}'.format(name, i))
                    inputs[i] = givens[a]

        # ========== collect outputs
        outputs = OrderedDict(enumerate(self.s_results.values()))

        theano_vars = {'inputs': inputs,
                       'outputs': outputs,
                       'givens': givens}

        return theano_vars

    def get_symbolic_arg(self, x):
        """
        Retrieve the symbolic version of x.

        x : python object or string

        If x is a string, it is matched to the names of the function arguments.
        If x is an object, it must have been traced by the Symbolic class.
        If x is a small int, raises an error.
        """
        if type(x) is int and -5 <= x <= 256:
            raise ValueError('Small integer arguments can not be traced '
                             'selectively. Either recast or redesign your '
                             'function.')
        elif isinstance(x, basestring):
            if x in self.s_args:
                return self.s_args[x]
            else:
                raise ValueError('Argument {0} has not been traced.'.format(x))
        else:
            if id(x) in self.s_vars:
                return self.s_vars[id(x)]
            else:
                raise ValueError('Object {0} has not been traced.'.format(x))

class Function(Symbolic):

    def __call__(self, *args, **kwargs):
        return self.call(*args, **kwargs)

    def compile_function(self, args, kwargs):
        theano_vars = self.get_theano_vars(args, kwargs)

        inputs = theano_vars['inputs'].values()
        outputs = theano_vars['outputs'].values()
        givens = theano_vars['givens']

        if len(outputs) == 1:
            outputs = outputs[0]

        # compile function
        fn = theano.function(inputs=inputs,
                             outputs=outputs,
                             givens=givens,
                             on_unused_input='ignore')

        # store in cache corresponding to the number of positional inputs
        argspec = getargspec(self.pyfn)
        callargs = utils.orderedcallargs(self.pyfn, *args, **kwargs)
        self.cache[len(callargs.get(argspec.varargs, ()))] = fn

        return fn

    def call(self, *args, **kwargs):
        argspec = getargspec(self.pyfn)
        callargs = utils.orderedcallargs(self.pyfn, *args, **kwargs)

        # try to retrieve function from cache; otherwise compile
        fn = self.cache.get(len(callargs.get(argspec.varargs, ())),
                            self.compile_function(args, kwargs))

        pos_args = [callargs[arg] for arg in argspec.args]
        pos_args.extend(callargs.get(argspec.varargs, ()))
        kw_args = callargs.get(argspec.keywords, {})

        return fn(*pos_args, **kw_args)
<|endoftext|>