<commit_before>import shutil
from os import symlink, path

# ngutil
from .template import _NGUtilTemplates
from .common import _NGUtilCommon, _NGUtilSELinux

class _NGUtilSite(_NGUtilCommon):
    """
    Class object for managing NGINX site definitions.
    """
    def __init__(self):
        super(_NGUtilSite, self).__init__()

        # Template manager
        self.template = _NGUtilTemplates()

        # SSL attributes
        self.ssl  = {
            'enable': False,
            'cert': None,
            'key': None
        }

        # Site properties
        self.properties = {}

        # Required / optional params
        self.params = {
            'required': ['fqdn'],
            'optional': ['default_doc', 'activate', 'ssl', 'ssl_cert', 'ssl_key']
        }

    def _setup_ssl_certs(self):
        """
        Setup SSL certificates if enabled.
        """
        if self.ssl['enable']:
            _sitename = self.properties['fqdn']
            _cert_dst = '/srv/www/{0}/ssl/{1}.crt'.format(_sitename)
            _key_dst  = '/srv/www/{0}/ssl/{1}.key'.format(_sitename)

            # Deploy the SSL certificate
            shutil.copy(self.properties['ssl_cert'], _cert_dst)
            self.feedback.success('Deployed SSL certificate -> {0}'.format(_cert_dst))

            # Deploy the SSL key
            shutil.copy(self.properties['ssl_key'], _key_dst)
            self.feedback.success('Deployed SSL key -> {0}'.format(_key_dst))

    def _create_dirs(self):
        """
        Create any required site directories.
        """

        # Site base directory
        site_base = '/srv/www/{0}'.format(self.properties['fqdn'])

        for dir in [
            '{0}/logs/php-fpm',
            '{0}/public',
            '{0}/session'
        ]:
            self.mkdir(dir.format(site_base))

        # If using SSL
        if self.ssl['enable']:
            self.mkdir('{0}/ssl'.format(site_base))

        # Setup directory permissions
        self.run_command('chown -R root:nginx {0}'.format(site_base))
        self.run_command('chmod -R 750 {0}'.format(site_base))

        # Set SELinux context
        self.selinux.chcon(site_base, 'unconfined_u:object_r:httpd_sys_content_t:s0', recursive=True)

    def _activate_site(self):
        """
        Optionally activate the site.
        """
        if self.properties['activate']:
            if not path.isfile(self.site_config['enabled']):
                symlink(self.site_config['available'], self.site_config['enabled'])
                self.feedback.success('Activated site -> {0}'.format(self.site_config['enabled']))
            else:
                self.feedback.info('Site already activated -> {0}'.format(self.site_config['enabled']))

    def _generate_nginx_config(self):
        """
        Generate NGINX config files for the new site.
        """

        # Setup the template
        self.template.setup(('NG_HTTPS' if self.ssl['enable'] else 'NG_HTTP'), self.site_config['available'])

        # Update placeholder variables
        self.template.setvars({
            'SITENAME': self.properties['fqdn'],
            'DEFAULTDOC': self.properties.get('default_doc', 'index.php')
        })

        # Deploy the configuration
        self.template.deploy(overwrite=True)

    def define(self, params):
        """
        Define attributes for creating a new site definition.
        """

        # SELinux manager
        self.selinux  = _NGUtilSELinux()

        # Make sure all required arguments are set
        for k in self.params['required']:
            if not params.get(k):
                self.die('Missing required argument \'{0}\''.format(k))

        # If using SSL
        if params.get('ssl'):
            self.ssl['enable'] = True

            # Make sure the certificate and key are set
            for k in ['ssl_cert', 'ssl_key']:
                if not params.get(k):
                    self.die('Missing required argument \'{0}\', must supply to enable SSL'.format(k))

                # Make sure the file exists
                ssl_file = params.get(k)
                if not os.path.isfile(ssl_file):
                    self.die('Could not locate \'{0}\' file \'{1}\''.format(k, ssl_file))

            # Set the SSL key and certificate
            self.ssl['key'] = params.get('ssl_key')
            self.ssl['cert'] = params.args.get('ssl_cert')
            self.feedback.info('Using SSL for site \'{0}\': cert={1}, key={2}'.format(params['fqdn'], params['ssl_cert'], param['ssl_key']))
        else:
            self.feedback.info('Not using SSL for site \'{0}\''.format(params['fqdn']))

        # Site configuration
        self.site_config = {
            'available': '/etc/nginx/sites-available/{0}.conf'.format(params['fqdn']),
            'enabled': '/etc/nginx/sites-enabled/{0}.conf'.format(params['fqdn'])
        }

        # Make sure the site isn't already defined
        if path.isfile(self.site_config['available']):
            self.die('Site \'{0}\' already defined in \'{1}\''.format(params['fqdn'], self.site_config['available']))

        # Merge with site properties
        self.properties = params

    def create(self):
        """
        Create the site definition.
        """
        self._create_dirs()
        self._setup_ssl_certs()
        self._generate_nginx_config()

        # Site created
        self.feedback.block([
            '\'{0}\' configuration complete!'.format(self.properties['fqdn']),
            'You will need to restart NGINX/PHP-FPM to make the site available:',
            '> service nginx restart',
            '> service php-fpm restart'
        ], 'COMPLETE')
<commit_msg>Set default_doc to index.php if None<commit_after>import shutil
from os import symlink, path

# ngutil
from .template import _NGUtilTemplates
from .common import _NGUtilCommon, _NGUtilSELinux

class _NGUtilSite(_NGUtilCommon):
    """
    Class object for managing NGINX site definitions.
    """
    def __init__(self):
        super(_NGUtilSite, self).__init__()

        # Template manager
        self.template = _NGUtilTemplates()

        # SSL attributes
        self.ssl  = {
            'enable': False,
            'cert': None,
            'key': None
        }

        # Site properties
        self.properties = {}

        # Required / optional params
        self.params = {
            'required': ['fqdn'],
            'optional': ['default_doc', 'activate', 'ssl', 'ssl_cert', 'ssl_key']
        }

    def _setup_ssl_certs(self):
        """
        Setup SSL certificates if enabled.
        """
        if self.ssl['enable']:
            _sitename = self.properties['fqdn']
            _cert_dst = '/srv/www/{0}/ssl/{1}.crt'.format(_sitename)
            _key_dst  = '/srv/www/{0}/ssl/{1}.key'.format(_sitename)

            # Deploy the SSL certificate
            shutil.copy(self.properties['ssl_cert'], _cert_dst)
            self.feedback.success('Deployed SSL certificate -> {0}'.format(_cert_dst))

            # Deploy the SSL key
            shutil.copy(self.properties['ssl_key'], _key_dst)
            self.feedback.success('Deployed SSL key -> {0}'.format(_key_dst))

    def _create_dirs(self):
        """
        Create any required site directories.
        """

        # Site base directory
        site_base = '/srv/www/{0}'.format(self.properties['fqdn'])

        for dir in [
            '{0}/logs/php-fpm',
            '{0}/public',
            '{0}/session'
        ]:
            self.mkdir(dir.format(site_base))

        # If using SSL
        if self.ssl['enable']:
            self.mkdir('{0}/ssl'.format(site_base))

        # Setup directory permissions
        self.run_command('chown -R root:nginx {0}'.format(site_base))
        self.run_command('chmod -R 750 {0}'.format(site_base))

        # Set SELinux context
        self.selinux.chcon(site_base, 'unconfined_u:object_r:httpd_sys_content_t:s0', recursive=True)

    def _activate_site(self):
        """
        Optionally activate the site.
        """
        if self.properties['activate']:
            if not path.isfile(self.site_config['enabled']):
                symlink(self.site_config['available'], self.site_config['enabled'])
                self.feedback.success('Activated site -> {0}'.format(self.site_config['enabled']))
            else:
                self.feedback.info('Site already activated -> {0}'.format(self.site_config['enabled']))

    def _generate_nginx_config(self):
        """
        Generate NGINX config files for the new site.
        """

        # Setup the template
        self.template.setup(('NG_HTTPS' if self.ssl['enable'] else 'NG_HTTP'), self.site_config['available'])

        # Update placeholder variables
        self.template.setvars({
            'SITENAME': self.properties['fqdn'],
            'DEFAULTDOC': 'index.php' if not self.properties['fqdn'] else self.properties['fqdn']
        })

        # Deploy the configuration
        self.template.deploy(overwrite=True)

    def define(self, params):
        """
        Define attributes for creating a new site definition.
        """

        # SELinux manager
        self.selinux  = _NGUtilSELinux()

        # Make sure all required arguments are set
        for k in self.params['required']:
            if not params.get(k):
                self.die('Missing required argument \'{0}\''.format(k))

        # If using SSL
        if params.get('ssl'):
            self.ssl['enable'] = True

            # Make sure the certificate and key are set
            for k in ['ssl_cert', 'ssl_key']:
                if not params.get(k):
                    self.die('Missing required argument \'{0}\', must supply to enable SSL'.format(k))

                # Make sure the file exists
                ssl_file = params.get(k)
                if not os.path.isfile(ssl_file):
                    self.die('Could not locate \'{0}\' file \'{1}\''.format(k, ssl_file))

            # Set the SSL key and certificate
            self.ssl['key'] = params.get('ssl_key')
            self.ssl['cert'] = params.args.get('ssl_cert')
            self.feedback.info('Using SSL for site \'{0}\': cert={1}, key={2}'.format(params['fqdn'], params['ssl_cert'], param['ssl_key']))
        else:
            self.feedback.info('Not using SSL for site \'{0}\''.format(params['fqdn']))

        # Site configuration
        self.site_config = {
            'available': '/etc/nginx/sites-available/{0}.conf'.format(params['fqdn']),
            'enabled': '/etc/nginx/sites-enabled/{0}.conf'.format(params['fqdn'])
        }

        # Make sure the site isn't already defined
        if path.isfile(self.site_config['available']):
            self.die('Site \'{0}\' already defined in \'{1}\''.format(params['fqdn'], self.site_config['available']))

        # Merge with site properties
        print params
        self.properties = params

    def create(self):
        """
        Create the site definition.
        """
        self._create_dirs()
        self._setup_ssl_certs()
        self._generate_nginx_config()

        # Site created
        self.feedback.block([
            '\'{0}\' configuration complete!'.format(self.properties['fqdn']),
            'You will need to restart NGINX/PHP-FPM to make the site available:',
            '> service nginx restart',
            '> service php-fpm restart'
        ], 'COMPLETE')
<|endoftext|>


<commit_before># python-libjit, Copyright 2014 Niklas Koep
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from functools import wraps
import inspect
import ctypes
from _ctypes import CFuncPtr, FUNCFLAG_CDECL

from _jit import *

# XXX: This would be nice to have as a regular instance method on
#      jit.Context. We cannot monkey-patch the type though since this is not
#      allowed for extension types. Implementing a decorator in C is kind of
#      a hassle on the other hand. Maybe we could use a static extension method
#      in C which just stores the function object of `builds_function' in
#      the extension code and then delegate the call to the proper
#      `builds_function' instance method to the one defined here. Another
#      possibility would be to let the jit.Context c'tor accept a single
#      argument if said delegation object hasn't been set yet. Since we set
#      it here, the type will always be initialized properly. The only problem
#      is that users could theoretically import _jit.Context manually.
def builds_function(context, signature):
    function = jit.Function(context, signature)
    num_params = signature.num_params()
    args = [function.value_get_param(i) for i in range(num_params)]

    def decorator(f):
        argspec = inspect.getargspec(f)
        num_params_accepted = len(argspec.args)
        if num_params_accepted != num_params:
            numerus = "" if num_params == 1 else "s"
            raise ValueError(
                "decorated function must accept exactly %d argument%s, not %d"
                % (num_params, numerus, num_params_accepted))

        if argspec.varargs is not None:
            raise ValueError(
                "decorated function must not use varargs")

        if argspec.keywords is not None:
            raise ValueError(
                "decorated function must not use keyword arguments")

        # This seems like an arbitrary constraint. However, it'd be too easy
        # for the user to misinterpret default arguments as something that gets
        # passed to the JIT'ed function when it's called when in fact they are
        # only used during construction of the JIT function body. The unusual
        # thing about this decorator is that it calls the decorated function
        # immediately and actually returns the JIT-compiled version instead of
        # deferring the call until its actual invocation.
        if argspec.defaults is not None:
            raise ValueError(
                "decorated function must not use default arguments")

        function.insn_return(f(*args))
        function.compile_()
        try:
            # Assigning to `function' here shadows the original assignment and
            # raises an UnboundLocalError exception.
            jit_function = Closure(function)
        except ValueError:
            jit_function = function

        @wraps(f)
        def wrapper(*args):
            return jit_function(*args)
        return wrapper
    return decorator

class Closure(CFuncPtr):
    # XXX: This should depend on the calling convention of the wrapped
    #      function. Unfortunately it means we have to create the wrapper class
    #      dynamically since _flags_ has to be set during class creation. We
    #      could just cache it under the appropriate jit.ABI_* key.
    _flags_ = FUNCFLAG_CDECL

    def __new__(cls, function):
        if not isinstance(function, jit.Function):
            raise TypeError("function must be an instance of jit.Function")
        if not function.is_compiled():
            function.compile_()
        addr = function.to_closure()
        if addr is None:
            raise ValueError("failed to obtain closure from function object")
        return super(Closure, cls).__new__(cls, addr)

    def __init__(self, function):
        super(Closure, self).__init__()
        self._function = function
        # TODO: Use the function signature to annotate the wrapped FuncPtr.
        self.restype = ctypes.c_int
        self.argtypes = [ctypes.c_int]

<commit_msg>Fix NameError exception in jit package<commit_after># python-libjit, Copyright 2014 Niklas Koep
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from functools import wraps
import inspect
import ctypes
from _ctypes import CFuncPtr, FUNCFLAG_CDECL

from _jit import *

# XXX: This would be nice to have as a regular instance method on
#      jit.Context. We cannot monkey-patch the type though since this is not
#      allowed for extension types. Implementing a decorator in C is kind of
#      a hassle on the other hand. Maybe we could use a static extension method
#      in C which just stores the function object of `builds_function' in
#      the extension code and then delegate the call to the proper
#      `builds_function' instance method to the one defined here. Another
#      possibility would be to let the jit.Context c'tor accept a single
#      argument if said delegation object hasn't been set yet. Since we set
#      it here, the type will always be initialized properly. The only problem
#      is that users could theoretically import _jit.Context manually.
def builds_function(context, signature):
    function = Function(context, signature)
    num_params = signature.num_params()
    args = [function.value_get_param(i) for i in range(num_params)]

    def decorator(f):
        argspec = inspect.getargspec(f)
        num_params_accepted = len(argspec.args)
        if num_params_accepted != num_params:
            numerus = "" if num_params == 1 else "s"
            raise ValueError(
                "decorated function must accept exactly %d argument%s, not %d"
                % (num_params, numerus, num_params_accepted))

        if argspec.varargs is not None:
            raise ValueError(
                "decorated function must not use varargs")

        if argspec.keywords is not None:
            raise ValueError(
                "decorated function must not use keyword arguments")

        # This seems like an arbitrary constraint. However, it'd be too easy
        # for the user to misinterpret default arguments as something that gets
        # passed to the JIT'ed function when it's called when in fact they are
        # only used during construction of the JIT function body. The unusual
        # thing about this decorator is that it calls the decorated function
        # immediately and actually returns the JIT-compiled version instead of
        # deferring the call until its actual invocation.
        if argspec.defaults is not None:
            raise ValueError(
                "decorated function must not use default arguments")

        function.insn_return(f(*args))
        function.compile_()
        try:
            # Assigning to `function' here shadows the original assignment and
            # raises an UnboundLocalError exception.
            jit_function = Closure(function)
        except ValueError:
            jit_function = function

        @wraps(f)
        def wrapper(*args):
            return jit_function(*args)
        return wrapper
    return decorator

class Closure(CFuncPtr):
    # XXX: This should depend on the calling convention of the wrapped
    #      function. Unfortunately it means we have to create the wrapper class
    #      dynamically since _flags_ has to be set during class creation. We
    #      could just cache it under the appropriate jit.ABI_* key.
    _flags_ = FUNCFLAG_CDECL

    def __new__(cls, function):
        if not isinstance(function, Function):
            raise TypeError("function must be an instance of jit.Function")
        if not function.is_compiled():
            function.compile_()
        addr = function.to_closure()
        if addr is None:
            raise ValueError("failed to obtain closure from function object")
        return super(Closure, cls).__new__(cls, addr)

    def __init__(self, function):
        super(Closure, self).__init__()
        self._function = function
        # TODO: Use the function signature to annotate the wrapped FuncPtr.
        self.restype = ctypes.c_int
        self.argtypes = [ctypes.c_int]

<|endoftext|>


<commit_before>import re
import logging
from os.path import (join,
                     dirname,
                     realpath)

from typing import (Any,
                    Union,
                    Callable,
                    Iterable,
                    Optional)
from sklearn.cluster import MiniBatchKMeans
from sklearn.naive_bayes import (BernoulliNB,
                                 MultinomialNB)
from sklearn.feature_extraction import (FeatureHasher,
                                        DictVectorizer)
from sklearn.linear_model import (Perceptron,
                                  PassiveAggressiveRegressor)

from data import APPID_DICT

project_dir = dirname(dirname(realpath(__file__)))
data_dir = join(project_dir, 'data')
log_dir = join(project_dir, 'logs')
reports_dir = join(project_dir, 'reports')
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

# Type aliases
Learner = Union[Perceptron,
                MiniBatchKMeans,
                BernoulliNB,
                MultinomialNB,
                PassiveAggressiveRegressor]
Vectorizer = Union[DictVectorizer, FeatureHasher]
Numeric = Union[int, float]
Scorer = Optional[Union[str, Callable[[Iterable[Any], Iterable[Any]]]]]

# Seed for random state
SEED = 123456789

# Define default parameter grids
DEFAULT_PARAM_GRIDS = \
    {MiniBatchKMeans: {'n_clusters': [3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
                       'init' : ['k-means++', 'random'],
                       'random_state': [SEED]},
     BernoulliNB: {'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]},
     MultinomialNB: {'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]},
     Perceptron: {'penalty': [None, 'l2', 'l1', 'elasticnet'],
                  'alpha': [0.0001, 0.001, 0.01, 0.1],
                  'n_iter': [5, 10],
                  'random_state': [SEED]},
     PassiveAggressiveRegressor:
         {'C': [0.01, 0.1, 1.0, 10.0, 100.0],
          'n_iter': [5, 10],
          'random_state': [SEED],
          'loss': ['epsilon_insensitive',
                   'squared_epsilon_insensitive']}}

# Learners
LEARNER_ABBRS_DICT = {'mbkm': 'MiniBatchKMeans',
                      'bnb': 'BernoulliNB',
                      'mnb': 'MultinomialNB',
                      'perc': 'Perceptron',
                      'pagr': 'PassiveAggressiveRegressor'}
LEARNER_DICT_KEYS = frozenset(LEARNER_ABBRS_DICT.keys())
LEARNER_DICT = {k: eval(LEARNER_ABBRS_DICT[k]) for k in LEARNER_DICT_KEYS}
LEARNER_ABBRS_STRING = ', '.join(['"{0}" ({1})'.format(abbr, learner)
                                  for abbr, learner in LEARNER_ABBRS_DICT.items()])

# Objective functions
OBJ_FUNC_ABBRS_DICT = {'pearson_r': "Pearson's r",
                       'average_precision': "average precision",
                       'precision_macro': 'precision (macro)',
                       'precision_weighted': 'precision (weighted)',
                       'recall': "recall",
                       'f1_macro': 'f1 (macro)',
                       'f1_weighted': 'f1 (weighted)',
                       'accuracy': 'accuracy',
                       'uwk': 'unweighted kappa',
                       'uwk_off_by_one': 'unweighted kappa (off by one)',
                       'qwk': 'quadratic weighted kappa',
                       'qwk_off_by_one':
                           'quadratic weighted kappa (off by one)',
                       'lwk': 'linear weighted kappa',
                       'lwk_off_by_one': 'linear weighted kappa (off by one)'}
OBJ_FUNC_ABBRS_STRING = \
    ', '.join(['"{0}"{1}'
               .format(abbr, ' ({0})'.format(func) if abbr != func else '')
               for abbr, func in OBJ_FUNC_ABBRS_DICT.items()])

# Feature names
LABELS = frozenset({'num_guides', 'num_games_owned', 'num_friends',
                    'num_voted_helpfulness', 'num_groups',
                    'num_workshop_items', 'num_reviews', 'num_found_funny',
                    'friend_player_level', 'num_badges', 'num_found_helpful',
                    'num_screenshots', 'num_found_unhelpful',
                    'found_helpful_percentage', 'num_comments',
                    'total_game_hours', 'total_game_hours_bin',
                    'total_game_hours_last_two_weeks',
                    'num_achievements_percentage', 'num_achievements_attained',
                    'num_achievements_possible'})
LABELS_STRING = ', '.join(LABELS)
TIME_LABELS = frozenset({'total_game_hours', 'total_game_hours_bin',
                         'total_game_hours_last_two_weeks'})
FRIENDS_LABELS = frozenset({'num_friends', 'friend_player_level'})
HELPFUL_LABELS = frozenset({'num_voted_helpfulness', 'num_found_helpful',
                            'found_helpful_percentage', 'num_found_unhelpful'})
ACHIEVEMENTS_LABELS = frozenset({'num_achievements_percentage',
                                 'num_achievements_attained',
                                 'num_achievements_possible'})
LABELS_WITH_PCT_VALUES = frozenset({'num_achievements_percentage',
                                    'found_helpful_percentage'})

# Valid games
VALID_GAMES = frozenset([game for game in list(APPID_DICT) if game != 'sample'])

# Regular expressions
FLOAT_ONE_DEC = re.compile(r'^\-?\d+\.\d$')
test_float_decimal_places = FLOAT_ONE_DEC.search

# Data-scraping regular expressions
SPACE = re.compile(r'[\s]+')
space_sub = SPACE.sub
BREAKS_REGEX = re.compile(r'\
')
breaks_sub = BREAKS_REGEX.sub
COMMA = re.compile(r',')
comma_sub = COMMA.sub
HELPFUL_OR_FUNNY = re.compile('(helpful|funny)')
helpful_or_funny_search = HELPFUL_OR_FUNNY.search
DATE_END_WITH_YEAR_STRING = re.compile(r', \d{4}$')
date_end_with_year_string_search = DATE_END_WITH_YEAR_STRING.search
COMMENT_RE_1 = re.compile(r'<span id="commentthread[^<]+')
comment_re_1_search = COMMENT_RE_1.search
COMMENT_RE_2 = re.compile(r'>(\d*)$')
comment_re_2_search = COMMENT_RE_2.search
UNDERSCORE = re.compile(r'_')
underscore_sub = UNDERSCORE.sub
QUOTES = re.compile(r'\'|"')
quotes_sub = QUOTES.sub
BACKSLASH = re.compile(r'\\')
backslash_sub = BACKSLASH.sub

def find_default_param_grid(learner: str,
                            param_grids_dict: dict = DEFAULT_PARAM_GRIDS) -> dict:
    """
    Finds the default parameter grid for the specified learner.

    :param learner: abbreviated string representation of a learner
    :type learner: str
    :param param_grids_dict: dictionary of learner classes mapped to
                             parameter grids
    :type param_grids_dict: dict

    :returns: parameter grid
    :rtype: dict

    :raises ValueError: if an unrecognized learner abbreviation is used
    """

    for key_cls, grid in param_grids_dict.items():
        if issubclass(LEARNER_DICT[learner], key_cls):
            return grid
    raise ValueError('Unrecognized learner abbreviation: {0}'.format(learner))

def parse_learners_string(learners_string: str) -> set:
    """
    Parse command-line argument consisting of a set of learners to
    use (or the value "all" for all possible learners).

    :param learners_string: comma-separated list of learner
                            abbreviations (or "all" for all possible
                            learners)
    :type learners_string: str

    :returns: set of learner abbreviations
    :rtype: set

    :raises ValueError: if unrecognized learner(s) are included in the
                        input
    """

    if learners_string == 'all':
        learners = set(LEARNER_DICT_KEYS)
    else:
        learners = set(learners_string.split(','))
        if not learners.issubset(LEARNER_DICT_KEYS):
            raise ValueError('Found unrecognized learner(s) in list of '
                             'passed-in learners: {0}. Available learners: {1}.'
                             .format(', '.join(learners), LEARNER_ABBRS_STRING))

    return learners

def parse_non_nlp_features_string(features_string: str,
                                  prediction_label: str) -> set:
    """
    Parse command-line argument consisting of a set of non-NLP features
    (or one of the values "all" or "none" for all or none of the
    possible non-NLP features).

    If the prediction label is one of the time features, take other
    time features out of the set of non-NLP features since the
    information could be duplicated.

    :param features_string: comma-separated list of non-NLP features
                            (or "all"/"none" for all/none of the
                            possible non-NLP features)
    :type features_string: str
    :param prediction_label: the feature that is being predicted
    :type prediction_label: str

    :returns: set of non-NLP features to use
    :rtype: set

    :raises ValueError: if unrecognized features are found in the input
                        or there is a conflict between features
                        included in the input and the feature used as
                        the prediction label
    """

    if features_string == 'all':
        non_nlp_features = set(LABELS)

        # Remove time-related labels if the prediction label is also
        # time-related
        if prediction_label in TIME_LABELS:
            [non_nlp_features.remove(label) for label in TIME_LABELS]

        # Remove friends-related labels if the prediction label is also
        # friends-related
        elif prediction_label in FRIENDS_LABELS:
            [non_nlp_features.remove(label) for label in FRIENDS_LABELS]

        # Remove helpful-related labels if the prediction label is also
        # helpful-related
        elif prediction_label in HELPFUL_LABELS:
            [non_nlp_features.remove(label) for label in HELPFUL_LABELS]

        # Remove achievements-related labels if the prediction label is
        # also achievements-related
        elif prediction_label in ACHIEVEMENTS_LABELS:
            [non_nlp_features.remove(label) for label in ACHIEVEMENTS_LABELS]

        # Otherwise, just remove the prediction label
        else:
            non_nlp_features.remove(prediction_label)

    elif features_string == 'none':
        non_nlp_features = set()
    else:
        non_nlp_features = set(features_string.split(','))

        # Raise an exception if unrecognized labels are found
        if not non_nlp_features.issubset(LABELS):
            raise ValueError('Found unrecognized feature(s) in the list of '
                             'passed-in non-NLP features: {0}. Available '
                             'features: {1}.'
                             .format(', '.join(non_nlp_features), LABELS_STRING))

        # Raise an exception if there are conflicts between the
        # prediction label and the set of non-NLP labels to use
        if (prediction_label in TIME_LABELS
            and non_nlp_features.intersection(TIME_LABELS)):
            raise ValueError('The list of non-NLP features should not '
                             'contain any of the time-related features if '
                             'the prediction label is itself a '
                             'time-related feature.')
        elif (prediction_label in FRIENDS_LABELS
              and non_nlp_features.intersection(FRIENDS_LABELS)):
            raise ValueError('The list of non-NLP features should not '
                             'contain any of the friends-related features if '
                             'the prediction label is itself a '
                             'friends-related feature.')
        elif (prediction_label in HELPFUL_LABELS
              and non_nlp_features.intersection(HELPFUL_LABELS)):
            raise ValueError('The list of non-NLP features should not '
                             'contain any of the helpful-related features if '
                             'the prediction label is itself a '
                             'helpful-related feature.')
        elif (prediction_label in ACHIEVEMENTS_LABELS
              and non_nlp_features.intersection(ACHIEVEMENTS_LABELS)):
            raise ValueError('The list of non-NLP features should not '
                             'contain any of the achievements-related '
                             'features if the prediction label is itself an '
                             'achievements-related feature.')

    return non_nlp_features

def parse_games_string(games_string: str) -> set:
    """
    Parse games string passed in via the command-line into a set of
    valid games (or the value "all" for all games).

    :param games_string: comma-separated list of games (or "all")
    :type games_string: str

    :returns: set of games
    :rtype: set

    :raises ValueError: if unrecognized game IDs are found in the input
    """

    # Return empty set for empty string
    if not games_string:
        return set()

    # Return the full list of games if 'all' is used
    if games_string == 'all':
        return set(VALID_GAMES)

    # Parse string
    specified_games = games_string.split(',')

    # Raise exception if the list contains unrecognized games
    if any(game not in VALID_GAMES for game in specified_games):
        raise ValueError('Found unrecognized games in the list of specified '
                         'games: {0}. These are the valid games (in addition '
                         'to using "all" for all games): {1}.'
                         .format(', '.join(specified_games),
                                 ', '.join(VALID_GAMES)))
    return set(specified_games)
<commit_msg>Fix bug in the way Callable is used<commit_after>import re
import logging
from os.path import (join,
                     dirname,
                     realpath)

from typing import (Any,
                    Union,
                    Callable,
                    Iterable,
                    Optional)
from sklearn.cluster import MiniBatchKMeans
from sklearn.naive_bayes import (BernoulliNB,
                                 MultinomialNB)
from sklearn.feature_extraction import (FeatureHasher,
                                        DictVectorizer)
from sklearn.linear_model import (Perceptron,
                                  PassiveAggressiveRegressor)

from data import APPID_DICT

project_dir = dirname(dirname(realpath(__file__)))
data_dir = join(project_dir, 'data')
log_dir = join(project_dir, 'logs')
reports_dir = join(project_dir, 'reports')
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

# Type aliases
Learner = Union[Perceptron,
                MiniBatchKMeans,
                BernoulliNB,
                MultinomialNB,
                PassiveAggressiveRegressor]
Vectorizer = Union[DictVectorizer, FeatureHasher]
Numeric = Union[int, float]
Scorer = Optional[Union[str, Callable[[Iterable[Any], Iterable[Any]], Any]]]

# Seed for random state
SEED = 123456789

# Define default parameter grids
DEFAULT_PARAM_GRIDS = \
    {MiniBatchKMeans: {'n_clusters': [3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
                       'init' : ['k-means++', 'random'],
                       'random_state': [SEED]},
     BernoulliNB: {'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]},
     MultinomialNB: {'alpha': [0.1, 0.25, 0.5, 0.75, 1.0]},
     Perceptron: {'penalty': [None, 'l2', 'l1', 'elasticnet'],
                  'alpha': [0.0001, 0.001, 0.01, 0.1],
                  'n_iter': [5, 10],
                  'random_state': [SEED]},
     PassiveAggressiveRegressor:
         {'C': [0.01, 0.1, 1.0, 10.0, 100.0],
          'n_iter': [5, 10],
          'random_state': [SEED],
          'loss': ['epsilon_insensitive',
                   'squared_epsilon_insensitive']}}

# Learners
LEARNER_ABBRS_DICT = {'mbkm': 'MiniBatchKMeans',
                      'bnb': 'BernoulliNB',
                      'mnb': 'MultinomialNB',
                      'perc': 'Perceptron',
                      'pagr': 'PassiveAggressiveRegressor'}
LEARNER_DICT_KEYS = frozenset(LEARNER_ABBRS_DICT.keys())
LEARNER_DICT = {k: eval(LEARNER_ABBRS_DICT[k]) for k in LEARNER_DICT_KEYS}
LEARNER_ABBRS_STRING = ', '.join(['"{0}" ({1})'.format(abbr, learner)
                                  for abbr, learner in LEARNER_ABBRS_DICT.items()])

# Objective functions
OBJ_FUNC_ABBRS_DICT = {'pearson_r': "Pearson's r",
                       'average_precision': "average precision",
                       'precision_macro': 'precision (macro)',
                       'precision_weighted': 'precision (weighted)',
                       'recall': "recall",
                       'f1_macro': 'f1 (macro)',
                       'f1_weighted': 'f1 (weighted)',
                       'accuracy': 'accuracy',
                       'uwk': 'unweighted kappa',
                       'uwk_off_by_one': 'unweighted kappa (off by one)',
                       'qwk': 'quadratic weighted kappa',
                       'qwk_off_by_one':
                           'quadratic weighted kappa (off by one)',
                       'lwk': 'linear weighted kappa',
                       'lwk_off_by_one': 'linear weighted kappa (off by one)'}
OBJ_FUNC_ABBRS_STRING = \
    ', '.join(['"{0}"{1}'
               .format(abbr, ' ({0})'.format(func) if abbr != func else '')
               for abbr, func in OBJ_FUNC_ABBRS_DICT.items()])

# Feature names
LABELS = frozenset({'num_guides', 'num_games_owned', 'num_friends',
                    'num_voted_helpfulness', 'num_groups',
                    'num_workshop_items', 'num_reviews', 'num_found_funny',
                    'friend_player_level', 'num_badges', 'num_found_helpful',
                    'num_screenshots', 'num_found_unhelpful',
                    'found_helpful_percentage', 'num_comments',
                    'total_game_hours', 'total_game_hours_bin',
                    'total_game_hours_last_two_weeks',
                    'num_achievements_percentage', 'num_achievements_attained',
                    'num_achievements_possible'})
LABELS_STRING = ', '.join(LABELS)
TIME_LABELS = frozenset({'total_game_hours', 'total_game_hours_bin',
                         'total_game_hours_last_two_weeks'})
FRIENDS_LABELS = frozenset({'num_friends', 'friend_player_level'})
HELPFUL_LABELS = frozenset({'num_voted_helpfulness', 'num_found_helpful',
                            'found_helpful_percentage', 'num_found_unhelpful'})
ACHIEVEMENTS_LABELS = frozenset({'num_achievements_percentage',
                                 'num_achievements_attained',
                                 'num_achievements_possible'})
LABELS_WITH_PCT_VALUES = frozenset({'num_achievements_percentage',
                                    'found_helpful_percentage'})

# Valid games
VALID_GAMES = frozenset([game for game in list(APPID_DICT) if game != 'sample'])

# Regular expressions
FLOAT_ONE_DEC = re.compile(r'^\-?\d+\.\d$')
test_float_decimal_places = FLOAT_ONE_DEC.search

# Data-scraping regular expressions
SPACE = re.compile(r'[\s]+')
space_sub = SPACE.sub
BREAKS_REGEX = re.compile(r'\
')
breaks_sub = BREAKS_REGEX.sub
COMMA = re.compile(r',')
comma_sub = COMMA.sub
HELPFUL_OR_FUNNY = re.compile('(helpful|funny)')
helpful_or_funny_search = HELPFUL_OR_FUNNY.search
DATE_END_WITH_YEAR_STRING = re.compile(r', \d{4}$')
date_end_with_year_string_search = DATE_END_WITH_YEAR_STRING.search
COMMENT_RE_1 = re.compile(r'<span id="commentthread[^<]+')
comment_re_1_search = COMMENT_RE_1.search
COMMENT_RE_2 = re.compile(r'>(\d*)$')
comment_re_2_search = COMMENT_RE_2.search
UNDERSCORE = re.compile(r'_')
underscore_sub = UNDERSCORE.sub
QUOTES = re.compile(r'\'|"')
quotes_sub = QUOTES.sub
BACKSLASH = re.compile(r'\\')
backslash_sub = BACKSLASH.sub

def find_default_param_grid(learner: str,
                            param_grids_dict: dict = DEFAULT_PARAM_GRIDS) -> dict:
    """
    Finds the default parameter grid for the specified learner.

    :param learner: abbreviated string representation of a learner
    :type learner: str
    :param param_grids_dict: dictionary of learner classes mapped to
                             parameter grids
    :type param_grids_dict: dict

    :returns: parameter grid
    :rtype: dict

    :raises ValueError: if an unrecognized learner abbreviation is used
    """

    for key_cls, grid in param_grids_dict.items():
        if issubclass(LEARNER_DICT[learner], key_cls):
            return grid
    raise ValueError('Unrecognized learner abbreviation: {0}'.format(learner))

def parse_learners_string(learners_string: str) -> set:
    """
    Parse command-line argument consisting of a set of learners to
    use (or the value "all" for all possible learners).

    :param learners_string: comma-separated list of learner
                            abbreviations (or "all" for all possible
                            learners)
    :type learners_string: str

    :returns: set of learner abbreviations
    :rtype: set

    :raises ValueError: if unrecognized learner(s) are included in the
                        input
    """

    if learners_string == 'all':
        learners = set(LEARNER_DICT_KEYS)
    else:
        learners = set(learners_string.split(','))
        if not learners.issubset(LEARNER_DICT_KEYS):
            raise ValueError('Found unrecognized learner(s) in list of '
                             'passed-in learners: {0}. Available learners: {1}.'
                             .format(', '.join(learners), LEARNER_ABBRS_STRING))

    return learners

def parse_non_nlp_features_string(features_string: str,
                                  prediction_label: str) -> set:
    """
    Parse command-line argument consisting of a set of non-NLP features
    (or one of the values "all" or "none" for all or none of the
    possible non-NLP features).

    If the prediction label is one of the time features, take other
    time features out of the set of non-NLP features since the
    information could be duplicated.

    :param features_string: comma-separated list of non-NLP features
                            (or "all"/"none" for all/none of the
                            possible non-NLP features)
    :type features_string: str
    :param prediction_label: the feature that is being predicted
    :type prediction_label: str

    :returns: set of non-NLP features to use
    :rtype: set

    :raises ValueError: if unrecognized features are found in the input
                        or there is a conflict between features
                        included in the input and the feature used as
                        the prediction label
    """

    if features_string == 'all':
        non_nlp_features = set(LABELS)

        # Remove time-related labels if the prediction label is also
        # time-related
        if prediction_label in TIME_LABELS:
            [non_nlp_features.remove(label) for label in TIME_LABELS]

        # Remove friends-related labels if the prediction label is also
        # friends-related
        elif prediction_label in FRIENDS_LABELS:
            [non_nlp_features.remove(label) for label in FRIENDS_LABELS]

        # Remove helpful-related labels if the prediction label is also
        # helpful-related
        elif prediction_label in HELPFUL_LABELS:
            [non_nlp_features.remove(label) for label in HELPFUL_LABELS]

        # Remove achievements-related labels if the prediction label is
        # also achievements-related
        elif prediction_label in ACHIEVEMENTS_LABELS:
            [non_nlp_features.remove(label) for label in ACHIEVEMENTS_LABELS]

        # Otherwise, just remove the prediction label
        else:
            non_nlp_features.remove(prediction_label)

    elif features_string == 'none':
        non_nlp_features = set()
    else:
        non_nlp_features = set(features_string.split(','))

        # Raise an exception if unrecognized labels are found
        if not non_nlp_features.issubset(LABELS):
            raise ValueError('Found unrecognized feature(s) in the list of '
                             'passed-in non-NLP features: {0}. Available '
                             'features: {1}.'
                             .format(', '.join(non_nlp_features), LABELS_STRING))

        # Raise an exception if there are conflicts between the
        # prediction label and the set of non-NLP labels to use
        if (prediction_label in TIME_LABELS
            and non_nlp_features.intersection(TIME_LABELS)):
            raise ValueError('The list of non-NLP features should not '
                             'contain any of the time-related features if '
                             'the prediction label is itself a '
                             'time-related feature.')
        elif (prediction_label in FRIENDS_LABELS
              and non_nlp_features.intersection(FRIENDS_LABELS)):
            raise ValueError('The list of non-NLP features should not '
                             'contain any of the friends-related features if '
                             'the prediction label is itself a '
                             'friends-related feature.')
        elif (prediction_label in HELPFUL_LABELS
              and non_nlp_features.intersection(HELPFUL_LABELS)):
            raise ValueError('The list of non-NLP features should not '
                             'contain any of the helpful-related features if '
                             'the prediction label is itself a '
                             'helpful-related feature.')
        elif (prediction_label in ACHIEVEMENTS_LABELS
              and non_nlp_features.intersection(ACHIEVEMENTS_LABELS)):
            raise ValueError('The list of non-NLP features should not '
                             'contain any of the achievements-related '
                             'features if the prediction label is itself an '
                             'achievements-related feature.')

    return non_nlp_features

def parse_games_string(games_string: str) -> set:
    """
    Parse games string passed in via the command-line into a set of
    valid games (or the value "all" for all games).

    :param games_string: comma-separated list of games (or "all")
    :type games_string: str

    :returns: set of games
    :rtype: set

    :raises ValueError: if unrecognized game IDs are found in the input
    """

    # Return empty set for empty string
    if not games_string:
        return set()

    # Return the full list of games if 'all' is used
    if games_string == 'all':
        return set(VALID_GAMES)

    # Parse string
    specified_games = games_string.split(',')

    # Raise exception if the list contains unrecognized games
    if any(game not in VALID_GAMES for game in specified_games):
        raise ValueError('Found unrecognized games in the list of specified '
                         'games: {0}. These are the valid games (in addition '
                         'to using "all" for all games): {1}.'
                         .format(', '.join(specified_games),
                                 ', '.join(VALID_GAMES)))
    return set(specified_games)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Copyright 2017 Guenter Bartsch
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

#
# Abstraction layer for multiple speech recognition engines,
# kaldi-asr and pocketsphinx at the moment
#

import traceback
import logging
import time
import re
import struct
import numpy as np
import pocketsphinx

from base64             import b64encode
from kaldiasr.nnet3     import KaldiNNet3OnlineModel, KaldiNNet3OnlineDecoder

ASR_ENGINE_NNET3        = 'kaldi-nnet3'
ASR_ENGINE_POCKETSPHINX = 'pocketsphinx'

DEFAULT_ENGINE          = ASR_ENGINE_NNET3
DEFAULT_MODEL_DIR       = 'models/kaldi-nnet3-voxforge-en-latest'
DEFAULT_MODEL_NAME      = 'nnet_tdnn_a'
DEFAULT_STREAM_ID       = '__default__'

class ASR(object):

    def __init__(self,
                 engine      = DEFAULT_ENGINE,
                 model_dir   = DEFAULT_MODEL_DIR,
                 model_name  = DEFAULT_MODEL_NAME):

        self._engine      = engine
        self._model_dir   = model_dir
        self._model_name  = model_name
        self.asr_decoders = {} # stream_id -> decoder

        if self._engine == ASR_ENGINE_NNET3:

            logging.debug ('loading ASR model %s from %s...' % (self._model_name, self._model_dir))
            start_time = time.time()
            self.nnet3_model = KaldiNNet3OnlineModel ( self._model_dir, self._model_name )
            logging.debug ('ASR model loaded. took %fs' % (time.time() - start_time))

        elif self._engine == ASR_ENGINE_POCKETSPHINX:

            self.ps_config = pocketsphinx.Decoder.default_config()

            # determine CFG_N_TIED_STATES, CFG_WAVFILE_SRATE
            # cmusphinx-cont-voxforge-en-latest/etc/sphinx_train.cfg
            traincfg_fn        = '%s/etc/sphinx_train.cfg' % model_dir
            n_tied_states      = 6000
            self.ps_samplerate = 16000
            with open (traincfg_fn, 'r') as traincfg_f:
                for line in traincfg_f:
                    if not line:
                        break
                    # $CFG_N_TIED_STATES = 6000;
                    if 'CFG_N_TIED_STATES' in line:
                        # logging.debug ('parsing train cfg line %s' % line)
                        m = re.match (r"\$CFG_N_TIED_STATES\s+=\s+([0-9]+)\s*;", line.strip())
                        if m:
                            n_tied_states = int(m.group(1))
                            # logging.debug ('matched, n_tied_states=%d' % n_tied_states)

                    # $CFG_WAVFILE_SRATE = 16000.0;
                    if 'CFG_WAVFILE_SRATE' in line:
                        m = re.match (r"\$CFG_WAVFILE_SRATE\s+=\s+([0-9.]+)\s*;", line.strip())
                        if m:
                            self.ps_samplerate = int(float(m.group(1)))

            self.ps_config.set_string('-hmm', '%s/model_parameters/%s.cd_cont_%d' % (model_dir, model_name, n_tied_states))
            self.ps_config.set_float ('-lw', 10)
            self.ps_config.set_string('-feat', '1s_c_d_dd')
            self.ps_config.set_float ('-beam', 1e-80)
            self.ps_config.set_float ('-wbeam', 1e-40)
            self.ps_config.set_string('-dict', '%s/etc/%s.dic' % (model_dir, model_name))
            self.ps_config.set_float ('-wip', 0.2)
            self.ps_config.set_string('-agc', 'none')
            self.ps_config.set_string('-varnorm', 'no')
            self.ps_config.set_string('-cmn', 'current')
            self.ps_config.set_string('-lm', '%s/etc/%s.lm.bin' % (model_dir, model_name))

            self.ps_config.set_string('-logfn', "/dev/null")

            self.asr_in_utt = {} # stream_id -> Boolean

        else:
            raise Exception ('unknown ASR engine: %s' % self._engine)

    def decode (self, sample_rate, audio, do_finalize, stream_id = DEFAULT_STREAM_ID):

        if self._engine == ASR_ENGINE_NNET3:

            if not stream_id in self.asr_decoders:
                self.asr_decoders[stream_id] = KaldiNNet3OnlineDecoder (self.nnet3_model)

            decoder = self.asr_decoders[stream_id]
            decoder.decode(sample_rate, np.array(audio, dtype=np.float32), do_finalize)

            if not do_finalize:
                return None, 0.0

            hstr, confidence = decoder.get_decoded_string()

        elif self._engine == ASR_ENGINE_POCKETSPHINX:

            if sample_rate != self.ps_samplerate:
                raise Exception ('decode: samplerate does not match model: %d vs %d' % (sample_rate, self.ps_samplerate))

            if not stream_id in self.asr_decoders:
                self.asr_decoders[stream_id] = pocketsphinx.Decoder(self.ps_config)
                self.asr_in_utt[stream_id] = False

            decoder = self.asr_decoders[stream_id]
            if not self.asr_in_utt[stream_id]:
                decoder.start_utt()
                self.asr_in_utt[stream_id] = True

            audios = struct.pack('<%dh' % len(audio), *audio)

            decoder.process_raw(audios, False, False)

            if not do_finalize:
                return None, 0.0

            decoder.end_utt()
            self.asr_in_utt[stream_id] = False

            hypothesis = decoder.hyp()
            logmath = decoder.get_logmath()
            hstr = hypothesis.hypstr.decode('utf8').strip()
            confidence = logmath.exp(hypothesis.prob)

        else:
            raise Exception ('unknown ASR engine: %s' % self._engine)

        return hstr, confidence

    @property
    def engine(self):
        return self._engine
    # @engine.setter
    # def engine(self, v):
    #     self._engine = v

    @property
    def model_dir(self):
        return self._model_dir
    # @model_dir.setter
    # def model_dir(self, v):
    #     self._model_dir = v

    @property
    def model_name(self):
        return self._model_name
    # @model_name.setter
    # def model_name(self, v):
    #     self._model_name = v

<commit_msg>allow specification of beam, acoustic_scale and frame_subsampling_factor for chain models<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Copyright 2017 Guenter Bartsch
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

#
# Abstraction layer for multiple speech recognition engines,
# kaldi-asr and pocketsphinx at the moment
#

import traceback
import logging
import time
import re
import struct
import numpy as np
import pocketsphinx

from base64             import b64encode
from kaldiasr.nnet3     import KaldiNNet3OnlineModel, KaldiNNet3OnlineDecoder

ASR_ENGINE_NNET3        = 'kaldi-nnet3'
ASR_ENGINE_POCKETSPHINX = 'pocketsphinx'

DEFAULT_ENGINE          = ASR_ENGINE_NNET3
DEFAULT_MODEL_DIR       = 'models/kaldi-nnet3-voxforge-en-latest'
DEFAULT_MODEL_NAME      = 'nnet_tdnn_a'
DEFAULT_STREAM_ID       = '__default__'

DEFAULT_KALDI_BEAM                      = 15.0
DEFAULT_KALDI_ACOUSTIC_SCALE            = 0.1
DEFAULT_KALDI_FRAME_SUBSAMPLING_FACTOR  = 1

class ASR(object):

    def __init__(self,
                 engine      = DEFAULT_ENGINE,
                 model_dir   = DEFAULT_MODEL_DIR,
                 model_name  = DEFAULT_MODEL_NAME,

                 kaldi_beam                     = DEFAULT_KALDI_BEAM,
                 kaldi_acoustic_scale           = DEFAULT_KALDI_ACOUSTIC_SCALE,
                 kaldi_frame_subsampling_factor = DEFAULT_KALDI_FRAME_SUBSAMPLING_FACTOR,
                ):

        self._engine      = engine
        self._model_dir   = model_dir
        self._model_name  = model_name
        self.asr_decoders = {} # stream_id -> decoder

        if self._engine == ASR_ENGINE_NNET3:

            logging.debug ('loading ASR model %s from %s...' % (self._model_name, self._model_dir))
            start_time = time.time()
            self.nnet3_model = KaldiNNet3OnlineModel ( self._model_dir, self._model_name,
                                                       beam                     = kaldi_beam,
                                                       acoustic_scale           = kaldi_acoustic_scale,
                                                       frame_subsampling_factor = kaldi_frame_subsampling_factor)
            logging.debug ('ASR model loaded. took %fs' % (time.time() - start_time))

        elif self._engine == ASR_ENGINE_POCKETSPHINX:

            self.ps_config = pocketsphinx.Decoder.default_config()

            # determine CFG_N_TIED_STATES, CFG_WAVFILE_SRATE
            # cmusphinx-cont-voxforge-en-latest/etc/sphinx_train.cfg
            traincfg_fn        = '%s/etc/sphinx_train.cfg' % model_dir
            n_tied_states      = 6000
            self.ps_samplerate = 16000
            with open (traincfg_fn, 'r') as traincfg_f:
                for line in traincfg_f:
                    if not line:
                        break
                    # $CFG_N_TIED_STATES = 6000;
                    if 'CFG_N_TIED_STATES' in line:
                        # logging.debug ('parsing train cfg line %s' % line)
                        m = re.match (r"\$CFG_N_TIED_STATES\s+=\s+([0-9]+)\s*;", line.strip())
                        if m:
                            n_tied_states = int(m.group(1))
                            # logging.debug ('matched, n_tied_states=%d' % n_tied_states)

                    # $CFG_WAVFILE_SRATE = 16000.0;
                    if 'CFG_WAVFILE_SRATE' in line:
                        m = re.match (r"\$CFG_WAVFILE_SRATE\s+=\s+([0-9.]+)\s*;", line.strip())
                        if m:
                            self.ps_samplerate = int(float(m.group(1)))

            self.ps_config.set_string('-hmm', '%s/model_parameters/%s.cd_cont_%d' % (model_dir, model_name, n_tied_states))
            self.ps_config.set_float ('-lw', 10)
            self.ps_config.set_string('-feat', '1s_c_d_dd')
            self.ps_config.set_float ('-beam', 1e-80)
            self.ps_config.set_float ('-wbeam', 1e-40)
            self.ps_config.set_string('-dict', '%s/etc/%s.dic' % (model_dir, model_name))
            self.ps_config.set_float ('-wip', 0.2)
            self.ps_config.set_string('-agc', 'none')
            self.ps_config.set_string('-varnorm', 'no')
            self.ps_config.set_string('-cmn', 'current')
            self.ps_config.set_string('-lm', '%s/etc/%s.lm.bin' % (model_dir, model_name))

            self.ps_config.set_string('-logfn', "/dev/null")

            self.asr_in_utt = {} # stream_id -> Boolean

        else:
            raise Exception ('unknown ASR engine: %s' % self._engine)

    def decode (self, sample_rate, audio, do_finalize, stream_id = DEFAULT_STREAM_ID):

        if self._engine == ASR_ENGINE_NNET3:

            if not stream_id in self.asr_decoders:
                self.asr_decoders[stream_id] = KaldiNNet3OnlineDecoder (self.nnet3_model)

            decoder = self.asr_decoders[stream_id]
            decoder.decode(sample_rate, np.array(audio, dtype=np.float32), do_finalize)

            if not do_finalize:
                return None, 0.0

            hstr, confidence = decoder.get_decoded_string()

        elif self._engine == ASR_ENGINE_POCKETSPHINX:

            if sample_rate != self.ps_samplerate:
                raise Exception ('decode: samplerate does not match model: %d vs %d' % (sample_rate, self.ps_samplerate))

            if not stream_id in self.asr_decoders:
                self.asr_decoders[stream_id] = pocketsphinx.Decoder(self.ps_config)
                self.asr_in_utt[stream_id] = False

            decoder = self.asr_decoders[stream_id]
            if not self.asr_in_utt[stream_id]:
                decoder.start_utt()
                self.asr_in_utt[stream_id] = True

            audios = struct.pack('<%dh' % len(audio), *audio)

            decoder.process_raw(audios, False, False)

            if not do_finalize:
                return None, 0.0

            decoder.end_utt()
            self.asr_in_utt[stream_id] = False

            hypothesis = decoder.hyp()
            logmath = decoder.get_logmath()
            hstr = hypothesis.hypstr.decode('utf8').strip()
            confidence = logmath.exp(hypothesis.prob)

        else:
            raise Exception ('unknown ASR engine: %s' % self._engine)

        return hstr, confidence

    @property
    def engine(self):
        return self._engine
    # @engine.setter
    # def engine(self, v):
    #     self._engine = v

    @property
    def model_dir(self):
        return self._model_dir
    # @model_dir.setter
    # def model_dir(self, v):
    #     self._model_dir = v

    @property
    def model_name(self):
        return self._model_name
    # @model_name.setter
    # def model_name(self, v):
    #     self._model_name = v

<|endoftext|>


<commit_before>"""GCL standard library functions."""

import itertools
import functools

from os import path

from . import framework

def eager(x):
  """Force eager evaluation of a Thunk, and turn a Tuple into a dict eagerly.

  This forces that there are no unbound variables at parsing time (as opposed
  to later when the variables may be accessed).

  Only eagerly evaluates one level.
  """
  if not hasattr(x, 'items'):
    # The Thunk has been evaluated already, so that was easy :)
    return x

  return dict(x.items())

class StringInterpolationProxy(object):
  """Lazy stringification object from a tuple.

  Also support period-based element access, cuz it looks good.
  """
  def __init__(self, tup, key):
    self.tup = tup
    self.key = key

  def __getattr__(self, key):
    v = self.tup[self.key]
    # If this is a tuple, return another lazy proxy
    if isinstance(v, framework.TupleLike):
      return StringInterpolationProxy(v, key)
    return v[key]

  def __str__(self):
    return str(self.tup[self.key])

def fmt(str, args=None, env=None):
  """String interpolation.

  Normally, we'd just call str.format(**args), but we only want to evaluate
  values from the tuple which are actually used in the string interpolation,
  so we use proxy objects.

  If no args are given, we're able to take the current environment.
  """
  args = args or env
  proxies = {k: StringInterpolationProxy(args, k) for k in args.keys()}
  return str.format(**proxies)

def str_join(lst, sep=' '):
  """Behaves like string.join from Python 2."""
  return sep.join(str(x) for x in lst)

def compose_all(tups):
  """Compose all given tuples together."""
  from . import ast  # I weep for humanity
  return functools.reduce(lambda x, y: x.compose(y), map(ast.make_tuple, tups), ast.make_tuple({}))

def split(string, sep=' '):
  """Splits a string."""
  return string.split(sep)

def has_key(tup, key):
  """Return whether a given tuple has a key and the key is bound."""
  if isinstance(tup, framework.TupleLike):
    return tup.is_bound(key)
  if isinstance(tup, dict):
    return key in tup
  raise ValueError('Not a tuple-like object: %r' % tup)

def flatten(list_of_lists):
  """Flatten a list of lists."""
  ret = []
  for lst in list_of_lists:
    if not isinstance(lst, list):
      raise ValueError('%r is not a list' % lst)
    ret.extend(lst)
  return ret

builtin_functions = {
    'eager': eager,
    'path_join': path.join,
    'join': str_join,
    'fmt': framework.EnvironmentFunction(fmt),
    'sum': sum,
    'compose_all': compose_all,
    'sorted': sorted,
    'split': split,
    'has': has_key,
    'flatten': flatten
    }

# Binary operators, by precedence level
binary_operators = [
    {
      '*': lambda x, y: x * y,
      '/': lambda x, y: x / y,
      '%': lambda x, y: x % y,
    }, {
      '+': lambda x, y: x + y,
      '-': lambda x, y: x - y,
    }, {
      '==': lambda x, y: x == y,
      '!=': lambda x, y: x != y,
      '<': lambda x, y: x < y,
      '<=': lambda x, y: x <= y,
      '>': lambda x, y: x > y,
      '>=': lambda x, y: x >= y,
    }, {
      'and': lambda x, y: x and y,
      'or': lambda x, y: x or y,
    }]
all_binary_operators = {k: v for os in binary_operators for k, v in os.items()}

unary_operators = {
    '-': lambda x: -x,
    'not': lambda x: not x,
    }

<commit_msg>Make 'has' also work on lists (as indices)<commit_after>"""GCL standard library functions."""

import itertools
import functools

from os import path

from . import framework

def eager(x):
  """Force eager evaluation of a Thunk, and turn a Tuple into a dict eagerly.

  This forces that there are no unbound variables at parsing time (as opposed
  to later when the variables may be accessed).

  Only eagerly evaluates one level.
  """
  if not hasattr(x, 'items'):
    # The Thunk has been evaluated already, so that was easy :)
    return x

  return dict(x.items())

class StringInterpolationProxy(object):
  """Lazy stringification object from a tuple.

  Also support period-based element access, cuz it looks good.
  """
  def __init__(self, tup, key):
    self.tup = tup
    self.key = key

  def __getattr__(self, key):
    v = self.tup[self.key]
    # If this is a tuple, return another lazy proxy
    if isinstance(v, framework.TupleLike):
      return StringInterpolationProxy(v, key)
    return v[key]

  def __str__(self):
    return str(self.tup[self.key])

def fmt(str, args=None, env=None):
  """String interpolation.

  Normally, we'd just call str.format(**args), but we only want to evaluate
  values from the tuple which are actually used in the string interpolation,
  so we use proxy objects.

  If no args are given, we're able to take the current environment.
  """
  args = args or env
  proxies = {k: StringInterpolationProxy(args, k) for k in args.keys()}
  return str.format(**proxies)

def str_join(lst, sep=' '):
  """Behaves like string.join from Python 2."""
  return sep.join(str(x) for x in lst)

def compose_all(tups):
  """Compose all given tuples together."""
  from . import ast  # I weep for humanity
  return functools.reduce(lambda x, y: x.compose(y), map(ast.make_tuple, tups), ast.make_tuple({}))

def split(string, sep=' '):
  """Splits a string."""
  return string.split(sep)

def has_key(tup, key):
  """Return whether a given tuple has a key and the key is bound."""
  if isinstance(tup, framework.TupleLike):
    return tup.is_bound(key)
  if isinstance(tup, dict):
    return key in tup
  if isinstance(tup, list):
    if not isinstance(key, int):
      raise ValueError('Key must be integer when checking list index')
    return key < len(tup)
  raise ValueError('Not a tuple-like object: %r' % tup)

def flatten(list_of_lists):
  """Flatten a list of lists."""
  ret = []
  for lst in list_of_lists:
    if not isinstance(lst, list):
      raise ValueError('%r is not a list' % lst)
    ret.extend(lst)
  return ret

builtin_functions = {
    'eager': eager,
    'path_join': path.join,
    'join': str_join,
    'fmt': framework.EnvironmentFunction(fmt),
    'sum': sum,
    'compose_all': compose_all,
    'sorted': sorted,
    'split': split,
    'has': has_key,
    'flatten': flatten
    }

# Binary operators, by precedence level
binary_operators = [
    {
      '*': lambda x, y: x * y,
      '/': lambda x, y: x / y,
      '%': lambda x, y: x % y,
    }, {
      '+': lambda x, y: x + y,
      '-': lambda x, y: x - y,
    }, {
      '==': lambda x, y: x == y,
      '!=': lambda x, y: x != y,
      '<': lambda x, y: x < y,
      '<=': lambda x, y: x <= y,
      '>': lambda x, y: x > y,
      '>=': lambda x, y: x >= y,
    }, {
      'and': lambda x, y: x and y,
      'or': lambda x, y: x or y,
    }]
all_binary_operators = {k: v for os in binary_operators for k, v in os.items()}

unary_operators = {
    '-': lambda x: -x,
    'not': lambda x: not x,
    }

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
A class for parsing set of files containig nntp posts
"""
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

# A path to the directory containing the files
class Parser:

    def __init__(self, path):
        """
        returns a dictionary of parsed files where key = Message-ID and value = tuple
        """
        print path
        pass

    def __getFiles(self, path):
        """
        returns a list of files under the path
        """
        pass

    def __countFiles(self, filelist):
        """
        returns an integer reperesenting the total number of files
        """
        pass

    def __parseFile(self, a_file):
        """
        for a given file returns a tuple with Message-ID, author, date, previous post and content
        """
        pass

    def __getHeader(self, message):
        """
        returns a string with the whole header
        """
        pass

    def __getContent(self, message):
        """
        returns a string containing everything from the post except the header
        """
        pass

    def __clearCitations(self, content):
        """
        returns a string with all the lines begining with '>' deleted
        """
        pass

    def __getID(self, header):
        """
        returns a string with the value of Message-ID field from the header
        """
        pass

    def __getLastReference(self, header):
        """
        returns the Message-ID of the imediate predecessor of the post, or an empty string for the root
        """
        pass

    def __getDate(self, header):
        """
        returns the creation date of the post
        """
        pass

    def __getAuthor(self, header):
        """
        returns a string with the author's name
        """
    pass

nntp_parser = Parser("~/jakas_sciezka")
<commit_msg>getFiles method<commit_after>#!/usr/bin/env python
"""
A class for parsing set of files containig nntp posts
"""
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import os
import sys

class Parser:

    __filelist = []
    __file_no = 0

    def __init__(self, path):
        """
        initializes filelist
        """
        if not os.path.isdir(path):
            sys.exit(path + ' is not a valid directory path')
        f = self.__getFiles(path)
        self.__file_no = len(f)
        self.__filelist = f

    def __getFiles(self, path):
        """
        returns a list of files under the path
        """
        result = []
        for dirpath, dirnames, filenames in os.walk(path):
            for f in filenames:
                if f.endswith('.msg'):
                    result.append(os.path.join(dirpath, f))
        return result

    def __countFiles(self, filelist):
        """
        returns an integer reperesenting the total number of files
        """
        pass

    def __parseFile(self, a_file):
        """
        for a given file returns a tuple with Message-ID, author, date, previous post and content
        """
        pass

    def __getHeader(self, message):
        """
        returns a string with the whole header
        """
        pass

    def __getContent(self, message):
        """
        returns a string containing everything from the post except the header
        """
        pass

    def __clearCitations(self, content):
        """
        returns a string with all the lines begining with '>' deleted
        """
        pass

    def __getID(self, header):
        """
        returns a string with the value of Message-ID field from the header
        """
        pass

    def __getLastReference(self, header):
        """
        returns the Message-ID of the imediate predecessor of the post, or an empty string for the root
        """
        pass

    def __getDate(self, header):
        """
        returns the creation date of the post
        """
        pass

    def __getAuthor(self, header):
        """
        returns a string with the author's name
        """
    pass

nntp_parser = Parser(sys.argv[1])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

<commit_msg>add functions<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

def get_ast(url = None, webelement = None):
	"""
	Return an AST for one webpage.

	Each AST stores data about children, size of an element (width, heigth, x, y)
	"""

	pass

def get_workarea(urls = None, webelements = None):
	"""
	Analyze a given webpage and return xpath to a work area.

	:param urls: list with urls
	:type urls: list
	:param webelements: list of webelements (from selenium)
	:type driver: list
	:return: xpath of work area
	"""

	pass
<|endoftext|>


<commit_before>import inspect
import json
import os

from django.core.urlresolvers import reverse
from django.dispatch import receiver
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.utils.datastructures import MultiValueDictKeyError
from django.views.decorators.csrf import csrf_exempt

from django.conf import settings

from djeventstream.signals import event_received

from registry import event_handlers, request_handlers
from util import default_optional_kwargs

import auth
import util
from util import optional_parameter_call

import logging

log=logging.getLogger(__name__)

@auth.auth
def index(request):
    return HttpResponseRedirect("static/index.html")

@auth.auth
def event_properties(request):
    ''' Adds a view to advertise defined event properties '''
    from registry import event_property_registry
    items = []
    for key in event_property_registry.keys():
        items.append("<di>{name}</di>
{doc}
".format(**event_property_registry[key]))
    return HttpResponse("
"+"\n".join(items)+"
")

@auth.auth
def schema(request):
    ''' Returns all available views and queries as a JSON
    object.
    '''
    from registry import schema_helper
    endpoints = schema_helper()
    if request.GET.get("f", "") == "html":
        return HttpResponse("\n".join(sorted(["

{class}/{name} {category}

{doc}
".format(**rh) for rh in endpoints])))
    return HttpResponse(json.dumps(endpoints))

view_object=None
@auth.auth
def handle_view(request, name, **kwargs):
    ''' Handles generic view.
        Category is where this should be place (per student, per problem, etc.)
        Name is specific
    '''
    global view_object
    if not False:
        from djobject import embed
        view_object = embed('view')
    if name[0] == '_':
        raise SuspiciousOperation(name+' called')
    kwargs.update(request.POST.items())
    kwargs.update(request.GET.items())
    results = view_object.__getattr__(name)(**kwargs)
    return HttpResponse(results)

query_object = None
@auth.auth
def handle_query(request, name, **kwargs):
    ''' Handles generic view.
        Category is where this should be place (per student, per problem, etc.)
        Name is specific
    '''
    global query_object
    if not False:
        from djobject import embed
        query_object = embed('query')
    if name[0] == '_':
        raise SuspiciousOperation(name+' called')
    kwargs.update(request.POST.items())
    kwargs.update(request.GET.items())
    results = query_object.__getattr__(name)(**kwargs)
    try:
        results = json.dumps(results)
    except:
        pass
    return HttpResponse(results)

@receiver(event_received)
def handle_event(sender, **kwargs):
    ''' Receives either an event or a list of events, as sent by
    djeventstream (either from a Python logging HTTPHandler or
    SNSHandler.

    Forwards it along to the event handlers defined in the modules.

    This is not a view, but it is the moral equivalent.
    '''
    # Handle strings, lists, and dictionaries
    msg = kwargs['msg']
    if isinstance(msg,str) or isinstance(msg,unicode):
        msg = json.loads(msg)

    # If we get a batch of events, we need to load them.
    if isinstance(msg,list):
        for i in xrange(0,len(msg)):
            try:
                msg[i] = json.loads(msg[i])
            except:
                pass

    # Single event should still be passed as a list for API
    # compatibility between patched and unbatched.
    if not isinstance(msg, list):
        msg = [msg]

    from registry import StreamingEvent
    msg = map(StreamingEvent, msg)

    for e in event_handlers:
        event_func = e['function']
        batch = e['batch']
        if not batch: ## Message was a list of events, but handler cannot batch events
            for event in msg:
                try:
                    optional_parameter_call(event_func, default_optional_kwargs, {'events':[event]})
                except:
                    handle_event_exception(e['function'])
        else: ## Message was a list of events, and handler can batch events
            try:
                optional_parameter_call(event_func, default_optional_kwargs, {'events':msg})
            except:
                handle_event_exception(e['function'])

    return HttpResponse( "Success" )

def handle_event_exception(function):
    log.exception("Handler {0} failed".format(function))

<commit_msg>Fixed a TODO with 'is None.'<commit_after>import inspect
import json
import os

from django.core.urlresolvers import reverse
from django.dispatch import receiver
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.utils.datastructures import MultiValueDictKeyError
from django.views.decorators.csrf import csrf_exempt

from django.conf import settings

from djeventstream.signals import event_received

from registry import event_handlers, request_handlers
from util import default_optional_kwargs

import auth
import util
from util import optional_parameter_call

import logging

log=logging.getLogger(__name__)

@auth.auth
def index(request):
    return HttpResponseRedirect("static/index.html")

@auth.auth
def event_properties(request):
    ''' Adds a view to advertise defined event properties '''
    from registry import event_property_registry
    items = []
    for key in event_property_registry.keys():
        items.append("<di>{name}</di>
{doc}
".format(**event_property_registry[key]))
    return HttpResponse("
"+"\n".join(items)+"
")

@auth.auth
def schema(request):
    ''' Returns all available views and queries as a JSON
    object.
    '''
    from registry import schema_helper
    endpoints = schema_helper()
    if request.GET.get("f", "") == "html":
        return HttpResponse("\n".join(sorted(["

{class}/{name} {category}

{doc}
".format(**rh) for rh in endpoints])))
    return HttpResponse(json.dumps(endpoints))

view_object=None
@auth.auth
def handle_view(request, name, **kwargs):
    ''' Handles generic view.
        Category is where this should be place (per student, per problem, etc.)
        Name is specific
    '''
    global view_object
    if view_object is None:
        from djobject import embed
        view_object = embed('view')
    if name[0] == '_':
        raise SuspiciousOperation(name+' called')
    kwargs.update(request.POST.items())
    kwargs.update(request.GET.items())
    results = view_object.__getattr__(name)(**kwargs)
    return HttpResponse(results)

query_object = None
@auth.auth
def handle_query(request, name, **kwargs):
    ''' Handles generic view.
        Category is where this should be place (per student, per problem, etc.)
        Name is specific
    '''
    global query_object
    if query_object is None:
        from djobject import embed
        query_object = embed('query')
    if name[0] == '_':
        raise SuspiciousOperation(name+' called')
    kwargs.update(request.POST.items())
    kwargs.update(request.GET.items())
    results = query_object.__getattr__(name)(**kwargs)
    try:
        results = json.dumps(results)
    except:
        pass
    return HttpResponse(results)

@receiver(event_received)
def handle_event(sender, **kwargs):
    ''' Receives either an event or a list of events, as sent by
    djeventstream (either from a Python logging HTTPHandler or
    SNSHandler.

    Forwards it along to the event handlers defined in the modules.

    This is not a view, but it is the moral equivalent.
    '''
    # Handle strings, lists, and dictionaries
    msg = kwargs['msg']
    if isinstance(msg,str) or isinstance(msg,unicode):
        msg = json.loads(msg)

    # If we get a batch of events, we need to load them.
    if isinstance(msg,list):
        for i in xrange(0,len(msg)):
            try:
                msg[i] = json.loads(msg[i])
            except:
                pass

    # Single event should still be passed as a list for API
    # compatibility between patched and unbatched.
    if not isinstance(msg, list):
        msg = [msg]

    from registry import StreamingEvent
    msg = map(StreamingEvent, msg)

    for e in event_handlers:
        event_func = e['function']
        batch = e['batch']
        if not batch: ## Message was a list of events, but handler cannot batch events
            for event in msg:
                try:
                    optional_parameter_call(event_func, default_optional_kwargs, {'events':[event]})
                except:
                    handle_event_exception(e['function'])
        else: ## Message was a list of events, and handler can batch events
            try:
                optional_parameter_call(event_func, default_optional_kwargs, {'events':msg})
            except:
                handle_event_exception(e['function'])

    return HttpResponse( "Success" )

def handle_event_exception(function):
    log.exception("Handler {0} failed".format(function))

<|endoftext|>