<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

#
# OSV: Objects Services
#

import orm
import netsvc
import pooler
import copy
import sys
import traceback
import logging
from psycopg2 import IntegrityError
from tools.func import wraps

module_list = []
module_class_list = {}
class_pool = {}

class except_osv(Exception):
    def __init__(self, name, value, exc_type='warning'):
        self.name = name
        self.exc_type = exc_type
        self.value = value
        self.args = (exc_type, name)

class osv_pool(netsvc.Service):

    def check(f):
        @wraps(f)
        def wrapper(self, dbname, *args, **kwargs):
            try:
                if not pooler.get_pool(dbname)._ready:
                    raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.')
                return f(self, dbname, *args, **kwargs)
            except orm.except_orm, inst:
                if inst.name == 'AccessError':
                    self.logger.debug("AccessError", exc_info=True)
                self.abortResponse(1, inst.name, 'warning', inst.value)
            except except_osv, inst:
                self.abortResponse(1, inst.name, inst.exc_type, inst.value)
            except IntegrityError, inst:
                for key in self._sql_error.keys():
                    if key in inst[0]:
                        self.abortResponse(1, 'Constraint Error', 'warning', self._sql_error[key])
                self.abortResponse(1, 'Integrity Error', 'warning', inst[0])
            except Exception, e:
                self.logger.exception("Uncaught exception")
                raise

        return wrapper

    def __init__(self):
        self._ready = False
        self.obj_pool = {}
        self.module_object_list = {}
        self.created = []
        self._sql_error = {}
        self._store_function = {}
        self._init = True
        self._init_parent = {}
        self.logger = logging.getLogger("web-services")
        netsvc.Service.__init__(self, 'object_proxy', audience='')
        self.exportMethod(self.obj_list)
        self.exportMethod(self.exec_workflow)
        self.exportMethod(self.execute)

    def init_set(self, cr, mode):
        different = mode != self._init
        if different:
            if mode:
                self._init_parent = {}
            if not mode:
                for o in self._init_parent:
                    self.get(o)._parent_store_compute(cr)
            self._init = mode

        self._ready = True
        return different

    def execute_cr(self, cr, uid, obj, method, *args, **kw):
        object = pooler.get_pool(cr.dbname).get(obj)
        if not object:
            raise except_osv('Object Error', 'Object %s doesn\'t exist' % str(obj))
        return getattr(object, method)(cr, uid, *args, **kw)

    @check
    def execute(self, db, uid, obj, method, *args, **kw):
        db, pool = pooler.get_db_and_pool(db)
        cr = db.cursor()
        try:
            try:
                if method.startswith('_'):
                    raise except_osv('Access Denied', 'Private methods (such as %s) cannot be called remotely.' % (method,))
                res = pool.execute_cr(cr, uid, obj, method, *args, **kw)
                if res is None:
                    self.logger.warning('RPC methods cannot return `None` at the moment!')
                cr.commit()
            except Exception:
                cr.rollback()
                raise
        finally:
            cr.close()
        return res

    def exec_workflow_cr(self, cr, uid, obj, method, *args):
        wf_service = netsvc.LocalService("workflow")
        return wf_service.trg_validate(uid, obj, args[0], method, cr)

    @check
    def exec_workflow(self, db, uid, obj, method, *args):
        cr = pooler.get_db(db).cursor()
        try:
            try:
                res = self.exec_workflow_cr(cr, uid, obj, method, *args)
                cr.commit()
            except Exception:
                cr.rollback()
                raise
        finally:
            cr.close()
        return res

    def obj_list(self):
        return self.obj_pool.keys()

    # adds a new object instance to the object pool.
    # if it already existed, the instance is replaced
    def add(self, name, obj_inst):
        if name in self.obj_pool:
            del self.obj_pool[name]
        self.obj_pool[name] = obj_inst

        module = str(obj_inst.__class__)[6:]
        module = module[:len(module)-1]
        module = module.split('.')[0][2:]
        self.module_object_list.setdefault(module, []).append(obj_inst)

    # Return None if object does not exist
    def get(self, name):
        obj = self.obj_pool.get(name, None)
        return obj

    #TODO: pass a list of modules to load
    def instanciate(self, module, cr):
        res = []
        class_list = module_class_list.get(module, [])
        for klass in class_list:
            res.append(klass.createInstance(self, module, cr))
        return res

class osv_base(object):
    def __init__(self, pool, cr):
        pool.add(self._name, self)
        self.pool = pool
        super(osv_base, self).__init__(cr)

    def __new__(cls):
        module = str(cls)[6:]
        module = module[:len(module)-1]
        module = module.split('.')[0][2:]
        if not hasattr(cls, '_module'):
            cls._module = module
        module_class_list.setdefault(cls._module, []).append(cls)
        class_pool[cls._name] = cls
        if module not in module_list:
            module_list.append(cls._module)
        return None

class osv_memory(osv_base, orm.orm_memory):
    #
    # Goal: try to apply inheritancy at the instanciation level and
    #       put objects in the pool var
    #
    def createInstance(cls, pool, module, cr):
        parent_names = getattr(cls, '_inherit', None)
        if parent_names:
            if isinstance(parent_names, (str, unicode)):
                name = cls._name or parent_names
                parent_names = [parent_names]
            else:
                name = cls._name
            if not name:
                raise TypeError('_name is mandatory in case of multiple inheritance')

            for parent_name in ((type(parent_names)==list) and parent_names or [parent_names]):
                parent_class = pool.get(parent_name).__class__
                assert pool.get(parent_name), "parent class %s does not exist in module %s !" % (parent_name, module)
                nattr = {}
                for s in ('_columns', '_defaults'):
                    new = copy.copy(getattr(pool.get(parent_name), s))
                    if hasattr(new, 'update'):
                        new.update(cls.__dict__.get(s, {}))
                    else:
                        new.extend(cls.__dict__.get(s, []))
                    nattr[s] = new
                cls = type(name, (cls, parent_class), nattr)

        obj = object.__new__(cls)
        obj.__init__(pool, cr)
        return obj
    createInstance = classmethod(createInstance)

class osv(osv_base, orm.orm):
    #
    # Goal: try to apply inheritancy at the instanciation level and
    #       put objects in the pool var
    #
    def createInstance(cls, pool, module, cr):
        parent_names = getattr(cls, '_inherit', None)
        if parent_names:
            if isinstance(parent_names, (str, unicode)):
                name = cls._name or parent_names
                parent_names = [parent_names]
            else:
                name = cls._name
            if not name:
                raise TypeError('_name is mandatory in case of multiple inheritance')

            for parent_name in ((type(parent_names)==list) and parent_names or [parent_names]):
                parent_class = pool.get(parent_name).__class__
                assert pool.get(parent_name), "parent class %s does not exist in module %s !" % (parent_name, module)
                nattr = {}
                for s in ('_columns', '_defaults', '_inherits', '_constraints', '_sql_constraints'):
                    new = copy.copy(getattr(pool.get(parent_name), s))
                    if hasattr(new, 'update'):
                        new.update(cls.__dict__.get(s, {}))
                    else:
                        if s=='_constraints':
                            for c in cls.__dict__.get(s, []):
                                exist = False
                                for c2 in range(len(new)):
                                    if new[c2][2]==c[2]:
                                        new[c2] = c
                                        exist = True
                                        break
                                if not exist:
                                    new.append(c)
                        else:
                            new.extend(cls.__dict__.get(s, []))
                    nattr[s] = new
                cls = type(name, (cls, parent_class), nattr)
        obj = object.__new__(cls)
        obj.__init__(pool, cr)
        return obj
    createInstance = classmethod(createInstance)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>[FIX] _constraints : Multiple constraints on the same field should be allowed<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

#
# OSV: Objects Services
#

import orm
import netsvc
import pooler
import copy
import sys
import traceback
import logging
from psycopg2 import IntegrityError
from tools.func import wraps

module_list = []
module_class_list = {}
class_pool = {}

class except_osv(Exception):
    def __init__(self, name, value, exc_type='warning'):
        self.name = name
        self.exc_type = exc_type
        self.value = value
        self.args = (exc_type, name)

class osv_pool(netsvc.Service):

    def check(f):
        @wraps(f)
        def wrapper(self, dbname, *args, **kwargs):
            try:
                if not pooler.get_pool(dbname)._ready:
                    raise except_osv('Database not ready', 'Currently, this database is not fully loaded and can not be used.')
                return f(self, dbname, *args, **kwargs)
            except orm.except_orm, inst:
                if inst.name == 'AccessError':
                    self.logger.debug("AccessError", exc_info=True)
                self.abortResponse(1, inst.name, 'warning', inst.value)
            except except_osv, inst:
                self.abortResponse(1, inst.name, inst.exc_type, inst.value)
            except IntegrityError, inst:
                for key in self._sql_error.keys():
                    if key in inst[0]:
                        self.abortResponse(1, 'Constraint Error', 'warning', self._sql_error[key])
                self.abortResponse(1, 'Integrity Error', 'warning', inst[0])
            except Exception, e:
                self.logger.exception("Uncaught exception")
                raise

        return wrapper

    def __init__(self):
        self._ready = False
        self.obj_pool = {}
        self.module_object_list = {}
        self.created = []
        self._sql_error = {}
        self._store_function = {}
        self._init = True
        self._init_parent = {}
        self.logger = logging.getLogger("web-services")
        netsvc.Service.__init__(self, 'object_proxy', audience='')
        self.exportMethod(self.obj_list)
        self.exportMethod(self.exec_workflow)
        self.exportMethod(self.execute)

    def init_set(self, cr, mode):
        different = mode != self._init
        if different:
            if mode:
                self._init_parent = {}
            if not mode:
                for o in self._init_parent:
                    self.get(o)._parent_store_compute(cr)
            self._init = mode

        self._ready = True
        return different

    def execute_cr(self, cr, uid, obj, method, *args, **kw):
        object = pooler.get_pool(cr.dbname).get(obj)
        if not object:
            raise except_osv('Object Error', 'Object %s doesn\'t exist' % str(obj))
        return getattr(object, method)(cr, uid, *args, **kw)

    @check
    def execute(self, db, uid, obj, method, *args, **kw):
        db, pool = pooler.get_db_and_pool(db)
        cr = db.cursor()
        try:
            try:
                if method.startswith('_'):
                    raise except_osv('Access Denied', 'Private methods (such as %s) cannot be called remotely.' % (method,))
                res = pool.execute_cr(cr, uid, obj, method, *args, **kw)
                if res is None:
                    self.logger.warning('RPC methods cannot return `None` at the moment!')
                cr.commit()
            except Exception:
                cr.rollback()
                raise
        finally:
            cr.close()
        return res

    def exec_workflow_cr(self, cr, uid, obj, method, *args):
        wf_service = netsvc.LocalService("workflow")
        return wf_service.trg_validate(uid, obj, args[0], method, cr)

    @check
    def exec_workflow(self, db, uid, obj, method, *args):
        cr = pooler.get_db(db).cursor()
        try:
            try:
                res = self.exec_workflow_cr(cr, uid, obj, method, *args)
                cr.commit()
            except Exception:
                cr.rollback()
                raise
        finally:
            cr.close()
        return res

    def obj_list(self):
        return self.obj_pool.keys()

    # adds a new object instance to the object pool.
    # if it already existed, the instance is replaced
    def add(self, name, obj_inst):
        if name in self.obj_pool:
            del self.obj_pool[name]
        self.obj_pool[name] = obj_inst

        module = str(obj_inst.__class__)[6:]
        module = module[:len(module)-1]
        module = module.split('.')[0][2:]
        self.module_object_list.setdefault(module, []).append(obj_inst)

    # Return None if object does not exist
    def get(self, name):
        obj = self.obj_pool.get(name, None)
        return obj

    #TODO: pass a list of modules to load
    def instanciate(self, module, cr):
        res = []
        class_list = module_class_list.get(module, [])
        for klass in class_list:
            res.append(klass.createInstance(self, module, cr))
        return res

class osv_base(object):
    def __init__(self, pool, cr):
        pool.add(self._name, self)
        self.pool = pool
        super(osv_base, self).__init__(cr)

    def __new__(cls):
        module = str(cls)[6:]
        module = module[:len(module)-1]
        module = module.split('.')[0][2:]
        if not hasattr(cls, '_module'):
            cls._module = module
        module_class_list.setdefault(cls._module, []).append(cls)
        class_pool[cls._name] = cls
        if module not in module_list:
            module_list.append(cls._module)
        return None

class osv_memory(osv_base, orm.orm_memory):
    #
    # Goal: try to apply inheritancy at the instanciation level and
    #       put objects in the pool var
    #
    def createInstance(cls, pool, module, cr):
        parent_names = getattr(cls, '_inherit', None)
        if parent_names:
            if isinstance(parent_names, (str, unicode)):
                name = cls._name or parent_names
                parent_names = [parent_names]
            else:
                name = cls._name
            if not name:
                raise TypeError('_name is mandatory in case of multiple inheritance')

            for parent_name in ((type(parent_names)==list) and parent_names or [parent_names]):
                parent_class = pool.get(parent_name).__class__
                assert pool.get(parent_name), "parent class %s does not exist in module %s !" % (parent_name, module)
                nattr = {}
                for s in ('_columns', '_defaults'):
                    new = copy.copy(getattr(pool.get(parent_name), s))
                    if hasattr(new, 'update'):
                        new.update(cls.__dict__.get(s, {}))
                    else:
                        new.extend(cls.__dict__.get(s, []))
                    nattr[s] = new
                cls = type(name, (cls, parent_class), nattr)

        obj = object.__new__(cls)
        obj.__init__(pool, cr)
        return obj
    createInstance = classmethod(createInstance)

class osv(osv_base, orm.orm):
    #
    # Goal: try to apply inheritancy at the instanciation level and
    #       put objects in the pool var
    #
    def createInstance(cls, pool, module, cr):
        parent_names = getattr(cls, '_inherit', None)
        if parent_names:
            if isinstance(parent_names, (str, unicode)):
                name = cls._name or parent_names
                parent_names = [parent_names]
            else:
                name = cls._name
            if not name:
                raise TypeError('_name is mandatory in case of multiple inheritance')

            for parent_name in ((type(parent_names)==list) and parent_names or [parent_names]):
                parent_class = pool.get(parent_name).__class__
                assert pool.get(parent_name), "parent class %s does not exist in module %s !" % (parent_name, module)
                nattr = {}
                for s in ('_columns', '_defaults', '_inherits', '_constraints', '_sql_constraints'):
                    new = copy.copy(getattr(pool.get(parent_name), s))
                    if hasattr(new, 'update'):
                        new.update(cls.__dict__.get(s, {}))
                    else:
                        if s=='_constraints':
                            for c in cls.__dict__.get(s, []):
                                exist = False
                                for c2 in range(len(new)):
                                     #For _constraints, we should check field and methods as well
                                     if new[c2][2]==c[2] and new[c2][0]==c[0]:
                                        new[c2] = c
                                        exist = True
                                        break
                                if not exist:
                                    new.append(c)
                        else:
                            new.extend(cls.__dict__.get(s, []))
                    nattr[s] = new
                cls = type(name, (cls, parent_class), nattr)
        obj = object.__new__(cls)
        obj.__init__(pool, cr)
        return obj
    createInstance = classmethod(createInstance)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before>import collections
import numpy
import six

import chainer
from chainer.backends import cuda
from chainer import function
from chainer import utils
from chainer.utils import type_check
from chainer import variable

def _logsumexp(a, xp, axis=None):
    vmax = xp.amax(a, axis=axis, keepdims=True)
    if xp is numpy:
        vmax += xp.log(xp.sum(xp.exp(a - vmax),
                              axis=axis, keepdims=True, dtype=a.dtype))
    else:
        _logsumexp_impl = cuda.reduce(
            'T x, T vmax', 'T y',
            'exp(x - vmax)', 'a + b', 'y += log(a)', '0',
            'logsumexp_impl')
        _logsumexp_impl(a, vmax, vmax, axis=axis, keepdims=True)
    return xp.squeeze(vmax, axis=axis)

def _softmax(x, xp):
    val = xp.exp(x - xp.amax(x, axis=2, keepdims=True))
    val /= xp.sum(val, axis=2, keepdims=True)
    return val

def _label_to_path(labels, blank_symbol, xp):
    path = xp.full((len(labels), labels.shape[1] * 2 + 1),
                   blank_symbol, dtype=numpy.int32)
    path[:, 1::2] = labels
    return path

def _log_dot(prob, rr, xp):
    return _logsumexp(prob + xp.swapaxes(rr, 1, 2), xp, axis=2)

def _move_label_to_back(path, path_length, xp):
    s1 = path.shape[1]  # TODO(okuta): Change name
    index = (xp.arange(0, path.size, s1, dtype=numpy.int32)[:, None] +
             (xp.arange(s1) + path_length[:, None])[:, ::-1] % s1)
    return xp.take(path, index)

def _move_inputs(prob, input_length, xp):
    seq, batch, ch = prob.shape
    rotate = (xp.arange(seq)[:, None] + input_length) % seq
    index = rotate * batch + xp.arange(batch)
    return xp.take(prob.reshape(seq * batch, ch), index, axis=0)

class ConnectionistTemporalClassification(function.Function):

    """The implementation of Connectionist Temporal Classfication loss functions.

    To make it usable for real-world cases, this class has two policies below.
    1. This class computes forward and backward variables in the log domain.
    2. This class applies the softmax function to inputs. The Backward
    values of CTC loss is often overflows. This is avoided by computing
    backward values before the activation function is applied.
    """

    def __init__(self, blank_symbol, reduce='mean'):
        self.blank_symbol = blank_symbol
        self.zero_padding = -10000000000.0

        if reduce not in ('mean', 'no'):
            raise ValueError(
                "only 'mean' and 'no' are valid "
                "for 'reduce', but '%s' is given" % reduce)
        self.reduce = reduce

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() > 3)  # TODO(okuta): > 3?
        l_type = in_types[2]
        type_check.expect(l_type.dtype == numpy.int32)

        x_basetype = in_types[3]  # TODO(oktua): Check x_basetype size

        for i in six.moves.range(3, len(in_types)):
            x_type = in_types[i]
            type_check.expect(
                x_type.dtype == numpy.float32,
                x_type.shape == x_basetype.shape,
            )

    def log_matrix(self, x, xp):
        if xp == numpy:
            res = numpy.ma.log(x).filled(fill_value=self.zero_padding)
        else:
            create_recurrence_relation = cuda.cupy.ElementwiseKernel(
                'T x, T e', 'T y',
                'y = x == 0 ? e : log(x)',
                'create_recurrence_relation')
            res = create_recurrence_relation(x, self.zero_padding)
        return res.astype(numpy.float32)

    def recurrence_relation(self, label, path_length, max_length, dtype, xp):
        """Transition in forword and backword algorithms is represented as matrix.

        See also
        https://blog.wtf.sg/2014/10/06/connectionist-temporal-classification-ctc-with-theano/
        """
        batch, lab = label.shape
        repeat_mask = xp.ones((batch, lab * 2 + 1))
        repeat_mask[:, 1::2] = (label !=
                                xp.take(label, xp.arange(-1, lab - 1)
                                        % lab + xp.arange(0, batch * lab,
                                                          lab)[:, None]))
        repeat_mask[:, 1] = 1
        rr = (xp.eye(max_length, dtype=dtype)[None, :] +
              xp.eye(max_length, k=1, dtype=dtype)[None, :] +
              (xp.eye(max_length, k=2, dtype=dtype) *
               (xp.arange(max_length, dtype=dtype) % dtype(2))[None, :]
               * repeat_mask[:, None]))
        return self.log_matrix(
            rr * (path_length[:, None] > xp.arange(max_length))[..., None], xp)

    # path probablity to label probability
    def label_probability(self, label_size, path, path_length,
                          multiply_seq, xp):
        labels_prob = self.log_matrix(xp.zeros((len(path), label_size),
                                               dtype=multiply_seq.dtype), xp)
        ret = xp.empty(
            (len(multiply_seq),) + labels_prob.shape, dtype=labels_prob.dtype)
        ret[...] = labels_prob
        if xp == numpy:
            for b in six.moves.range(len(path)):
                target_path = path[b][0:path_length[b]]
                chars = {c for c in target_path}
                for c in chars:
                    ret[:, b, c] = _logsumexp(
                        multiply_seq[:, b, 0:path_length[b]]
                        [:, target_path == c], numpy, axis=1)
        else:
            for i, multiply in enumerate(multiply_seq):
                # TODO(okuta): remove loop
                cuda.cupy.ElementwiseKernel(
                    'raw T x, raw I y, raw I l, I b_max, I c_max',
                    'T z',
                    '''
                    T value = z;
                    I b = i / b_max;
                    I c = i - b * b_max;
                    int ind[2] = {b, -1};
                    for (int index = 0; index < c_max; ++index) {
                        ind[1] = index;
                        if (ind[1] < l[ind[0]] && y[ind] == c) {
                            T xvalue = x[ind];
                            T at = xvalue, bt = value;
                            if (value > xvalue) {
                                at = value;
                                bt = xvalue;
                            }
                            value = at + log1p(exp(bt - at));
                        }
                    }
                    z = value;
                    ''',
                    'reduce_probability')(multiply, path, path_length,
                                          labels_prob.shape[1],
                                          path.shape[1], ret[i])
        return ret

    def calc_trans(self, yseq, input_length,
                   label, label_length, path, path_length, xp):
        forward_prob = self.log_matrix(
            xp.eye(path.shape[1], dtype='f')[0], xp)[None, :]
        backward_prob = forward_prob
        offset = xp.arange(
            0, yseq[0].size, yseq[0].shape[1], dtype=path.dtype)[:, None]

        # prob[i] := forward[i] + backward[-i-1]
        index = offset + path
        frr = self.recurrence_relation(
            label, path_length, path.shape[1], numpy.float32, xp)
        prob = xp.empty(
            (len(yseq),) + index.shape, dtype=forward_prob.dtype)
        # forward computation.
        for i, y in enumerate(yseq):
            # calc forward probability in log scale
            forward_prob = xp.take(y, index) + _log_dot(
                forward_prob[:, None, :], frr, xp)
            prob[i] = forward_prob
        r_index = offset + _move_label_to_back(path, path_length, xp)

        # rotate yseq with path_length
        yseq_inv = _move_inputs(yseq, input_length, xp)[::-1]
        brr = self.recurrence_relation(
            _move_label_to_back(label, label_length, xp),
            path_length, path.shape[1], numpy.float32, xp)
        # move to back.
        prob = _move_inputs(prob, input_length, xp)

        # backward computation.
        ps1 = path.shape[1]
        backward_prob_index = (
            xp.arange(0, path.size, ps1, dtype=numpy.int32)[:, None] +
            (xp.arange(ps1) - path_length[:, None]) % ps1)
        for i, y_inv in enumerate(yseq_inv):
            # calc backward probability
            backward_prob = _log_dot(backward_prob[:, None, :], brr, xp)
            prob[-i - 1] += xp.take(
                backward_prob[:, ::-1], backward_prob_index)
            backward_prob += xp.take(y_inv, r_index)

        # move to front.
        return _move_inputs(prob, -self.input_length, xp)

    def forward(self, inputs):
        xp = cuda.get_array_module(inputs[0])
        self.input_length = inputs[0]
        label_length = inputs[1]
        t = inputs[2]
        xs = inputs[3:]

        if chainer.is_debug():
            # Batch size check.
            assert len(xs[0]) == len(t)
            assert len(xs[0]) == len(self.input_length)
            assert len(xs[0]) == len(label_length)

            # Length check.
            assert len(xs) >= xp.max(self.input_length)
            assert len(t[0]) >= xp.max(label_length)

        self.path_length = 2 * label_length + 1

        yseq_shape = (len(xs),) + xs[0].shape
        self.yseq = _softmax(xp.vstack(xs).reshape(yseq_shape), xp)
        log_yseq = self.log_matrix(self.yseq, xp)
        self.path = _label_to_path(t, self.blank_symbol, xp)
        self.prob_trans = self.calc_trans(
            log_yseq, self.input_length, t,
            label_length, self.path, self.path_length, xp)

        loss = -_logsumexp(self.prob_trans[0], xp, axis=1)
        if self.reduce == 'mean':
            loss = utils.force_array(xp.mean(loss))
        return loss,

    def backward(self, inputs, grad_output):
        xp = cuda.get_array_module(inputs[0])
        batch_size = len(inputs[2])

        total_probability = _logsumexp(self.prob_trans[0], xp, axis=1)
        label_prob = self.label_probability(
            self.yseq.shape[2], self.path, self.path_length,
            self.prob_trans, xp)
        self.yseq -= xp.exp(label_prob - total_probability[:, None])
        if self.reduce == 'mean':
            self.yseq *= grad_output[0] / batch_size
        else:
            self.yseq *= grad_output[0][..., None]
        # mask
        self.yseq *= (
            xp.arange(len(self.yseq))[:, None] < self.input_length)[..., None]
        return (None, None, None) + tuple([y for y in self.yseq])

def connectionist_temporal_classification(
        x, t, blank_symbol, input_length=None, label_length=None,
        reduce='mean'):
    """Connectionist Temporal Classification loss function.

    Connectionist Temporal Classification(CTC) [Graves2006]_ is a loss function
    of sequence labeling where the alignment between the inputs and target is
    unknown. See also [Graves2012]_

    The output is a variable whose value depends on the value of
    the option ``reduce``. If it is ``'no'``, it holds the samplewise
    loss values. If it is ``'mean'``, it takes the mean of loss values.

    Args:
        x (sequence of Variable): RNN output at each time. ``x`` must be a list
            of :class:`~chainer.Variable` s. Each element of ``x``, ``x[i]``
            is a :class:`~chainer.Variable` representing output of RNN at time
            ``i``.
        t (Variable): Expected label sequence.
        blank_symbol (int): Index of blank_symbol.
            This value must be non-negative.
        input_length (Variable): Length of valid sequence for each of mini
            batch ``x`` (optional). If input_length is skipped, It regards that
            all of ``x`` is valid input.
        label_length (Variable): Length of valid sequence for each of mini
            batch ``t`` (optional). If label_length is skipped, It regards that
            all of ``t`` is valid input.
        reduce (str): Reduction option. Its value must be either
            ``'mean'`` or ``'no'``. Otherwise,
            :class:`ValueError` is raised.

    Returns:
       ~chainer.Variable:
           A variable holding a scalar value of the CTC loss.
           If ``reduce`` is ``'no'``, the output variable holds array
           whose shape is `(B,)` where `B` is the number of samples.
           If it is ``'mean'``, it holds a scalar.

    .. note::
       You need to input ``x`` without applying to activation functions(e.g.
       softmax function), because this function applies softmax functions
       to ``x`` before calculating CTC loss to avoid numerical limitations.
       You also need to apply softmax function to forwarded values before you
       decode it.

    .. note::
       This function is differentiable only by ``x``.

    .. note::
       This function supports (batch, sequence, 1-dimensional input)-data.

    .. [Graves2006] Alex Graves, Santiago Fernandez,\
    Faustino Gomez, Jurgen Schmidhuber,\
    `Connectionist Temporal Classification: Labelling Unsegmented\
    Sequence Data with Recurrent Neural Networks\
    <ftp://ftp.idsia.ch/pub/juergen/icml2006.pdf>`_

    .. [Graves2012] Alex Graves,\
    `Supervised Sequence Labelling with Recurrent Neural Networks\
    <http://www.cs.toronto.edu/~graves/preprint.pdf>`_

    """
    if not isinstance(x, collections.Sequence):
        raise TypeError('x must be a list of Variables')
    if not isinstance(blank_symbol, int):
        raise TypeError('blank_symbol must be non-negative integer.')
    assert blank_symbol >= 0
    assert blank_symbol < x[0].shape[1]
    # This implementation only supports 1-dimensional data.
    # TODO(jnishi): Support d(>1)-dimentinal inputs.
    assert(len(x[0].shape) == 2)

    if input_length is None:
        xp = cuda.get_array_module(x[0].data)
        input_length = variable.Variable(
            xp.full((len(x[0].data),), len(x), dtype=numpy.int32))
        label_length = variable.Variable(
            xp.full((len(t.data),), len(t.data[0]), dtype=numpy.int32))

    return ConnectionistTemporalClassification(blank_symbol, reduce)(
        input_length, label_length, t, *x)
<commit_msg>Simplify CTC backward<commit_after>import collections
import numpy
import six

import chainer
from chainer.backends import cuda
from chainer import function
from chainer import utils
from chainer.utils import type_check
from chainer import variable

def _logsumexp(a, xp, axis=None):
    vmax = xp.amax(a, axis=axis, keepdims=True)
    if xp is numpy:
        vmax += xp.log(xp.sum(xp.exp(a - vmax),
                              axis=axis, keepdims=True, dtype=a.dtype))
    else:
        _logsumexp_impl = cuda.reduce(
            'T x, T vmax', 'T y',
            'exp(x - vmax)', 'a + b', 'y += log(a)', '0',
            'logsumexp_impl')
        _logsumexp_impl(a, vmax, vmax, axis=axis, keepdims=True)
    return xp.squeeze(vmax, axis=axis)

def _softmax(x, xp):
    val = xp.exp(x - xp.amax(x, axis=2, keepdims=True))
    val /= xp.sum(val, axis=2, keepdims=True)
    return val

def _label_to_path(labels, blank_symbol, xp):
    path = xp.full((len(labels), labels.shape[1] * 2 + 1),
                   blank_symbol, dtype=numpy.int32)
    path[:, 1::2] = labels
    return path

def _log_dot(prob, rr, xp):
    return _logsumexp(prob + xp.swapaxes(rr, 1, 2), xp, axis=2)

def _move_label_to_back(path, path_length, xp):
    s1 = path.shape[1]  # TODO(okuta): Change name
    index = (xp.arange(0, path.size, s1, dtype=numpy.int32)[:, None] +
             (xp.arange(s1) + path_length[:, None])[:, ::-1] % s1)
    return xp.take(path, index)

def _move_inputs(prob, input_length, xp):
    seq, batch, ch = prob.shape
    rotate = (xp.arange(seq)[:, None] + input_length) % seq
    index = rotate * batch + xp.arange(batch)
    return xp.take(prob.reshape(seq * batch, ch), index, axis=0)

class ConnectionistTemporalClassification(function.Function):

    """The implementation of Connectionist Temporal Classfication loss functions.

    To make it usable for real-world cases, this class has two policies below.
    1. This class computes forward and backward variables in the log domain.
    2. This class applies the softmax function to inputs. The Backward
    values of CTC loss is often overflows. This is avoided by computing
    backward values before the activation function is applied.
    """

    def __init__(self, blank_symbol, reduce='mean'):
        self.blank_symbol = blank_symbol
        self.zero_padding = -10000000000.0

        if reduce not in ('mean', 'no'):
            raise ValueError(
                "only 'mean' and 'no' are valid "
                "for 'reduce', but '%s' is given" % reduce)
        self.reduce = reduce

    def check_type_forward(self, in_types):
        type_check.expect(in_types.size() > 3)  # TODO(okuta): > 3?
        l_type = in_types[2]
        type_check.expect(l_type.dtype == numpy.int32)

        x_basetype = in_types[3]  # TODO(oktua): Check x_basetype size

        for i in six.moves.range(3, len(in_types)):
            x_type = in_types[i]
            type_check.expect(
                x_type.dtype == numpy.float32,
                x_type.shape == x_basetype.shape,
            )

    def log_matrix(self, x, xp):
        if xp == numpy:
            res = numpy.ma.log(x).filled(fill_value=self.zero_padding)
        else:
            create_recurrence_relation = cuda.cupy.ElementwiseKernel(
                'T x, T e', 'T y',
                'y = x == 0 ? e : log(x)',
                'create_recurrence_relation')
            res = create_recurrence_relation(x, self.zero_padding)
        return res.astype(numpy.float32)

    def recurrence_relation(self, label, path_length, max_length, dtype, xp):
        """Transition in forword and backword algorithms is represented as matrix.

        See also
        https://blog.wtf.sg/2014/10/06/connectionist-temporal-classification-ctc-with-theano/
        """
        batch, lab = label.shape
        repeat_mask = xp.ones((batch, lab * 2 + 1))
        repeat_mask[:, 1::2] = (label !=
                                xp.take(label, xp.arange(-1, lab - 1)
                                        % lab + xp.arange(0, batch * lab,
                                                          lab)[:, None]))
        repeat_mask[:, 1] = 1
        rr = (xp.eye(max_length, dtype=dtype)[None, :] +
              xp.eye(max_length, k=1, dtype=dtype)[None, :] +
              (xp.eye(max_length, k=2, dtype=dtype) *
               (xp.arange(max_length, dtype=dtype) % dtype(2))[None, :]
               * repeat_mask[:, None]))
        return self.log_matrix(
            rr * (path_length[:, None] > xp.arange(max_length))[..., None], xp)

    # path probablity to label probability
    def label_probability(self, label_size, path, path_length,
                          multiply_seq, xp):
        labels_prob = self.log_matrix(xp.zeros((len(path), label_size),
                                               dtype=multiply_seq.dtype), xp)
        ret = xp.empty(
            (len(multiply_seq),) + labels_prob.shape, dtype=labels_prob.dtype)
        ret[...] = labels_prob
        if xp == numpy:
            for b in six.moves.range(len(path)):
                target_path = path[b][0:path_length[b]]
                chars = {c for c in target_path}
                for c in chars:
                    ret[:, b, c] = _logsumexp(
                        multiply_seq[:, b, 0:path_length[b]]
                        [:, target_path == c], numpy, axis=1)
        else:
            for i, multiply in enumerate(multiply_seq):
                # TODO(okuta): remove loop
                cuda.cupy.ElementwiseKernel(
                    'raw T prob, raw I path, I path_length, I label_size',
                    'T z',
                    '''
                    T value = z;
                    I b = i / label_size;
                    I c = i - b * label_size;
                    for (int index = 0; index < path_length; ++index) {
                        int ind[] = {b, index};
                        if (path[ind] == c) {
                            T xvalue = prob[ind];
                            T at = xvalue, bt = value;
                            if (value > xvalue) {
                                at = value;
                                bt = xvalue;
                            }
                            value = at + log1p(exp(bt - at));
                        }
                    }
                    z = value;
                    ''',
                    'reduce_probability')(multiply, path, path_length[:, None],
                                          label_size, ret[i])
        return ret

    def calc_trans(self, yseq, input_length,
                   label, label_length, path, path_length, xp):
        forward_prob = self.log_matrix(
            xp.eye(path.shape[1], dtype='f')[0], xp)[None, :]
        backward_prob = forward_prob
        offset = xp.arange(
            0, yseq[0].size, yseq[0].shape[1], dtype=path.dtype)[:, None]

        # prob[i] := forward[i] + backward[-i-1]
        index = offset + path
        frr = self.recurrence_relation(
            label, path_length, path.shape[1], numpy.float32, xp)
        prob = xp.empty(
            (len(yseq),) + index.shape, dtype=forward_prob.dtype)
        # forward computation.
        for i, y in enumerate(yseq):
            # calc forward probability in log scale
            forward_prob = xp.take(y, index) + _log_dot(
                forward_prob[:, None, :], frr, xp)
            prob[i] = forward_prob
        r_index = offset + _move_label_to_back(path, path_length, xp)

        # rotate yseq with path_length
        yseq_inv = _move_inputs(yseq, input_length, xp)[::-1]
        brr = self.recurrence_relation(
            _move_label_to_back(label, label_length, xp),
            path_length, path.shape[1], numpy.float32, xp)
        # move to back.
        prob = _move_inputs(prob, input_length, xp)

        # backward computation.
        ps1 = path.shape[1]
        backward_prob_index = (
            xp.arange(0, path.size, ps1, dtype=numpy.int32)[:, None] +
            (xp.arange(ps1) - path_length[:, None]) % ps1)
        for i, y_inv in enumerate(yseq_inv):
            # calc backward probability
            backward_prob = _log_dot(backward_prob[:, None, :], brr, xp)
            prob[-i - 1] += xp.take(
                backward_prob[:, ::-1], backward_prob_index)
            backward_prob += xp.take(y_inv, r_index)

        # move to front.
        return _move_inputs(prob, -self.input_length, xp)

    def forward(self, inputs):
        xp = cuda.get_array_module(inputs[0])
        self.input_length = inputs[0]
        label_length = inputs[1]
        t = inputs[2]
        xs = inputs[3:]

        if chainer.is_debug():
            # Batch size check.
            assert len(xs[0]) == len(t)
            assert len(xs[0]) == len(self.input_length)
            assert len(xs[0]) == len(label_length)

            # Length check.
            assert len(xs) >= xp.max(self.input_length)
            assert len(t[0]) >= xp.max(label_length)

        self.path_length = 2 * label_length + 1

        yseq_shape = (len(xs),) + xs[0].shape
        self.yseq = _softmax(xp.vstack(xs).reshape(yseq_shape), xp)
        log_yseq = self.log_matrix(self.yseq, xp)
        self.path = _label_to_path(t, self.blank_symbol, xp)
        self.prob_trans = self.calc_trans(
            log_yseq, self.input_length, t,
            label_length, self.path, self.path_length, xp)

        loss = -_logsumexp(self.prob_trans[0], xp, axis=1)
        if self.reduce == 'mean':
            loss = utils.force_array(xp.mean(loss))
        return loss,

    def backward(self, inputs, grad_output):
        xp = cuda.get_array_module(inputs[0])
        batch_size = len(inputs[2])

        total_probability = _logsumexp(self.prob_trans[0], xp, axis=1)
        label_prob = self.label_probability(
            self.yseq.shape[2], self.path, self.path_length,
            self.prob_trans, xp)
        self.yseq -= xp.exp(label_prob - total_probability[:, None])
        if self.reduce == 'mean':
            self.yseq *= grad_output[0] / batch_size
        else:
            self.yseq *= grad_output[0][..., None]
        # mask
        self.yseq *= (
            xp.arange(len(self.yseq))[:, None] < self.input_length)[..., None]
        return (None, None, None) + tuple([y for y in self.yseq])

def connectionist_temporal_classification(
        x, t, blank_symbol, input_length=None, label_length=None,
        reduce='mean'):
    """Connectionist Temporal Classification loss function.

    Connectionist Temporal Classification(CTC) [Graves2006]_ is a loss function
    of sequence labeling where the alignment between the inputs and target is
    unknown. See also [Graves2012]_

    The output is a variable whose value depends on the value of
    the option ``reduce``. If it is ``'no'``, it holds the samplewise
    loss values. If it is ``'mean'``, it takes the mean of loss values.

    Args:
        x (sequence of Variable): RNN output at each time. ``x`` must be a list
            of :class:`~chainer.Variable` s. Each element of ``x``, ``x[i]``
            is a :class:`~chainer.Variable` representing output of RNN at time
            ``i``.
        t (Variable): Expected label sequence.
        blank_symbol (int): Index of blank_symbol.
            This value must be non-negative.
        input_length (Variable): Length of valid sequence for each of mini
            batch ``x`` (optional). If input_length is skipped, It regards that
            all of ``x`` is valid input.
        label_length (Variable): Length of valid sequence for each of mini
            batch ``t`` (optional). If label_length is skipped, It regards that
            all of ``t`` is valid input.
        reduce (str): Reduction option. Its value must be either
            ``'mean'`` or ``'no'``. Otherwise,
            :class:`ValueError` is raised.

    Returns:
       ~chainer.Variable:
           A variable holding a scalar value of the CTC loss.
           If ``reduce`` is ``'no'``, the output variable holds array
           whose shape is `(B,)` where `B` is the number of samples.
           If it is ``'mean'``, it holds a scalar.

    .. note::
       You need to input ``x`` without applying to activation functions(e.g.
       softmax function), because this function applies softmax functions
       to ``x`` before calculating CTC loss to avoid numerical limitations.
       You also need to apply softmax function to forwarded values before you
       decode it.

    .. note::
       This function is differentiable only by ``x``.

    .. note::
       This function supports (batch, sequence, 1-dimensional input)-data.

    .. [Graves2006] Alex Graves, Santiago Fernandez,\
    Faustino Gomez, Jurgen Schmidhuber,\
    `Connectionist Temporal Classification: Labelling Unsegmented\
    Sequence Data with Recurrent Neural Networks\
    <ftp://ftp.idsia.ch/pub/juergen/icml2006.pdf>`_

    .. [Graves2012] Alex Graves,\
    `Supervised Sequence Labelling with Recurrent Neural Networks\
    <http://www.cs.toronto.edu/~graves/preprint.pdf>`_

    """
    if not isinstance(x, collections.Sequence):
        raise TypeError('x must be a list of Variables')
    if not isinstance(blank_symbol, int):
        raise TypeError('blank_symbol must be non-negative integer.')
    assert blank_symbol >= 0
    assert blank_symbol < x[0].shape[1]
    # This implementation only supports 1-dimensional data.
    # TODO(jnishi): Support d(>1)-dimentinal inputs.
    assert(len(x[0].shape) == 2)

    if input_length is None:
        xp = cuda.get_array_module(x[0].data)
        input_length = variable.Variable(
            xp.full((len(x[0].data),), len(x), dtype=numpy.int32))
        label_length = variable.Variable(
            xp.full((len(t.data),), len(t.data[0]), dtype=numpy.int32))

    return ConnectionistTemporalClassification(blank_symbol, reduce)(
        input_length, label_length, t, *x)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Volume control.

Configuration parameters:
    button_down: button to decrease volume (default 5)
    button_mute: button to toggle mute (default 1)
    button_up: button to increase volume (default 4)
    cache_timeout: how often we refresh this module in seconds.
        (default 10)
    card: Card to use. amixer supports this. (default None)
    channel: channel to track. Default value is backend dependent.
        (default None)
    command: Choose between "amixer", "pamixer" or "pactl".
        If None, try to guess based on available commands.
        (default None)
    device: Device to use. Defaults value is backend dependent
        (default None)
    format: Format of the output.
        (default '[\?if=is_input 😮|♪]: {percentage}%')
    format_muted: Format of the output when the volume is muted.
        (default '[\?if=is_input 😶|♪]: muted')
    is_input: Is this an input device or an output device?
        (default False)
    max_volume: Allow the volume to be increased past 100% if available.
        pactl and pamixer supports this. (default 120)
    thresholds: Threshold for percent volume.
        (default [(0, 'bad'), (20, 'degraded'), (50, 'good')])
    volume_delta: Percentage amount that the volume is increased or
        decreased by when volume buttons pressed.
        (default 5)

Format placeholders:
    {percentage} Percentage volume

Color options:
    color_muted: Volume is muted, if not supplied color_bad is used
        if set to `None` then the threshold color will be used.

Requires:
    alsa-utils: an alternative implementation of linux sound support
    pamixer: pulseaudio command-line mixer like amixer

Notes:
    If you are changing volume state by external scripts etc and
    want to refresh the module quicker than the i3status interval,
    send a USR1 signal to py3status in the keybinding.
    Example: killall -s USR1 py3status

Examples:
```
# Set thresholds to rainbow colors
volume_status {
    thresholds = [
        (0, "#FF0000"),
        (10, "#E2571E"),
        (20, "#FF7F00"),
        (30, "#FFFF00"),
        (40, "#00FF00"),
        (50, "#96BF33"),
        (60, "#0000FF"),
        (70, "#4B0082"),
        (80, "#8B00FF"),
        (90, "#FFFFFF")
    ]
}
```

@author <Jan T> <jans.tuomi@gmail.com>
@license BSD

SAMPLE OUTPUT
{'color': '#00FF00', 'full_text': u'\u266a: 95%'}

mute
{'color': '#FF0000', 'full_text': u'\u266a: muted'}
"""

import re
from py3status.exceptions import CommandError

STRING_ERROR = "invalid command `%s`"
STRING_NOT_AVAILABLE = "no available binary"
COMMAND_NOT_INSTALLED = "command `%s` not installed"

class AudioBackend:
    def __init__(self, parent):
        self.card = parent.card
        self.channel = parent.channel
        self.device = parent.device
        self.is_input = parent.is_input
        self.max_volume = parent.max_volume
        self.parent = parent
        self.setup(parent)

    def setup(self, parent):
        raise NotImplementedError

    def run_cmd(self, cmd):
        return self.parent.py3.command_run(cmd)

    def command_output(self, cmd):
        return self.parent.py3.command_output(cmd)

class AmixerBackend(AudioBackend):
    def setup(self, parent):
        if self.card is None:
            self.card = "0"
        if self.channel is None:
            self.channel = "Capture" if self.is_input else "Master"
        if self.device is None:
            self.device = "default"
        self.cmd = [
            "amixer",
            "-q",
            "-D",
            self.device,
            "-c",
            self.card,
            "sset",
            self.channel,
        ]
        self.get_volume_cmd = [
            "amixer",
            "-M",
            "-D",
            self.device,
            "-c",
            self.card,
            "sget",
            self.channel,
        ]

    def get_volume(self):
        output = self.command_output(self.get_volume_cmd)

        # find percentage and status
        p = re.compile(r"\[(\d{1,3})%\].*\[(\w{2,3})\]")
        perc, muted = p.search(output).groups()

        # muted should be 'on' or 'off'
        if muted in ["on", "off"]:
            muted = muted == "off"
        else:
            muted = False

        return perc, muted

    def volume_up(self, delta):
        self.run_cmd(self.cmd + ["{}%+".format(delta)])

    def volume_down(self, delta):
        self.run_cmd(self.cmd + ["{}%-".format(delta)])

    def toggle_mute(self):
        self.run_cmd(self.cmd + ["toggle"])

class PamixerBackend(AudioBackend):
    def setup(self, parent):
        is_input = "--source" if self.is_input else "--sink"
        self.cmd = ["pamixer", "--allow-boost", is_input, self.device or "0"]

    def get_volume(self):
        try:
            line = self.command_output(self.cmd + ["--get-mute", "--get-volume"])
        except CommandError as ce:
            # pamixer throws error on zero percent. see #1135
            line = ce.output
        try:
            muted, perc = line.split()
            muted = muted == "true"
        except ValueError:
            muted, perc = None, None
        return perc, muted

    def volume_up(self, delta):
        perc, muted = self.get_volume()
        if int(perc) + delta >= self.max_volume:
            options = ["--set-volume", str(self.max_volume)]
        else:
            options = ["--increase", str(delta)]
        self.run_cmd(self.cmd + options)

    def volume_down(self, delta):
        self.run_cmd(self.cmd + ["--decrease", str(delta)])

    def toggle_mute(self):
        self.run_cmd(self.cmd + ["--toggle-mute"])

class PactlBackend(AudioBackend):
    def setup(self, parent):
        # get available device number if not specified
        self.device_type = "source" if self.is_input else "sink"
        self.device_type_pl = self.device_type + "s"
        self.device_type_cap = self.device_type[0].upper() + self.device_type[1:]

        self.reinit_device = self.device is None
        if self.device is None:
            self.device = self.get_default_device()
        self.update_device()

    def update_device(self):
        self.re_volume = re.compile(
            r"{} \#{}.*?State: (\w+).*?Mute: (\w{{2,3}}).*?Volume:.*?(\d{{1,3}})\%".format(
                self.device_type_cap, self.device
            ),
            re.M | re.DOTALL,
        )

    def get_default_device(self):
        device_id = None

        # Find the default device for the device type
        default_dev_pattern = re.compile(
            r"^Default {}: (.*)$".format(self.device_type_cap)
        )
        output = self.command_output(["pactl", "info"])
        for info_line in output.splitlines():
            default_dev_match = default_dev_pattern.match(info_line)
            if default_dev_match is not None:
                device_id = default_dev_match.groups()[0]
                break

        # with the long gross id, find the associated number
        if device_id is not None:
            output = self.command_output(
                ["pactl", "list", "short", self.device_type_pl]
            )
            for line in output.splitlines():
                parts = line.split()
                if len(parts) < 2:
                    continue
                if parts[1] == device_id:
                    return parts[0]

        raise RuntimeError(
            "Failed to find default {} device.  Looked for {}".format(
                "input" if self.is_input else "output", device_id
            )
        )

    def get_volume(self):
        output = self.command_output(["pactl", "list", self.device_type_pl]).strip()
        try:
            state, muted, perc = self.re_volume.search(output).groups()
        except AttributeError:
            state, muted, perc = None, False, 0
            # if device is unset, try again with possibly
            # a new default device, otherwise print 0
        if self.reinit_device and state != "RUNNING":
            self.device = self.get_default_device()
            self.update_device()

        # muted should be 'on' or 'off'
        if muted in ["yes", "no"]:
            muted = muted == "yes"
        else:
            muted = False

        return perc, muted

    def volume_up(self, delta):
        perc, muted = self.get_volume()
        if int(perc) + delta >= self.max_volume:
            change = "{}%".format(self.max_volume)
        else:
            change = "+{}%".format(delta)
        self.run_cmd(
            [
                "pactl",
                "--",
                "set-{}-volume".format(self.device_type),
                self.device,
                change,
            ]
        )

    def volume_down(self, delta):
        self.run_cmd(
            [
                "pactl",
                "--",
                "set-{}-volume".format(self.device_type),
                self.device,
                "-{}%".format(delta),
            ]
        )

    def toggle_mute(self):
        self.run_cmd(
            ["pactl", "set-{}-mute".format(self.device_type), self.device, "toggle"]
        )

class Py3status:
    """
    """

    # available configuration parameters
    button_down = 5
    button_mute = 1
    button_up = 4
    cache_timeout = 10
    card = None
    channel = None
    command = None
    device = None
    format = u"[\?if=is_input 😮|♪]: {percentage}%"
    format_muted = u"[\?if=is_input 😶|♪]: muted"
    is_input = False
    max_volume = 120
    thresholds = [(0, "bad"), (20, "degraded"), (50, "good")]
    volume_delta = 5

    class Meta:
        def deprecate_function(config):
            # support old thresholds
            return {
                "thresholds": [
                    (0, "bad"),
                    (config.get("threshold_bad", 20), "degraded"),
                    (config.get("threshold_degraded", 50), "good"),
                ]
            }

        deprecated = {
            "function": [{"function": deprecate_function}],
            "remove": [
                {
                    "param": "threshold_bad",
                    "msg": "obsolete set using thresholds parameter",
                },
                {
                    "param": "threshold_degraded",
                    "msg": "obsolete set using thresholds parameter",
                },
            ],
        }

    def post_config_hook(self):
        if not self.command:
            commands = ["pamixer", "pactl", "amixer"]
            # pamixer, pactl requires pulseaudio to work
            if not self.py3.check_commands("pulseaudio"):
                commands = ["amixer"]
            self.command = self.py3.check_commands(commands)
        elif self.command not in ["amixer", "pamixer", "pactl"]:
            raise Exception(STRING_ERROR % self.command)
        elif not self.py3.check_commands(self.command):
            raise Exception(COMMAND_NOT_INSTALLED % self.command)
        if not self.command:
            raise Exception(STRING_NOT_AVAILABLE)

        # turn integers to strings
        if self.card is not None:
            self.card = "%s" % self.card
        if self.device is not None:
            self.device = "%s" % self.device
        self.volume_delta = int(self.volume_delta)

        if self.command == "amixer":
            self.backend = AmixerBackend(self)
        elif self.command == "pamixer":
            self.backend = PamixerBackend(self)
        elif self.command == "pactl":
            self.backend = PactlBackend(self)

    # compares current volume to the thresholds, returns a color code
    def _perc_to_color(self, string):
        return self.py3.threshold_get_color(string)

    # return the format string formatted with available variables
    def _format_output(self, format, percentage):
        text = self.py3.safe_format(format, {"percentage": percentage})
        return text

    def volume_status(self):
        # call backend
        perc, muted = self.backend.get_volume()

        color = None
        if muted:
            color = self.py3.COLOR_MUTED or self.py3.COLOR_BAD
        if not self.py3.is_color(color):
            # determine the color based on the current volume level
            color = self._perc_to_color(perc)

        # format the output
        text = self._format_output(self.format_muted if muted else self.format, perc)
        # create response dict
        response = {
            "cached_until": self.py3.time_in(self.cache_timeout),
            "color": color,
            "full_text": text,
        }
        return response

    def on_click(self, event):
        """
        Volume up/down and toggle mute.
        """
        button = event["button"]
        # volume up
        if button == self.button_up:
            try:
                self.backend.volume_up(self.volume_delta)
            except TypeError:
                pass
        # volume down
        elif button == self.button_down:
            self.backend.volume_down(self.volume_delta)
        # toggle mute
        elif button == self.button_mute:
            self.backend.toggle_mute()

if __name__ == "__main__":
    """
    Run module in test mode.
    """
    from py3status.module_test import module_test

    module_test(Py3status)
<commit_msg>volume_status module: pactl improvements (#1765)<commit_after># -*- coding: utf-8 -*-
"""
Volume control.

Configuration parameters:
    button_down: button to decrease volume (default 5)
    button_mute: button to toggle mute (default 1)
    button_up: button to increase volume (default 4)
    cache_timeout: how often we refresh this module in seconds.
        (default 10)
    card: Card to use. amixer supports this. (default None)
    channel: channel to track. Default value is backend dependent.
        (default None)
    command: Choose between "amixer", "pamixer" or "pactl".
        If None, try to guess based on available commands.
        (default None)
    device: Device to use. Defaults value is backend dependent
        (default None)
    format: Format of the output.
        (default '[\?if=is_input 😮|♪]: {percentage}%')
    format_muted: Format of the output when the volume is muted.
        (default '[\?if=is_input 😶|♪]: muted')
    is_input: Is this an input device or an output device?
        (default False)
    max_volume: Allow the volume to be increased past 100% if available.
        pactl and pamixer supports this. (default 120)
    thresholds: Threshold for percent volume.
        (default [(0, 'bad'), (20, 'degraded'), (50, 'good')])
    volume_delta: Percentage amount that the volume is increased or
        decreased by when volume buttons pressed.
        (default 5)

Format placeholders:
    {percentage} Percentage volume

Color options:
    color_muted: Volume is muted, if not supplied color_bad is used
        if set to `None` then the threshold color will be used.

Requires:
    alsa-utils: an alternative implementation of linux sound support
    pamixer: pulseaudio command-line mixer like amixer

Notes:
    If you are changing volume state by external scripts etc and
    want to refresh the module quicker than the i3status interval,
    send a USR1 signal to py3status in the keybinding.
    Example: killall -s USR1 py3status

Examples:
```
# Set thresholds to rainbow colors
volume_status {
    thresholds = [
        (0, "#FF0000"),
        (10, "#E2571E"),
        (20, "#FF7F00"),
        (30, "#FFFF00"),
        (40, "#00FF00"),
        (50, "#96BF33"),
        (60, "#0000FF"),
        (70, "#4B0082"),
        (80, "#8B00FF"),
        (90, "#FFFFFF")
    ]
}
```

@author <Jan T> <jans.tuomi@gmail.com>
@license BSD

SAMPLE OUTPUT
{'color': '#00FF00', 'full_text': u'\u266a: 95%'}

mute
{'color': '#FF0000', 'full_text': u'\u266a: muted'}
"""

import re
from py3status.exceptions import CommandError

STRING_ERROR = "invalid command `%s`"
STRING_NOT_AVAILABLE = "no available binary"
COMMAND_NOT_INSTALLED = "command `%s` not installed"

class AudioBackend:
    def __init__(self, parent):
        self.card = parent.card
        self.channel = parent.channel
        self.device = parent.device
        self.is_input = parent.is_input
        self.max_volume = parent.max_volume
        self.parent = parent
        self.setup(parent)

    def setup(self, parent):
        raise NotImplementedError

    def run_cmd(self, cmd):
        return self.parent.py3.command_run(cmd)

    def command_output(self, cmd):
        return self.parent.py3.command_output(cmd)

class AmixerBackend(AudioBackend):
    def setup(self, parent):
        if self.card is None:
            self.card = "0"
        if self.channel is None:
            self.channel = "Capture" if self.is_input else "Master"
        if self.device is None:
            self.device = "default"
        self.cmd = [
            "amixer",
            "-q",
            "-D",
            self.device,
            "-c",
            self.card,
            "sset",
            self.channel,
        ]
        self.get_volume_cmd = [
            "amixer",
            "-M",
            "-D",
            self.device,
            "-c",
            self.card,
            "sget",
            self.channel,
        ]

    def get_volume(self):
        output = self.command_output(self.get_volume_cmd)

        # find percentage and status
        p = re.compile(r"\[(\d{1,3})%\].*\[(\w{2,3})\]")
        perc, muted = p.search(output).groups()

        # muted should be 'on' or 'off'
        if muted in ["on", "off"]:
            muted = muted == "off"
        else:
            muted = False

        return perc, muted

    def volume_up(self, delta):
        self.run_cmd(self.cmd + ["{}%+".format(delta)])

    def volume_down(self, delta):
        self.run_cmd(self.cmd + ["{}%-".format(delta)])

    def toggle_mute(self):
        self.run_cmd(self.cmd + ["toggle"])

class PamixerBackend(AudioBackend):
    def setup(self, parent):
        is_input = "--source" if self.is_input else "--sink"
        self.cmd = ["pamixer", "--allow-boost", is_input, self.device or "0"]

    def get_volume(self):
        try:
            line = self.command_output(self.cmd + ["--get-mute", "--get-volume"])
        except CommandError as ce:
            # pamixer throws error on zero percent. see #1135
            line = ce.output
        try:
            muted, perc = line.split()
            muted = muted == "true"
        except ValueError:
            muted, perc = None, None
        return perc, muted

    def volume_up(self, delta):
        perc, muted = self.get_volume()
        if int(perc) + delta >= self.max_volume:
            options = ["--set-volume", str(self.max_volume)]
        else:
            options = ["--increase", str(delta)]
        self.run_cmd(self.cmd + options)

    def volume_down(self, delta):
        self.run_cmd(self.cmd + ["--decrease", str(delta)])

    def toggle_mute(self):
        self.run_cmd(self.cmd + ["--toggle-mute"])

class PactlBackend(AudioBackend):
    def setup(self, parent):
        # get available device number if not specified
        self.device_type = "source" if self.is_input else "sink"
        self.device_type_pl = self.device_type + "s"
        self.device_type_cap = self.device_type[0].upper() + self.device_type[1:]

        self.use_default_device = self.device is None
        if self.use_default_device:
            self.device = self.get_default_device()
        self.update_device()

    def update_device(self):
        self.re_volume = re.compile(
            r"{} (?:\#{}|.*?Name: {}).*?Mute: (\w{{2,3}}).*?Volume:.*?(\d{{1,3}})\%".format(
                self.device_type_cap, self.device, self.device
            ),
            re.M | re.DOTALL,
        )

    def get_default_device(self):
        device_id = None

        # Find the default device for the device type
        default_dev_pattern = re.compile(
            r"^Default {}: (.*)$".format(self.device_type_cap)
        )
        output = self.command_output(["pactl", "info"])
        for info_line in output.splitlines():
            default_dev_match = default_dev_pattern.match(info_line)
            if default_dev_match is not None:
                device_id = default_dev_match.groups()[0]
                break

        # with the long gross id, find the associated number
        if device_id is not None:
            output = self.command_output(
                ["pactl", "list", "short", self.device_type_pl]
            )
            for line in output.splitlines():
                parts = line.split()
                if len(parts) < 2:
                    continue
                if parts[1] == device_id:
                    return parts[0]

        raise RuntimeError(
            "Failed to find default {} device.  Looked for {}".format(
                "input" if self.is_input else "output", device_id
            )
        )

    def get_volume(self):
        output = self.command_output(["pactl", "list", self.device_type_pl]).strip()
        if self.use_default_device:
            self.device = self.get_default_device()
            self.update_device()
        try:
            muted, perc = self.re_volume.search(output).groups()
            muted = muted == "yes"
        except AttributeError:
            muted, perc = None, None
        return perc, muted

    def volume_up(self, delta):
        perc, muted = self.get_volume()
        if int(perc) + delta >= self.max_volume:
            change = "{}%".format(self.max_volume)
        else:
            change = "+{}%".format(delta)
        self.run_cmd(
            [
                "pactl",
                "--",
                "set-{}-volume".format(self.device_type),
                self.device,
                change,
            ]
        )

    def volume_down(self, delta):
        self.run_cmd(
            [
                "pactl",
                "--",
                "set-{}-volume".format(self.device_type),
                self.device,
                "-{}%".format(delta),
            ]
        )

    def toggle_mute(self):
        self.run_cmd(
            ["pactl", "set-{}-mute".format(self.device_type), self.device, "toggle"]
        )

class Py3status:
    """
    """

    # available configuration parameters
    button_down = 5
    button_mute = 1
    button_up = 4
    cache_timeout = 10
    card = None
    channel = None
    command = None
    device = None
    format = u"[\?if=is_input 😮|♪]: {percentage}%"
    format_muted = u"[\?if=is_input 😶|♪]: muted"
    is_input = False
    max_volume = 120
    thresholds = [(0, "bad"), (20, "degraded"), (50, "good")]
    volume_delta = 5

    class Meta:
        def deprecate_function(config):
            # support old thresholds
            return {
                "thresholds": [
                    (0, "bad"),
                    (config.get("threshold_bad", 20), "degraded"),
                    (config.get("threshold_degraded", 50), "good"),
                ]
            }

        deprecated = {
            "function": [{"function": deprecate_function}],
            "remove": [
                {
                    "param": "threshold_bad",
                    "msg": "obsolete set using thresholds parameter",
                },
                {
                    "param": "threshold_degraded",
                    "msg": "obsolete set using thresholds parameter",
                },
            ],
        }

    def post_config_hook(self):
        if not self.command:
            commands = ["pamixer", "pactl", "amixer"]
            # pamixer, pactl requires pulseaudio to work
            if not self.py3.check_commands("pulseaudio"):
                commands = ["amixer"]
            self.command = self.py3.check_commands(commands)
        elif self.command not in ["amixer", "pamixer", "pactl"]:
            raise Exception(STRING_ERROR % self.command)
        elif not self.py3.check_commands(self.command):
            raise Exception(COMMAND_NOT_INSTALLED % self.command)
        if not self.command:
            raise Exception(STRING_NOT_AVAILABLE)

        # turn integers to strings
        if self.card is not None:
            self.card = "%s" % self.card
        if self.device is not None:
            self.device = "%s" % self.device
        self.volume_delta = int(self.volume_delta)

        if self.command == "amixer":
            self.backend = AmixerBackend(self)
        elif self.command == "pamixer":
            self.backend = PamixerBackend(self)
        elif self.command == "pactl":
            self.backend = PactlBackend(self)

    # compares current volume to the thresholds, returns a color code
    def _perc_to_color(self, string):
        return self.py3.threshold_get_color(string)

    # return the format string formatted with available variables
    def _format_output(self, format, percentage):
        text = self.py3.safe_format(format, {"percentage": percentage})
        return text

    def volume_status(self):
        # call backend
        perc, muted = self.backend.get_volume()

        color = None
        if perc is None or muted is None:
            color = self.py3.COLOR_BAD
            perc = "?"
        else:
            if muted:
                color = self.py3.COLOR_MUTED or self.py3.COLOR_BAD
            if not self.py3.is_color(color):
                # determine the color based on the current volume level
                color = self._perc_to_color(perc)

        # format the output
        text = self._format_output(self.format_muted if muted else self.format, perc)

        # create response dict
        response = {
            "cached_until": self.py3.time_in(self.cache_timeout),
            "color": color,
            "full_text": text,
        }
        return response

    def on_click(self, event):
        """
        Volume up/down and toggle mute.
        """
        button = event["button"]
        # volume up
        if button == self.button_up:
            try:
                self.backend.volume_up(self.volume_delta)
            except TypeError:
                pass
        # volume down
        elif button == self.button_down:
            self.backend.volume_down(self.volume_delta)
        # toggle mute
        elif button == self.button_mute:
            self.backend.toggle_mute()

if __name__ == "__main__":
    """
    Run module in test mode.
    """
    from py3status.module_test import module_test

    module_test(Py3status)
<|endoftext|>