<commit_before># -*- coding: utf-8 -*-
'''
Management of NFS exports
===============================================

To ensure an NFS export exists:

.. code-block:: yaml

    add_simple_export:
      nfs_export.present:
        - name:     '/srv/nfs'
        - hosts:    '10.0.2.0/24'
        - options:
          - 'rw'

This creates the following in /etc/exports:

.. code-block:: bash

    /srv/nfs 10.0.2.0/24(rw)

For more complex exports with multiple groups of hosts, use 'clients':

.. code-block:: yaml

    add_complex_export:
      nfs_export.present:
        - name: '/srv/nfs'
        - clients:
          # First export, same as simple one above
          - hosts: '10.0.2.0/24'
            options:
              - 'rw'
          # Second export
          - hosts: '*.example.com'
            options:
              - 'ro'
              - 'subtree_check'

This creates the following in /etc/exports:

.. code-block:: bash

    /srv/nfs 10.0.2.0/24(rw) 192.168.0.0/24,172.19.0.0/16(ro,subtree_check)

Any export of the given path will be modified to match the one specified.

To ensure an NFS export is absent:

.. code-block:: yaml

    delete_export:
      nfs_export.absent:
        - name: '/srv/nfs'

'''

def present(name, clients=None, hosts=None, options=None, exports='/etc/exports'):
    '''
    Ensure that the named export is present with the given options

    name
        The export path to configure

    clients
        A list of hosts and the options applied to them.
        This option may not be used in combination with
        the 'hosts' or 'options' shortcuts.

    .. code-block:: yaml

        - clients:
          # First export
          - hosts: '10.0.2.0/24'
            options:
              - 'rw'
          # Second export
          - hosts: '*.example.com'
            options:
              - 'ro'
              - 'subtree_check'

    hosts
        A string matching a number of hosts, for example:

    .. code-block:: yaml

        hosts: '10.0.2.123'

        hosts: '10.0.2.0/24'

        hosts: 'minion1.example.com'

        hosts: '*.example.com'

        hosts: '*'

    options
        A list of NFS options, for example:

    .. code-block:: yaml

        options:
          - 'rw'
          - 'subtree_check'

    '''
    path = name
    ret = {'name': name,
           'changes': {},
           'result': None,
           'comment': ''}

    if not clients:
        if not hosts:
            ret['result']   = False
            ret['comment']  = 'Either \'clients\' or \'hosts\' must be defined'
            return ret
        # options being None is handled by add_export()
        clients = [{'hosts': hosts, 'options': options}]

    old = __salt__['nfs3.list_exports'](exports)
    if path in old:
        if old[path] == clients:
            ret['result']   = True
            ret['comment']  = 'Export {0} already configured'.format(path)
            return ret

        ret['changes']['new'] = clients
        ret['changes']['old'] = old[path]
        if __opts__['test']:
            ret['result'] = None
            ret['comment']  = 'Export {0} would be changed'.format(path)
            return ret

        __salt__['nfs3.del_export'](exports, path)

    else:
        ret['changes']['old'] = None
        ret['changes']['new'] = clients
        if __opts__['test']:
            ret['result'] = None
            ret['comment']  = 'Export {0} would be added'.format(path)
            return ret

    for export in clients:
        __salt__['nfs3.add_export'](exports, path, export['hosts'], export['options'])

    ret['changes']['new'] = clients

    export_attempt = __salt__['nfs3.reload_exports']()
    ret['comment'] = export_attempt['stdout'] + "\n" + export_attempt['stderr']
    ret['result'] = export_attempt['result']
    return ret

def absent(name, exports='/etc/exports'):
    '''
    Ensure that the named path is not exported

    name
        The export path to remove
    '''

    path = name
    ret = {'name': name,
           'changes': {},
           'result': None,
           'comment': ''}

    old = __salt__['nfs3.list_exports'](exports)
    if path in old:
        if __opts__['test']:
            ret['comment']  = 'Export {0} would be removed'.format(path)
            ret['changes'][path] = old[path]
            ret['result']   = None
            return ret

        __salt__['nfs3.del_export'](exports, path)
        ret['comment']  = 'Export {0} removed'.format(path)
        ret['changes'][path] = old[path]
        ret['result']   = True
    else:
        ret['comment'] = 'Export {0} already absent'.format(path)
        ret['result']   = True

    return ret
<commit_msg>nfs_exports: add reload to absent() with error handling<commit_after># -*- coding: utf-8 -*-
'''
Management of NFS exports
===============================================

To ensure an NFS export exists:

.. code-block:: yaml

    add_simple_export:
      nfs_export.present:
        - name:     '/srv/nfs'
        - hosts:    '10.0.2.0/24'
        - options:
          - 'rw'

This creates the following in /etc/exports:

.. code-block:: bash

    /srv/nfs 10.0.2.0/24(rw)

For more complex exports with multiple groups of hosts, use 'clients':

.. code-block:: yaml

    add_complex_export:
      nfs_export.present:
        - name: '/srv/nfs'
        - clients:
          # First export, same as simple one above
          - hosts: '10.0.2.0/24'
            options:
              - 'rw'
          # Second export
          - hosts: '*.example.com'
            options:
              - 'ro'
              - 'subtree_check'

This creates the following in /etc/exports:

.. code-block:: bash

    /srv/nfs 10.0.2.0/24(rw) 192.168.0.0/24,172.19.0.0/16(ro,subtree_check)

Any export of the given path will be modified to match the one specified.

To ensure an NFS export is absent:

.. code-block:: yaml

    delete_export:
      nfs_export.absent:
        - name: '/srv/nfs'

'''

def present(name, clients=None, hosts=None, options=None, exports='/etc/exports'):
    '''
    Ensure that the named export is present with the given options

    name
        The export path to configure

    clients
        A list of hosts and the options applied to them.
        This option may not be used in combination with
        the 'hosts' or 'options' shortcuts.

    .. code-block:: yaml

        - clients:
          # First export
          - hosts: '10.0.2.0/24'
            options:
              - 'rw'
          # Second export
          - hosts: '*.example.com'
            options:
              - 'ro'
              - 'subtree_check'

    hosts
        A string matching a number of hosts, for example:

    .. code-block:: yaml

        hosts: '10.0.2.123'

        hosts: '10.0.2.0/24'

        hosts: 'minion1.example.com'

        hosts: '*.example.com'

        hosts: '*'

    options
        A list of NFS options, for example:

    .. code-block:: yaml

        options:
          - 'rw'
          - 'subtree_check'

    '''
    path = name
    ret = {'name': name,
           'changes': {},
           'result': None,
           'comment': ''}

    if not clients:
        if not hosts:
            ret['result']   = False
            ret['comment']  = 'Either \'clients\' or \'hosts\' must be defined'
            return ret
        # options being None is handled by add_export()
        clients = [{'hosts': hosts, 'options': options}]

    old = __salt__['nfs3.list_exports'](exports)
    if path in old:
        if old[path] == clients:
            ret['result']   = True
            ret['comment']  = 'Export {0} already configured'.format(path)
            return ret

        ret['changes']['new'] = clients
        ret['changes']['old'] = old[path]
        if __opts__['test']:
            ret['result'] = None
            ret['comment']  = 'Export {0} would be changed'.format(path)
            return ret

        __salt__['nfs3.del_export'](exports, path)

    else:
        ret['changes']['old'] = None
        ret['changes']['new'] = clients
        if __opts__['test']:
            ret['result'] = None
            ret['comment']  = 'Export {0} would be added'.format(path)
            return ret

    for export in clients:
        __salt__['nfs3.add_export'](exports, path, export['hosts'], export['options'])

    ret['changes']['new'] = clients

    export_attempt = __salt__['nfs3.reload_exports']()
    ret['comment'] = export_attempt['stdout'] + "\n" + export_attempt['stderr']
    ret['result'] = export_attempt['result']
    return ret

def absent(name, exports='/etc/exports'):
    '''
    Ensure that the named path is not exported

    name
        The export path to remove
    '''

    path = name
    ret = {'name': name,
           'changes': {},
           'result': None,
           'comment': ''}

    old = __salt__['nfs3.list_exports'](exports)
    if path in old:
        if __opts__['test']:
            ret['comment']  = 'Export {0} would be removed'.format(path)
            ret['changes'][path] = old[path]
            ret['result']   = None
            return ret

        __salt__['nfs3.del_export'](exports, path)
        export_attempt = __salt__['nfs3.reload_exports']()
        if not export_attempt['result']:
            ret['comment'] = export_attempt['stdout'] + "\n" + export_attempt['stderr']
        else:
            ret['comment']  = 'Export {0} removed'.format(path)

        ret['result'] = export_attempt['result']
        ret['changes'][path] = old[path]
    else:
        ret['comment'] = 'Export {0} already absent'.format(path)
        ret['result']   = True

    return ret
<|endoftext|>


<commit_before>import numpy as nm

import sfepy.linalg as la
from extmods.mappings import CSurfaceMapping

def prepare_remap(indices, n_full):
    """
    Prepare vector for remapping range `[0, n_full]` to its subset given
    by `indices`.
    """
    remap = nm.empty((n_full,), dtype=nm.int32)
    remap.fill(-1)
    remap[indices] = nm.arange(indices.shape[0], dtype=nm.int32)

    return remap

def invert_remap(remap):
    """
    Return the inverse of `remap`, i.e. a mapping from a sub-range
    indices to a full range, see :func:`prepare_remap()`.
    """
    if remap is not None:
        inverse = nm.where(remap >= 0)[0].astype(nm.int32)

    else:
        inverse = None

    return inverse

def compute_nodal_normals(nodes, region, field, return_imap=False):
    """Nodal normals are computed by simple averaging of element normals of
    elements every node is contained in. """
    dim = field.shape[0]

    region.select_cells_of_surface()

    normals = nm.zeros( (nodes.shape[0], dim),
                        dtype = nm.float64 )
    mask = nm.zeros( (nodes.max()+1,), dtype = nm.int32 )
    imap = nm.empty_like( mask )
    imap.fill( nodes.shape[0] ) # out-of-range index for normals.
    imap[nodes] = nm.arange( nodes.shape[0], dtype = nm.int32 )

    for ig, fis in region.fis.iteritems():
        ap = field.aps[ig]
        n_fa = fis.shape[0]
        n_fp = ap.efaces.shape[1]
        face_type = 's%d' % n_fp

        faces = ap.efaces[fis[:,1]]
        ee = ap.econn[fis[:,0]]
        econn = nm.empty( faces.shape, dtype = nm.int32 )
        for ir, face in enumerate( faces ):
            econn[ir] = ee[ir,face]
        mask[econn] += 1
        # Unit normals -> weights = ones.
        ps = ap.interp.poly_spaces[face_type]
        weights = nm.ones((n_fp,), dtype=nm.float64)

        coors = ps.node_coors
        bf_sg = ps.eval_base(coors, diff=True)

        cmap = CSurfaceMapping(n_fa, n_fp, dim, n_fp)
        cmap.describe(field.get_coor(), econn, bf_sg, weights)

        e_normals = cmap.normal.squeeze()

        # normals[imap[econn]] += e_normals
        im = imap[econn]
        for ii, en in enumerate( e_normals ):
            normals[im[ii]] += en

    # All nodes must have a normal.
    if not nm.all( mask[nodes] > 0 ):
        raise ValueError( 'region %s has not complete faces!' % region.name )

    normals /= la.norm_l2_along_axis( normals )[:,nm.newaxis]

    if return_imap:
        return normals, imap

    else:
        return normals

def extend_cell_data( data, domain, rname, val = None ):
    """Extend cell data defined in a region rname to the whole domain using the
    value val, or the smallest value in data if val is None."""
    n_el = domain.shape.n_el
    if data.shape[0] == n_el: return data

    if val is None:
        if data.shape[2] > 1: # Vector.
            val = nm.amin( nm.abs( data ) )
        else: # Scalar.
            val = nm.amin( data )

    edata = nm.empty( (n_el,) + data.shape[1:], dtype = nm.float64 )
    edata.fill( val )

    region = domain.regions[rname]
    offs = region.get_cell_offsets()
    eoffs = domain.get_cell_offsets()
##     print offs
##     print eoffs
##     print domain.mat_ids_to_i_gs
##     pause()

    for group in domain.iter_groups():
        ig = group.ig
        ii = eoffs[ig]
        if ig in region.igs:
            n_cell = region.shape[ig].n_cell
            ir = offs[ig]
            edata[ii+region.cells[ig]] = data[ir:ir+n_cell]
    return edata

def refine_mesh(filename, level):
    """
    Uniformly refine `level`-times a mesh given by `filename`.

    The refined mesh is saved to a file with name constructed from base
    name of `filename` and `level`-times appended `'_r'` suffix.

    Parameters
    ----------
    filename : str
        The mesh file name.
    level : int
        The refinement level.
    """
    import os
    from sfepy.base.base import output
    from sfepy.fem import Mesh, Domain

    if level > 0:
        mesh = Mesh.from_file(filename)
        domain = Domain(mesh.name, mesh)
        for ii in range(level):
            output('refine %d...' % ii)
            domain = domain.refine()
            output('... %d nodes %d elements'
                   % (domain.shape.n_nod, domain.shape.n_el))

        suffix = os.path.splitext(filename)[1]
        filename = domain.name + suffix

        domain.mesh.write(filename, io='auto')

    return filename

<commit_msg>update extend_cell_data() for surface regions<commit_after>import numpy as nm

import sfepy.linalg as la
from extmods.mappings import CSurfaceMapping

def prepare_remap(indices, n_full):
    """
    Prepare vector for remapping range `[0, n_full]` to its subset given
    by `indices`.
    """
    remap = nm.empty((n_full,), dtype=nm.int32)
    remap.fill(-1)
    remap[indices] = nm.arange(indices.shape[0], dtype=nm.int32)

    return remap

def invert_remap(remap):
    """
    Return the inverse of `remap`, i.e. a mapping from a sub-range
    indices to a full range, see :func:`prepare_remap()`.
    """
    if remap is not None:
        inverse = nm.where(remap >= 0)[0].astype(nm.int32)

    else:
        inverse = None

    return inverse

def compute_nodal_normals(nodes, region, field, return_imap=False):
    """Nodal normals are computed by simple averaging of element normals of
    elements every node is contained in. """
    dim = field.shape[0]

    region.select_cells_of_surface()

    normals = nm.zeros( (nodes.shape[0], dim),
                        dtype = nm.float64 )
    mask = nm.zeros( (nodes.max()+1,), dtype = nm.int32 )
    imap = nm.empty_like( mask )
    imap.fill( nodes.shape[0] ) # out-of-range index for normals.
    imap[nodes] = nm.arange( nodes.shape[0], dtype = nm.int32 )

    for ig, fis in region.fis.iteritems():
        ap = field.aps[ig]
        n_fa = fis.shape[0]
        n_fp = ap.efaces.shape[1]
        face_type = 's%d' % n_fp

        faces = ap.efaces[fis[:,1]]
        ee = ap.econn[fis[:,0]]
        econn = nm.empty( faces.shape, dtype = nm.int32 )
        for ir, face in enumerate( faces ):
            econn[ir] = ee[ir,face]
        mask[econn] += 1
        # Unit normals -> weights = ones.
        ps = ap.interp.poly_spaces[face_type]
        weights = nm.ones((n_fp,), dtype=nm.float64)

        coors = ps.node_coors
        bf_sg = ps.eval_base(coors, diff=True)

        cmap = CSurfaceMapping(n_fa, n_fp, dim, n_fp)
        cmap.describe(field.get_coor(), econn, bf_sg, weights)

        e_normals = cmap.normal.squeeze()

        # normals[imap[econn]] += e_normals
        im = imap[econn]
        for ii, en in enumerate( e_normals ):
            normals[im[ii]] += en

    # All nodes must have a normal.
    if not nm.all( mask[nodes] > 0 ):
        raise ValueError( 'region %s has not complete faces!' % region.name )

    normals /= la.norm_l2_along_axis( normals )[:,nm.newaxis]

    if return_imap:
        return normals, imap

    else:
        return normals

def extend_cell_data(data, domain, rname, val=None, is_surface=False):
    """
    Extend cell data defined in a region to the whole domain.

    Parameters
    ----------
    data : array
        The data defined in the region.
    domain : Domain instance
        The FE domain.
    rname : str
        The region name.
    val : float, optional
        The value for filling cells not covered by the region. If not given,
        the smallest value in data is used.
    is_surface : bool
        If True, the data are defined on a surface region. In that case the
        values are averaged into the cells containing the region surface faces.

    Returns
    -------
    edata : array
        The data extended to all domain elements.
    """
    n_el = domain.shape.n_el
    if data.shape[0] == n_el: return data

    if val is None:
        if data.shape[2] > 1: # Vector.
            val = nm.amin(nm.abs(data))
        else: # Scalar.
            val = nm.amin(data)

    edata = nm.empty((n_el,) + data.shape[1:], dtype = nm.float64)
    edata.fill(val)

    region = domain.regions[rname]
    eoffs = domain.get_cell_offsets()

    if not is_surface:
        offs = region.get_cell_offsets()
        for group in domain.iter_groups():
            ig = group.ig
            ii = eoffs[ig]
            if ig in region.igs:
                n_cell = region.shape[ig].n_cell
                ir = offs[ig]
                edata[ii+region.cells[ig]] = data[ir:ir+n_cell]

    else:
        for group in domain.iter_groups():
            ig = group.ig
            ii = eoffs[ig]
            if ig in region.igs:
                cells = region.cells[ig]
                ucells = nm.unique(cells)

                avg = nm.bincount(cells, minlength=group.shape.n_el)[ucells]
                for ic in xrange(data.shape[2]):
                    evals = nm.bincount(cells, weights=data[:, 0, ic, 0],
                                        minlength=group.shape.n_el)[ucells]

                    edata[ii+ucells, 0, ic, 0] = evals / avg

    return edata

def refine_mesh(filename, level):
    """
    Uniformly refine `level`-times a mesh given by `filename`.

    The refined mesh is saved to a file with name constructed from base
    name of `filename` and `level`-times appended `'_r'` suffix.

    Parameters
    ----------
    filename : str
        The mesh file name.
    level : int
        The refinement level.
    """
    import os
    from sfepy.base.base import output
    from sfepy.fem import Mesh, Domain

    if level > 0:
        mesh = Mesh.from_file(filename)
        domain = Domain(mesh.name, mesh)
        for ii in range(level):
            output('refine %d...' % ii)
            domain = domain.refine()
            output('... %d nodes %d elements'
                   % (domain.shape.n_nod, domain.shape.n_el))

        suffix = os.path.splitext(filename)[1]
        filename = domain.name + suffix

        domain.mesh.write(filename, io='auto')

    return filename

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright 2012 Camptocamp SA (Guewen Baconnier)
#    Copyright (C) 2010   Sébastien Beau
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
from openerp.osv.orm import Model, AbstractModel
from openerp.osv import fields
from openerp.tools.translate import _
from openerp.tools import DEFAULT_SERVER_DATETIME_FORMAT

class easy_reconcile_options(AbstractModel):
    """Options of a reconciliation profile, columns
    shared by the configuration of methods and by the
    reconciliation wizards. This allows decoupling
    of the methods with the wizards and allows to
    launch the wizards alone
    """

    _name = 'easy.reconcile.options'

    def _get_rec_base_date(self, cr, uid, context=None):
        return [('end_period_last_credit', 'End of period of most recent credit'),
                ('newest', 'Most recent move line'),
                ('actual', 'Today'),
                ('end_period', 'End of period of most recent move line'),
                ('newest_credit', 'Date of most recent credit'),
                ('newest_debit', 'Date of most recent debit')]

    _columns = {
            'write_off': fields.float('Write off allowed'),
            'account_lost_id': fields.many2one('account.account', 'Account Lost'),
            'account_profit_id': fields.many2one('account.account', 'Account Profit'),
            'journal_id': fields.many2one('account.journal', 'Journal'),
            'date_base_on': fields.selection(_get_rec_base_date,
                required=True,
                string='Date of reconcilation'),
            'filter': fields.char('Filter', size=128),
    }

    _defaults = {
        'write_off': 0.,
        'date_base_on': 'end_period_last_credit',
    }

class account_easy_reconcile_method(Model):

    _name = 'account.easy.reconcile.method'
    _description = 'reconcile method for account_easy_reconcile'

    _inherit = 'easy.reconcile.options'
    _auto = True  # restore property set to False by AbstractModel

    _order = 'sequence'

    def _get_all_rec_method(self, cr, uid, context=None):
        return [
            ('easy.reconcile.simple.name', 'Simple. Amount and Name'),
            ('easy.reconcile.simple.partner', 'Simple. Amount and Partner'),
            ('easy.reconcile.simple.reference', 'Simple. Amount and Reference'),
            ]

    def _get_rec_method(self, cr, uid, context=None):
        return self._get_all_rec_method(cr, uid, context=None)

    _columns = {
            'name': fields.selection(_get_rec_method, 'Type', size=128, required=True),
            'sequence': fields.integer('Sequence', required=True,
                help="The sequence field is used to order the reconcile method"),
            'task_id': fields.many2one('account.easy.reconcile', 'Task',
                required=True, ondelete='cascade'),
    }

    _defaults = {
        'sequence': 1,
    }

    def init(self, cr):
        """ Migration stuff, name is not anymore methods names
        but models name"""
        cr.execute("""
        UPDATE account_easy_reconcile_method
        SET name = 'easy.reconcile.simple.partner'
        WHERE name = 'action_rec_auto_partner'
        """)
        cr.execute("""
        UPDATE account_easy_reconcile_method
        SET name = 'easy.reconcile.simple.name'
        WHERE name = 'action_rec_auto_name'
        """)

class account_easy_reconcile(Model):

    _name = 'account.easy.reconcile'
    _description = 'account easy reconcile'

    def _get_total_unrec(self, cr, uid, ids, name, arg, context=None):
        obj_move_line = self.pool.get('account.move.line')
        res = {}
        for task in self.browse(cr, uid, ids, context=context):
            res[task.id] = len(obj_move_line.search(
                cr, uid,
                [('account_id', '=', task.account.id),
                 ('reconcile_id', '=', False),
                 ('reconcile_partial_id', '=', False)],
                context=context))
        return res

    def _get_partial_rec(self, cr, uid, ids, name, arg, context=None):
        obj_move_line = self.pool.get('account.move.line')
        res = {}
        for task in self.browse(cr, uid, ids, context=context):
            res[task.id] = len(obj_move_line.search(
                cr, uid,
                [('account_id', '=', task.account.id),
                 ('reconcile_id', '=', False),
                 ('reconcile_partial_id', '!=', False)],
                context=context))
        return res

    _columns = {
        'name': fields.char('Name', size=64, required=True),
        'account': fields.many2one('account.account', 'Account', required=True),
        'reconcile_method': fields.one2many('account.easy.reconcile.method', 'task_id', 'Method'),
        'scheduler': fields.many2one('ir.cron', 'scheduler', readonly=True),
        'unreconciled_count': fields.function(_get_total_unrec,
            type='integer', string='Unreconciled Entries'),
        'reconciled_partial_count': fields.function(_get_partial_rec,
            type='integer', string='Partially Reconciled Entries'),
        'history_ids': fields.one2many(
            'easy.reconcile.history',
            'easy_reconcile_id',
            string='History'),
    }

    def copy_data(self, cr, uid, id, default=None, context=None):
        if default is None:
            default = {}
        default = dict(default, scheduler=False)
        return super(account_easy_reconcile, self).copy_data(
            cr, uid, id, default=default, context=context)

    def _prepare_run_transient(self, cr, uid, rec_method, context=None):
        return {'account_id': rec_method.task_id.account.id,
                'write_off': rec_method.write_off,
                'account_lost_id': rec_method.account_lost_id and \
                        rec_method.account_lost_id.id,
                'account_profit_id': rec_method.account_profit_id and \
                        rec_method.account_profit_id.id,
                'journal_id': rec_method.journal_id and rec_method.journal_id.id,
                'date_base_on': rec_method.date_base_on,
                'filter': rec_method.filter}

    def run_reconcile(self, cr, uid, ids, context=None):
        def find_reconcile_ids(fieldname, move_line_ids):
            if not move_line_ids:
                return []
            sql = ("SELECT DISTINCT " + fieldname +
                   " FROM account_move_line "
                   " WHERE id in %s "
                   " AND " + fieldname + " IS NOT NULL")
            cr.execute(sql, (tuple(move_line_ids),))
            res = cr.fetchall()
            return [row[0] for row in res]

        if context is None:
            context = {}
        for rec in self.browse(cr, uid, ids, context=context):
            all_ml_rec_ids = []
            all_ml_partial_ids = []

            for method in rec.reconcile_method:
                rec_model = self.pool.get(method.name)
                auto_rec_id = rec_model.create(
                    cr, uid,
                    self._prepare_run_transient(cr, uid, method, context=context),
                    context=context)

                ml_rec_ids, ml_partial_ids = rec_model.automatic_reconcile(
                    cr, uid, auto_rec_id, context=context)

                all_ml_rec_ids += ml_rec_ids
                all_ml_partial_ids += ml_partial_ids

            reconcile_ids = find_reconcile_ids(
                    'reconcile_id', all_ml_rec_ids)
            partial_ids = find_reconcile_ids(
                    'reconcile_partial_id', all_ml_partial_ids)

            self.pool.get('easy.reconcile.history').create(
                cr,
                uid,
                {'easy_reconcile_id': rec.id,
                 'date': fields.datetime.now(),
                 'reconcile_ids': [(4, rid) for rid in reconcile_ids],
                 'reconcile_partial_ids': [(4, rid) for rid in partial_ids]},
                context=context)
        return True

    def last_history_reconcile(self, cr, uid, rec_id, context=None):
        """ Get the last history record for this reconciliation profile
        and return the action which opens move lines reconciled
        """
        if isinstance(rec_id, (tuple, list)):
            assert len(rec_id) == 1, \
                    "Only 1 id expected"
            rec_id = rec_id[0]
        rec = self.browse(cr, uid, rec_id, context=context)
        # the history is ordered by date desc
        last_history = rec.history_ids[0]
        return last_history.open_reconcile()

    def last_history_partial(self, cr, uid, rec_id, context=None):
        """ Get the last history record for this reconciliation profile
        and return the action which opens move lines reconciled
        """
        if isinstance(rec_id, (tuple, list)):
            assert len(rec_id) == 1, \
                    "Only 1 id expected"
            rec_id = rec_id[0]
        rec = self.browse(cr, uid, rec_id, context=context)
        # the history is ordered by date desc
        last_history = rec.history_ids[0]
        return last_history.open_partial()
<commit_msg>[IMP] add a field last_history<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Copyright 2012 Camptocamp SA (Guewen Baconnier)
#    Copyright (C) 2010   Sébastien Beau
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import time
from openerp.osv.orm import Model, AbstractModel
from openerp.osv import fields
from openerp.tools.translate import _
from openerp.tools import DEFAULT_SERVER_DATETIME_FORMAT

class easy_reconcile_options(AbstractModel):
    """Options of a reconciliation profile, columns
    shared by the configuration of methods and by the
    reconciliation wizards. This allows decoupling
    of the methods with the wizards and allows to
    launch the wizards alone
    """

    _name = 'easy.reconcile.options'

    def _get_rec_base_date(self, cr, uid, context=None):
        return [('end_period_last_credit', 'End of period of most recent credit'),
                ('newest', 'Most recent move line'),
                ('actual', 'Today'),
                ('end_period', 'End of period of most recent move line'),
                ('newest_credit', 'Date of most recent credit'),
                ('newest_debit', 'Date of most recent debit')]

    _columns = {
            'write_off': fields.float('Write off allowed'),
            'account_lost_id': fields.many2one('account.account', 'Account Lost'),
            'account_profit_id': fields.many2one('account.account', 'Account Profit'),
            'journal_id': fields.many2one('account.journal', 'Journal'),
            'date_base_on': fields.selection(_get_rec_base_date,
                required=True,
                string='Date of reconcilation'),
            'filter': fields.char('Filter', size=128),
    }

    _defaults = {
        'write_off': 0.,
        'date_base_on': 'end_period_last_credit',
    }

class account_easy_reconcile_method(Model):

    _name = 'account.easy.reconcile.method'
    _description = 'reconcile method for account_easy_reconcile'

    _inherit = 'easy.reconcile.options'
    _auto = True  # restore property set to False by AbstractModel

    _order = 'sequence'

    def _get_all_rec_method(self, cr, uid, context=None):
        return [
            ('easy.reconcile.simple.name', 'Simple. Amount and Name'),
            ('easy.reconcile.simple.partner', 'Simple. Amount and Partner'),
            ('easy.reconcile.simple.reference', 'Simple. Amount and Reference'),
            ]

    def _get_rec_method(self, cr, uid, context=None):
        return self._get_all_rec_method(cr, uid, context=None)

    _columns = {
            'name': fields.selection(_get_rec_method, 'Type', size=128, required=True),
            'sequence': fields.integer('Sequence', required=True,
                help="The sequence field is used to order the reconcile method"),
            'task_id': fields.many2one('account.easy.reconcile', 'Task',
                required=True, ondelete='cascade'),
    }

    _defaults = {
        'sequence': 1,
    }

    def init(self, cr):
        """ Migration stuff, name is not anymore methods names
        but models name"""
        cr.execute("""
        UPDATE account_easy_reconcile_method
        SET name = 'easy.reconcile.simple.partner'
        WHERE name = 'action_rec_auto_partner'
        """)
        cr.execute("""
        UPDATE account_easy_reconcile_method
        SET name = 'easy.reconcile.simple.name'
        WHERE name = 'action_rec_auto_name'
        """)

class account_easy_reconcile(Model):

    _name = 'account.easy.reconcile'
    _description = 'account easy reconcile'

    def _get_total_unrec(self, cr, uid, ids, name, arg, context=None):
        obj_move_line = self.pool.get('account.move.line')
        res = {}
        for task in self.browse(cr, uid, ids, context=context):
            res[task.id] = len(obj_move_line.search(
                cr, uid,
                [('account_id', '=', task.account.id),
                 ('reconcile_id', '=', False),
                 ('reconcile_partial_id', '=', False)],
                context=context))
        return res

    def _get_partial_rec(self, cr, uid, ids, name, arg, context=None):
        obj_move_line = self.pool.get('account.move.line')
        res = {}
        for task in self.browse(cr, uid, ids, context=context):
            res[task.id] = len(obj_move_line.search(
                cr, uid,
                [('account_id', '=', task.account.id),
                 ('reconcile_id', '=', False),
                 ('reconcile_partial_id', '!=', False)],
                context=context))
        return res

    def _last_history(self, cr, uid, ids, name, args, context=None):
        result = {}
        for history in self.browse(cr, uid, ids, context=context):
            # history is sorted by date desc
            result[history.id] = history.history_ids[0].id
        return result

    _columns = {
        'name': fields.char('Name', size=64, required=True),
        'account': fields.many2one('account.account', 'Account', required=True),
        'reconcile_method': fields.one2many('account.easy.reconcile.method', 'task_id', 'Method'),
        'scheduler': fields.many2one('ir.cron', 'scheduler', readonly=True),
        'unreconciled_count': fields.function(_get_total_unrec,
            type='integer', string='Unreconciled Entries'),
        'reconciled_partial_count': fields.function(_get_partial_rec,
            type='integer', string='Partially Reconciled Entries'),
        'history_ids': fields.one2many(
            'easy.reconcile.history',
            'easy_reconcile_id',
            string='History'),
        'last_history':
            fields.function(
                _last_history,
                string='Last History',
                type='many2one',
                relation='easy.reconcile.history',
                readonly=True),
    }

    def copy_data(self, cr, uid, id, default=None, context=None):
        if default is None:
            default = {}
        default = dict(default, scheduler=False)
        return super(account_easy_reconcile, self).copy_data(
            cr, uid, id, default=default, context=context)

    def _prepare_run_transient(self, cr, uid, rec_method, context=None):
        return {'account_id': rec_method.task_id.account.id,
                'write_off': rec_method.write_off,
                'account_lost_id': rec_method.account_lost_id and \
                        rec_method.account_lost_id.id,
                'account_profit_id': rec_method.account_profit_id and \
                        rec_method.account_profit_id.id,
                'journal_id': rec_method.journal_id and rec_method.journal_id.id,
                'date_base_on': rec_method.date_base_on,
                'filter': rec_method.filter}

    def run_reconcile(self, cr, uid, ids, context=None):
        def find_reconcile_ids(fieldname, move_line_ids):
            if not move_line_ids:
                return []
            sql = ("SELECT DISTINCT " + fieldname +
                   " FROM account_move_line "
                   " WHERE id in %s "
                   " AND " + fieldname + " IS NOT NULL")
            cr.execute(sql, (tuple(move_line_ids),))
            res = cr.fetchall()
            return [row[0] for row in res]

        if context is None:
            context = {}
        for rec in self.browse(cr, uid, ids, context=context):
            all_ml_rec_ids = []
            all_ml_partial_ids = []

            for method in rec.reconcile_method:
                rec_model = self.pool.get(method.name)
                auto_rec_id = rec_model.create(
                    cr, uid,
                    self._prepare_run_transient(cr, uid, method, context=context),
                    context=context)

                ml_rec_ids, ml_partial_ids = rec_model.automatic_reconcile(
                    cr, uid, auto_rec_id, context=context)

                all_ml_rec_ids += ml_rec_ids
                all_ml_partial_ids += ml_partial_ids

            reconcile_ids = find_reconcile_ids(
                    'reconcile_id', all_ml_rec_ids)
            partial_ids = find_reconcile_ids(
                    'reconcile_partial_id', all_ml_partial_ids)

            self.pool.get('easy.reconcile.history').create(
                cr,
                uid,
                {'easy_reconcile_id': rec.id,
                 'date': fields.datetime.now(),
                 'reconcile_ids': [(4, rid) for rid in reconcile_ids],
                 'reconcile_partial_ids': [(4, rid) for rid in partial_ids]},
                context=context)
        return True

    def last_history_reconcile(self, cr, uid, rec_id, context=None):
        """ Get the last history record for this reconciliation profile
        and return the action which opens move lines reconciled
        """
        if isinstance(rec_id, (tuple, list)):
            assert len(rec_id) == 1, \
                    "Only 1 id expected"
            rec_id = rec_id[0]
        rec = self.browse(cr, uid, rec_id, context=context)
        return rec.last_history.open_reconcile()

    def last_history_partial(self, cr, uid, rec_id, context=None):
        """ Get the last history record for this reconciliation profile
        and return the action which opens move lines reconciled
        """
        if isinstance(rec_id, (tuple, list)):
            assert len(rec_id) == 1, \
                    "Only 1 id expected"
            rec_id = rec_id[0]
        rec = self.browse(cr, uid, rec_id, context=context)
        return rec.last_history.open_partial()
<|endoftext|>


<commit_before># Author: Damien Crier
# Author: Julien Coux
# Copyright 2016 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
{
    'name': 'Account Financial Reports',
    'version': '11.0.2.1.0',
    'category': 'Reporting',
    'summary': 'OCA Financial Reports',
    'author': 'Camptocamp SA,'
              'initOS GmbH,'
              'redCOR AG,'
              'Odoo Community Association (OCA)',
    "website": "https://odoo-community.org/",
    'depends': [
        'account',
        'account_invoicing',
        'date_range',
        'report_xlsx',
    ],
    'data': [
        'wizard/aged_partner_balance_wizard_view.xml',
        'wizard/general_ledger_wizard_view.xml',
        'wizard/journal_ledger_wizard_view.xml',
        'wizard/open_items_wizard_view.xml',
        'wizard/trial_balance_wizard_view.xml',
        'wizard/vat_report_wizard_view.xml',
        'menuitems.xml',
        'reports.xml',
        'report/templates/layouts.xml',
        'report/templates/aged_partner_balance.xml',
        'report/templates/general_ledger.xml',
        'report/templates/journal_ledger.xml',
        'report/templates/open_items.xml',
        'report/templates/trial_balance.xml',
        'report/templates/vat_report.xml',
        'view/account_view.xml',
        'view/report_template.xml',
        'view/report_general_ledger.xml',
        'view/report_journal_ledger.xml',
        'view/report_trial_balance.xml',
        'view/report_open_items.xml',
        'view/report_aged_partner_balance.xml',
        'view/report_vat_report.xml',
    ],
    'installable': True,
    'application': True,
    'auto_install': False,
    'license': 'AGPL-3',
}
<commit_msg>Increasing version because of #PR403<commit_after># Author: Damien Crier
# Author: Julien Coux
# Copyright 2016 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
{
    'name': 'Account Financial Reports',
    'version': '11.0.2.2.0',
    'category': 'Reporting',
    'summary': 'OCA Financial Reports',
    'author': 'Camptocamp SA,'
              'initOS GmbH,'
              'redCOR AG,'
              'Odoo Community Association (OCA)',
    "website": "https://odoo-community.org/",
    'depends': [
        'account',
        'account_invoicing',
        'date_range',
        'report_xlsx',
    ],
    'data': [
        'wizard/aged_partner_balance_wizard_view.xml',
        'wizard/general_ledger_wizard_view.xml',
        'wizard/journal_ledger_wizard_view.xml',
        'wizard/open_items_wizard_view.xml',
        'wizard/trial_balance_wizard_view.xml',
        'wizard/vat_report_wizard_view.xml',
        'menuitems.xml',
        'reports.xml',
        'report/templates/layouts.xml',
        'report/templates/aged_partner_balance.xml',
        'report/templates/general_ledger.xml',
        'report/templates/journal_ledger.xml',
        'report/templates/open_items.xml',
        'report/templates/trial_balance.xml',
        'report/templates/vat_report.xml',
        'view/account_view.xml',
        'view/report_template.xml',
        'view/report_general_ledger.xml',
        'view/report_journal_ledger.xml',
        'view/report_trial_balance.xml',
        'view/report_open_items.xml',
        'view/report_aged_partner_balance.xml',
        'view/report_vat_report.xml',
    ],
    'installable': True,
    'application': True,
    'auto_install': False,
    'license': 'AGPL-3',
}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import sys
import os
import shutil
import ConfigParser
from datetime import datetime

# -----------------------------------------------------------------------------
# Utility:
# -----------------------------------------------------------------------------
def clean_text(text, length, uppercase=False, error=None):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []

    text = text.strip()
    if len(text) > length:
        error.append('Text: %s > %s' % (text, length))
    if uppercase:
        return text.upper()
    return text

def clean_date(italian_date, separator='', out_format='iso', error=None):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []
    italian_date = italian_date.split(' ')[0] # remove hour block
    if len(italian_date) != 10:
        error.append('Error not italian date: %s' % italian_date)
        # not stopped
    if out_format == 'iso':
        return '%s%s%s%s%s' % (
            italian_date[-4:],
            separator,
            italian_date[3:5],
            separator,
            italian_date[:2],
            )
    elif out_format == 'italian':
        return '%s%s%s%s%s' % (
            italian_date[:2],
            separator,
            italian_date[3:5],
            separator,
            italian_date[-4:],
            )
    elif out_format == 'english':
        return '%s%s%s%s%s' % (
            italian_date[3:5],
            separator,
            italian_date[:2],
            separator,
            italian_date[-4:],
            )
    else: # incorrect format:
        return italian_date # nothing todo

def clean_float(value, length, decimal=3, separator='.', error=None):
    ''' Clean float and return float format
    '''
    if error is None:
        error = []
    try:
        value = value.replace(',', '.')
        float_value = float(value.strip())
    except:
        error.append('Not a float: %s' % value)
        float_value = 0.0

    mask = '%%%s.%sf' % (length, decimal)
    res = mask % float_value
    res = res.replace('.', separator)
    return res

def log_on_file(message, mode='INFO', file_list=None, verbose=True):
    ''' Write message in file list passed
        mode: INFO WARNING ERROR
    '''
    if file_list is None:
        return False

    message_log = '%s. [%s] %s\n' % (datetime.now(), mode, message)
    if verbose:
        print message_log

    for f_log in file_list:
        if not f_log:
            continue
        f_log.write(message_log.strip())
    return True

# -----------------------------------------------------------------------------
# Read configuration parameter:
# -----------------------------------------------------------------------------
try:
    company = sys.argv[1]
    if len(company) > 3:
        print 'Run: python ./pre_process.py CMP (must be max 3 char length)'
        sys.exit()
except:
    print 'Launch program as:\n\npython ./pre_process.py CMP'

try:
    cfg_file = os.path.expanduser('./%s.cfg' % company)
except:
    print 'Config file not found: %s.cfg' % company

config = ConfigParser.ConfigParser()
config.read([cfg_file])

# In parameters:
in_check = os.path.expanduser(config.get('in', 'check'))
in_path = os.path.expanduser(config.get('in', 'path'))
in_history = os.path.expanduser(config.get('in', 'history'))
in_log = os.path.expanduser(config.get('in', 'log'))
in_schedule = os.path.expanduser(config.get('in', 'schedule'))

# Outp arameters:
out_check = os.path.expanduser(config.get('out', 'check'))
out_path = os.path.expanduser(config.get('out', 'path'))
out_log = os.path.expanduser(config.get('out', 'log'))
out_schedule = os.path.expanduser(config.get('out', 'schedule'))
out_original = os.path.expanduser(config.get('out', 'original'))

# Mail parameters:
mail_error = config.get('mail', 'error')
mail_info = config.get('mail', 'info')

# TODO put in config file:
separator = config.get('file', 'separator')
tot_col = eval(config.get('file', 'tot_col'))

# Calculated parameters:
f_in_schedule = False # open(in_schedule, 'a')
f_in_log = False #open(in_log, 'a')

f_out_schedule = open(out_schedule, 'a')
f_out_log = open(out_log, 'a')

# -----------------------------------------------------------------------------
# Check mount folder:
# -----------------------------------------------------------------------------
if in_check:
    if not os.path.exists(in_check):
        log_on_file(
            'Cannot mount IN folder: %s' % in_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

if out_check:
    if not os.path.exists(out_check):
        log_on_file(
            'Cannot mount OUT folder: %s' % out_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

# -----------------------------------------------------------------------------
# Read IN folder:
# -----------------------------------------------------------------------------
log_on_file(
    'Start import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

for root, dirs, files in os.walk(in_path):
    log_on_file(
        'Read root folder: %s [%s]' % (root, company),
        mode='INFO',
        file_list=[f_in_log, f_out_log])

    for f in files:
        # Fullname needed:
        file_in = os.path.join(root, f)
        file_history = os.path.join(in_history, f)
        file_original = os.path.join(out_original, f)
        file_out = os.path.join(out_path, '%s' % f) # TODO change name

        # ---------------------------------------------------------------------
        # Read input file:
        # ---------------------------------------------------------------------
        error = []
        f_in = open(file_in, 'r')
        row_out = []
        log_on_file(
            'Parsing file: %s [%s]' % (file_in, company),
            mode='INFO',
            file_list=[f_in_log, f_out_log])
        for line in f_in:
            row = line.split(separator)
            if len(row) != tot_col:
                log_on_file(
                    'Different number columns: %s [%s]' % (file_in, company),
                    mode='INFO',
                    file_list=[f_in_log, f_out_log])
                break

            # -----------------------------------------------------------------
            # Read fields:
            # -----------------------------------------------------------------
            deadline = clean_date(row[0], separator='', out_format='iso',
                error=error)

            # Char:
            code = clean_text(row[1], 9, error=error)
            cook_center = clean_text(row[2], 60, error=error) # XXX check
            address = clean_text(row[3], 60, error=error)
            cap = clean_text(row[4], 5, error=error) # XXX Check
            city = clean_text(row[5], 60, error=error) # XXX Check
            province = clean_text(row[6], 4, uppercase=True, error=error)
            order = clean_text(row[7], 10, error=error)
            default_code = clean_text(row[8], 16, uppercase=True, error=error)
            default_code_supplier = clean_text(
                row[9], 16, uppercase=True, error=error)
            name = clean_text(row[10], 60, error=error)
            um = clean_text(row[11], 2, uppercase=True, error=error)

            # Float:
            quantity = clean_float(row[12], 15, 2, error=error)
            price = clean_float(row[13], 15, 3, error=error)
            cost = clean_float(row[14], 15, 3, error=error)

            # -----------------------------------------------------------------
            # Convert row input file:
            # -----------------------------------------------------------------
            row_out.append(
                '%3s|%-8s|%-9s|%-10s|%-60s|%-5s|%-60s|%-60s|%-10s|%-16s|%-16s|'
                '%-60s|%-2s|%-15s|%-15s|%-15s\r\n' % (
                    company, # depend on parameter
                    deadline,
                    code,
                    cook_center,
                    address,
                    cap,
                    city,
                    province,
                    order,
                    default_code,
                    default_code_supplier,
                    name,
                    um,
                    quantity,
                    price,
                    cost,
                    ))
        f_in.close()

        if error:
            log_on_file(
                'Error reading file: %s [%s]' % (error, company),
                mode='ERROR',
                file_list=[f_in_log, f_out_log])
            continue

        # ---------------------------------------------------------------------
        # Save file out:
        # ---------------------------------------------------------------------
        error = []
        # Convert file:
        f_out = open(file_out, 'w')
        for line in row_out:
            f_out.write(line)
        f_out.close()
        if error:
            log_on_file(
                'Error generating out file: %s [%s]' % (error, company),
                mode='ERROR',
                file_list=[f_in_log, f_out_log])
            continue

        # ---------------------------------------------------------------------
        # History the in file:
        # ---------------------------------------------------------------------
        error = ''
        try:
            shutil.copy(file_in, file_original)
        except:
            error += 'Error copy orginal folder %s >> %s' % (
                file_in, file_original)
        try:
            shutil.move(file_in, file_history)
        except:
            error += 'Error moving file %s >> %s' % (file_in, file_history)

        if error:
            log_on_file(
                error,
                mode='ERROR',
                file_list=[f_in_log, f_out_log])
            # XXX TODO delete OUT file
            continue
    break # only root folder is read

log_on_file(
    'End import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

# -----------------------------------------------------------------------------
# History the in file:
# -----------------------------------------------------------------------------
try:
    if f_out_log:
        f_out_log.close()
    if f_in_schedule:
        f_in_schedule.close()
    if f_in_log:
        f_in_log.close()
    if f_out_schedule:
        f_out_schedule.close()
except:
    print 'Error closing log file'
<commit_msg>generic bugfix (fast message)<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import sys
import os
import shutil
import ConfigParser
from datetime import datetime

# -----------------------------------------------------------------------------
# Utility:
# -----------------------------------------------------------------------------
def clean_text(text, length, uppercase=False, error=None):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []

    text = text.strip()
    if len(text) > length:
        error.append('Text: %s > %s' % (text, length))
    if uppercase:
        return text.upper()
    return text

def clean_date(italian_date, separator='', out_format='iso', error=None):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []
    italian_date = italian_date.split(' ')[0] # remove hour block
    if len(italian_date) != 10:
        error.append('Error not italian date: %s' % italian_date)
        # not stopped
    if out_format == 'iso':
        return '%s%s%s%s%s' % (
            italian_date[-4:],
            separator,
            italian_date[3:5],
            separator,
            italian_date[:2],
            )
    elif out_format == 'italian':
        return '%s%s%s%s%s' % (
            italian_date[:2],
            separator,
            italian_date[3:5],
            separator,
            italian_date[-4:],
            )
    elif out_format == 'english':
        return '%s%s%s%s%s' % (
            italian_date[3:5],
            separator,
            italian_date[:2],
            separator,
            italian_date[-4:],
            )
    else: # incorrect format:
        return italian_date # nothing todo

def clean_float(value, length, decimal=3, separator='.', error=None):
    ''' Clean float and return float format
    '''
    if error is None:
        error = []
    try:
        value = value.replace(',', '.')
        float_value = float(value.strip())
    except:
        error.append('Not a float: %s' % value)
        float_value = 0.0

    mask = '%%%s.%sf' % (length, decimal)
    res = mask % float_value
    res = res.replace('.', separator)
    return res

def log_on_file(message, mode='INFO', file_list=None, verbose=True):
    ''' Write message in file list passed
        mode: INFO WARNING ERROR
    '''
    if file_list is None:
        return False

    message_log = '%s. [%s] %s\n' % (datetime.now(), mode, message)
    if verbose:
        print message_log

    for f_log in file_list:
        if not f_log:
            continue
        f_log.write(message_log.strip())
    return True

# -----------------------------------------------------------------------------
# Read configuration parameter:
# -----------------------------------------------------------------------------
try:
    company = sys.argv[1]
    if len(company) > 3:
        print 'Run: python ./pre_process.py CMP (must be max 3 char length)'
        sys.exit()
except:
    print 'Launch program as:\n\npython ./pre_process.py CMP'

try:
    cfg_file = os.path.expanduser('./%s.cfg' % company)
except:
    print 'Config file not found: %s.cfg' % company

config = ConfigParser.ConfigParser()
config.read([cfg_file])

# In parameters:
in_check = os.path.expanduser(config.get('in', 'check'))
in_path = os.path.expanduser(config.get('in', 'path'))
in_history = os.path.expanduser(config.get('in', 'history'))
in_log = os.path.expanduser(config.get('in', 'log'))
in_schedule = os.path.expanduser(config.get('in', 'schedule'))

# Outp arameters:
out_check = os.path.expanduser(config.get('out', 'check'))
out_path = os.path.expanduser(config.get('out', 'path'))
out_log = os.path.expanduser(config.get('out', 'log'))
out_schedule = os.path.expanduser(config.get('out', 'schedule'))
out_original = os.path.expanduser(config.get('out', 'original'))

# Mail parameters:
mail_error = config.get('mail', 'error')
mail_info = config.get('mail', 'info')

# TODO put in config file:
separator = config.get('file', 'separator')
tot_col = eval(config.get('file', 'tot_col'))

# Calculated parameters:
f_in_schedule = False # open(in_schedule, 'a')
f_in_log = False #open(in_log, 'a')

f_out_schedule = open(out_schedule, 'a')
f_out_log = open(out_log, 'a')

# -----------------------------------------------------------------------------
# Check mount folder:
# -----------------------------------------------------------------------------
if in_check:
    if not os.path.exists(in_check):
        log_on_file(
            'Cannot mount IN folder: %s' % in_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

if out_check:
    if not os.path.exists(out_check):
        log_on_file(
            'Cannot mount OUT folder: %s' % out_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

# -----------------------------------------------------------------------------
# Read IN folder:
# -----------------------------------------------------------------------------
log_on_file(
    'Start import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

for root, dirs, files in os.walk(in_path):
    log_on_file(
        'Read root folder: %s [%s]' % (root, company),
        mode='INFO',
        file_list=[f_in_log, f_out_log])

    for f in files:
        # Fullname needed:
        file_in = os.path.join(root, f)
        file_history = os.path.join(in_history, f)
        file_original = os.path.join(out_original, f)
        file_out = os.path.join(out_path, '%s' % f) # TODO change name

        # ---------------------------------------------------------------------
        # Read input file:
        # ---------------------------------------------------------------------
        error = []
        f_in = open(file_in, 'r')
        row_out = []
        log_on_file(
            'Parsing file: %s [%s]' % (file_in, company),
            mode='INFO',
            file_list=[f_in_log, f_out_log])
        for line in f_in:
            row = line.split(separator)
            if len(row) != tot_col:
                log_on_file(
                    'Different number columns: %s [%s]' % (file_in, company),
                    mode='INFO',
                    file_list=[f_in_log, f_out_log])
                break

            # -----------------------------------------------------------------
            # Read fields:
            # -----------------------------------------------------------------
            deadline = clean_date(row[0], separator='', out_format='iso',
                error=error)

            # Char:
            code = clean_text(row[1], 9, error=error)
            cook_center = clean_text(row[2], 60, error=error) # XXX check
            address = clean_text(row[3], 60, error=error)
            cap = clean_text(row[4], 5, error=error) # XXX Check
            city = clean_text(row[5], 60, error=error) # XXX Check
            province = clean_text(row[6], 4, uppercase=True, error=error)
            order = clean_text(row[7], 10, error=error)
            default_code = clean_text(row[8], 16, uppercase=True, error=error)
            default_code_supplier = clean_text(
                row[9], 16, uppercase=True, error=error)
            name = clean_text(row[10], 60, error=error)
            um = clean_text(row[11], 2, uppercase=True, error=error)

            # Float:
            quantity = clean_float(row[12], 15, 2, error=error)
            price = clean_float(row[13], 15, 3, error=error)
            cost = clean_float(row[14], 15, 3, error=error)

            # -----------------------------------------------------------------
            # Convert row input file:
            # -----------------------------------------------------------------
            row_out.append(
                '%3s|%-8s|%-9s|%-10s|%-60s|%-5s|%-60s|%-60s|%-10s|%-16s|%-16s|'
                '%-60s|%-2s|%-15s|%-15s|%-15s\r\n' % (
                    company, # depend on parameter
                    deadline,
                    code,
                    cook_center,
                    address,
                    cap,
                    city,
                    province,
                    order,
                    default_code,
                    default_code_supplier,
                    name,
                    um,
                    quantity,
                    price,
                    cost,
                    ))
        f_in.close()

        if error:
            log_on_file(
                'Error reading file: %s [%s]' % (error, company),
                mode='ERROR',
                file_list=[f_in_log, f_out_log])
            continue

        # ---------------------------------------------------------------------
        # Save file out:
        # ---------------------------------------------------------------------
        error = []
        # Convert file:
        f_out = open(file_out, 'w')
        for line in row_out:
            f_out.write(line)
        f_out.close()
        if error:
            log_on_file(
                'Error generating out file: %s [%s]' % (error, company),
                mode='ERROR',
                file_list=[f_in_log, f_out_log])
            continue

        # ---------------------------------------------------------------------
        # History the in file:
        # ---------------------------------------------------------------------
        error = ''
        try:
            shutil.copy(file_in, file_original)
        except:
            error += 'Error copy orginal folder %s >> %s' % (
                file_in, file_original)
        try:
            os.remove(file_history)
        except:
            pass
        try:
            shutil.move(file_in, file_history)
        except:
            error += 'Error moving file %s >> %s' % (file_in, file_history)

        if error:
            log_on_file(
                error,
                mode='ERROR',
                file_list=[f_in_log, f_out_log])
            # XXX TODO delete OUT file
            continue
    break # only root folder is read

log_on_file(
    'End import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

# -----------------------------------------------------------------------------
# History the in file:
# -----------------------------------------------------------------------------
try:
    if f_out_log:
        f_out_log.close()
    if f_in_schedule:
        f_in_schedule.close()
    if f_in_log:
        f_in_log.close()
    if f_out_schedule:
        f_out_schedule.close()
except:
    print 'Error closing log file'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import re
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.http import HttpResponse
from django.http import JsonResponse
from django.shortcuts import render, get_object_or_404
from data_center.models import Course, Department
from data_center.const import DEPT_CHOICE, GEC_CHOICE, \
    GE_CHOICE
from django.views.decorators.cache import cache_page
from django import forms

from haystack.query import SearchQuerySet
from haystack.inputs import AutoQuery

def group_words(s):
    """Split Chinese token for better search result"""
    regex = []

    # Match a whole word:
    regex += [ur'\w+']

    # Match a single CJK character:
    regex += [ur'[\u4e00-\ufaff]']

    # Match one of anything else, except for spaces:
    regex += [ur'[^\s]']

    regex = "|".join(regex)
    r = re.compile(regex)

    return r.findall(s)

def search(request):
    result = {}
    q = request.GET.get('q', '')
    q = ' '.join(group_words(q))
    page = request.GET.get('page', '')
    size = request.GET.get('size', '')
    code = request.GET.get('code', '')
    dept_required = request.GET.get('dept_required', '')
    sortby_param = request.GET.get('sort', '')
    reverse_param = request.GET.get('reverse', '')

    page_size = size or 10
    sortby = sortby_param or 'time_token'
    reverse = True if reverse_param == 'true' else False
    rev_sortby = '-' + sortby if reverse else sortby

    courses = SearchQuerySet()

    if dept_required:
        try:
            courses = Department.objects.get(
                dept_name=dept_required).required_course.all()
        except:
            pass
        if courses:
            result['type'] = 'required'
            page_size = courses.count()
    else:
        courses = courses.filter(content=AutoQuery(q))
        if code:
            courses = courses.filter(code__contains=code)

        if courses.count() > 300:
            return HttpResponse('TMD')  # Too many detail

        courses = Course.objects.filter(pk__in=[c.pk for c in courses])
        if code in ['GE', 'GEC']:
            core = request.GET.get(code.lower(), '')
            if core:
                courses = courses.filter(ge__contains=core)

    courses = courses.order_by(rev_sortby)
    paginator = Paginator(courses, page_size)

    try:
        courses_page = paginator.page(page)
    except PageNotAnInteger:
        # If page is not an integer, deliver first page.
        courses_page = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        courses_page = paginator.page(paginator.num_pages)

    courses_list = courses_page.object_list. \
        values('id', 'no', 'eng_title', 'chi_title', 'note', 'objective',
               'time', 'time_token', 'teacher', 'room', 'credit',
               'prerequisite', 'ge', 'code')

    result['total'] = courses.count()
    result['page'] = courses_page.number
    result['courses'] = list(courses_list)
    result['page_size'] = page_size

    return JsonResponse(result)

@cache_page(60 * 60)
def syllabus(request, id):
    course = get_object_or_404(Course, id=id)
    return render(request, 'syllabus.html',
                  {'course': course, 'syllabus_path': request.path})

def hit(request, id):
    course = get_object_or_404(Course, id=id)
    course.hit += 1
    course.save()
    return HttpResponse('')

def generate_dept_required_choice():
    choices = (('', '---'),)
    departments = Department.objects.all()
    for department in departments:
        dept_name = department.dept_name
        year = {'104': '一年級', '103': '二年級', '102': '三年級', '101': '四年級'}. \
            get(dept_name[4:7], '')
        degree = {'B': '大學部', 'D': '博士班', 'M': '碩士班', 'P': '專班'}. \
            get(dept_name[7], '')
        chi_dept_name = degree

        if dept_name[7] == 'B':
            chi_dept_name += year
            chi_dept_name += {'BA': '清班', 'BB': '華班', 'BC': '梅班'}. \
                get(dept_name[7:], '')

        choices += ((dept_name, chi_dept_name),)
    return sorted(choices)

class CourseSearchForm(forms.Form):
    DEPT_REQUIRED_CHOICE = generate_dept_required_choice()
    q = forms.CharField(label='關鍵字', required=False)
    code = forms.ChoiceField(label='開課代號', choices=DEPT_CHOICE, required=False)
    ge = forms.ChoiceField(label='向度', choices=GE_CHOICE, required=False)
    gec = forms.ChoiceField(label='向度', choices=GEC_CHOICE, required=False)
    dept_required = forms.ChoiceField(
        label='必選修', choices=DEPT_REQUIRED_CHOICE, required=False)

def table(request):
    render_data = {}
    render_data['search_filter'] = CourseSearchForm(request.GET)
    return render(request, 'table.html', render_data)
<commit_msg>Re-indent search/views.py<commit_after># -*- coding: utf-8 -*-
import re
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.http import HttpResponse
from django.http import JsonResponse
from django.shortcuts import render, get_object_or_404
from data_center.models import Course, Department
from data_center.const import DEPT_CHOICE, GEC_CHOICE, GE_CHOICE
from django.views.decorators.cache import cache_page
from django import forms

from haystack.query import SearchQuerySet
from haystack.inputs import AutoQuery

def group_words(s):
    """Split Chinese token for better search result"""
    regex = []

    # Match a whole word:
    regex += [ur'\w+']

    # Match a single CJK character:
    regex += [ur'[\u4e00-\ufaff]']

    # Match one of anything else, except for spaces:
    regex += [ur'[^\s]']

    regex = "|".join(regex)
    r = re.compile(regex)

    return r.findall(s)

def search(request):
    result = {}
    q = request.GET.get('q', '')
    q = ' '.join(group_words(q))
    page = request.GET.get('page', '')
    size = request.GET.get('size', '')
    code = request.GET.get('code', '')
    dept_required = request.GET.get('dept_required', '')
    sortby_param = request.GET.get('sort', '')
    reverse_param = request.GET.get('reverse', '')

    page_size = size or 10
    sortby = sortby_param or 'time_token'
    reverse = True if reverse_param == 'true' else False
    rev_sortby = '-' + sortby if reverse else sortby

    courses = SearchQuerySet()

    if dept_required:
        try:
            courses = Department.objects.get(
                dept_name=dept_required).required_course.all()
        except:
            pass
        if courses:
            result['type'] = 'required'
            page_size = courses.count()
    else:
        courses = courses.filter(content=AutoQuery(q))
        if code:
            courses = courses.filter(code__contains=code)

        if courses.count() > 300:
            return HttpResponse('TMD')  # Too many detail

        courses = Course.objects.filter(pk__in=[c.pk for c in courses])
        if code in ['GE', 'GEC']:
            core = request.GET.get(code.lower(), '')
            if core:
                courses = courses.filter(ge__contains=core)

    courses = courses.order_by(rev_sortby)
    paginator = Paginator(courses, page_size)

    try:
        courses_page = paginator.page(page)
    except PageNotAnInteger:
        # If page is not an integer, deliver first page.
        courses_page = paginator.page(1)
    except EmptyPage:
        # If page is out of range (e.g. 9999), deliver last page of results.
        courses_page = paginator.page(paginator.num_pages)

    courses_list = courses_page.object_list. \
        values('id', 'no', 'eng_title', 'chi_title', 'note', 'objective',
               'time', 'time_token', 'teacher', 'room', 'credit',
               'prerequisite', 'ge', 'code')

    result['total'] = courses.count()
    result['page'] = courses_page.number
    result['courses'] = list(courses_list)
    result['page_size'] = page_size

    return JsonResponse(result)

@cache_page(60 * 60)
def syllabus(request, id):
    course = get_object_or_404(Course, id=id)
    return render(request, 'syllabus.html',
                  {'course': course, 'syllabus_path': request.path})

def hit(request, id):
    course = get_object_or_404(Course, id=id)
    course.hit += 1
    course.save()
    return HttpResponse('')

def generate_dept_required_choice():
    choices = (('', '---'),)
    departments = Department.objects.all()
    for department in departments:
        dept_name = department.dept_name
        year = {'104': '一年級', '103': '二年級', '102': '三年級', '101': '四年級'}. \
            get(dept_name[4:7], '')
        degree = {'B': '大學部', 'D': '博士班', 'M': '碩士班', 'P': '專班'}. \
            get(dept_name[7], '')
        chi_dept_name = degree

        if dept_name[7] == 'B':
            chi_dept_name += year
            chi_dept_name += {'BA': '清班', 'BB': '華班', 'BC': '梅班'}. \
                get(dept_name[7:], '')

        choices += ((dept_name, chi_dept_name),)
    return sorted(choices)

class CourseSearchForm(forms.Form):
    DEPT_REQUIRED_CHOICE = generate_dept_required_choice()
    q = forms.CharField(label='關鍵字', required=False)
    code = forms.ChoiceField(label='開課代號', choices=DEPT_CHOICE, required=False)
    ge = forms.ChoiceField(label='向度', choices=GE_CHOICE, required=False)
    gec = forms.ChoiceField(label='向度', choices=GEC_CHOICE, required=False)
    dept_required = forms.ChoiceField(
        label='必選修', choices=DEPT_REQUIRED_CHOICE, required=False)

def table(request):
    render_data = {}
    render_data['search_filter'] = CourseSearchForm(request.GET)
    return render(request, 'table.html', render_data)
<|endoftext|>