<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#
#    Copyright (C) 2010-2012 OpenERP s.a. (<http://openerp.com>).
#    Copyright (c) 2014 Noviat nv/sa (www.noviat.com). All rights reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import fields, orm
from openerp.tools.translate import _
import logging
_logger = logging.getLogger(__name__)

# List of move's fields that can't be modified if move is linked
# with a depreciation line
FIELDS_AFFECTS_ASSET_MOVE = set(['period_id', 'journal_id', 'date'])
# List of move line's fields that can't be modified if move is linked
# with a depreciation line
FIELDS_AFFECTS_ASSET_MOVE_LINE = \
    set(['credit', 'debit', 'account_id', 'journal_id', 'date'])

class account_move(orm.Model):
    _inherit = 'account.move'

    def unlink(self, cr, uid, ids, context=None, check=True):
        if not context:
            context = {}
        depr_obj = self.pool.get('account.asset.depreciation.line')
        for move_id in ids:
            depr_ids = depr_obj.search(
                cr, uid,
                [('move_id', '=', move_id),
                 ('type', 'in', ['depreciate', 'remove'])])
            if depr_ids and not context.get('unlink_from_asset'):
                raise orm.except_orm(
                    _('Error!'),
                    _("You are not allowed to remove an accounting entry "
                      "linked to an asset."
                      "\nYou should remove such entries from the asset."))
            # trigger store function
            depr_obj.write(cr, uid, depr_ids, {'move_id': False}, context)
        return super(account_move, self).unlink(
            cr, uid, ids, context=context, check=check)

    def write(self, cr, uid, ids, vals, context=None):
        if set(vals).intersection(FIELDS_AFFECTS_ASSET_MOVE):
            if isinstance(ids, (int, long)):
                ids = [ids]
            depr_obj = self.pool.get('account.asset.depreciation.line')
            for move_id in ids:
                depr_ids = depr_obj.search(
                    cr, uid,
                    [('move_id', '=', move_id), ('type', '=', 'depreciate')])
                if depr_ids:
                    raise orm.except_orm(
                        _('Error!'),
                        _("You cannot change an accounting entry "
                          "linked to an asset depreciation line."))
        return super(account_move, self).write(cr, uid, ids, vals, context)

class account_move_line(orm.Model):
    _inherit = 'account.move.line'

    _columns = {
        'asset_category_id': fields.many2one(
            'account.asset.category', 'Asset Category'),
    }

    def onchange_account_id(self, cr, uid, ids,
                            account_id=False, partner_id=False, context=None):
        res = super(account_move_line, self).onchange_account_id(
            cr, uid, ids, account_id, partner_id, context)
        account_obj = self.pool.get('account.account')
        if account_id:
            account = account_obj.browse(cr, uid, account_id)
            asset_category = account.asset_category_id
            if asset_category:
                res['value'].update({'asset_category_id': asset_category.id})
        return res

    def create(self, cr, uid, vals, context=None, check=True):
        if not context:
            context = {}
        if vals.get('asset_id') and not context.get('allow_asset'):
            raise orm.except_orm(_(
                'Error!'),
                _("You are not allowed to link "
                  "an accounting entry to an asset."
                  "\nYou should generate such entries from the asset."))
        if vals.get('asset_category_id'):
            asset_obj = self.pool.get('account.asset.asset')
            # create asset
            move = self.pool.get('account.move').browse(
                cr, uid, vals['move_id'])
            asset_value = vals['debit'] or -vals['credit']
            asset_vals = {
                'name': vals['name'],
                'category_id': vals['asset_category_id'],
                'purchase_value': asset_value,
                'partner_id': vals['partner_id'],
                'date_start': move.date,
            }
            if context.get('company_id'):
                asset_vals['company_id'] = context['company_id']
            changed_vals = asset_obj.onchange_category_id(
                cr, uid, [], vals['asset_category_id'], context=context)
            asset_vals.update(changed_vals['value'])
            ctx = dict(context, create_asset_from_move_line=True,
                       move_id=vals['move_id'])
            asset_id = asset_obj.create(cr, uid, asset_vals, context=ctx)
            vals['asset_id'] = asset_id

        return super(account_move_line, self).create(
            cr, uid, vals, context, check)

    def write(self, cr, uid, ids, vals,
              context=None, check=True, update_check=True):
        for move_line in self.browse(cr, uid, ids, context=context):
            if move_line.asset_id.id:
                if set(vals).intersection(FIELDS_AFFECTS_ASSET_MOVE_LINE):
                    raise orm.except_orm(
                        _('Error!'),
                        _("You cannot change an accounting item "
                          "linked to an asset depreciation line."))
        if vals.get('asset_id'):
            raise orm.except_orm(
                _('Error!'),
                _("You are not allowed to link "
                  "an accounting entry to an asset."
                  "\nYou should generate such entries from the asset."))
        if vals.get('asset_category_id'):
            assert len(ids) == 1, \
                'This option should only be used for a single id at a time.'
            asset_obj = self.pool.get('account.asset.asset')
            for aml in self.browse(cr, uid, ids, context):
                if vals['asset_category_id'] == aml.asset_category_id.id:
                    continue
                # create asset
                debit = 'debit' in vals and vals.get('debit', 0.0) or aml.debit
                credit = 'credit' in vals and \
                    vals.get('credit', 0.0) or aml.credit
                asset_value = debit - credit
                partner_id = 'partner' in vals and \
                    vals.get('partner', False) or aml.partner_id.id
                date_start = 'date' in vals and \
                    vals.get('date', False) or aml.date
                asset_vals = {
                    'name': vals.get('name') or aml.name,
                    'category_id': vals['asset_category_id'],
                    'purchase_value': asset_value,
                    'partner_id': partner_id,
                    'date_start': date_start,
                    'company_id': vals.get('company_id') or aml.company_id.id,
                }
                changed_vals = asset_obj.onchange_category_id(
                    cr, uid, [], vals['asset_category_id'], context=context)
                asset_vals.update(changed_vals['value'])
                ctx = dict(context, create_asset_from_move_line=True,
                           move_id=aml.move_id.id)
                asset_id = asset_obj.create(cr, uid, asset_vals, context=ctx)
                vals['asset_id'] = asset_id

        return super(account_move_line, self).write(
            cr, uid, ids, vals, context, check, update_check)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[FIX][account_asset_management] Add items on affects fields list<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#
#    Copyright (C) 2010-2012 OpenERP s.a. (<http://openerp.com>).
#    Copyright (c) 2014 Noviat nv/sa (www.noviat.com). All rights reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp.osv import fields, orm
from openerp.tools.translate import _
import logging
_logger = logging.getLogger(__name__)

# List of move's fields that can't be modified if move is linked
# with a depreciation line
FIELDS_AFFECTS_ASSET_MOVE = set(['period_id', 'journal_id', 'date'])
# List of move line's fields that can't be modified if move is linked
# with a depreciation line
FIELDS_AFFECTS_ASSET_MOVE_LINE = \
    set(['credit', 'debit', 'account_id', 'journal_id', 'date',
         'asset_category_id', 'asset_id', 'tax_code_id', 'tax_amount'])

class account_move(orm.Model):
    _inherit = 'account.move'

    def unlink(self, cr, uid, ids, context=None, check=True):
        if not context:
            context = {}
        depr_obj = self.pool.get('account.asset.depreciation.line')
        for move_id in ids:
            depr_ids = depr_obj.search(
                cr, uid,
                [('move_id', '=', move_id),
                 ('type', 'in', ['depreciate', 'remove'])])
            if depr_ids and not context.get('unlink_from_asset'):
                raise orm.except_orm(
                    _('Error!'),
                    _("You are not allowed to remove an accounting entry "
                      "linked to an asset."
                      "\nYou should remove such entries from the asset."))
            # trigger store function
            depr_obj.write(cr, uid, depr_ids, {'move_id': False}, context)
        return super(account_move, self).unlink(
            cr, uid, ids, context=context, check=check)

    def write(self, cr, uid, ids, vals, context=None):
        if set(vals).intersection(FIELDS_AFFECTS_ASSET_MOVE):
            if isinstance(ids, (int, long)):
                ids = [ids]
            depr_obj = self.pool.get('account.asset.depreciation.line')
            for move_id in ids:
                depr_ids = depr_obj.search(
                    cr, uid,
                    [('move_id', '=', move_id), ('type', '=', 'depreciate')])
                if depr_ids:
                    raise orm.except_orm(
                        _('Error!'),
                        _("You cannot change an accounting entry "
                          "linked to an asset depreciation line."))
        return super(account_move, self).write(cr, uid, ids, vals, context)

class account_move_line(orm.Model):
    _inherit = 'account.move.line'

    _columns = {
        'asset_category_id': fields.many2one(
            'account.asset.category', 'Asset Category'),
    }

    def onchange_account_id(self, cr, uid, ids,
                            account_id=False, partner_id=False, context=None):
        res = super(account_move_line, self).onchange_account_id(
            cr, uid, ids, account_id, partner_id, context)
        account_obj = self.pool.get('account.account')
        if account_id:
            account = account_obj.browse(cr, uid, account_id)
            asset_category = account.asset_category_id
            if asset_category:
                res['value'].update({'asset_category_id': asset_category.id})
        return res

    def create(self, cr, uid, vals, context=None, check=True):
        if not context:
            context = {}
        if vals.get('asset_id') and not context.get('allow_asset'):
            raise orm.except_orm(_(
                'Error!'),
                _("You are not allowed to link "
                  "an accounting entry to an asset."
                  "\nYou should generate such entries from the asset."))
        if vals.get('asset_category_id'):
            asset_obj = self.pool.get('account.asset.asset')
            # create asset
            move = self.pool.get('account.move').browse(
                cr, uid, vals['move_id'])
            asset_value = vals['debit'] or -vals['credit']
            asset_vals = {
                'name': vals['name'],
                'category_id': vals['asset_category_id'],
                'purchase_value': asset_value,
                'partner_id': vals['partner_id'],
                'date_start': move.date,
            }
            if context.get('company_id'):
                asset_vals['company_id'] = context['company_id']
            changed_vals = asset_obj.onchange_category_id(
                cr, uid, [], vals['asset_category_id'], context=context)
            asset_vals.update(changed_vals['value'])
            ctx = dict(context, create_asset_from_move_line=True,
                       move_id=vals['move_id'])
            asset_id = asset_obj.create(cr, uid, asset_vals, context=ctx)
            vals['asset_id'] = asset_id

        return super(account_move_line, self).create(
            cr, uid, vals, context, check)

    def write(self, cr, uid, ids, vals,
              context=None, check=True, update_check=True):
        for move_line in self.browse(cr, uid, ids, context=context):
            if move_line.asset_id.id:
                if set(vals).intersection(FIELDS_AFFECTS_ASSET_MOVE_LINE):
                    raise orm.except_orm(
                        _('Error!'),
                        _("You cannot change an accounting item "
                          "linked to an asset depreciation line."))
        if vals.get('asset_id'):
            raise orm.except_orm(
                _('Error!'),
                _("You are not allowed to link "
                  "an accounting entry to an asset."
                  "\nYou should generate such entries from the asset."))
        if vals.get('asset_category_id'):
            assert len(ids) == 1, \
                'This option should only be used for a single id at a time.'
            asset_obj = self.pool.get('account.asset.asset')
            for aml in self.browse(cr, uid, ids, context):
                if vals['asset_category_id'] == aml.asset_category_id.id:
                    continue
                # create asset
                debit = 'debit' in vals and vals.get('debit', 0.0) or aml.debit
                credit = 'credit' in vals and \
                    vals.get('credit', 0.0) or aml.credit
                asset_value = debit - credit
                partner_id = 'partner' in vals and \
                    vals.get('partner', False) or aml.partner_id.id
                date_start = 'date' in vals and \
                    vals.get('date', False) or aml.date
                asset_vals = {
                    'name': vals.get('name') or aml.name,
                    'category_id': vals['asset_category_id'],
                    'purchase_value': asset_value,
                    'partner_id': partner_id,
                    'date_start': date_start,
                    'company_id': vals.get('company_id') or aml.company_id.id,
                }
                changed_vals = asset_obj.onchange_category_id(
                    cr, uid, [], vals['asset_category_id'], context=context)
                asset_vals.update(changed_vals['value'])
                ctx = dict(context, create_asset_from_move_line=True,
                           move_id=aml.move_id.id)
                asset_id = asset_obj.create(cr, uid, asset_vals, context=ctx)
                vals['asset_id'] = asset_id

        return super(account_move_line, self).write(
            cr, uid, ids, vals, context, check, update_check)

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re
from base64 import b64decode
import hashlib
import random
from time import sleep

from module.plugins.Hoster import Hoster
from module.network.Request import getURL
from module.plugins.Plugin import chunks

def getInfo(urls):
    api_url_base = "http://www.share-online.biz/linkcheck/linkcheck.php"

    for chunk in chunks(urls, 90):
        api_param_file = {"links": "\n".join(x.replace("http://www.share-online.biz/dl/","").rstrip("/") for x in chunk)} #api only supports old style links
        src = getURL(api_url_base, post=api_param_file)
        result = []
        for i, res in enumerate(src.split("\n")):
            if not res:
                continue
            fields = res.split(";")

            if fields[1] == "OK":
                status = 2
            elif fields[1] in ("DELETED", "NOT FOUND"):
                status = 1
            else:
                status = 3

            result.append((fields[2], int(fields[3]), status, chunk[i]))
        yield result

class ShareonlineBiz(Hoster):
    __name__ = "ShareonlineBiz"
    __type__ = "hoster"
    __pattern__ = r"(?:http://)?(?:www.)?share-online.biz/(download.php\?id=|dl/)"
    __version__ = "0.2"
    __description__ = """Shareonline.biz Download Hoster"""
    __author_name__ = ("spoob", "mkaay")
    __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de")

    def setup(self):
        #self.req.canContinue = self.multiDL = True if self.account else False
        # range request not working?
        self.multiDL = True if self.account else False

    def process(self, pyfile):
        self.convertURL()
        self.downloadAPIData()
        pyfile.name = self.api_data["filename"]
        pyfile.sync()

        self.downloadHTML()

        self.download(self.getFileUrl(), cookies=True)

    def downloadAPIData(self):
        api_url_base = "http://www.share-online.biz/linkcheck/linkcheck.php?md5=1"
        api_param_file = {"links": self.pyfile.url.replace("http://www.share-online.biz/dl/","")} #api only supports old style links
        src = self.load(api_url_base, cookies=False, post=api_param_file)

        fields = src.split(";")
        self.api_data = {}
        self.api_data["fileid"] = fields[0]
        self.api_data["status"] = fields[1]
        if not self.api_data["status"] == "OK":
            self.offline()
        self.api_data["filename"] = fields[2]
        self.api_data["size"] = fields[3] # in bytes
        self.api_data["checksum"] = fields[4].strip().lower().replace("\n\n", "") # md5

    def downloadHTML(self):
        self.html = self.load(self.pyfile.url, cookies=True)
        if not self.account:
            html = self.load("%s/free/" % self.pyfile.url, post={"dl_free":"1"}, cookies=True)
            if re.search(r"/failure/full/1", self.req.lastEffectiveURL):
                self.setWait(120)
                self.log.info("%s: no free slots, waiting 120 seconds" % (self.__name__))
                self.wait()
                self.retry()

            return True

            # captcha = self.decryptCaptcha("http://www.share-online.biz/captcha.php", get={"rand":"0.%s" % random.randint(10**15,10**16)}, cookies=True)

            # self.log.debug("%s Captcha: %s" % (self.__name__, captcha))
            # sleep(3)

            # html = self.load(self.pyfile.url, post={"captchacode": captcha}, cookies=True)
            # return True

            #m = re.search("var timeout='(\d+)';", self.html[1])
            #self.waitUntil = time() + int(m.group(1)) if m else 30
            # if r"Der Download ist Ihnen zu langsam" in html:
                # return True

            # if r"The download is too slow for you" in html:
                # return True

            self.retry()
        else:
            if r"Die Nummer ist leider nicht richtig oder ausgelaufen!" in self.html:
                self.retry()
            return True

    def convertURL(self):
        if self.account:
            self.pyfile.url = self.pyfile.url.replace("http://www.share-online.biz/dl/", "http://www.share-online.biz/download.php?id=")
            self.pyfile.url = self.pyfile.url.replace("http://www.share-online.biz/dl/", "http://share-online.biz/download.php?id=")
        else:
            self.pyfile.url = self.pyfile.url.replace("http://www.share-online.biz/download.php?id=", "http://www.share-online.biz/dl/")
            self.pyfile.url = self.pyfile.url.replace("http://share-online.biz/download.php?id=", "http://www.share-online.biz/dl/")

    def getFileUrl(self):
        """ returns the absolute downloadable filepath
        """
        if self.account:
            return re.search('The following link contains a ticket to a valid mirror for your desired file\..*?<a href="(.*?)" onmouseout', self.html, re.S).group(1)
        file_url_pattern = r'var\sdl="(.*?)"'
        return b64decode(re.search(file_url_pattern, self.html).group(1))

    def checksum(self, local_file):
        if self.api_data and self.api_data["checksum"]:
            h = hashlib.md5()
            f = open(local_file, "rb")
            h.update(f.read())
            f.close()
            hexd = h.hexdigest()
            if hexd == self.api_data["checksum"]:
                return (True, 0)
            else:
                return (False, 1)
        else:
            return (True, 5)
<commit_msg>share-online, try fix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re
from base64 import b64decode
import hashlib
import random
from time import sleep

from module.plugins.Hoster import Hoster
from module.network.Request import getURL
from module.plugins.Plugin import chunks

def getInfo(urls):
    api_url_base = "http://www.share-online.biz/linkcheck/linkcheck.php"

    for chunk in chunks(urls, 90):
        api_param_file = {"links": "\n".join(x.replace("http://www.share-online.biz/dl/","").rstrip("/") for x in chunk)} #api only supports old style links
        src = getURL(api_url_base, post=api_param_file)
        result = []
        for i, res in enumerate(src.split("\n")):
            if not res:
                continue
            fields = res.split(";")

            if fields[1] == "OK":
                status = 2
            elif fields[1] in ("DELETED", "NOT FOUND"):
                status = 1
            else:
                status = 3

            result.append((fields[2], int(fields[3]), status, chunk[i]))
        yield result

class ShareonlineBiz(Hoster):
    __name__ = "ShareonlineBiz"
    __type__ = "hoster"
    __pattern__ = r"(?:http://)?(?:www.)?share-online.biz/(download.php\?id=|dl/)"
    __version__ = "0.2"
    __description__ = """Shareonline.biz Download Hoster"""
    __author_name__ = ("spoob", "mkaay")
    __author_mail__ = ("spoob@pyload.org", "mkaay@mkaay.de")

    def setup(self):
        #self.req.canContinue = self.multiDL = True if self.account else False
        # range request not working?
        self.multiDL = True if self.account else False

    def process(self, pyfile):
        self.convertURL()
        self.downloadAPIData()
        pyfile.name = self.api_data["filename"]
        pyfile.sync()

        self.downloadHTML()

        self.download(self.getFileUrl(), cookies=True)

    def downloadAPIData(self):
        api_url_base = "http://www.share-online.biz/linkcheck/linkcheck.php?md5=1"
        api_param_file = {"links": self.pyfile.url.replace("http://www.share-online.biz/dl/","")} #api only supports old style links
        src = self.load(api_url_base, cookies=False, post=api_param_file)

        fields = src.split(";")
        self.api_data = {}
        self.api_data["fileid"] = fields[0]
        self.api_data["status"] = fields[1]
        if not self.api_data["status"] == "OK":
            self.offline()
        self.api_data["filename"] = fields[2]
        self.api_data["size"] = fields[3] # in bytes
        self.api_data["checksum"] = fields[4].strip().lower().replace("\n\n", "") # md5

    def downloadHTML(self):
        self.html = self.load(self.pyfile.url, cookies=True)
        if not self.account:
            html = self.load("%s/free/" % self.pyfile.url, post={"dl_free":"1"}, cookies=True)
            if re.search(r"/failure/full/1", self.req.lastEffectiveURL):
                self.setWait(120)
                self.log.info("%s: no free slots, waiting 120 seconds" % (self.__name__))
                self.wait()
                self.retry()

            if "Captcha number error or expired" in html:
                captcha = self.decryptCaptcha("http://www.share-online.biz/captcha.php", get={"rand":"0.%s" % random.randint(10**15,10**16)}, cookies=True)

                self.log.debug("%s Captcha: %s" % (self.__name__, captcha))
                sleep(3)

                html = self.load(self.pyfile.url, post={"captchacode": captcha}, cookies=True)

                if r"Der Download ist Ihnen zu langsam" not in html and r"The download is too slow for you" not in html:
                    self.fail("Plugin defect. Save dumps and report.")

            m = re.search("var timeout='(\d+)';", self.html[1])
            wait_time = int(m.group(1)) if m else 30
            self.setWait(wait_time)
            self.log.debug("%s: Waiting %d seconds." % (self.__name__, wait_time))
            self.wait()

            return True

        else:
            if r"Die Nummer ist leider nicht richtig oder ausgelaufen!" in self.html:
                self.retry()
            return True

    def convertURL(self):
        if self.account:
            self.pyfile.url = self.pyfile.url.replace("http://www.share-online.biz/dl/", "http://www.share-online.biz/download.php?id=")
            self.pyfile.url = self.pyfile.url.replace("http://www.share-online.biz/dl/", "http://share-online.biz/download.php?id=")
        else:
            self.pyfile.url = self.pyfile.url.replace("http://www.share-online.biz/download.php?id=", "http://www.share-online.biz/dl/")
            self.pyfile.url = self.pyfile.url.replace("http://share-online.biz/download.php?id=", "http://www.share-online.biz/dl/")

    def getFileUrl(self):
        """ returns the absolute downloadable filepath
        """
        if self.account:
            return re.search('The following link contains a ticket to a valid mirror for your desired file\..*?<a href="(.*?)" onmouseout', self.html, re.S).group(1)
        file_url_pattern = r'var\sdl="(.*?)"'
        return b64decode(re.search(file_url_pattern, self.html).group(1))

    def checksum(self, local_file):
        if self.api_data and self.api_data["checksum"]:
            h = hashlib.md5()
            f = open(local_file, "rb")
            h.update(f.read())
            f.close()
            hexd = h.hexdigest()
            if hexd == self.api_data["checksum"]:
                return (True, 0)
            else:
                return (False, 1)
        else:
            return (True, 5)
<|endoftext|>


<commit_before>#!/usr/local/bin/python

import sys
import math
import csv

def calculate_vm_lifetime(file_name):
        ifile  = open(file_name, "rt")
        reader = csv.reader(ifile)
        total_time=0
        last=0
        for row in reader:
                prevLast = last
                last=int(row[0].partition(" ")[0])
                total_time = total_time + last
        total_time=total_time-last-prevLast
        sys.stdout.write("Final Total Time: %d   \r" % (total_time) )
        ifile.close()
        return total_time

if __name__ == "__main__":
    calculate_vm_lifetime("VM_Task_1.csv")
<commit_msg>Added end time calculation<commit_after>#!/usr/local/bin/python

import sys
import math
import csv
import time

#==============================================================================
# Variables
#==============================================================================
#name                = "vmplacementandscaling"
#version             = "0.1"
#long_description    = """vmplacementandscaling is a set of API's/tools written to create virtual machines for cloud users efficiently."""
#url                 = "https://github.com/dineshappavoo/VMPlacementAndScaling"
#license             = ""

#==============================================================================

data_folder_path='../com.vmplacement.data/vms'

def calculate_vm_lifetime(file_name):
        ifile  = open(file_name, "rt")
        reader = csv.reader(ifile)
        total_time=0
        last=0
        #prevLast=0
        for row in reader:
                #secprevLast = prevLast
                prevLast = last
                last=int(row[0].partition(" ")[0])
                total_time = total_time + last
        total_time=total_time-last-prevLast #-secprevLast
        ifile.close()
        return total_time

def calculate_vm_endtime(vm_id, start_time):
        lifetime = calculate_vm_lifetime(data_folder_path+"/"+vm_id+".csv")
        end_time = (lifetime+ (start_time/60))*60
        sys.stdout.write("End time: %d   \r" % (end_time) )
        return end_time

if __name__ == "__main__":
    calculate_vm_endtime('VM_Task_1', time.time())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import sys
import os
import shutil
import ConfigParser
from datetime import datetime

# -----------------------------------------------------------------------------
# Utility:
# -----------------------------------------------------------------------------
def sort_line(row):
    ''' Generate order depend on start code, the row is the string generated
        for ERP (code 4th field)
    '''
    code = row.slit('|')[3].upper()
    start_1 = code[:1]
    start_2 = code[:2]

    # Test based on 1 or 2 start char:
    if start_1 in 'OH' or start_2 == 'SP': # XXX has S - SP problem so first
        return 3 # dry
    elif start_1 in 'VCPDSF': # XXX has S - SP problem so second test
        return 1 # freeze
    elif start_1 in 'ILTB':
        return 2 # fresh
    elif start_1 in 'G':
        return 4 # pasta
    else: # Error list
        # Log error:
        f_error = ('./sort_error.log', 'w')
        f_error.write('Char not found %s - %s\n' % (start_1, start_2))
        f_error.close()
        return 5

def clean_text(text, length, uppercase=False, error=None, truncate=False):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []

    text = text.strip()
    if len(text) > length:
        if truncate:
            text = text[:length]
        else:
            error.append('Text: %s > %s' % (text, length))
    if uppercase:
        return text.upper()
    return text

def clean_date(italian_date, separator='', out_format='iso', error=None):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []
    italian_date = italian_date.split(' ')[0] # remove hour block
    if len(italian_date) != 10:
        error.append('Error not italian date: %s' % italian_date)
        # not stopped
    if out_format == 'iso':
        return '%s%s%s%s%s' % (
            italian_date[-4:],
            separator,
            italian_date[3:5],
            separator,
            italian_date[:2],
            )
    elif out_format == 'italian':
        return '%s%s%s%s%s' % (
            italian_date[:2],
            separator,
            italian_date[3:5],
            separator,
            italian_date[-4:],
            )
    elif out_format == 'english':
        return '%s%s%s%s%s' % (
            italian_date[3:5],
            separator,
            italian_date[:2],
            separator,
            italian_date[-4:],
            )
    else: # incorrect format:
        return italian_date # nothing todo

def clean_float(value, length, decimal=3, separator='.', error=None):
    ''' Clean float and return float format
    '''
    if error is None:
        error = []
    try:
        value = value.replace(',', '.')
        float_value = float(value.strip())
    except:
        error.append('Not a float: %s' % value)
        float_value = 0.0

    mask = '%%%s.%sf' % (length, decimal)
    res = mask % float_value
    res = res.replace('.', separator)
    return res

def log_on_file(message, mode='INFO', file_list=None, verbose=True):
    ''' Write message in file list passed
        mode: INFO WARNING ERROR
    '''
    if file_list is None:
        return False

    message_log = '%s. [%s] %s\n' % (datetime.now(), mode, message)
    if verbose:
        print message_log.strip()

    for f_log in file_list:
        if not f_log:
            continue
        f_log.write(message_log)
    return True

# -----------------------------------------------------------------------------
# Read configuration parameter:
# -----------------------------------------------------------------------------
try:
    company = sys.argv[1]
    if len(company) > 3:
        print 'Run: python ./pre_process.py CMP (must be max 3 char length)'
        sys.exit()
except:
    print 'Launch program as:\n\npython ./pre_process.py CMP'

try:
    cfg_file = os.path.expanduser('./%s.cfg' % company)
except:
    print 'Config file not found: %s.cfg' % company

config = ConfigParser.ConfigParser()
config.read([cfg_file])

# In parameters:
in_check = os.path.expanduser(config.get('in', 'check'))
in_path = os.path.expanduser(config.get('in', 'path'))
in_history = os.path.expanduser(config.get('in', 'history'))
in_log = os.path.expanduser(config.get('in', 'log'))
in_schedule = os.path.expanduser(config.get('in', 'schedule'))

# Outp arameters:
out_check = os.path.expanduser(config.get('out', 'check'))
out_path = os.path.expanduser(config.get('out', 'path'))
out_log = os.path.expanduser(config.get('out', 'log'))
out_schedule = os.path.expanduser(config.get('out', 'schedule'))
out_original = os.path.expanduser(config.get('out', 'original'))

# Mail parameters:
mail_error = config.get('mail', 'error')
mail_info = config.get('mail', 'info')

# TODO put in config file:
separator = config.get('file', 'separator')
tot_col = eval(config.get('file', 'tot_col'))

# Calculated parameters:
f_in_schedule = False # open(in_schedule, 'a')
f_in_log = False #open(in_log, 'a')

f_out_schedule = open(out_schedule, 'a')
f_out_log = open(out_log, 'a')

# -----------------------------------------------------------------------------
# Check mount folder:
# -----------------------------------------------------------------------------
if in_check:
    if not os.path.exists(in_check):
        log_on_file(
            'Cannot mount IN folder: %s' % in_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

if out_check:
    if not os.path.exists(out_check):
        log_on_file(
            'Cannot mount OUT folder: %s' % out_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

# -----------------------------------------------------------------------------
# Read IN folder:
# -----------------------------------------------------------------------------
log_on_file(
    'Start import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

for root, dirs, files in os.walk(in_path):
    log_on_file(
        'Read root folder: %s [%s]' % (root, company),
        mode='INFO',
        file_list=[f_in_log, f_out_log])

    for f in files:
        file_part = f.split('_')
        command = (file_part[2][:3]).upper() # New, Upd, Can

        # Fullname needed:
        file_in = os.path.join(root, f)
        file_history = os.path.join(in_history, f)
        file_original = os.path.join(out_original, f)

        create_date = datetime.fromtimestamp(
            os.path.getctime(file_in)
            ).strftime('%Y%m%d_%H%M%S')

        file_out = os.path.join(
            out_path, '%s' % (
                '%s_%s' % (create_date, f),
                )) # TODO change name

        # ---------------------------------------------------------------------
        # Read input file:
        # ---------------------------------------------------------------------
        error = []
        f_in = open(file_in, 'r')
        row_out = []
        log_on_file(
            'Parsing file: %s [%s]' % (file_in, company),
            mode='INFO',
            file_list=[f_in_log, f_out_log])
        for line in f_in:
            row = line.split(separator)
            if len(row) != tot_col:
                log_on_file(
                    'Different number columns: %s [%s]' % (file_in, company),
                    mode='INFO', file_list=[f_in_log, f_out_log])
                break

            # -----------------------------------------------------------------
            # Read fields:
            # -----------------------------------------------------------------
            date = create_date[:8] # From create date

            deadline = clean_date(row[0], separator='', out_format='iso',
                error=error)

            # Char:
            code = clean_text(row[1], 9, error=error)
            cook_center = clean_text(row[2], 110, error=error)[:60]# XXX Trunc
            address = clean_text(row[3], 60, error=error)
            cap = clean_text(row[4], 5, error=error) # XXX Check
            city = clean_text(row[5], 60, error=error) # XXX Check
            province = clean_text(row[6], 4, uppercase=True, error=error)
            order = clean_text(row[7], 10, error=error)
            default_code = clean_text(row[8], 16, uppercase=True, error=error)
            default_code_supplier = clean_text(
                row[9], 16, uppercase=True, error=error)
            name = clean_text(row[10], 60, error=error, truncate=True)
            um = clean_text(row[11], 2, uppercase=True, error=error)

            # Float:
            quantity = clean_float(row[12], 15, 2, error=error)
            price = clean_float(row[13], 15, 3, error=error)
            cost = clean_float(row[14], 15, 3, error=error)

            # -----------------------------------------------------------------
            # Convert row input file:
            # -----------------------------------------------------------------
            row_out.append(
                '%3s|%3s|%-8s|%-9s|%-60s|%-60s|%-5s|%-60s|%-4s|%-10s|%-16s|'
                '%-16s|%-60s|%-2s|%-15s|%-15s|%-15s|%-8s\r\n' % (
                    # Header:
                    company, # depend on parameter
                    command,
                    deadline,
                    code,
                    cook_center,
                    address,
                    cap,
                    city,
                    province,
                    order,

                    # Row:
                    default_code,
                    default_code_supplier,
                    name,
                    um,
                    quantity,
                    price,
                    cost,
                    date,
                    ))
        f_in.close()

        if error:
            log_on_file(
                'Error reading file: %s [%s]' % (error, company),
                mode='ERROR', file_list=[f_in_log, f_out_log])
            continue

        # ---------------------------------------------------------------------
        # Save file out:
        # ---------------------------------------------------------------------
        error = []
        # Convert file:
        f_out = open(file_out, 'w')

        for line in sorted(row_out, key=lambda x: sort_line(x)):
            f_out.write(line)
        f_out.close()

        if error:
            log_on_file(
                'Error generating out file: %s [%s]' % (error, company),
                mode='ERROR', file_list=[f_in_log, f_out_log])
            continue
        else:
            log_on_file(
                'Created file: %s [%s]' % (file_out, company),
                mode='INFO', file_list=[f_in_log, f_out_log])

        # ---------------------------------------------------------------------
        # History the in file:
        # ---------------------------------------------------------------------
        error = ''
        try:
            os.remove(file_original)
        except:
            pass
        try:
            shutil.copy(file_in, file_original)
            log_on_file(
                'History company file: %s [%s]' % (file_original, company),
                mode='INFO', file_list=[f_in_log, f_out_log])
        except:
            error += 'Error copy orginal folder %s >> %s' % (
                file_in, file_original)
        try:
            os.remove(file_history)
        except:
            pass
        try:
            shutil.move(file_in, file_history)
            log_on_file(
                'History supplier file: %s [%s]' % (file_history, company),
                mode='INFO', file_list=[f_in_log, f_out_log])
        except:
            error += 'Error moving file %s >> %s' % (file_in, file_history)

        if error:
            log_on_file(
                error, mode='ERROR', file_list=[f_in_log, f_out_log])
            # XXX TODO delete OUT file
            continue
    break # only root folder is read

log_on_file(
    'End import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

# -----------------------------------------------------------------------------
# History the in file:
# -----------------------------------------------------------------------------
try:
    if f_out_log:
        f_out_log.close()
    if f_in_schedule:
        f_in_schedule.close()
    if f_in_log:
        f_in_log.close()
    if f_out_schedule:
        f_out_schedule.close()
except:
    print 'Error closing log file'
<commit_msg>generic bugfix (fast message)<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import sys
import os
import shutil
import ConfigParser
from datetime import datetime

# -----------------------------------------------------------------------------
# Utility:
# -----------------------------------------------------------------------------
def sort_line(row):
    ''' Generate order depend on start code, the row is the string generated
        for ERP (code 4th field)
    '''
    code = row.split('|')[3].upper()
    start_1 = code[:1]
    start_2 = code[:2]

    # Test based on 1 or 2 start char:
    if start_1 in 'OH' or start_2 == 'SP': # XXX has S - SP problem so first
        return 3 # dry
    elif start_1 in 'VCPDSF': # XXX has S - SP problem so second test
        return 1 # freeze
    elif start_1 in 'ILTB':
        return 2 # fresh
    elif start_1 in 'G':
        return 4 # pasta
    else: # Error list
        # Log error:
        f_error = ('./sort_error.log', 'w')
        f_error.write('Char not found %s - %s\n' % (start_1, start_2))
        f_error.close()
        return 5

def clean_text(text, length, uppercase=False, error=None, truncate=False):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []

    text = text.strip()
    if len(text) > length:
        if truncate:
            text = text[:length]
        else:
            error.append('Text: %s > %s' % (text, length))
    if uppercase:
        return text.upper()
    return text

def clean_date(italian_date, separator='', out_format='iso', error=None):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []
    italian_date = italian_date.split(' ')[0] # remove hour block
    if len(italian_date) != 10:
        error.append('Error not italian date: %s' % italian_date)
        # not stopped
    if out_format == 'iso':
        return '%s%s%s%s%s' % (
            italian_date[-4:],
            separator,
            italian_date[3:5],
            separator,
            italian_date[:2],
            )
    elif out_format == 'italian':
        return '%s%s%s%s%s' % (
            italian_date[:2],
            separator,
            italian_date[3:5],
            separator,
            italian_date[-4:],
            )
    elif out_format == 'english':
        return '%s%s%s%s%s' % (
            italian_date[3:5],
            separator,
            italian_date[:2],
            separator,
            italian_date[-4:],
            )
    else: # incorrect format:
        return italian_date # nothing todo

def clean_float(value, length, decimal=3, separator='.', error=None):
    ''' Clean float and return float format
    '''
    if error is None:
        error = []
    try:
        value = value.replace(',', '.')
        float_value = float(value.strip())
    except:
        error.append('Not a float: %s' % value)
        float_value = 0.0

    mask = '%%%s.%sf' % (length, decimal)
    res = mask % float_value
    res = res.replace('.', separator)
    return res

def log_on_file(message, mode='INFO', file_list=None, verbose=True):
    ''' Write message in file list passed
        mode: INFO WARNING ERROR
    '''
    if file_list is None:
        return False

    message_log = '%s. [%s] %s\n' % (datetime.now(), mode, message)
    if verbose:
        print message_log.strip()

    for f_log in file_list:
        if not f_log:
            continue
        f_log.write(message_log)
    return True

# -----------------------------------------------------------------------------
# Read configuration parameter:
# -----------------------------------------------------------------------------
try:
    company = sys.argv[1]
    if len(company) > 3:
        print 'Run: python ./pre_process.py CMP (must be max 3 char length)'
        sys.exit()
except:
    print 'Launch program as:\n\npython ./pre_process.py CMP'

try:
    cfg_file = os.path.expanduser('./%s.cfg' % company)
except:
    print 'Config file not found: %s.cfg' % company

config = ConfigParser.ConfigParser()
config.read([cfg_file])

# In parameters:
in_check = os.path.expanduser(config.get('in', 'check'))
in_path = os.path.expanduser(config.get('in', 'path'))
in_history = os.path.expanduser(config.get('in', 'history'))
in_log = os.path.expanduser(config.get('in', 'log'))
in_schedule = os.path.expanduser(config.get('in', 'schedule'))

# Outp arameters:
out_check = os.path.expanduser(config.get('out', 'check'))
out_path = os.path.expanduser(config.get('out', 'path'))
out_log = os.path.expanduser(config.get('out', 'log'))
out_schedule = os.path.expanduser(config.get('out', 'schedule'))
out_original = os.path.expanduser(config.get('out', 'original'))

# Mail parameters:
mail_error = config.get('mail', 'error')
mail_info = config.get('mail', 'info')

# TODO put in config file:
separator = config.get('file', 'separator')
tot_col = eval(config.get('file', 'tot_col'))

# Calculated parameters:
f_in_schedule = False # open(in_schedule, 'a')
f_in_log = False #open(in_log, 'a')

f_out_schedule = open(out_schedule, 'a')
f_out_log = open(out_log, 'a')

# -----------------------------------------------------------------------------
# Check mount folder:
# -----------------------------------------------------------------------------
if in_check:
    if not os.path.exists(in_check):
        log_on_file(
            'Cannot mount IN folder: %s' % in_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

if out_check:
    if not os.path.exists(out_check):
        log_on_file(
            'Cannot mount OUT folder: %s' % out_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

# -----------------------------------------------------------------------------
# Read IN folder:
# -----------------------------------------------------------------------------
log_on_file(
    'Start import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

for root, dirs, files in os.walk(in_path):
    log_on_file(
        'Read root folder: %s [%s]' % (root, company),
        mode='INFO',
        file_list=[f_in_log, f_out_log])

    for f in files:
        file_part = f.split('_')
        command = (file_part[2][:3]).upper() # New, Upd, Can

        # Fullname needed:
        file_in = os.path.join(root, f)
        file_history = os.path.join(in_history, f)
        file_original = os.path.join(out_original, f)

        create_date = datetime.fromtimestamp(
            os.path.getctime(file_in)
            ).strftime('%Y%m%d_%H%M%S')

        file_out = os.path.join(
            out_path, '%s' % (
                '%s_%s' % (create_date, f),
                )) # TODO change name

        # ---------------------------------------------------------------------
        # Read input file:
        # ---------------------------------------------------------------------
        error = []
        f_in = open(file_in, 'r')
        row_out = []
        log_on_file(
            'Parsing file: %s [%s]' % (file_in, company),
            mode='INFO',
            file_list=[f_in_log, f_out_log])
        for line in f_in:
            row = line.split(separator)
            if len(row) != tot_col:
                log_on_file(
                    'Different number columns: %s [%s]' % (file_in, company),
                    mode='INFO', file_list=[f_in_log, f_out_log])
                break

            # -----------------------------------------------------------------
            # Read fields:
            # -----------------------------------------------------------------
            date = create_date[:8] # From create date

            deadline = clean_date(row[0], separator='', out_format='iso',
                error=error)

            # Char:
            code = clean_text(row[1], 9, error=error)
            cook_center = clean_text(row[2], 110, error=error)[:60]# XXX Trunc
            address = clean_text(row[3], 60, error=error)
            cap = clean_text(row[4], 5, error=error) # XXX Check
            city = clean_text(row[5], 60, error=error) # XXX Check
            province = clean_text(row[6], 4, uppercase=True, error=error)
            order = clean_text(row[7], 10, error=error)
            default_code = clean_text(row[8], 16, uppercase=True, error=error)
            default_code_supplier = clean_text(
                row[9], 16, uppercase=True, error=error)
            name = clean_text(row[10], 60, error=error, truncate=True)
            um = clean_text(row[11], 2, uppercase=True, error=error)

            # Float:
            quantity = clean_float(row[12], 15, 2, error=error)
            price = clean_float(row[13], 15, 3, error=error)
            cost = clean_float(row[14], 15, 3, error=error)

            # -----------------------------------------------------------------
            # Convert row input file:
            # -----------------------------------------------------------------
            row_out.append(
                '%3s|%3s|%-8s|%-9s|%-60s|%-60s|%-5s|%-60s|%-4s|%-10s|%-16s|'
                '%-16s|%-60s|%-2s|%-15s|%-15s|%-15s|%-8s\r\n' % (
                    # Header:
                    company, # depend on parameter
                    command,
                    deadline,
                    code,
                    cook_center,
                    address,
                    cap,
                    city,
                    province,
                    order,

                    # Row:
                    default_code,
                    default_code_supplier,
                    name,
                    um,
                    quantity,
                    price,
                    cost,
                    date,
                    ))
        f_in.close()

        if error:
            log_on_file(
                'Error reading file: %s [%s]' % (error, company),
                mode='ERROR', file_list=[f_in_log, f_out_log])
            continue

        # ---------------------------------------------------------------------
        # Save file out:
        # ---------------------------------------------------------------------
        error = []
        # Convert file:
        f_out = open(file_out, 'w')

        for line in sorted(row_out, key=lambda x: sort_line(x)):
            f_out.write(line)
        f_out.close()

        if error:
            log_on_file(
                'Error generating out file: %s [%s]' % (error, company),
                mode='ERROR', file_list=[f_in_log, f_out_log])
            continue
        else:
            log_on_file(
                'Created file: %s [%s]' % (file_out, company),
                mode='INFO', file_list=[f_in_log, f_out_log])

        # ---------------------------------------------------------------------
        # History the in file:
        # ---------------------------------------------------------------------
        error = ''
        try:
            os.remove(file_original)
        except:
            pass
        try:
            shutil.copy(file_in, file_original)
            log_on_file(
                'History company file: %s [%s]' % (file_original, company),
                mode='INFO', file_list=[f_in_log, f_out_log])
        except:
            error += 'Error copy orginal folder %s >> %s' % (
                file_in, file_original)
        try:
            os.remove(file_history)
        except:
            pass
        try:
            shutil.move(file_in, file_history)
            log_on_file(
                'History supplier file: %s [%s]' % (file_history, company),
                mode='INFO', file_list=[f_in_log, f_out_log])
        except:
            error += 'Error moving file %s >> %s' % (file_in, file_history)

        if error:
            log_on_file(
                error, mode='ERROR', file_list=[f_in_log, f_out_log])
            # XXX TODO delete OUT file
            continue
    break # only root folder is read

log_on_file(
    'End import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

# -----------------------------------------------------------------------------
# History the in file:
# -----------------------------------------------------------------------------
try:
    if f_out_log:
        f_out_log.close()
    if f_in_schedule:
        f_in_schedule.close()
    if f_in_log:
        f_in_log.close()
    if f_out_schedule:
        f_out_schedule.close()
except:
    print 'Error closing log file'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import sys
import os
import shutil
import ConfigParser
from datetime import datetime

# -----------------------------------------------------------------------------
# Utility:
# -----------------------------------------------------------------------------
def sort_line(row):
    ''' Generate order depend on start code, the row is the string generated
        for ERP (code 4th field)
    '''
    code = row.slit('|')[3].upper()
    start_1 = code[:1]
    start_2 = code[:2]

    # Test based on 1 or 2 start char:
    if start_1 in 'OH' or start_2 == 'SP': # XXX has S - SP problem so first
        return 3 # dry
    elif start_1 in 'VCPDSF': # XXX has S - SP problem so second test
        return 1 # freeze
    elif start_1 in 'ILTB':
        return 2 # fresh
    elif start_1 in 'G':
        return 4 # pasta
    else: # Error list
        # Log error:
        f_error = ('./sort_error.log', 'w')
        f_error.write('Char not found %s - %s\n' % (start_1, start_2))
        f_error.close()
        return 5

def clean_text(text, length, uppercase=False, error=None, truncate=False):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []

    text = text.strip()
    if len(text) > length:
        if truncate:
            text = text[:length]
        else:
            error.append('Text: %s > %s' % (text, length))
    if uppercase:
        return text.upper()
    return text

def clean_date(italian_date, separator='', out_format='iso', error=None):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []
    italian_date = italian_date.split(' ')[0] # remove hour block
    if len(italian_date) != 10:
        error.append('Error not italian date: %s' % italian_date)
        # not stopped
    if out_format == 'iso':
        return '%s%s%s%s%s' % (
            italian_date[-4:],
            separator,
            italian_date[3:5],
            separator,
            italian_date[:2],
            )
    elif out_format == 'italian':
        return '%s%s%s%s%s' % (
            italian_date[:2],
            separator,
            italian_date[3:5],
            separator,
            italian_date[-4:],
            )
    elif out_format == 'english':
        return '%s%s%s%s%s' % (
            italian_date[3:5],
            separator,
            italian_date[:2],
            separator,
            italian_date[-4:],
            )
    else: # incorrect format:
        return italian_date # nothing todo

def clean_float(value, length, decimal=3, separator='.', error=None):
    ''' Clean float and return float format
    '''
    if error is None:
        error = []
    try:
        value = value.replace(',', '.')
        float_value = float(value.strip())
    except:
        error.append('Not a float: %s' % value)
        float_value = 0.0

    mask = '%%%s.%sf' % (length, decimal)
    res = mask % float_value
    res = res.replace('.', separator)
    return res

def log_on_file(message, mode='INFO', file_list=None, verbose=True):
    ''' Write message in file list passed
        mode: INFO WARNING ERROR
    '''
    if file_list is None:
        return False

    message_log = '%s. [%s] %s\n' % (datetime.now(), mode, message)
    if verbose:
        print message_log.strip()

    for f_log in file_list:
        if not f_log:
            continue
        f_log.write(message_log)
    return True

# -----------------------------------------------------------------------------
# Read configuration parameter:
# -----------------------------------------------------------------------------
try:
    company = sys.argv[1]
    if len(company) > 3:
        print 'Run: python ./pre_process.py CMP (must be max 3 char length)'
        sys.exit()
except:
    print 'Launch program as:\n\npython ./pre_process.py CMP'

try:
    cfg_file = os.path.expanduser('./%s.cfg' % company)
except:
    print 'Config file not found: %s.cfg' % company

config = ConfigParser.ConfigParser()
config.read([cfg_file])

# In parameters:
in_check = os.path.expanduser(config.get('in', 'check'))
in_path = os.path.expanduser(config.get('in', 'path'))
in_history = os.path.expanduser(config.get('in', 'history'))
in_log = os.path.expanduser(config.get('in', 'log'))
in_schedule = os.path.expanduser(config.get('in', 'schedule'))

# Outp arameters:
out_check = os.path.expanduser(config.get('out', 'check'))
out_path = os.path.expanduser(config.get('out', 'path'))
out_log = os.path.expanduser(config.get('out', 'log'))
out_schedule = os.path.expanduser(config.get('out', 'schedule'))
out_original = os.path.expanduser(config.get('out', 'original'))

# Mail parameters:
mail_error = config.get('mail', 'error')
mail_info = config.get('mail', 'info')

# TODO put in config file:
separator = config.get('file', 'separator')
tot_col = eval(config.get('file', 'tot_col'))

# Calculated parameters:
f_in_schedule = False # open(in_schedule, 'a')
f_in_log = False #open(in_log, 'a')

f_out_schedule = open(out_schedule, 'a')
f_out_log = open(out_log, 'a')

# -----------------------------------------------------------------------------
# Check mount folder:
# -----------------------------------------------------------------------------
if in_check:
    if not os.path.exists(in_check):
        log_on_file(
            'Cannot mount IN folder: %s' % in_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

if out_check:
    if not os.path.exists(out_check):
        log_on_file(
            'Cannot mount OUT folder: %s' % out_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

# -----------------------------------------------------------------------------
# Read IN folder:
# -----------------------------------------------------------------------------
log_on_file(
    'Start import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

for root, dirs, files in os.walk(in_path):
    log_on_file(
        'Read root folder: %s [%s]' % (root, company),
        mode='INFO',
        file_list=[f_in_log, f_out_log])

    for f in files:
        file_part = f.split('_')
        command = (file_part[2][:3]).upper() # New, Upd, Can

        # Fullname needed:
        file_in = os.path.join(root, f)
        file_history = os.path.join(in_history, f)
        file_original = os.path.join(out_original, f)

        create_date = datetime.fromtimestamp(
            os.path.getctime(file_in)
            ).strftime('%Y%m%d_%H%M%S')

        file_out = os.path.join(
            out_path, '%s' % (
                '%s_%s' % (create_date, f),
                )) # TODO change name

        # ---------------------------------------------------------------------
        # Read input file:
        # ---------------------------------------------------------------------
        error = []
        f_in = open(file_in, 'r')
        row_out = []
        log_on_file(
            'Parsing file: %s [%s]' % (file_in, company),
            mode='INFO',
            file_list=[f_in_log, f_out_log])
        for line in f_in:
            row = line.split(separator)
            if len(row) != tot_col:
                log_on_file(
                    'Different number columns: %s [%s]' % (file_in, company),
                    mode='INFO', file_list=[f_in_log, f_out_log])
                break

            # -----------------------------------------------------------------
            # Read fields:
            # -----------------------------------------------------------------
            date = create_date[:8] # From create date

            deadline = clean_date(row[0], separator='', out_format='iso',
                error=error)

            # Char:
            code = clean_text(row[1], 9, error=error)
            cook_center = clean_text(row[2], 110, error=error)[:60]# XXX Trunc
            address = clean_text(row[3], 60, error=error)
            cap = clean_text(row[4], 5, error=error) # XXX Check
            city = clean_text(row[5], 60, error=error) # XXX Check
            province = clean_text(row[6], 4, uppercase=True, error=error)
            order = clean_text(row[7], 10, error=error)
            default_code = clean_text(row[8], 16, uppercase=True, error=error)
            default_code_supplier = clean_text(
                row[9], 16, uppercase=True, error=error)
            name = clean_text(row[10], 60, error=error, truncate=True)
            um = clean_text(row[11], 2, uppercase=True, error=error)

            # Float:
            quantity = clean_float(row[12], 15, 2, error=error)
            price = clean_float(row[13], 15, 3, error=error)
            cost = clean_float(row[14], 15, 3, error=error)

            # -----------------------------------------------------------------
            # Convert row input file:
            # -----------------------------------------------------------------
            row_out.append(
                '%3s|%3s|%-8s|%-9s|%-60s|%-60s|%-5s|%-60s|%-4s|%-10s|%-16s|'
                '%-16s|%-60s|%-2s|%-15s|%-15s|%-15s|%-8s\r\n' % (
                    # Header:
                    company, # depend on parameter
                    command,
                    deadline,
                    code,
                    cook_center,
                    address,
                    cap,
                    city,
                    province,
                    order,

                    # Row:
                    default_code,
                    default_code_supplier,
                    name,
                    um,
                    quantity,
                    price,
                    cost,
                    date,
                    ))
        f_in.close()

        if error:
            log_on_file(
                'Error reading file: %s [%s]' % (error, company),
                mode='ERROR', file_list=[f_in_log, f_out_log])
            continue

        # ---------------------------------------------------------------------
        # Save file out:
        # ---------------------------------------------------------------------
        error = []
        # Convert file:
        f_out = open(file_out, 'w')

        for line in sorted(row_out, key=lambda x: sort_line(s)):
            f_out.write(line)
        f_out.close()

        if error:
            log_on_file(
                'Error generating out file: %s [%s]' % (error, company),
                mode='ERROR', file_list=[f_in_log, f_out_log])
            continue
        else:
            log_on_file(
                'Created file: %s [%s]' % (file_out, company),
                mode='INFO', file_list=[f_in_log, f_out_log])

        # ---------------------------------------------------------------------
        # History the in file:
        # ---------------------------------------------------------------------
        error = ''
        try:
            os.remove(file_original)
        except:
            pass
        try:
            shutil.copy(file_in, file_original)
            log_on_file(
                'History company file: %s [%s]' % (file_original, company),
                mode='INFO', file_list=[f_in_log, f_out_log])
        except:
            error += 'Error copy orginal folder %s >> %s' % (
                file_in, file_original)
        try:
            os.remove(file_history)
        except:
            pass
        try:
            shutil.move(file_in, file_history)
            log_on_file(
                'History supplier file: %s [%s]' % (file_history, company),
                mode='INFO', file_list=[f_in_log, f_out_log])
        except:
            error += 'Error moving file %s >> %s' % (file_in, file_history)

        if error:
            log_on_file(
                error, mode='ERROR', file_list=[f_in_log, f_out_log])
            # XXX TODO delete OUT file
            continue
    break # only root folder is read

log_on_file(
    'End import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

# -----------------------------------------------------------------------------
# History the in file:
# -----------------------------------------------------------------------------
try:
    if f_out_log:
        f_out_log.close()
    if f_in_schedule:
        f_in_schedule.close()
    if f_in_log:
        f_in_log.close()
    if f_out_schedule:
        f_out_schedule.close()
except:
    print 'Error closing log file'
<commit_msg>generic bugfix (fast message)<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import sys
import os
import shutil
import ConfigParser
from datetime import datetime

# -----------------------------------------------------------------------------
# Utility:
# -----------------------------------------------------------------------------
def sort_line(row):
    ''' Generate order depend on start code, the row is the string generated
        for ERP (code 4th field)
    '''
    code = row.slit('|')[3].upper()
    start_1 = code[:1]
    start_2 = code[:2]

    # Test based on 1 or 2 start char:
    if start_1 in 'OH' or start_2 == 'SP': # XXX has S - SP problem so first
        return 3 # dry
    elif start_1 in 'VCPDSF': # XXX has S - SP problem so second test
        return 1 # freeze
    elif start_1 in 'ILTB':
        return 2 # fresh
    elif start_1 in 'G':
        return 4 # pasta
    else: # Error list
        # Log error:
        f_error = ('./sort_error.log', 'w')
        f_error.write('Char not found %s - %s\n' % (start_1, start_2))
        f_error.close()
        return 5

def clean_text(text, length, uppercase=False, error=None, truncate=False):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []

    text = text.strip()
    if len(text) > length:
        if truncate:
            text = text[:length]
        else:
            error.append('Text: %s > %s' % (text, length))
    if uppercase:
        return text.upper()
    return text

def clean_date(italian_date, separator='', out_format='iso', error=None):
    ''' Return clean text with limit cut
        Log in error if over length
    '''
    if error is None:
        error = []
    italian_date = italian_date.split(' ')[0] # remove hour block
    if len(italian_date) != 10:
        error.append('Error not italian date: %s' % italian_date)
        # not stopped
    if out_format == 'iso':
        return '%s%s%s%s%s' % (
            italian_date[-4:],
            separator,
            italian_date[3:5],
            separator,
            italian_date[:2],
            )
    elif out_format == 'italian':
        return '%s%s%s%s%s' % (
            italian_date[:2],
            separator,
            italian_date[3:5],
            separator,
            italian_date[-4:],
            )
    elif out_format == 'english':
        return '%s%s%s%s%s' % (
            italian_date[3:5],
            separator,
            italian_date[:2],
            separator,
            italian_date[-4:],
            )
    else: # incorrect format:
        return italian_date # nothing todo

def clean_float(value, length, decimal=3, separator='.', error=None):
    ''' Clean float and return float format
    '''
    if error is None:
        error = []
    try:
        value = value.replace(',', '.')
        float_value = float(value.strip())
    except:
        error.append('Not a float: %s' % value)
        float_value = 0.0

    mask = '%%%s.%sf' % (length, decimal)
    res = mask % float_value
    res = res.replace('.', separator)
    return res

def log_on_file(message, mode='INFO', file_list=None, verbose=True):
    ''' Write message in file list passed
        mode: INFO WARNING ERROR
    '''
    if file_list is None:
        return False

    message_log = '%s. [%s] %s\n' % (datetime.now(), mode, message)
    if verbose:
        print message_log.strip()

    for f_log in file_list:
        if not f_log:
            continue
        f_log.write(message_log)
    return True

# -----------------------------------------------------------------------------
# Read configuration parameter:
# -----------------------------------------------------------------------------
try:
    company = sys.argv[1]
    if len(company) > 3:
        print 'Run: python ./pre_process.py CMP (must be max 3 char length)'
        sys.exit()
except:
    print 'Launch program as:\n\npython ./pre_process.py CMP'

try:
    cfg_file = os.path.expanduser('./%s.cfg' % company)
except:
    print 'Config file not found: %s.cfg' % company

config = ConfigParser.ConfigParser()
config.read([cfg_file])

# In parameters:
in_check = os.path.expanduser(config.get('in', 'check'))
in_path = os.path.expanduser(config.get('in', 'path'))
in_history = os.path.expanduser(config.get('in', 'history'))
in_log = os.path.expanduser(config.get('in', 'log'))
in_schedule = os.path.expanduser(config.get('in', 'schedule'))

# Outp arameters:
out_check = os.path.expanduser(config.get('out', 'check'))
out_path = os.path.expanduser(config.get('out', 'path'))
out_log = os.path.expanduser(config.get('out', 'log'))
out_schedule = os.path.expanduser(config.get('out', 'schedule'))
out_original = os.path.expanduser(config.get('out', 'original'))

# Mail parameters:
mail_error = config.get('mail', 'error')
mail_info = config.get('mail', 'info')

# TODO put in config file:
separator = config.get('file', 'separator')
tot_col = eval(config.get('file', 'tot_col'))

# Calculated parameters:
f_in_schedule = False # open(in_schedule, 'a')
f_in_log = False #open(in_log, 'a')

f_out_schedule = open(out_schedule, 'a')
f_out_log = open(out_log, 'a')

# -----------------------------------------------------------------------------
# Check mount folder:
# -----------------------------------------------------------------------------
if in_check:
    if not os.path.exists(in_check):
        log_on_file(
            'Cannot mount IN folder: %s' % in_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

if out_check:
    if not os.path.exists(out_check):
        log_on_file(
            'Cannot mount OUT folder: %s' % out_path, mode='ERROR', file_list=[
                f_in_schedule, f_out_schedule])
        sys.exit()

# -----------------------------------------------------------------------------
# Read IN folder:
# -----------------------------------------------------------------------------
log_on_file(
    'Start import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

for root, dirs, files in os.walk(in_path):
    log_on_file(
        'Read root folder: %s [%s]' % (root, company),
        mode='INFO',
        file_list=[f_in_log, f_out_log])

    for f in files:
        file_part = f.split('_')
        command = (file_part[2][:3]).upper() # New, Upd, Can

        # Fullname needed:
        file_in = os.path.join(root, f)
        file_history = os.path.join(in_history, f)
        file_original = os.path.join(out_original, f)

        create_date = datetime.fromtimestamp(
            os.path.getctime(file_in)
            ).strftime('%Y%m%d_%H%M%S')

        file_out = os.path.join(
            out_path, '%s' % (
                '%s_%s' % (create_date, f),
                )) # TODO change name

        # ---------------------------------------------------------------------
        # Read input file:
        # ---------------------------------------------------------------------
        error = []
        f_in = open(file_in, 'r')
        row_out = []
        log_on_file(
            'Parsing file: %s [%s]' % (file_in, company),
            mode='INFO',
            file_list=[f_in_log, f_out_log])
        for line in f_in:
            row = line.split(separator)
            if len(row) != tot_col:
                log_on_file(
                    'Different number columns: %s [%s]' % (file_in, company),
                    mode='INFO', file_list=[f_in_log, f_out_log])
                break

            # -----------------------------------------------------------------
            # Read fields:
            # -----------------------------------------------------------------
            date = create_date[:8] # From create date

            deadline = clean_date(row[0], separator='', out_format='iso',
                error=error)

            # Char:
            code = clean_text(row[1], 9, error=error)
            cook_center = clean_text(row[2], 110, error=error)[:60]# XXX Trunc
            address = clean_text(row[3], 60, error=error)
            cap = clean_text(row[4], 5, error=error) # XXX Check
            city = clean_text(row[5], 60, error=error) # XXX Check
            province = clean_text(row[6], 4, uppercase=True, error=error)
            order = clean_text(row[7], 10, error=error)
            default_code = clean_text(row[8], 16, uppercase=True, error=error)
            default_code_supplier = clean_text(
                row[9], 16, uppercase=True, error=error)
            name = clean_text(row[10], 60, error=error, truncate=True)
            um = clean_text(row[11], 2, uppercase=True, error=error)

            # Float:
            quantity = clean_float(row[12], 15, 2, error=error)
            price = clean_float(row[13], 15, 3, error=error)
            cost = clean_float(row[14], 15, 3, error=error)

            # -----------------------------------------------------------------
            # Convert row input file:
            # -----------------------------------------------------------------
            row_out.append(
                '%3s|%3s|%-8s|%-9s|%-60s|%-60s|%-5s|%-60s|%-4s|%-10s|%-16s|'
                '%-16s|%-60s|%-2s|%-15s|%-15s|%-15s|%-8s\r\n' % (
                    # Header:
                    company, # depend on parameter
                    command,
                    deadline,
                    code,
                    cook_center,
                    address,
                    cap,
                    city,
                    province,
                    order,

                    # Row:
                    default_code,
                    default_code_supplier,
                    name,
                    um,
                    quantity,
                    price,
                    cost,
                    date,
                    ))
        f_in.close()

        if error:
            log_on_file(
                'Error reading file: %s [%s]' % (error, company),
                mode='ERROR', file_list=[f_in_log, f_out_log])
            continue

        # ---------------------------------------------------------------------
        # Save file out:
        # ---------------------------------------------------------------------
        error = []
        # Convert file:
        f_out = open(file_out, 'w')

        for line in sorted(row_out, key=lambda x: sort_line(x)):
            f_out.write(line)
        f_out.close()

        if error:
            log_on_file(
                'Error generating out file: %s [%s]' % (error, company),
                mode='ERROR', file_list=[f_in_log, f_out_log])
            continue
        else:
            log_on_file(
                'Created file: %s [%s]' % (file_out, company),
                mode='INFO', file_list=[f_in_log, f_out_log])

        # ---------------------------------------------------------------------
        # History the in file:
        # ---------------------------------------------------------------------
        error = ''
        try:
            os.remove(file_original)
        except:
            pass
        try:
            shutil.copy(file_in, file_original)
            log_on_file(
                'History company file: %s [%s]' % (file_original, company),
                mode='INFO', file_list=[f_in_log, f_out_log])
        except:
            error += 'Error copy orginal folder %s >> %s' % (
                file_in, file_original)
        try:
            os.remove(file_history)
        except:
            pass
        try:
            shutil.move(file_in, file_history)
            log_on_file(
                'History supplier file: %s [%s]' % (file_history, company),
                mode='INFO', file_list=[f_in_log, f_out_log])
        except:
            error += 'Error moving file %s >> %s' % (file_in, file_history)

        if error:
            log_on_file(
                error, mode='ERROR', file_list=[f_in_log, f_out_log])
            # XXX TODO delete OUT file
            continue
    break # only root folder is read

log_on_file(
    'End import order mode: %s' % company, mode='INFO', file_list=[
        f_in_schedule, f_out_schedule])

# -----------------------------------------------------------------------------
# History the in file:
# -----------------------------------------------------------------------------
try:
    if f_out_log:
        f_out_log.close()
    if f_in_schedule:
        f_in_schedule.close()
    if f_in_log:
        f_in_log.close()
    if f_out_schedule:
        f_out_schedule.close()
except:
    print 'Error closing log file'
<|endoftext|>