<commit_before>import gametheory.base.simulation as simulation

import cPickle
import os
import random
import string

from nose.tools import eq_ as assert_equal
from optparse import OptionParser

def filename_generator(size=6, chars=string.ascii_uppercase + string.digits):
    return ''.join(random.choice(chars) for x in range(size))

class Sim(simulation.Simulation):
    def _run(self):
        return "runs"

class Sim2(simulation.Simulation):
    def _run(self):
        print >>self._out, "runs"

        return "runs"

class Batch(simulation.SimulationBatch):
    def _set_options(self):
        self._oparser.add_option("-t", "--test", action="store_true", dest="test", default=False, help="Testing")

    def _check_options(self):
        if not self._options.test:
            self._oparser.error("Test flag not passed")

    def _set_data(self):
        self._data['test'] = self._options.test

    def _when_done(self):
        return "test"

class TestSimulation:

    def setUp(self):
        self.sim = Sim(1, 2, 3)

    def tearDown(self):
        self.sim = None

    def test_simulation_init(self):
        assert self.sim is not None, "Sim is not set up"
        assert_equal(self.sim._data, 1)
        assert_equal(self.sim._num, 2)
        assert_equal(self.sim._outfile, 3)

        assert self.sim._out is not None, "Sim._out is not set up"
        assert_equal(self.sim._out_opened, True)

    def test_simulation_set_outfile(self):
        self.sim.set_output_file("/tmp/test")
        assert_equal(self.sim._outfile, "/tmp/test")
        assert self.sim._out is not None, "Sim._out is not set up"

        self.sim._close_out_fd()
        assert self.sim._out is None, "Sim._out was not closed"
        assert_equal(self.sim._out_opened, False)

        self.sim._open_out_fd()
        assert self.sim._out is not None, "Sim._out was not opened"
        assert_equal(self.sim._out_opened, True)

        self.sim.set_output_file("/tmp/test2")
        self.sim._open_out_fd()
        assert self.sim._out is not None, "Sim._out was not opened"
        assert_equal(self.sim._out_opened, True)

    def test_simulation_run(self):
        assert_equal(self.sim._out_opened, True)

        self.sim.set_output_file(False)

        result = self.sim.run()
        assert_equal(self.sim.result, "runs")
        assert_equal(result, "runs")

        assert_equal(self.sim._out_opened, False)

class TestSimulationBatch:

    def setUp(self):
        self.dir = "/tmp/" + filename_generator(8)
        self.batch = Batch(Sim2)

    def tearDown(self):
        self.batch = None
        if os.path.isdir(self.dir):
            files = os.listdir(self.dir)
            for f in files:
                if f == "." or f == "..": continue
                if f[-8:] == ".testout":
                    os.remove(self.dir + os.sep + f)
            os.rmdir(self.dir)

    def test_batch_init(self):
        assert self.batch is not None, "Batch is not set up"
        assert isinstance(self.batch._oparser, OptionParser), "Option parser is not initialized"
        assert self.batch._options is None, "Options is initialized"
        assert self.batch._args is None, "Args is initialized"
        assert_equal(self.batch._data, {})
        assert_equal(self.batch._task_dup_num, False)

    def test_batch_option_setup(self):
        assert self.batch._oparser.has_option("-D"), "No -D option"
        assert self.batch._oparser.has_option("--nofiledump"), "No --nofiledump option"
        assert self.batch._oparser.has_option("-F"), "No -F option"
        assert self.batch._oparser.has_option("--filename"), "No --filename option"
        assert self.batch._oparser.has_option("-N"), "No -N option"
        assert self.batch._oparser.has_option("--duplications"), "No --duplications option"
        assert self.batch._oparser.has_option("-O"), "No -O option"
        assert self.batch._oparser.has_option("--output"), "No --output option"
        assert self.batch._oparser.has_option("-P"), "No -P option"
        assert self.batch._oparser.has_option("--poolsize"), "No --poolsize option"
        assert self.batch._oparser.has_option("-Q"), "No -Q option"
        assert self.batch._oparser.has_option("--quiet"), "No --quiet option"
        assert self.batch._oparser.has_option("-S"), "No -S option"
        assert self.batch._oparser.has_option("--statsfile"), "No --statsfile option"
        assert self.batch._oparser.has_option("-t"), "No -t option"
        assert self.batch._oparser.has_option("--test"), "No --test option"

    def test_batch_go(self):
        args = ["-F",  "iter_{0}.testout", "-N", "4", "-P", "2", "-O", self.dir, "-S", "results.testout", "--test"]
        assert_equal(self.batch.go(args), "test")
        assert_equal(self.batch._options.test, True)
        assert_equal(self.batch._options.dup, 4)
        assert_equal(self.batch._options.output_dir, self.dir)
        assert_equal(self.batch._options.output_file, "iter_{0}.testout")
        assert_equal(self.batch._options.file_dump, True)
        assert_equal(self.batch._options.stats_file, "results.testout")
        assert_equal(self.batch._options.pool_size, 2)
        assert_equal(self.batch._options.quiet, False)

        assert_equal(self.batch._data['test'], True)

        for i in range(4):
            assert os.path.isfile(self.dir + os.sep + 'iter_{0}.testout'.format(i + 1)), "Dup file {0} is missing".format(i + 1)
        assert os.path.isfile(self.dir + os.sep + 'results.testout'), "Results file is missing"

        for i in range(4):
            with open(self.dir + os.sep + 'iter_{0}.testout'.format(i + 1), "r") as dup_file:
                assert_equal(dup_file.read(), "runs\n")

        with open(self.dir + os.sep + 'results.testout', "r") as results_file:
            should_be = ''
            should_be += cPickle.dumps(self.batch._options) + "\n"
            should_be += "\n"
            for _ in range(4):
                should_be += cPickle.dumps("runs") + "\n"
                should_be += "\n"
            assert_equal(results_file.read(), should_be)

    def test_batch_go2(self):
        args = ["-N", "6", "-P", "2", "-O", self.dir, "-S", "results.testout", "-Q", "--test", "-D"]
        assert_equal(self.batch.go(args), "test")
        assert_equal(self.batch._options.test, True)
        assert_equal(self.batch._options.dup, 6)
        assert_equal(self.batch._options.output_dir, self.dir)
        assert_equal(self.batch._options.output_file, "duplication_{0}")
        assert_equal(self.batch._options.file_dump, False)
        assert_equal(self.batch._options.stats_file, "results.testout")
        assert_equal(self.batch._options.pool_size, 2)
        assert_equal(self.batch._options.quiet, True)

        assert_equal(self.batch._data['test'], True)

        for i in range(6):
            assert not os.path.isfile(self.dir + os.sep + 'iter_{0}.testout'.format(i + 1)), "Dup file {0} is missing".format(i + 1)
        assert os.path.isfile(self.dir + os.sep + 'results.testout'), "Results file is missing"

        with open(self.dir + os.sep + 'results.testout', "r") as results_file:
            should_be = ''
            should_be += cPickle.dumps(self.batch._options) + "\n"
            should_be += "\n"
            for _ in range(6):
                should_be += cPickle.dumps("runs") + "\n"
                should_be += "\n"
            assert_equal(results_file.read(), should_be)

<commit_msg>Fixing tests that got accidentally removed<commit_after>import gametheory.base.simulation as simulation

import cPickle
import os
import random
import string

from nose.tools import assert_equal
from nose.tools import assert_raises
from optparse import OptionParser

def filename_generator(size=6, chars=string.ascii_uppercase + string.digits):
    return ''.join(random.choice(chars) for x in range(size))

class Sim(simulation.Simulation):
    def _run(self):
        return "runs"

class Sim2(simulation.Simulation):
    def _run(self):
        print >>self._out, "runs"

        return "runs"

class Batch(simulation.SimulationBatch):
    def _set_options(self):
        self._oparser.add_option("-t", "--test", action="store_true", dest="test", default=False, help="Testing")

    def _check_options(self):
        if not self._options.test:
            self._oparser.error("Test flag not passed")

    def _set_data(self):
        self._data['test'] = self._options.test

    def _when_done(self):
        return "test"

class TestSimulation:

    def setUp(self):
        self.sim = Sim(1, 2, 3)

    def tearDown(self):
        self.sim = None

    def test_simulation_init(self):
        assert self.sim is not None, "Sim is not set up"
        assert_equal(self.sim._data, 1)
        assert_equal(self.sim._num, 2)
        assert_equal(self.sim._outfile, 3)

        assert self.sim._out is not None, "Sim._out is not set up"
        assert_equal(self.sim._out_opened, True)

    def test_simulation_set_outfile(self):
        self.sim.set_output_file("/tmp/test")
        assert_equal(self.sim._outfile, "/tmp/test")
        assert self.sim._out is not None, "Sim._out is not set up"

        self.sim._close_out_fd()
        assert self.sim._out is None, "Sim._out was not closed"
        assert_equal(self.sim._out_opened, False)

        self.sim._open_out_fd()
        assert self.sim._out is not None, "Sim._out was not opened"
        assert_equal(self.sim._out_opened, True)

        self.sim.set_output_file("/tmp/test2")
        self.sim._open_out_fd()
        assert self.sim._out is not None, "Sim._out was not opened"
        assert_equal(self.sim._out_opened, True)

    def test_simulation_run(self):
        assert_equal(self.sim._out_opened, True)

        self.sim.set_output_file(False)

        result = self.sim.run()
        assert_equal(self.sim.result, "runs")
        assert_equal(result, "runs")

        assert_equal(self.sim._out_opened, False)

        assert simulation.Simulation._run(self.sim) is None

    def test_delegation_method(self):
        self.sim.set_output_file(None)
        assert_equal(simulation._run_simulation(self.sim), "runs")

class TestSimulationBatch:

    def setUp(self):
        self.dir = "/tmp/" + filename_generator(8)
        self.batch = Batch(Sim2)

    def tearDown(self):
        self.batch = None
        if os.path.isdir(self.dir):
            files = os.listdir(self.dir)
            for f in files:
                if f == "." or f == "..": continue
                if f[-8:] == ".testout":
                    os.remove(self.dir + os.sep + f)
            os.rmdir(self.dir)

    def test_batch_init(self):
        assert self.batch is not None, "Batch is not set up"
        assert isinstance(self.batch._oparser, OptionParser), "Option parser is not initialized"
        assert self.batch._options is None, "Options is initialized"
        assert self.batch._args is None, "Args is initialized"
        assert_equal(self.batch._data, {})
        assert_equal(self.batch._task_dup_num, False)

    def test_batch_option_setup(self):
        assert self.batch._oparser.has_option("-D"), "No -D option"
        assert self.batch._oparser.has_option("--nofiledump"), "No --nofiledump option"
        assert self.batch._oparser.has_option("-F"), "No -F option"
        assert self.batch._oparser.has_option("--filename"), "No --filename option"
        assert self.batch._oparser.has_option("-N"), "No -N option"
        assert self.batch._oparser.has_option("--duplications"), "No --duplications option"
        assert self.batch._oparser.has_option("-O"), "No -O option"
        assert self.batch._oparser.has_option("--output"), "No --output option"
        assert self.batch._oparser.has_option("-P"), "No -P option"
        assert self.batch._oparser.has_option("--poolsize"), "No --poolsize option"
        assert self.batch._oparser.has_option("-Q"), "No -Q option"
        assert self.batch._oparser.has_option("--quiet"), "No --quiet option"
        assert self.batch._oparser.has_option("-S"), "No -S option"
        assert self.batch._oparser.has_option("--statsfile"), "No --statsfile option"
        assert self.batch._oparser.has_option("-t"), "No -t option"
        assert self.batch._oparser.has_option("--test"), "No --test option"

    def test_batch_go(self):
        args = ["-F",  "iter_{0}.testout", "-N", "4", "-P", "2", "-O", self.dir, "-S", "results.testout", "--test"]
        assert_equal(self.batch.go(args), "test")
        assert_equal(self.batch._options.test, True)
        assert_equal(self.batch._options.dup, 4)
        assert_equal(self.batch._options.output_dir, self.dir)
        assert_equal(self.batch._options.output_file, "iter_{0}.testout")
        assert_equal(self.batch._options.file_dump, True)
        assert_equal(self.batch._options.stats_file, "results.testout")
        assert_equal(self.batch._options.pool_size, 2)
        assert_equal(self.batch._options.quiet, False)

        assert_equal(self.batch._data['test'], True)

        for i in range(4):
            assert os.path.isfile(self.dir + os.sep + 'iter_{0}.testout'.format(i + 1)), "Dup file {0} is missing".format(i + 1)
        assert os.path.isfile(self.dir + os.sep + 'results.testout'), "Results file is missing"

        for i in range(4):
            with open(self.dir + os.sep + 'iter_{0}.testout'.format(i + 1), "r") as dup_file:
                assert_equal(dup_file.read(), "runs\n")

        with open(self.dir + os.sep + 'results.testout', "r") as results_file:
            should_be = ''
            should_be += cPickle.dumps(self.batch._options) + "\n"
            should_be += "\n"
            for _ in range(4):
                should_be += cPickle.dumps("runs") + "\n"
                should_be += "\n"
            assert_equal(results_file.read(), should_be)

    def test_batch_go2(self):
        args = ["-N", "6", "-P", "2", "-O", self.dir, "-S", "results.testout", "-Q", "--test", "-D"]
        assert_equal(self.batch.go(args), "test")
        assert_equal(self.batch._options.test, True)
        assert_equal(self.batch._options.dup, 6)
        assert_equal(self.batch._options.output_dir, self.dir)
        assert_equal(self.batch._options.output_file, "duplication_{0}")
        assert_equal(self.batch._options.file_dump, False)
        assert_equal(self.batch._options.stats_file, "results.testout")
        assert_equal(self.batch._options.pool_size, 2)
        assert_equal(self.batch._options.quiet, True)

        assert_equal(self.batch._data['test'], True)

        for i in range(6):
            assert not os.path.isfile(self.dir + os.sep + 'iter_{0}.testout'.format(i + 1)), "Dup file {0} is missing".format(i + 1)
        assert os.path.isfile(self.dir + os.sep + 'results.testout'), "Results file is missing"

        with open(self.dir + os.sep + 'results.testout', "r") as results_file:
            should_be = ''
            should_be += cPickle.dumps(self.batch._options) + "\n"
            should_be += "\n"
            for _ in range(6):
                should_be += cPickle.dumps("runs") + "\n"
                should_be += "\n"
            assert_equal(results_file.read(), should_be)

    def test_option_failure(self):
        args = ["-N", "-6", "-P", "2", "-O", self.dir, "-S", "results.testout", "-Q", "-D", "--test"]

        assert_raises(SystemExit, self.batch.go, args)

    def test_option_failure2(self):
        args = ["-N", "6", "-P", "2", "-O", self.dir, "-S", "results.testout", "-Q", "-D"]

        assert_raises(SystemExit, self.batch.go, args)

    def test_dummies(self):
        assert simulation.SimulationBatch._set_options(self.batch) is None
        assert simulation.SimulationBatch._check_options(self.batch) is None
        assert simulation.SimulationBatch._set_data(self.batch) is None
        assert simulation.SimulationBatch._when_done(self.batch) is None
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import time
import netsvc
from osv import fields, osv
import tools

from tools.misc import currency
from tools.translate import _

class res_currency(osv.osv):
    def _current_rate(self, cr, uid, ids, name, arg, context=None):
        currency_rate_obj = self.pool.get('res.currency.rate')
        if context is None:
            context = {}
        res = {}
        if 'date' in context:
            date = context['date']
        else:
            date = time.strftime('%Y-%m-%d')
        date = date or time.strftime('%Y-%m-%d')
        domain = [('currency_id','in',ids), ('name','<=',date)]
        currency_rate_type_id = context.get('currency_rate_type_id', False)
        if currency_rate_type_id:
            domain.append(('currency_rate_type_id','=',currency_rate_type_id))
        curr_rate_ids = currency_rate_obj.search(cr, uid, domain, context=context)
        for id in ids:
            if curr_rate_ids:
                for cur in currency_rate_obj.browse(cr, uid, curr_rate_ids, context=context):
                    if cur.currency_id.id == id:
                        res[id] = cur.rate
            else:
                res[id] = 0

        return res
    _name = "res.currency"
    _description = "Currency"
    _columns = {
        # Note: 'code' column was removed as of v6.0, the 'name' should now hold the ISO code.
        'name': fields.char('Currency', size=32, required=True, help="Currency Code (ISO 4217)"),
        'symbol': fields.char('Symbol', size=3, help="Currency sign, to be used when printing amounts."),
        'rate': fields.function(_current_rate, method=True, string='Current Rate', digits=(12,6),
            help='The rate of the currency to the currency of rate 1.'),
        'rate_ids': fields.one2many('res.currency.rate', 'currency_id', 'Rates'),
        'accuracy': fields.integer('Computational Accuracy'),
        'rounding': fields.float('Rounding Factor', digits=(12,6)),
        'active': fields.boolean('Active'),
        'company_id':fields.many2one('res.company', 'Company'),
        'date': fields.date('Date'),
        'base': fields.boolean('Base')

    }
    _defaults = {
        'active': lambda *a: 1,
        'company_id': lambda self,cr,uid,c: self.pool.get('res.company')._company_default_get(cr, uid, 'res.currency', context=c)
    }
    _order = "name"

    def read(self, cr, user, ids, fields=None, context=None, load='_classic_read'):
        res = super(osv.osv, self).read(cr, user, ids, fields, context, load)
        for r in res:
            if r.__contains__('rate_ids'):
                rates=r['rate_ids']
                if rates:
                    currency_rate_obj=  self.pool.get('res.currency.rate')
                    currency_date = currency_rate_obj.read(cr,user,rates[0],['name'])['name']
                    r['date'] = currency_date

        return res

    def name_get(self, cr, uid, ids, context=None):
        if not ids:
            return []
        if isinstance(ids, (int, long)):
            ids = [ids]
        reads = self.read(cr, uid, ids, ['name','symbol'], context=context, load='_classic_write')
        return [(x['id'], tools.ustr(x['name']) + (x['symbol'] and (' (' + tools.ustr(x['symbol']) + ')') or '')) for x in reads]

    def round(self, cr, uid, currency, amount):
        if currency.rounding == 0:
            return 0.0
        else:
            # /!\ First member below must be rounded to full unit!
            # Do not pass a rounding digits value to round()
            return round(amount / currency.rounding) * currency.rounding

    def is_zero(self, cr, uid, currency, amount):
        return abs(self.round(cr, uid, currency, amount)) < currency.rounding

    def _get_conversion_rate(self, cr, uid, from_currency, to_currency, context=None):
        if context is None:
            context = {}
        if from_currency['rate'] == 0 or to_currency['rate'] == 0:
            date = context.get('date', time.strftime('%Y-%m-%d'))
            if from_currency['rate'] == 0:
                currency_symbol = from_currency.symbol
            else:
                currency_symbol = to_currency.symbol
            raise osv.except_osv(_('Error'), _('No rate found \n' \
                    'for the currency: %s \n' \
                    'at the date: %s') % (currency_symbol, date))

        return to_currency.rate/from_currency.rate

    def compute(self, cr, uid, from_currency_id, to_currency_id, from_amount, round=True, rate_type=False, context=None):
        if not from_currency_id:
            from_currency_id = to_currency_id
        if not to_currency_id:
            to_currency_id = from_currency_id

        if rate_type:
            context.update({'currency_rate_type_id': rate_type})
        xc = self.browse(cr, uid, [from_currency_id,to_currency_id], context=context)
        from_currency = (xc[0].id == from_currency_id and xc[0]) or xc[1]
        to_currency = (xc[0].id == to_currency_id and xc[0]) or xc[1]
        if to_currency_id == from_currency_id:
            if round:
                return self.round(cr, uid, to_currency, from_amount)
            else:
                return from_amount
        else:
            rate = self._get_conversion_rate(cr, uid, from_currency, to_currency, context=context)
            if round:
                return self.round(cr, uid, to_currency, from_amount * rate)
            else:
                return (from_amount * rate)

res_currency()

class res_currency_rate_type(osv.osv):
    _name = "res.currency.rate.type"
    _description = "Currency Rate Type"
    _columns = {
        'name': fields.char('Name', size=32, required=True),
    }

res_currency_rate_type()

class res_currency_rate(osv.osv):
    _name = "res.currency.rate"
    _description = "Currency Rate"
    _columns = {
        'name': fields.date('Date', required=True, select=True),
        'rate': fields.float('Rate', digits=(12,6), required=True,
            help='The rate of the currency to the currency of rate 1'),
        'currency_id': fields.many2one('res.currency', 'Currency', readonly=True),
        'currency_rate_type_id': fields.many2one('res.currency.rate.type', 'Currency Rate Type'),
    }
    _defaults = {
        'name': lambda *a: time.strftime('%Y-%m-%d'),
    }
    _order = "name desc"

res_currency_rate()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>[IMP] base: Improved _currency_rate function.<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import time
import netsvc
from osv import fields, osv
import tools

from tools.misc import currency
from tools.translate import _

class res_currency(osv.osv):
    def _current_rate(self, cr, uid, ids, name, arg, context=None):
        currency_rate_obj = self.pool.get('res.currency.rate')
        if context is None:
            context = {}
        res = {}
        if 'date' in context:
            date = context['date']
        else:
            date = time.strftime('%Y-%m-%d')
        date = date or time.strftime('%Y-%m-%d')
        domain = [('currency_id','in',ids), ('name','<=',date)]
        currency_rate_type_id = context.get('currency_rate_type_id', False)
        if currency_rate_type_id:
            domain.append(('currency_rate_type_id','=',currency_rate_type_id))
        curr_rate_ids = currency_rate_obj.search(cr, uid, domain, order='name desc', context=context)
        curr_rates = currency_rate_obj.browse(cr, uid, curr_rate_ids, context=context)
        for id in ids:
            res[id] = 0
            for cur in curr_rates:
                if cur.currency_id.id == id:
                    res[id] = cur.rate
                    break

        return res
    _name = "res.currency"
    _description = "Currency"
    _columns = {
        # Note: 'code' column was removed as of v6.0, the 'name' should now hold the ISO code.
        'name': fields.char('Currency', size=32, required=True, help="Currency Code (ISO 4217)"),
        'symbol': fields.char('Symbol', size=3, help="Currency sign, to be used when printing amounts."),
        'rate': fields.function(_current_rate, method=True, string='Current Rate', digits=(12,6),
            help='The rate of the currency to the currency of rate 1.'),
        'rate_ids': fields.one2many('res.currency.rate', 'currency_id', 'Rates'),
        'accuracy': fields.integer('Computational Accuracy'),
        'rounding': fields.float('Rounding Factor', digits=(12,6)),
        'active': fields.boolean('Active'),
        'company_id':fields.many2one('res.company', 'Company'),
        'date': fields.date('Date'),
        'base': fields.boolean('Base')

    }
    _defaults = {
        'active': lambda *a: 1,
        'company_id': lambda self,cr,uid,c: self.pool.get('res.company')._company_default_get(cr, uid, 'res.currency', context=c)
    }
    _order = "name"

    def read(self, cr, user, ids, fields=None, context=None, load='_classic_read'):
        res = super(osv.osv, self).read(cr, user, ids, fields, context, load)
        for r in res:
            if r.__contains__('rate_ids'):
                rates=r['rate_ids']
                if rates:
                    currency_rate_obj=  self.pool.get('res.currency.rate')
                    currency_date = currency_rate_obj.read(cr,user,rates[0],['name'])['name']
                    r['date'] = currency_date

        return res

    def name_get(self, cr, uid, ids, context=None):
        if not ids:
            return []
        if isinstance(ids, (int, long)):
            ids = [ids]
        reads = self.read(cr, uid, ids, ['name','symbol'], context=context, load='_classic_write')
        return [(x['id'], tools.ustr(x['name']) + (x['symbol'] and (' (' + tools.ustr(x['symbol']) + ')') or '')) for x in reads]

    def round(self, cr, uid, currency, amount):
        if currency.rounding == 0:
            return 0.0
        else:
            # /!\ First member below must be rounded to full unit!
            # Do not pass a rounding digits value to round()
            return round(amount / currency.rounding) * currency.rounding

    def is_zero(self, cr, uid, currency, amount):
        return abs(self.round(cr, uid, currency, amount)) < currency.rounding

    def _get_conversion_rate(self, cr, uid, from_currency, to_currency, context=None):
        if context is None:
            context = {}
        if from_currency['rate'] == 0 or to_currency['rate'] == 0:
            date = context.get('date', time.strftime('%Y-%m-%d'))
            if from_currency['rate'] == 0:
                currency_symbol = from_currency.symbol
            else:
                currency_symbol = to_currency.symbol
            raise osv.except_osv(_('Error'), _('No rate found \n' \
                    'for the currency: %s \n' \
                    'at the date: %s') % (currency_symbol, date))

        return to_currency.rate/from_currency.rate

    def compute(self, cr, uid, from_currency_id, to_currency_id, from_amount, round=True, rate_type=False, context=None):
        if not from_currency_id:
            from_currency_id = to_currency_id
        if not to_currency_id:
            to_currency_id = from_currency_id

        if rate_type:
            context.update({'currency_rate_type_id': rate_type})
        xc = self.browse(cr, uid, [from_currency_id,to_currency_id], context=context)
        from_currency = (xc[0].id == from_currency_id and xc[0]) or xc[1]
        to_currency = (xc[0].id == to_currency_id and xc[0]) or xc[1]
        if to_currency_id == from_currency_id:
            if round:
                return self.round(cr, uid, to_currency, from_amount)
            else:
                return from_amount
        else:
            rate = self._get_conversion_rate(cr, uid, from_currency, to_currency, context=context)
            if round:
                return self.round(cr, uid, to_currency, from_amount * rate)
            else:
                return (from_amount * rate)

res_currency()

class res_currency_rate_type(osv.osv):
    _name = "res.currency.rate.type"
    _description = "Currency Rate Type"
    _columns = {
        'name': fields.char('Name', size=32, required=True),
    }

res_currency_rate_type()

class res_currency_rate(osv.osv):
    _name = "res.currency.rate"
    _description = "Currency Rate"
    _columns = {
        'name': fields.date('Date', required=True, select=True),
        'rate': fields.float('Rate', digits=(12,6), required=True,
            help='The rate of the currency to the currency of rate 1'),
        'currency_id': fields.many2one('res.currency', 'Currency', readonly=True),
        'currency_rate_type_id': fields.many2one('res.currency.rate.type', 'Currency Rate Type'),
    }
    _defaults = {
        'name': lambda *a: time.strftime('%Y-%m-%d'),
    }
    _order = "name desc"

res_currency_rate()

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    tests.unit.utils.event_test
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: © 2012 UfSoft.org - :email:`Pedro Algarvio (pedro@algarvio.me)`
    :license: Apache 2.0, see LICENSE for more details.
'''

import os
import integration
import hashlib
from saltunittest import TestCase, TestLoader, TextTestRunner

from salt.utils import event

SOCK_DIR = os.path.join(integration.TMP, 'test-socks')

class TestSaltEvent(TestCase):

    def test_master_event(self):
        me = event.MasterEvent(SOCK_DIR)
        self.assertEqual(
            me.puburi,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'master_event_pub.ipc')
            )
        )
        self.assertEqual(
            me.pulluri,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'master_event_pull.ipc')
            )
        )

    def test_minion_event(self):
        opts = dict(id='foo', sock_dir=SOCK_DIR)
        id_hash = hashlib.md5(opts['id']).hexdigest()
        me = event.MinionEvent(**opts)
        self.assertEqual(
            me.puburi,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'minion_event_{0}_pub.ipc'.format(id_hash))
            )
        )
        self.assertEqual(
            me.pulluri,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'minion_event_{0}_pull.ipc'.format(id_hash))
            )
        )

    def test_minion_event_tcp_ipc_mode(self):
        opts = dict(id='foo', ipc_mode='tcp')
        me = event.MinionEvent(**opts)
        self.assertEqual(me.puburi, 'tcp://127.0.0.1:4510')
        self.assertEqual(me.pulluri, 'tcp://127.0.0.1:4511')

    def test_minion_event_no_id(self):
        me = event.MinionEvent(sock_dir=SOCK_DIR)
        self.assertEqual(
            me.puburi,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'minion_event__pub.ipc')
            )
        )
        self.assertEqual(
            me.pulluri,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'minion_event__pull.ipc')
            )
        )

if __name__ == "__main__":
    loader = TestLoader()
    tests = loader.loadTestsFromTestCase(TestSaltEvent)
    TextTestRunner(verbosity=1).run(tests)
<commit_msg>Fix another test for new hased minion ids<commit_after># -*- coding: utf-8 -*-
'''
    tests.unit.utils.event_test
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~

    :copyright: © 2012 UfSoft.org - :email:`Pedro Algarvio (pedro@algarvio.me)`
    :license: Apache 2.0, see LICENSE for more details.
'''

import os
import integration
import hashlib
from saltunittest import TestCase, TestLoader, TextTestRunner

from salt.utils import event

SOCK_DIR = os.path.join(integration.TMP, 'test-socks')

class TestSaltEvent(TestCase):

    def test_master_event(self):
        me = event.MasterEvent(SOCK_DIR)
        self.assertEqual(
            me.puburi,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'master_event_pub.ipc')
            )
        )
        self.assertEqual(
            me.pulluri,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'master_event_pull.ipc')
            )
        )

    def test_minion_event(self):
        opts = dict(id='foo', sock_dir=SOCK_DIR)
        id_hash = hashlib.md5(opts['id']).hexdigest()
        me = event.MinionEvent(**opts)
        self.assertEqual(
            me.puburi,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'minion_event_{0}_pub.ipc'.format(id_hash))
            )
        )
        self.assertEqual(
            me.pulluri,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'minion_event_{0}_pull.ipc'.format(id_hash))
            )
        )

    def test_minion_event_tcp_ipc_mode(self):
        opts = dict(id='foo', ipc_mode='tcp')
        me = event.MinionEvent(**opts)
        self.assertEqual(me.puburi, 'tcp://127.0.0.1:4510')
        self.assertEqual(me.pulluri, 'tcp://127.0.0.1:4511')

    def test_minion_event_no_id(self):
        me = event.MinionEvent(sock_dir=SOCK_DIR)
        id_hash = hashlib.md5('').hexdigest()
        self.assertEqual(
            me.puburi,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'minion_event_{0}_pub.ipc'.format(id_hash))
            )
        )
        self.assertEqual(
            me.pulluri,
            'ipc://{0}'.format(
                os.path.join(SOCK_DIR, 'minion_event_{0}_pull.ipc'.format(id_hash))
            )
        )

if __name__ == "__main__":
    loader = TestLoader()
    tests = loader.loadTestsFromTestCase(TestSaltEvent)
    TextTestRunner(verbosity=1).run(tests)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# check if a file has the proper license in it
#
# USAGE: check-license.py [-C] file1 file2 ... fileN
#
# A 'file' may in fact be a directory, in which case it is recursively
# searched.
#
# If the license cannot be found, then the filename is printed to stdout.
# Typical usage:
#    $ check-license.py . > bad-files
#
# -C switch is used to change licenses.
# Typical usage:
#    $ check-license.py -C file1 file2 ... fileN
#

import sys, os, re

OLD_LICENSE = '''\
 \* ====================================================================
 \* Copyright \(c\) (200[0-9]|200[0-9]-200[0-9]) CollabNet.  All rights reserved.
 \*
 \* This software is licensed as described in the file COPYING, which
 \* you should have received as part of this distribution\.  The terms
 \* are also available at http://subversion.tigris.org/license-1\.html\.
 \* If newer versions of this license are posted there, you may use a
 \* newer version instead, at your option\.
 \*
 \* This software consists of voluntary contributions made by many
 \* individuals\.  For exact contribution history, see the revision
 \* history and logs, available at http://subversion.tigris.org/\.
 \* ====================================================================
'''

SH_OLD_LICENSE = re.subn(r'(?m)^ \\\*', '#', OLD_LICENSE)[0]

# Remember not to do regexp quoting for NEW_LICENSE.  Only OLD_LICENSE
# is used for matching; NEW_LICENSE is inserted as-is.
NEW_LICENSE = '''\
 * ====================================================================
 * Copyright (c) 2000-2009 CollabNet.  All rights reserved.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution.  The terms
 * are also available at http://subversion.tigris.org/license-1.html.
 * If newer versions of this license are posted there, you may use a
 * newer version instead, at your option.
 *
 * This software consists of voluntary contributions made by many
 * individuals.  For exact contribution history, see the revision
 * history and logs, available at http://subversion.tigris.org/.
 * ====================================================================
'''

SH_NEW_LICENSE = re.subn(r'(?m)^ \*', '#', NEW_LICENSE)[0]

re_OLD = re.compile(OLD_LICENSE)
re_SH_OLD = re.compile(SH_OLD_LICENSE)
re_EXCLUDE = re.compile(
    r'automatically generated by SWIG'
    + r'|Generated from configure\.in'
    + r'|placed into the public domain'
    )

c_comment_suffices = ('.c', '.java', '.h', '.cpp', '.hw', '.pas')

# Yes, this is an empty tuple. No types that fit in this category uniformly
# have a copyright block.
# Possible types to add here:
# ('.bat', '.py', '.pl', '.in')
sh_comment_suffices = ()

def check_file(fname, old_re, new_lic):
  s = open(fname).read()
  if (not old_re.search(s)
      and not re_EXCLUDE.search(s)):
    print(fname)

def change_license(fname, old_re, new_lic):
  s = open(fname).read()
  m = old_re.search(s)
  if not m:
    print('ERROR: missing old license: %s' % fname)
  else:
    s = s[:m.start()] + new_lic + s[m.end():]
    open(fname, 'w').write(s)
    print('Changed: %s' % fname)

def visit(baton, dirname, dircontents):
  file_func = baton
  for i in dircontents:
    # Don't recurse into certain directories
    if i in ('.svn', '.libs'):
      dircontents.remove(i)
      continue

    extension = os.path.splitext(i)[1]
    fullname = os.path.join(dirname, i)

    if os.path.isdir(fullname):
      continue

    if extension in c_comment_suffices:
      file_func(fullname, re_OLD, NEW_LICENSE)
    elif extension in sh_comment_suffices:
      file_func(fullname, re_SH_OLD, SH_NEW_LICENSE)

def main():
  file_func = check_file
  if sys.argv[1] == '-C':
    print('Changing license text...')
    del sys.argv[1]
    file_func = change_license

  for f in sys.argv[1:]:
    if os.path.isdir(f):
      baton = file_func
      for dirpath, dirs, files in os.walk(f):
        visit(baton, dirpath, dirs + files)
    else:
      baton = file_func
      dir, i = os.path.split(f)
      visit(baton, dir, i)

if __name__ == '__main__':
  main()
<commit_msg>* tools/dev/check-license.py   (OLD_LICENSE, NEW_LICENSE): Update, following up to r38370.<commit_after>#!/usr/bin/env python
#
# check if a file has the proper license in it
#
# USAGE: check-license.py [-C] file1 file2 ... fileN
#
# A 'file' may in fact be a directory, in which case it is recursively
# searched.
#
# If the license cannot be found, then the filename is printed to stdout.
# Typical usage:
#    $ check-license.py . > bad-files
#
# -C switch is used to change licenses.
# Typical usage:
#    $ check-license.py -C file1 file2 ... fileN
#

import sys, os, re

# Note: Right now, OLD_LICENSE and NEW_LICENSE are the same, because
# r38370 updated all the license blocks.  In the future, if we update
# the license block again, change just NEW_LICENSE and use this script.

OLD_LICENSE = '''\
 \* ====================================================================
 \*    Licensed to the Subversion Corporation \(SVN Corp\.\) under one
 \*    or more contributor license agreements\.  See the NOTICE file
 \*    distributed with this work for additional information
 \*    regarding copyright ownership\.  The SVN Corp\. licenses this file
 \*    to you under the Apache License, Version 2\.0 \(the
 \*    "License"\); you may not use this file except in compliance
 \*    with the License\.  You may obtain a copy of the License at
 \*
 \*      http://www\.apache\.org/licenses/LICENSE-2\.0
 \*
 \*    Unless required by applicable law or agreed to in writing,
 \*    software distributed under the License is distributed on an
 \*    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 \*    KIND, either express or implied\.  See the License for the
 \*    specific language governing permissions and limitations
 \*    under the License\.
 \* ====================================================================
'''

SH_OLD_LICENSE = re.subn(r'(?m)^ \\\*', '#', OLD_LICENSE)[0]

# Remember not to do regexp quoting for NEW_LICENSE.  Only OLD_LICENSE
# is used for matching; NEW_LICENSE is inserted as-is.
NEW_LICENSE = '''\
 * ====================================================================
 *    Licensed to the Subversion Corporation (SVN Corp.) under one
 *    or more contributor license agreements.  See the NOTICE file
 *    distributed with this work for additional information
 *    regarding copyright ownership.  The SVN Corp. licenses this file
 *    to you under the Apache License, Version 2.0 (the
 *    "License"); you may not use this file except in compliance
 *    with the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing,
 *    software distributed under the License is distributed on an
 *    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 *    KIND, either express or implied.  See the License for the
 *    specific language governing permissions and limitations
 *    under the License.
 * ====================================================================
'''

SH_NEW_LICENSE = re.subn(r'(?m)^ \*', '#', NEW_LICENSE)[0]

re_OLD = re.compile(OLD_LICENSE)
re_SH_OLD = re.compile(SH_OLD_LICENSE)
re_EXCLUDE = re.compile(
    r'automatically generated by SWIG'
    + r'|Generated from configure\.in'
    + r'|placed into the public domain'
    )

c_comment_suffices = ('.c', '.java', '.h', '.cpp', '.hw', '.pas')

# Yes, this is an empty tuple. No types that fit in this category uniformly
# have a copyright block.
# Possible types to add here:
# ('.bat', '.py', '.pl', '.in')
sh_comment_suffices = ()

def check_file(fname, old_re, new_lic):
  s = open(fname).read()
  if (not old_re.search(s)
      and not re_EXCLUDE.search(s)):
    print(fname)

def change_license(fname, old_re, new_lic):
  s = open(fname).read()
  m = old_re.search(s)
  if not m:
    print('ERROR: missing old license: %s' % fname)
  else:
    s = s[:m.start()] + new_lic + s[m.end():]
    open(fname, 'w').write(s)
    print('Changed: %s' % fname)

def visit(baton, dirname, dircontents):
  file_func = baton
  for i in dircontents:
    # Don't recurse into certain directories
    if i in ('.svn', '.libs'):
      dircontents.remove(i)
      continue

    extension = os.path.splitext(i)[1]
    fullname = os.path.join(dirname, i)

    if os.path.isdir(fullname):
      continue

    if extension in c_comment_suffices:
      file_func(fullname, re_OLD, NEW_LICENSE)
    elif extension in sh_comment_suffices:
      file_func(fullname, re_SH_OLD, SH_NEW_LICENSE)

def main():
  file_func = check_file
  if sys.argv[1] == '-C':
    print('Changing license text...')
    del sys.argv[1]
    file_func = change_license

  for f in sys.argv[1:]:
    if os.path.isdir(f):
      baton = file_func
      for dirpath, dirs, files in os.walk(f):
        visit(baton, dirpath, dirs + files)
    else:
      baton = file_func
      dir, i = os.path.split(f)
      visit(baton, dir, i)

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>import unittest
from datetime import datetime

from django.test import SimpleTestCase, ignore_warnings
from django.utils.datastructures import MultiValueDict
from django.utils.deprecation import RemovedInDjango40Warning
from django.utils.http import (
    base36_to_int, escape_leading_slashes, http_date, int_to_base36,
    is_safe_url, is_same_domain, parse_etags, parse_http_date, quote_etag,
    urlencode, urlquote, urlquote_plus, urlsafe_base64_decode,
    urlsafe_base64_encode, urlunquote, urlunquote_plus,
)

class URLEncodeTests(SimpleTestCase):
    cannot_encode_none_msg = (
        'Cannot encode None in a query string. Did you mean to pass an '
        'empty string or omit the value?'
    )

    def test_tuples(self):
        self.assertEqual(urlencode((('a', 1), ('b', 2), ('c', 3))), 'a=1&b=2&c=3')

    def test_dict(self):
        result = urlencode({'a': 1, 'b': 2, 'c': 3})
        # Dictionaries are treated as unordered.
        self.assertIn(result, [
            'a=1&b=2&c=3',
            'a=1&c=3&b=2',
            'b=2&a=1&c=3',
            'b=2&c=3&a=1',
            'c=3&a=1&b=2',
            'c=3&b=2&a=1',
        ])

    def test_dict_containing_sequence_not_doseq(self):
        self.assertEqual(urlencode({'a': [1, 2]}, doseq=False), 'a=%5B%271%27%2C+%272%27%5D')

    def test_dict_containing_sequence_doseq(self):
        self.assertEqual(urlencode({'a': [1, 2]}, doseq=True), 'a=1&a=2')

    def test_dict_containing_empty_sequence_doseq(self):
        self.assertEqual(urlencode({'a': []}, doseq=True), '')

    def test_multivaluedict(self):
        result = urlencode(MultiValueDict({
            'name': ['Adrian', 'Simon'],
            'position': ['Developer'],
        }), doseq=True)
        # MultiValueDicts are similarly unordered.
        self.assertIn(result, [
            'name=Adrian&name=Simon&position=Developer',
            'position=Developer&name=Adrian&name=Simon',
        ])

    def test_dict_with_bytes_values(self):
        self.assertEqual(urlencode({'a': b'abc'}, doseq=True), 'a=abc')

    def test_dict_with_sequence_of_bytes(self):
        self.assertEqual(urlencode({'a': [b'spam', b'eggs', b'bacon']}, doseq=True), 'a=spam&a=eggs&a=bacon')

    def test_dict_with_bytearray(self):
        self.assertEqual(urlencode({'a': bytearray(range(2))}, doseq=True), 'a=0&a=1')
        self.assertEqual(urlencode({'a': bytearray(range(2))}, doseq=False), 'a=%5B%270%27%2C+%271%27%5D')

    def test_generator(self):
        def gen():
            yield from range(2)

        self.assertEqual(urlencode({'a': gen()}, doseq=True), 'a=0&a=1')
        self.assertEqual(urlencode({'a': gen()}, doseq=False), 'a=%5B%270%27%2C+%271%27%5D')

    def test_none(self):
        with self.assertRaisesMessage(TypeError, self.cannot_encode_none_msg):
            urlencode({'a': None})

    def test_none_in_sequence(self):
        with self.assertRaisesMessage(TypeError, self.cannot_encode_none_msg):
            urlencode({'a': [None]}, doseq=True)

    def test_none_in_generator(self):
        def gen():
            yield None
        with self.assertRaisesMessage(TypeError, self.cannot_encode_none_msg):
            urlencode({'a': gen()}, doseq=True)

class Base36IntTests(SimpleTestCase):
    def test_roundtrip(self):
        for n in [0, 1, 1000, 1000000]:
            self.assertEqual(n, base36_to_int(int_to_base36(n)))

    def test_negative_input(self):
        with self.assertRaisesMessage(ValueError, 'Negative base36 conversion input.'):
            int_to_base36(-1)

    def test_to_base36_errors(self):
        for n in ['1', 'foo', {1: 2}, (1, 2, 3), 3.141]:
            with self.assertRaises(TypeError):
                int_to_base36(n)

    def test_invalid_literal(self):
        for n in ['#', ' ']:
            with self.assertRaisesMessage(ValueError, "invalid literal for int() with base 36: '%s'" % n):
                base36_to_int(n)

    def test_input_too_large(self):
        with self.assertRaisesMessage(ValueError, 'Base36 input too large'):
            base36_to_int('1' * 14)

    def test_to_int_errors(self):
        for n in [123, {1: 2}, (1, 2, 3), 3.141]:
            with self.assertRaises(TypeError):
                base36_to_int(n)

    def test_values(self):
        for n, b36 in [(0, '0'), (1, '1'), (42, '16'), (818469960, 'django')]:
            self.assertEqual(int_to_base36(n), b36)
            self.assertEqual(base36_to_int(b36), n)

class IsSafeURLTests(unittest.TestCase):
    def test_bad_urls(self):
        bad_urls = (
            'http://example.com',
            'http:///example.com',
            'https://example.com',
            'ftp://example.com',
            r'\\example.com',
            r'\\\example.com',
            r'/\\/example.com',
            r'\\\example.com',
            r'\\example.com',
            r'\\//example.com',
            r'/\/example.com',
            r'\/example.com',
            r'/\example.com',
            'http:///example.com',
            r'http:/\//example.com',
            r'http:\/example.com',
            r'http:/\example.com',
            'javascript:alert("XSS")',
            '\njavascript:alert(x)',
            '\x08//example.com',
            r'http://otherserver\@example.com',
            r'http:\\testserver\@example.com',
            r'http://testserver\me:pass@example.com',
            r'http://testserver\@example.com',
            r'http:\\testserver\confirm\me@example.com',
            'http:999999999',
            'ftp:9999999999',
            '\n',
            'http://[2001:cdba:0000:0000:0000:0000:3257:9652/',
            'http://2001:cdba:0000:0000:0000:0000:3257:9652]/',
        )
        for bad_url in bad_urls:
            with self.subTest(url=bad_url):
                self.assertIs(is_safe_url(bad_url, allowed_hosts={'testserver', 'testserver2'}), False)

    def test_good_urls(self):
        good_urls = (
            '/view/?param=http://example.com',
            '/view/?param=https://example.com',
            '/view?param=ftp://example.com',
            'view/?param=//example.com',
            'https://testserver/',
            'HTTPS://testserver/',
            '//testserver/',
            'http://testserver/confirm?email=me@example.com',
            '/url%20with%20spaces/',
            'path/http:2222222222',
        )
        for good_url in good_urls:
            with self.subTest(url=good_url):
                self.assertIs(is_safe_url(good_url, allowed_hosts={'otherserver', 'testserver'}), True)

    def test_basic_auth(self):
        # Valid basic auth credentials are allowed.
        self.assertIs(is_safe_url(r'http://user:pass@testserver/', allowed_hosts={'user:pass@testserver'}), True)

    def test_no_allowed_hosts(self):
        # A path without host is allowed.
        self.assertIs(is_safe_url('/confirm/me@example.com', allowed_hosts=None), True)
        # Basic auth without host is not allowed.
        self.assertIs(is_safe_url(r'http://testserver\@example.com', allowed_hosts=None), False)

    def test_allowed_hosts_str(self):
        self.assertIs(is_safe_url('http://good.com/good', allowed_hosts='good.com'), True)
        self.assertIs(is_safe_url('http://good.co/evil', allowed_hosts='good.com'), False)

    def test_secure_param_https_urls(self):
        secure_urls = (
            'https://example.com/p',
            'HTTPS://example.com/p',
            '/view/?param=http://example.com',
        )
        for url in secure_urls:
            with self.subTest(url=url):
                self.assertIs(is_safe_url(url, allowed_hosts={'example.com'}, require_https=True), True)

    def test_secure_param_non_https_urls(self):
        insecure_urls = (
            'http://example.com/p',
            'ftp://example.com/p',
            '//example.com/p',
        )
        for url in insecure_urls:
            with self.subTest(url=url):
                self.assertIs(is_safe_url(url, allowed_hosts={'example.com'}, require_https=True), False)

class URLSafeBase64Tests(unittest.TestCase):
    def test_roundtrip(self):
        bytestring = b'foo'
        encoded = urlsafe_base64_encode(bytestring)
        decoded = urlsafe_base64_decode(encoded)
        self.assertEqual(bytestring, decoded)

@ignore_warnings(category=RemovedInDjango40Warning)
class URLQuoteTests(unittest.TestCase):
    def test_quote(self):
        self.assertEqual(urlquote('Paris & Orl\xe9ans'), 'Paris%20%26%20Orl%C3%A9ans')
        self.assertEqual(urlquote('Paris & Orl\xe9ans', safe="&"), 'Paris%20&%20Orl%C3%A9ans')

    def test_unquote(self):
        self.assertEqual(urlunquote('Paris%20%26%20Orl%C3%A9ans'), 'Paris & Orl\xe9ans')
        self.assertEqual(urlunquote('Paris%20&%20Orl%C3%A9ans'), 'Paris & Orl\xe9ans')

    def test_quote_plus(self):
        self.assertEqual(urlquote_plus('Paris & Orl\xe9ans'), 'Paris+%26+Orl%C3%A9ans')
        self.assertEqual(urlquote_plus('Paris & Orl\xe9ans', safe="&"), 'Paris+&+Orl%C3%A9ans')

    def test_unquote_plus(self):
        self.assertEqual(urlunquote_plus('Paris+%26+Orl%C3%A9ans'), 'Paris & Orl\xe9ans')
        self.assertEqual(urlunquote_plus('Paris+&+Orl%C3%A9ans'), 'Paris & Orl\xe9ans')

class IsSameDomainTests(unittest.TestCase):
    def test_good(self):
        for pair in (
            ('example.com', 'example.com'),
            ('example.com', '.example.com'),
            ('foo.example.com', '.example.com'),
            ('example.com:8888', 'example.com:8888'),
            ('example.com:8888', '.example.com:8888'),
            ('foo.example.com:8888', '.example.com:8888'),
        ):
            self.assertIs(is_same_domain(*pair), True)

    def test_bad(self):
        for pair in (
            ('example2.com', 'example.com'),
            ('foo.example.com', 'example.com'),
            ('example.com:9999', 'example.com:8888'),
            ('foo.example.com:8888', ''),
        ):
            self.assertIs(is_same_domain(*pair), False)

class ETagProcessingTests(unittest.TestCase):
    def test_parsing(self):
        self.assertEqual(
            parse_etags(r'"" ,  "etag", "e\\tag", W/"weak"'),
            ['""', '"etag"', r'"e\\tag"', 'W/"weak"']
        )
        self.assertEqual(parse_etags('*'), ['*'])

        # Ignore RFC 2616 ETags that are invalid according to RFC 7232.
        self.assertEqual(parse_etags(r'"etag", "e\"t\"ag"'), ['"etag"'])

    def test_quoting(self):
        self.assertEqual(quote_etag('etag'), '"etag"')  # unquoted
        self.assertEqual(quote_etag('"etag"'), '"etag"')  # quoted
        self.assertEqual(quote_etag('W/"etag"'), 'W/"etag"')  # quoted, weak

class HttpDateProcessingTests(unittest.TestCase):
    def test_http_date(self):
        t = 1167616461.0
        self.assertEqual(http_date(t), 'Mon, 01 Jan 2007 01:54:21 GMT')

    def test_parsing_rfc1123(self):
        parsed = parse_http_date('Sun, 06 Nov 1994 08:49:37 GMT')
        self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))

    def test_parsing_rfc850(self):
        parsed = parse_http_date('Sunday, 06-Nov-94 08:49:37 GMT')
        self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))

    def test_parsing_asctime(self):
        parsed = parse_http_date('Sun Nov  6 08:49:37 1994')
        self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))

    def test_parsing_year_less_than_70(self):
        parsed = parse_http_date('Sun Nov  6 08:49:37 0050')
        self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(2050, 11, 6, 8, 49, 37))

class EscapeLeadingSlashesTests(unittest.TestCase):
    def test(self):
        tests = (
            ('//example.com', '/%2Fexample.com'),
            ('//', '/%2F'),
        )
        for url, expected in tests:
            with self.subTest(url=url):
                self.assertEqual(escape_leading_slashes(url), expected)
<commit_msg>Fixed #30264 -- Fixed crash of test_parsing_year_less_than_70() on 32-bit systems.<commit_after>import unittest
from datetime import datetime

from django.test import SimpleTestCase, ignore_warnings
from django.utils.datastructures import MultiValueDict
from django.utils.deprecation import RemovedInDjango40Warning
from django.utils.http import (
    base36_to_int, escape_leading_slashes, http_date, int_to_base36,
    is_safe_url, is_same_domain, parse_etags, parse_http_date, quote_etag,
    urlencode, urlquote, urlquote_plus, urlsafe_base64_decode,
    urlsafe_base64_encode, urlunquote, urlunquote_plus,
)

class URLEncodeTests(SimpleTestCase):
    cannot_encode_none_msg = (
        'Cannot encode None in a query string. Did you mean to pass an '
        'empty string or omit the value?'
    )

    def test_tuples(self):
        self.assertEqual(urlencode((('a', 1), ('b', 2), ('c', 3))), 'a=1&b=2&c=3')

    def test_dict(self):
        result = urlencode({'a': 1, 'b': 2, 'c': 3})
        # Dictionaries are treated as unordered.
        self.assertIn(result, [
            'a=1&b=2&c=3',
            'a=1&c=3&b=2',
            'b=2&a=1&c=3',
            'b=2&c=3&a=1',
            'c=3&a=1&b=2',
            'c=3&b=2&a=1',
        ])

    def test_dict_containing_sequence_not_doseq(self):
        self.assertEqual(urlencode({'a': [1, 2]}, doseq=False), 'a=%5B%271%27%2C+%272%27%5D')

    def test_dict_containing_sequence_doseq(self):
        self.assertEqual(urlencode({'a': [1, 2]}, doseq=True), 'a=1&a=2')

    def test_dict_containing_empty_sequence_doseq(self):
        self.assertEqual(urlencode({'a': []}, doseq=True), '')

    def test_multivaluedict(self):
        result = urlencode(MultiValueDict({
            'name': ['Adrian', 'Simon'],
            'position': ['Developer'],
        }), doseq=True)
        # MultiValueDicts are similarly unordered.
        self.assertIn(result, [
            'name=Adrian&name=Simon&position=Developer',
            'position=Developer&name=Adrian&name=Simon',
        ])

    def test_dict_with_bytes_values(self):
        self.assertEqual(urlencode({'a': b'abc'}, doseq=True), 'a=abc')

    def test_dict_with_sequence_of_bytes(self):
        self.assertEqual(urlencode({'a': [b'spam', b'eggs', b'bacon']}, doseq=True), 'a=spam&a=eggs&a=bacon')

    def test_dict_with_bytearray(self):
        self.assertEqual(urlencode({'a': bytearray(range(2))}, doseq=True), 'a=0&a=1')
        self.assertEqual(urlencode({'a': bytearray(range(2))}, doseq=False), 'a=%5B%270%27%2C+%271%27%5D')

    def test_generator(self):
        def gen():
            yield from range(2)

        self.assertEqual(urlencode({'a': gen()}, doseq=True), 'a=0&a=1')
        self.assertEqual(urlencode({'a': gen()}, doseq=False), 'a=%5B%270%27%2C+%271%27%5D')

    def test_none(self):
        with self.assertRaisesMessage(TypeError, self.cannot_encode_none_msg):
            urlencode({'a': None})

    def test_none_in_sequence(self):
        with self.assertRaisesMessage(TypeError, self.cannot_encode_none_msg):
            urlencode({'a': [None]}, doseq=True)

    def test_none_in_generator(self):
        def gen():
            yield None
        with self.assertRaisesMessage(TypeError, self.cannot_encode_none_msg):
            urlencode({'a': gen()}, doseq=True)

class Base36IntTests(SimpleTestCase):
    def test_roundtrip(self):
        for n in [0, 1, 1000, 1000000]:
            self.assertEqual(n, base36_to_int(int_to_base36(n)))

    def test_negative_input(self):
        with self.assertRaisesMessage(ValueError, 'Negative base36 conversion input.'):
            int_to_base36(-1)

    def test_to_base36_errors(self):
        for n in ['1', 'foo', {1: 2}, (1, 2, 3), 3.141]:
            with self.assertRaises(TypeError):
                int_to_base36(n)

    def test_invalid_literal(self):
        for n in ['#', ' ']:
            with self.assertRaisesMessage(ValueError, "invalid literal for int() with base 36: '%s'" % n):
                base36_to_int(n)

    def test_input_too_large(self):
        with self.assertRaisesMessage(ValueError, 'Base36 input too large'):
            base36_to_int('1' * 14)

    def test_to_int_errors(self):
        for n in [123, {1: 2}, (1, 2, 3), 3.141]:
            with self.assertRaises(TypeError):
                base36_to_int(n)

    def test_values(self):
        for n, b36 in [(0, '0'), (1, '1'), (42, '16'), (818469960, 'django')]:
            self.assertEqual(int_to_base36(n), b36)
            self.assertEqual(base36_to_int(b36), n)

class IsSafeURLTests(unittest.TestCase):
    def test_bad_urls(self):
        bad_urls = (
            'http://example.com',
            'http:///example.com',
            'https://example.com',
            'ftp://example.com',
            r'\\example.com',
            r'\\\example.com',
            r'/\\/example.com',
            r'\\\example.com',
            r'\\example.com',
            r'\\//example.com',
            r'/\/example.com',
            r'\/example.com',
            r'/\example.com',
            'http:///example.com',
            r'http:/\//example.com',
            r'http:\/example.com',
            r'http:/\example.com',
            'javascript:alert("XSS")',
            '\njavascript:alert(x)',
            '\x08//example.com',
            r'http://otherserver\@example.com',
            r'http:\\testserver\@example.com',
            r'http://testserver\me:pass@example.com',
            r'http://testserver\@example.com',
            r'http:\\testserver\confirm\me@example.com',
            'http:999999999',
            'ftp:9999999999',
            '\n',
            'http://[2001:cdba:0000:0000:0000:0000:3257:9652/',
            'http://2001:cdba:0000:0000:0000:0000:3257:9652]/',
        )
        for bad_url in bad_urls:
            with self.subTest(url=bad_url):
                self.assertIs(is_safe_url(bad_url, allowed_hosts={'testserver', 'testserver2'}), False)

    def test_good_urls(self):
        good_urls = (
            '/view/?param=http://example.com',
            '/view/?param=https://example.com',
            '/view?param=ftp://example.com',
            'view/?param=//example.com',
            'https://testserver/',
            'HTTPS://testserver/',
            '//testserver/',
            'http://testserver/confirm?email=me@example.com',
            '/url%20with%20spaces/',
            'path/http:2222222222',
        )
        for good_url in good_urls:
            with self.subTest(url=good_url):
                self.assertIs(is_safe_url(good_url, allowed_hosts={'otherserver', 'testserver'}), True)

    def test_basic_auth(self):
        # Valid basic auth credentials are allowed.
        self.assertIs(is_safe_url(r'http://user:pass@testserver/', allowed_hosts={'user:pass@testserver'}), True)

    def test_no_allowed_hosts(self):
        # A path without host is allowed.
        self.assertIs(is_safe_url('/confirm/me@example.com', allowed_hosts=None), True)
        # Basic auth without host is not allowed.
        self.assertIs(is_safe_url(r'http://testserver\@example.com', allowed_hosts=None), False)

    def test_allowed_hosts_str(self):
        self.assertIs(is_safe_url('http://good.com/good', allowed_hosts='good.com'), True)
        self.assertIs(is_safe_url('http://good.co/evil', allowed_hosts='good.com'), False)

    def test_secure_param_https_urls(self):
        secure_urls = (
            'https://example.com/p',
            'HTTPS://example.com/p',
            '/view/?param=http://example.com',
        )
        for url in secure_urls:
            with self.subTest(url=url):
                self.assertIs(is_safe_url(url, allowed_hosts={'example.com'}, require_https=True), True)

    def test_secure_param_non_https_urls(self):
        insecure_urls = (
            'http://example.com/p',
            'ftp://example.com/p',
            '//example.com/p',
        )
        for url in insecure_urls:
            with self.subTest(url=url):
                self.assertIs(is_safe_url(url, allowed_hosts={'example.com'}, require_https=True), False)

class URLSafeBase64Tests(unittest.TestCase):
    def test_roundtrip(self):
        bytestring = b'foo'
        encoded = urlsafe_base64_encode(bytestring)
        decoded = urlsafe_base64_decode(encoded)
        self.assertEqual(bytestring, decoded)

@ignore_warnings(category=RemovedInDjango40Warning)
class URLQuoteTests(unittest.TestCase):
    def test_quote(self):
        self.assertEqual(urlquote('Paris & Orl\xe9ans'), 'Paris%20%26%20Orl%C3%A9ans')
        self.assertEqual(urlquote('Paris & Orl\xe9ans', safe="&"), 'Paris%20&%20Orl%C3%A9ans')

    def test_unquote(self):
        self.assertEqual(urlunquote('Paris%20%26%20Orl%C3%A9ans'), 'Paris & Orl\xe9ans')
        self.assertEqual(urlunquote('Paris%20&%20Orl%C3%A9ans'), 'Paris & Orl\xe9ans')

    def test_quote_plus(self):
        self.assertEqual(urlquote_plus('Paris & Orl\xe9ans'), 'Paris+%26+Orl%C3%A9ans')
        self.assertEqual(urlquote_plus('Paris & Orl\xe9ans', safe="&"), 'Paris+&+Orl%C3%A9ans')

    def test_unquote_plus(self):
        self.assertEqual(urlunquote_plus('Paris+%26+Orl%C3%A9ans'), 'Paris & Orl\xe9ans')
        self.assertEqual(urlunquote_plus('Paris+&+Orl%C3%A9ans'), 'Paris & Orl\xe9ans')

class IsSameDomainTests(unittest.TestCase):
    def test_good(self):
        for pair in (
            ('example.com', 'example.com'),
            ('example.com', '.example.com'),
            ('foo.example.com', '.example.com'),
            ('example.com:8888', 'example.com:8888'),
            ('example.com:8888', '.example.com:8888'),
            ('foo.example.com:8888', '.example.com:8888'),
        ):
            self.assertIs(is_same_domain(*pair), True)

    def test_bad(self):
        for pair in (
            ('example2.com', 'example.com'),
            ('foo.example.com', 'example.com'),
            ('example.com:9999', 'example.com:8888'),
            ('foo.example.com:8888', ''),
        ):
            self.assertIs(is_same_domain(*pair), False)

class ETagProcessingTests(unittest.TestCase):
    def test_parsing(self):
        self.assertEqual(
            parse_etags(r'"" ,  "etag", "e\\tag", W/"weak"'),
            ['""', '"etag"', r'"e\\tag"', 'W/"weak"']
        )
        self.assertEqual(parse_etags('*'), ['*'])

        # Ignore RFC 2616 ETags that are invalid according to RFC 7232.
        self.assertEqual(parse_etags(r'"etag", "e\"t\"ag"'), ['"etag"'])

    def test_quoting(self):
        self.assertEqual(quote_etag('etag'), '"etag"')  # unquoted
        self.assertEqual(quote_etag('"etag"'), '"etag"')  # quoted
        self.assertEqual(quote_etag('W/"etag"'), 'W/"etag"')  # quoted, weak

class HttpDateProcessingTests(unittest.TestCase):
    def test_http_date(self):
        t = 1167616461.0
        self.assertEqual(http_date(t), 'Mon, 01 Jan 2007 01:54:21 GMT')

    def test_parsing_rfc1123(self):
        parsed = parse_http_date('Sun, 06 Nov 1994 08:49:37 GMT')
        self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))

    def test_parsing_rfc850(self):
        parsed = parse_http_date('Sunday, 06-Nov-94 08:49:37 GMT')
        self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))

    def test_parsing_asctime(self):
        parsed = parse_http_date('Sun Nov  6 08:49:37 1994')
        self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(1994, 11, 6, 8, 49, 37))

    def test_parsing_year_less_than_70(self):
        parsed = parse_http_date('Sun Nov  6 08:49:37 0037')
        self.assertEqual(datetime.utcfromtimestamp(parsed), datetime(2037, 11, 6, 8, 49, 37))

class EscapeLeadingSlashesTests(unittest.TestCase):
    def test(self):
        tests = (
            ('//example.com', '/%2Fexample.com'),
            ('//', '/%2F'),
        )
        for url, expected in tests:
            with self.subTest(url=url):
                self.assertEqual(escape_leading_slashes(url), expected)
<|endoftext|>


<commit_before>import asyncio
import json
import base64

from cryptography import fernet

from . import AbstractStorage, Session

class EncryptedCookieStorage(AbstractStorage):
    """Encrypted JSON storage.
    """

    def __init__(self, secret_key, *, cookie_name="AIOHTTP_SESSION",
                 domain=None, max_age=None, path='/',
                 secure=None, httponly=True):
        super().__init__(cookie_name=cookie_name, domain=domain,
                         max_age=max_age, path=path, secure=secure,
                         httponly=httponly)

        if isinstance(secret_key, str):
            pass
        elif isinstance(secret_key, (bytes, bytearray)):
            secret_key = base64.urlsafe_b64encode(secret_key)
        self._fernet = fernet.Fernet(secret_key)

    @asyncio.coroutine
    def load_session(self, request):
        cookie = self.load_cookie(request)
        if cookie is None:
            return Session(None, data=None, new=True)
        else:
            data = json.loads(
                self._fernet.decrypt(cookie.encode('utf-8')).decode('utf-8')
            )
            return Session(None, data=data, new=False)

    @asyncio.coroutine
    def save_session(self, request, response, session):
        if session.empty:
            return self.save_cookie(response, session._mapping,
                                    max_age=session.max_age)

        cookie_data = json.dumps(
            self._get_session_data(session)
        ).encode('utf-8')
        self.save_cookie(
            response,
            self._fernet.encrypt(cookie_data).decode('utf-8'),
            max_age=session.max_age
        )
<commit_msg>Handles cryptography.fernet.InvalidToken exception<commit_after>import asyncio
import json
import base64

from cryptography import fernet
from cryptography.fernet import InvalidToken

from . import AbstractStorage, Session

class EncryptedCookieStorage(AbstractStorage):
    """Encrypted JSON storage.
    """

    def __init__(self, secret_key, *, cookie_name="AIOHTTP_SESSION",
                 domain=None, max_age=None, path='/',
                 secure=None, httponly=True):
        super().__init__(cookie_name=cookie_name, domain=domain,
                         max_age=max_age, path=path, secure=secure,
                         httponly=httponly)

        if isinstance(secret_key, str):
            pass
        elif isinstance(secret_key, (bytes, bytearray)):
            secret_key = base64.urlsafe_b64encode(secret_key)
        self._fernet = fernet.Fernet(secret_key)

    @asyncio.coroutine
    def load_session(self, request):
        cookie = self.load_cookie(request)
        if cookie is None:
            return Session(None, data=None, new=True)
        else:
            try:
                data = json.loads(
                    self._fernet.decrypt(cookie.encode('utf-8')).decode('utf-8')
                )
                return Session(None, data=data, new=False)
            except InvalidToken:
                return Session(None, data=None, new=True)

    @asyncio.coroutine
    def save_session(self, request, response, session):
        if session.empty:
            return self.save_cookie(response, session._mapping,
                                    max_age=session.max_age)

        cookie_data = json.dumps(
            self._get_session_data(session)
        ).encode('utf-8')
        self.save_cookie(
            response,
            self._fernet.encrypt(cookie_data).decode('utf-8'),
            max_age=session.max_age
        )
<|endoftext|>


<commit_before>import oelite.fetch
import oelite.util
import os
import urlgrabber
import urlgrabber.progress
import hashlib
from oebakery import die, err, warn, info, debug

class UrlFetcher():

    SUPPORTED_SCHEMES = ("http", "https", "ftp")

    def __init__(self, uri, d):
        if not uri.scheme in self.SUPPORTED_SCHEMES:
            raise Exception(
                "Scheme %s not supported by oelite.fetch.UrlFetcher"%(
                    uri.scheme))
        self.url = "%s://%s"%(uri.scheme, uri.location)
        self.localname = os.path.basename(uri.location)
        self.localpath = os.path.join(uri.ingredients, uri.isubdir,
                                      self.localname)
        self.signatures = d.get("FILE") + ".sig"
        self.uri = uri
        self.fetch_signatures = d["__fetch_signatures"]
        self.proxies = self.get_proxy(d)
        self.ftpmode = self.get_disable_ftp_epsv(d)
        return

    def signature(self):
        try:
            self._signature = self.fetch_signatures[self.localname]
            return self._signature
        except KeyError:
            raise oelite.fetch.NoSignature(self.uri, "signature unknown")

    def fetch(self):
        localdir = os.path.dirname(self.localpath)
        oelite.util.makedirs(localdir)

        if os.path.exists(self.localpath):
            if "_signature" in dir(self):
                signature = self.localsignature()
                if signature == self._signature:
                    return True
            print "Removing unverifiable ingredient:", \
                os.path.join(self.uri.isubdir, self.localname)
            os.unlink(self.localpath)

        grabbed = False
        for url in self.uri.premirrors + [self.url] + self.uri.mirrors:
            assert not os.path.exists(self.localpath)
            if not isinstance(url, basestring):
                url = "".join(url)
            if not self.uri.allow_url(url):
                print "Skipping", url
                continue
            if grab(url, self.localpath, proxy=self.proxies, ftpmode=self.ftpmode):
                grabbed = True
                break
            if os.path.exists(self.localpath):
                print "Removing ingredient littering:", \
                    os.path.join(self.uri.isubdir, self.localname)
                os.unlink(self.localpath)
        if not grabbed:
            return False
        assert os.path.exists(self.localpath)

        signature = self.localsignature()
        if not "_signature" in dir(self):
            return (self.localname, signature)
        if signature != self._signature:
            print "Ingredient signature mismatch:", \
                os.path.join(self.uri.isubdir, self.localname)
            print "  expected: %s"%self._signature
            print "  obtained: %s"%signature
            return False
        else:
            return True

    def localsignature(self):
        m = hashlib.sha1()
        m.update(open(self.localpath, "r").read())
        return m.hexdigest()

    def get_proxy(self, d):
        proxy = d.get("http_proxy")
        proxies = None
        if not proxy is None:
            proxies = {}
            proxies["http"] = proxy

            ftpproxy = d.get("ftp_proxy")
            if ftpproxy is None:
                proxies["ftp"] = proxy
            else:
                proxies["ftp"] = ftpproxy

            httpsproxy = d.get("https_proxy")
            if httpsproxy is None:
                proxies["https"] = proxy
            else:
                proxies["https"] = httpsproxy
        return proxies

    def get_disable_ftp_epsv(self, d):
        val = d.get("DISABLE_FTP_EXTENDED_PASSIVE_MODE") or None
        if val is not None:
            if val == "0":
                return False
            if val == "1":
                return True
        return False

class SimpleProgress(urlgrabber.progress.BaseMeter):
    def _do_end(self, amount_read, now=None):
        print "grabbed %d bytes in %.2f seconds" %(amount_read,self.re.elapsed_time())

def grab(url, filename, timeout=120, retry=5, proxy=None, ftpmode=False):
    print "Grabbing", url
    def grab_fail_callback(data):
        # Only print debug here when non fatal retries, debug in other cases
        # is already printed
        if (data.exception.errno in retrycodes) and (data.tries != data.retry):
            print "grabbing retry %d/%d, exception %s"%(
                data.tries, data.retry, data.exception)
    try:
        retrycodes = urlgrabber.grabber.URLGrabberOptions().retrycodes
        if 12 not in retrycodes:
            retrycodes.append(12)
        if not os.path.exists(os.path.dirname(filename)):
            os.makedirs(os.path.dirname(filename))
        downloaded_file = urlgrabber.urlgrab(
            url, filename,timeout=timeout,retry=retry, retrycodes=retrycodes,
            progress_obj=SimpleProgress(), failure_callback=grab_fail_callback,
            copy_local=True, proxies=proxy, ftp_disable_epsv=ftpmode)
        if not downloaded_file:
            return False
    except urlgrabber.grabber.URLGrabError as e:
        warn('URLGrabError %i: %s' % (e.errno, e.strerror))
        if os.path.exists(filename):
            os.unlink(filename)
        return False
    return True
<commit_msg>lib/oelite/fetch/url: traverse mirror list while signature does not match<commit_after>import oelite.fetch
import oelite.util
import os
import urlgrabber
import urlgrabber.progress
import hashlib
from oebakery import die, err, warn, info, debug

class UrlFetcher():

    SUPPORTED_SCHEMES = ("http", "https", "ftp")

    def __init__(self, uri, d):
        if not uri.scheme in self.SUPPORTED_SCHEMES:
            raise Exception(
                "Scheme %s not supported by oelite.fetch.UrlFetcher"%(
                    uri.scheme))
        self.url = "%s://%s"%(uri.scheme, uri.location)
        self.localname = os.path.basename(uri.location)
        self.localpath = os.path.join(uri.ingredients, uri.isubdir,
                                      self.localname)
        self.signatures = d.get("FILE") + ".sig"
        self.uri = uri
        self.fetch_signatures = d["__fetch_signatures"]
        self.proxies = self.get_proxy(d)
        self.ftpmode = self.get_disable_ftp_epsv(d)
        return

    def signature(self):
        try:
            self._signature = self.fetch_signatures[self.localname]
            return self._signature
        except KeyError:
            raise oelite.fetch.NoSignature(self.uri, "signature unknown")

    def fetch(self):
        localdir = os.path.dirname(self.localpath)
        oelite.util.makedirs(localdir)

        if os.path.exists(self.localpath):
            if "_signature" in dir(self):
                signature = self.localsignature()
                if signature == self._signature:
                    return True
            print "Removing unverifiable ingredient:", \
                os.path.join(self.uri.isubdir, self.localname)
            os.unlink(self.localpath)

        grabbed = False
        for url in self.uri.premirrors + [self.url] + self.uri.mirrors:
            assert not os.path.exists(self.localpath)
            if not isinstance(url, basestring):
                url = "".join(url)
            if not self.uri.allow_url(url):
                print "Skipping", url
                continue
            if grab(url, self.localpath, proxy=self.proxies, ftpmode=self.ftpmode):
                if self.grabbedsignature():
                    grabbed = True
                    break
            if os.path.exists(self.localpath):
                print "Removing ingredient littering:", \
                    os.path.join(self.uri.isubdir, self.localname)
                os.unlink(self.localpath)
        if not grabbed:
            return False
        assert os.path.exists(self.localpath)
        return self.grabbedsignature()

    def grabbedsignature(self):
        signature = self.localsignature()
        if not "_signature" in dir(self):
            return (self.localname, signature)
        if signature != self._signature:
            print "Ingredient signature mismatch:", \
                os.path.join(self.uri.isubdir, self.localname)
            print "  expected: %s"%self._signature
            print "  obtained: %s"%signature
            return False
        else:
            return True

    def localsignature(self):
        m = hashlib.sha1()
        m.update(open(self.localpath, "r").read())
        return m.hexdigest()

    def get_proxy(self, d):
        proxy = d.get("http_proxy")
        proxies = None
        if not proxy is None:
            proxies = {}
            proxies["http"] = proxy

            ftpproxy = d.get("ftp_proxy")
            if ftpproxy is None:
                proxies["ftp"] = proxy
            else:
                proxies["ftp"] = ftpproxy

            httpsproxy = d.get("https_proxy")
            if httpsproxy is None:
                proxies["https"] = proxy
            else:
                proxies["https"] = httpsproxy
        return proxies

    def get_disable_ftp_epsv(self, d):
        val = d.get("DISABLE_FTP_EXTENDED_PASSIVE_MODE") or None
        if val is not None:
            if val == "0":
                return False
            if val == "1":
                return True
        return False

class SimpleProgress(urlgrabber.progress.BaseMeter):
    def _do_end(self, amount_read, now=None):
        print "grabbed %d bytes in %.2f seconds" %(amount_read,self.re.elapsed_time())

def grab(url, filename, timeout=120, retry=5, proxy=None, ftpmode=False):
    print "Grabbing", url
    def grab_fail_callback(data):
        # Only print debug here when non fatal retries, debug in other cases
        # is already printed
        if (data.exception.errno in retrycodes) and (data.tries != data.retry):
            print "grabbing retry %d/%d, exception %s"%(
                data.tries, data.retry, data.exception)
    try:
        retrycodes = urlgrabber.grabber.URLGrabberOptions().retrycodes
        if 12 not in retrycodes:
            retrycodes.append(12)
        if not os.path.exists(os.path.dirname(filename)):
            os.makedirs(os.path.dirname(filename))
        downloaded_file = urlgrabber.urlgrab(
            url, filename,timeout=timeout,retry=retry, retrycodes=retrycodes,
            progress_obj=SimpleProgress(), failure_callback=grab_fail_callback,
            copy_local=True, proxies=proxy, ftp_disable_epsv=ftpmode)
        if not downloaded_file:
            return False
    except urlgrabber.grabber.URLGrabError as e:
        warn('URLGrabError %i: %s' % (e.errno, e.strerror))
        if os.path.exists(filename):
            os.unlink(filename)
        return False
    return True
<|endoftext|>