<commit_before>import copy
from pprint import pprint

from django.db.models import Exists, OuterRef

from fo2.connections import db_cursor_so

from base.forms.forms2 import ModeloForm2
from base.views import O2BaseGetPostView
from geral.functions import config_get_value
from utils.functions.dictlist.dictlist_to_grade import dictlist_to_grade_qtd
from utils.functions.dictlist.operacoes_grade import OperacoesGrade
from utils.views import totalize_data

import comercial.models
import produto.models
import produto.queries
from cd.queries.novo_modulo import refs_em_palets
from comercial.views.estoque import grade_meta_estoque

import lotes.models
import lotes.queries.op
import lotes.queries.pedido
from lotes.views.parametros_functions import grade_meta_giro

class ProduzirGradeEmpenho(O2BaseGetPostView):

    def __init__(self, *args, **kwargs):
        super(ProduzirGradeEmpenho, self).__init__(*args, **kwargs)
        self.Form_class = ModeloForm2
        self.template_name = 'lotes/analise/produzir_grade_empenho.html'
        self.title_name = 'A produzir, por grade, empenho e carteira'
        self.get_args = ['modelo']

    def mount_context(self):
        cursor = db_cursor_so(self.request)
        og = OperacoesGrade()

        modelo = self.form.cleaned_data['modelo']
        self.context.update({
            'modelo': modelo,
        })

        data = produto.queries.modelo_inform(cursor, modelo)
        if len(data) == 0:
            self.context.update({
                'msg_erro': 'Modelo não encontrado',
            })
            return

        row = data[0]
        colecao = row['CODIGO_COLECAO']
        self.context.update({
            'colecao': row['COLECAO'],
            'descr': row['DESCR'],
        })

        lm_tam = 0
        lm_cor = 0
        try:
            LC = lotes.models.RegraColecao.objects.get(colecao=colecao)
            lm_tam = LC.lm_tam
            lm_cor = LC.lm_cor
        except lotes.models.RegraColecao.DoesNotExist:
            pass

        metas = comercial.models.MetaEstoque.objects
        metas = metas.annotate(antiga=Exists(
            comercial.models.MetaEstoque.objects.filter(
                modelo=OuterRef('modelo'),
                data__gt=OuterRef('data')
            )
        ))
        metas = metas.filter(antiga=False, modelo=modelo)
        metas = metas.order_by('-meta_estoque')
        if len(metas) == 0:
            self.context.update({
                'msg_meta_estoque': 'Modelo sem meta de estoque definida',
                'msg_meta_giro': 'Modelo sem meta de giro definida',
            })
            return
        else:
            meta = metas[0]

        gzerada = None

        calcula_grade = False
        gme = None
        if meta.meta_estoque == 0:
            self.context.update({
                'msg_meta_estoque': 'Modelo com meta de estoque zerada',
            })
        else:
            gme = grade_meta_estoque(meta)
            calcula_grade = True
            gzerada = og.update_gzerada(gzerada, gme)

        lead = produto.queries.lead_de_modelo(cursor, modelo)
        gmg = None
        if meta.meta_giro == 0:
            self.context.update({
                'msg_meta_giro': 'Modelo com meta de giro zerada',
            })
        else:
            gmg = grade_meta_giro(meta, lead, show_distrib=False)
            calcula_grade = True
            gzerada = og.update_gzerada(gzerada, gmg)

        if not calcula_grade:
            return

        g_header, g_fields, g_data, g_style, total_opa = \
            lotes.queries.op.op_sortimentos(
                cursor, tipo='a', descr_sort=False, modelo=modelo,
                situacao='a', tipo_ref='v', tipo_alt='p', total='Total')

        gopa = None
        if total_opa != 0:
            gopa = {
                'headers': g_header,
                'fields': g_fields,
                'data': g_data,
                'style': g_style,
            }
            gzerada = og.update_gzerada(gzerada, gopa)

        referencias = refs_em_palets.query(
            cursor,
            modelo=modelo,
        )

        filtra_ref = [
            row['ref']
            for row in referencias
        ]

        empenhado = refs_em_palets.query(
            cursor,
            fields='all',
            ref=filtra_ref,
        )

        for row in empenhado:
            row['qtd'] = row['qtd_emp'] + row['qtd_sol']

        grade_empenhado = dictlist_to_grade_qtd(
            empenhado,
            field_linha='cor',
            field_coluna='tam',
            facade_coluna='Tamanho',
            field_ordem_coluna='ordem_tam',
            field_quantidade='qtd',
        )
        total_sol = grade_empenhado['total']

        gsol = None
        if total_sol != 0:
            gsol = {
                'headers': grade_empenhado['headers'],
                'fields': grade_empenhado['fields'],
                'data': grade_empenhado['data'],
                'style': grade_empenhado['style'],
            }
            gzerada = og.update_gzerada(gzerada, gsol)

        dias_alem_lead = config_get_value('DIAS-ALEM-LEAD', default=7)
        self.context.update({
            'dias_alem_lead': dias_alem_lead,
        })

        if lead == 0:
            periodo = ''
        else:
            periodo = lead + dias_alem_lead

        gp_header, gp_fields, gp_data, gp_style, total_ped = \
            lotes.queries.pedido.sortimento(
                cursor, tipo_sort='c', descr_sort=False, modelo=modelo,
                cancelado='n', faturavel='f', total='Total', solicitado='n',
                periodo=':{}'.format(periodo))

        gped = None
        if total_ped != 0:
            gped = {
                'headers': gp_header,
                'fields': gp_fields,
                'data': gp_data,
                'style': gp_style,
            }
            gzerada = og.update_gzerada(gzerada, gped)

        # Utiliza grade zerada para igualar cores e tamanhos das grades base
        # dos cálculos
        if gme is not None:
            gme = og.soma_grades(gzerada, gme)
            self.context.update({
                'gme': gme,
            })

        if gmg is not None:
            gmg = og.soma_grades(gzerada, gmg)
            self.context.update({
                'gmg': gmg,
            })

        if gopa is not None:
            gopa = og.soma_grades(gzerada, gopa)
            self.context.update({
                'gopa': gopa,
            })

        if gsol is not None:
            gsol = og.soma_grades(gzerada, gsol)
            self.context.update({
                'gsol': gsol,
            })

        if gped is not None:
            gped = og.soma_grades(gzerada, gped)
            self.context.update({
                'gped': gped,
            })

        gop = None
        total_op = 0
        conta_grade_op = 0
        if gopa is not None:
            conta_grade_op = 1
            gop = gopa
            total_op = total_opa

        self.context.update({
            'gop': gop,
            'conta_grade_op': conta_grade_op,
        })

        gm = None
        if meta.meta_estoque != 0 or meta.meta_giro != 0:
            if meta.meta_estoque == 0:
                gm = gmg
            elif meta.meta_giro == 0:
                gm = gme
            else:
                gm = og.soma_grades(gme, gmg)

            self.context.update({
                'gm': gm,
            })

        gopp1 = None
        if total_op != 0 or total_ped != 0:
            if total_ped == 0:
                gopp1 = gop
            elif total_op == 0:
                gopp1 = og.subtrai_grades(gzerada, gped)
            else:
                gopp1 = og.subtrai_grades(gop, gped)

        gopp2 = None
        if gopp1 or total_sol != 0:
            if total_sol == 0:
                gopp2 = gopp1
            elif gopp1 is None:
                gopp2 = og.subtrai_grades(gzerada, gsol)
            else:
                gopp2 = og.subtrai_grades(gopp1, gsol)

        gopp = None
        if gopp2:
            gopp = gopp2
            self.context.update({
                'gopp': gopp,
            })

        gresult = None
        if gopp is not None or gm is not None:
            if gopp is None:
                gresult = gm
            elif gm is None:
                gresult = gopp
            else:
                gresult = og.subtrai_grades(gm, gopp)

        glm = None
        glc = None

        if gresult is not None:
            gap = og.opera_grade(gresult, lambda x: x if x > 0 else 0)
            self.context.update({
                'gap': gap,
            })
            gex = og.opera_grade(gresult, lambda x: -x if x < 0 else 0)
            self.context.update({
                'gex': gex,
            })

            if lm_tam != 0 or lm_cor != 0:
                glm = copy.deepcopy(gap)

        if glm is not None:

            row_tot = glm['data'][-1]
            tam_conf = {}
            for row_cor in glm['data'][:-1]:
                field_tot = glm['fields'][-1]
                for tam in glm['fields'][1:-1]:
                    if tam not in tam_conf:
                        tam_conf[tam] = {
                            'min_para_lm': 0,
                            'lm_cor_sozinha': 's',
                        }
                        try:
                            RLM = lotes.models.RegraLMTamanho.objects.get(
                                tamanho=tam)
                            tam_conf[tam] = {
                                'min_para_lm': RLM.min_para_lm,
                                'lm_cor_sozinha': RLM.lm_cor_sozinha,
                            }
                        except lotes.models.RegraLMTamanho.DoesNotExist:
                            pass

                    if lm_tam != 0 and row_cor[tam] != 0:
                        lm_lim = round(
                            lm_tam * tam_conf[tam]['min_para_lm'] / 100, 0)
                        if row_cor[tam] < lm_tam:
                            if row_cor[tam] >= lm_lim:
                                row_tot[field_tot] += lm_tam - row_cor[tam]
                                row_tot[tam] += lm_tam - row_cor[tam]
                                row_cor[field_tot] += lm_tam - row_cor[tam]
                                row_cor[tam] = lm_tam
                            else:
                                row_tot[field_tot] += -row_cor[tam]
                                row_tot[tam] += -row_cor[tam]
                                row_cor[field_tot] += -row_cor[tam]
                                row_cor[tam] = 0
                            row_cor['{}|STYLE'.format(tam)] = \
                                'font-weight: bold; color: red'

            if glm != gap:
                self.context.update({
                    'glm': glm,
                })

            glc = copy.deepcopy(glm)

        if glc is not None:
            row_tot = glc['data'][-1]
            for row_cor in glc['data'][:-1]:
                tam_count = 0
                tam_tot = 0
                field_tot = glc['fields'][-1]
                for tam in glc['fields'][1:-1]:
                    if row_cor[tam] != 0:
                        tam_count += 1
                    tam_tot += row_cor[tam]

                lm_cor_acresc = 0
                if tam_tot != 0 and lm_cor != 0:
                    if tam_tot < lm_cor:
                        if tam_count > 1 or \
                                tam_conf[tam]['lm_cor_sozinha'] == 's':
                            lm_cor_acresc = lm_cor - tam_tot
                if lm_cor_acresc > 0:
                    tam_tot_final = 0
                    tam_ult = None
                    for tam in glc['fields'][1:-1]:
                        if row_cor[tam] > 0:
                            tam_ult = tam
                            acrescenta = round(
                                lm_cor_acresc / tam_tot * row_cor[tam],
                                0)
                            row_tot[field_tot] += acrescenta
                            row_tot[tam] += acrescenta
                            row_cor[field_tot] += acrescenta
                            row_cor[tam] += acrescenta
                            row_cor['{}|STYLE'.format(tam)] = \
                                'font-weight: bold; color: red'
                            tam_tot_final += row_cor[tam]

                    # em caso de distribuição do lm_cor por mais de uma cor
                    # o arredondamento pode não formar lm_cor. Se total ainda
                    # estiver abaico do lm_cor, acrescentar ao último tamanho
                    # alterado
                    lm_cor_acresc = lm_cor - tam_tot_final
                    if lm_cor_acresc > 0:
                        if tam_ult is not None:
                            row_tot[field_tot] += lm_cor_acresc
                            row_tot[tam] += lm_cor_acresc
                            row_cor[field_tot] += lm_cor_acresc
                            row_cor[tam] += lm_cor_acresc

            if glc != glm:
                self.context.update({
                    'glc': glc,
                })
<commit_msg>passando a utilizar parâmetro modelo de refs_em_palets.query<commit_after>import copy
from pprint import pprint

from django.db.models import Exists, OuterRef

from fo2.connections import db_cursor_so

from base.forms.forms2 import ModeloForm2
from base.views import O2BaseGetPostView
from geral.functions import config_get_value
from utils.functions.dictlist.dictlist_to_grade import dictlist_to_grade_qtd
from utils.functions.dictlist.operacoes_grade import OperacoesGrade
from utils.views import totalize_data

import comercial.models
import produto.models
import produto.queries
from cd.queries.novo_modulo import refs_em_palets
from comercial.views.estoque import grade_meta_estoque

import lotes.models
import lotes.queries.op
import lotes.queries.pedido
from lotes.views.parametros_functions import grade_meta_giro

class ProduzirGradeEmpenho(O2BaseGetPostView):

    def __init__(self, *args, **kwargs):
        super(ProduzirGradeEmpenho, self).__init__(*args, **kwargs)
        self.Form_class = ModeloForm2
        self.template_name = 'lotes/analise/produzir_grade_empenho.html'
        self.title_name = 'A produzir, por grade, empenho e carteira'
        self.get_args = ['modelo']

    def mount_context(self):
        cursor = db_cursor_so(self.request)
        og = OperacoesGrade()

        modelo = self.form.cleaned_data['modelo']
        self.context.update({
            'modelo': modelo,
        })

        data = produto.queries.modelo_inform(cursor, modelo)
        if len(data) == 0:
            self.context.update({
                'msg_erro': 'Modelo não encontrado',
            })
            return

        row = data[0]
        colecao = row['CODIGO_COLECAO']
        self.context.update({
            'colecao': row['COLECAO'],
            'descr': row['DESCR'],
        })

        lm_tam = 0
        lm_cor = 0
        try:
            LC = lotes.models.RegraColecao.objects.get(colecao=colecao)
            lm_tam = LC.lm_tam
            lm_cor = LC.lm_cor
        except lotes.models.RegraColecao.DoesNotExist:
            pass

        metas = comercial.models.MetaEstoque.objects
        metas = metas.annotate(antiga=Exists(
            comercial.models.MetaEstoque.objects.filter(
                modelo=OuterRef('modelo'),
                data__gt=OuterRef('data')
            )
        ))
        metas = metas.filter(antiga=False, modelo=modelo)
        metas = metas.order_by('-meta_estoque')
        if len(metas) == 0:
            self.context.update({
                'msg_meta_estoque': 'Modelo sem meta de estoque definida',
                'msg_meta_giro': 'Modelo sem meta de giro definida',
            })
            return
        else:
            meta = metas[0]

        gzerada = None

        calcula_grade = False
        gme = None
        if meta.meta_estoque == 0:
            self.context.update({
                'msg_meta_estoque': 'Modelo com meta de estoque zerada',
            })
        else:
            gme = grade_meta_estoque(meta)
            calcula_grade = True
            gzerada = og.update_gzerada(gzerada, gme)

        lead = produto.queries.lead_de_modelo(cursor, modelo)
        gmg = None
        if meta.meta_giro == 0:
            self.context.update({
                'msg_meta_giro': 'Modelo com meta de giro zerada',
            })
        else:
            gmg = grade_meta_giro(meta, lead, show_distrib=False)
            calcula_grade = True
            gzerada = og.update_gzerada(gzerada, gmg)

        if not calcula_grade:
            return

        g_header, g_fields, g_data, g_style, total_opa = \
            lotes.queries.op.op_sortimentos(
                cursor, tipo='a', descr_sort=False, modelo=modelo,
                situacao='a', tipo_ref='v', tipo_alt='p', total='Total')

        gopa = None
        if total_opa != 0:
            gopa = {
                'headers': g_header,
                'fields': g_fields,
                'data': g_data,
                'style': g_style,
            }
            gzerada = og.update_gzerada(gzerada, gopa)

        empenhado = refs_em_palets.query(
            cursor,
            fields='all',
            modelo=modelo,
        )

        for row in empenhado:
            row['qtd'] = row['qtd_emp'] + row['qtd_sol']

        grade_empenhado = dictlist_to_grade_qtd(
            empenhado,
            field_linha='cor',
            field_coluna='tam',
            facade_coluna='Tamanho',
            field_ordem_coluna='ordem_tam',
            field_quantidade='qtd',
        )
        total_sol = grade_empenhado['total']

        gsol = None
        if total_sol != 0:
            gsol = {
                'headers': grade_empenhado['headers'],
                'fields': grade_empenhado['fields'],
                'data': grade_empenhado['data'],
                'style': grade_empenhado['style'],
            }
            gzerada = og.update_gzerada(gzerada, gsol)

        dias_alem_lead = config_get_value('DIAS-ALEM-LEAD', default=7)
        self.context.update({
            'dias_alem_lead': dias_alem_lead,
        })

        if lead == 0:
            periodo = ''
        else:
            periodo = lead + dias_alem_lead

        gp_header, gp_fields, gp_data, gp_style, total_ped = \
            lotes.queries.pedido.sortimento(
                cursor, tipo_sort='c', descr_sort=False, modelo=modelo,
                cancelado='n', faturavel='f', total='Total', solicitado='n',
                periodo=':{}'.format(periodo))

        gped = None
        if total_ped != 0:
            gped = {
                'headers': gp_header,
                'fields': gp_fields,
                'data': gp_data,
                'style': gp_style,
            }
            gzerada = og.update_gzerada(gzerada, gped)

        # Utiliza grade zerada para igualar cores e tamanhos das grades base
        # dos cálculos
        if gme is not None:
            gme = og.soma_grades(gzerada, gme)
            self.context.update({
                'gme': gme,
            })

        if gmg is not None:
            gmg = og.soma_grades(gzerada, gmg)
            self.context.update({
                'gmg': gmg,
            })

        if gopa is not None:
            gopa = og.soma_grades(gzerada, gopa)
            self.context.update({
                'gopa': gopa,
            })

        if gsol is not None:
            gsol = og.soma_grades(gzerada, gsol)
            self.context.update({
                'gsol': gsol,
            })

        if gped is not None:
            gped = og.soma_grades(gzerada, gped)
            self.context.update({
                'gped': gped,
            })

        gop = None
        total_op = 0
        conta_grade_op = 0
        if gopa is not None:
            conta_grade_op = 1
            gop = gopa
            total_op = total_opa

        self.context.update({
            'gop': gop,
            'conta_grade_op': conta_grade_op,
        })

        gm = None
        if meta.meta_estoque != 0 or meta.meta_giro != 0:
            if meta.meta_estoque == 0:
                gm = gmg
            elif meta.meta_giro == 0:
                gm = gme
            else:
                gm = og.soma_grades(gme, gmg)

            self.context.update({
                'gm': gm,
            })

        gopp1 = None
        if total_op != 0 or total_ped != 0:
            if total_ped == 0:
                gopp1 = gop
            elif total_op == 0:
                gopp1 = og.subtrai_grades(gzerada, gped)
            else:
                gopp1 = og.subtrai_grades(gop, gped)

        gopp2 = None
        if gopp1 or total_sol != 0:
            if total_sol == 0:
                gopp2 = gopp1
            elif gopp1 is None:
                gopp2 = og.subtrai_grades(gzerada, gsol)
            else:
                gopp2 = og.subtrai_grades(gopp1, gsol)

        gopp = None
        if gopp2:
            gopp = gopp2
            self.context.update({
                'gopp': gopp,
            })

        gresult = None
        if gopp is not None or gm is not None:
            if gopp is None:
                gresult = gm
            elif gm is None:
                gresult = gopp
            else:
                gresult = og.subtrai_grades(gm, gopp)

        glm = None
        glc = None

        if gresult is not None:
            gap = og.opera_grade(gresult, lambda x: x if x > 0 else 0)
            self.context.update({
                'gap': gap,
            })
            gex = og.opera_grade(gresult, lambda x: -x if x < 0 else 0)
            self.context.update({
                'gex': gex,
            })

            if lm_tam != 0 or lm_cor != 0:
                glm = copy.deepcopy(gap)

        if glm is not None:

            row_tot = glm['data'][-1]
            tam_conf = {}
            for row_cor in glm['data'][:-1]:
                field_tot = glm['fields'][-1]
                for tam in glm['fields'][1:-1]:
                    if tam not in tam_conf:
                        tam_conf[tam] = {
                            'min_para_lm': 0,
                            'lm_cor_sozinha': 's',
                        }
                        try:
                            RLM = lotes.models.RegraLMTamanho.objects.get(
                                tamanho=tam)
                            tam_conf[tam] = {
                                'min_para_lm': RLM.min_para_lm,
                                'lm_cor_sozinha': RLM.lm_cor_sozinha,
                            }
                        except lotes.models.RegraLMTamanho.DoesNotExist:
                            pass

                    if lm_tam != 0 and row_cor[tam] != 0:
                        lm_lim = round(
                            lm_tam * tam_conf[tam]['min_para_lm'] / 100, 0)
                        if row_cor[tam] < lm_tam:
                            if row_cor[tam] >= lm_lim:
                                row_tot[field_tot] += lm_tam - row_cor[tam]
                                row_tot[tam] += lm_tam - row_cor[tam]
                                row_cor[field_tot] += lm_tam - row_cor[tam]
                                row_cor[tam] = lm_tam
                            else:
                                row_tot[field_tot] += -row_cor[tam]
                                row_tot[tam] += -row_cor[tam]
                                row_cor[field_tot] += -row_cor[tam]
                                row_cor[tam] = 0
                            row_cor['{}|STYLE'.format(tam)] = \
                                'font-weight: bold; color: red'

            if glm != gap:
                self.context.update({
                    'glm': glm,
                })

            glc = copy.deepcopy(glm)

        if glc is not None:
            row_tot = glc['data'][-1]
            for row_cor in glc['data'][:-1]:
                tam_count = 0
                tam_tot = 0
                field_tot = glc['fields'][-1]
                for tam in glc['fields'][1:-1]:
                    if row_cor[tam] != 0:
                        tam_count += 1
                    tam_tot += row_cor[tam]

                lm_cor_acresc = 0
                if tam_tot != 0 and lm_cor != 0:
                    if tam_tot < lm_cor:
                        if tam_count > 1 or \
                                tam_conf[tam]['lm_cor_sozinha'] == 's':
                            lm_cor_acresc = lm_cor - tam_tot
                if lm_cor_acresc > 0:
                    tam_tot_final = 0
                    tam_ult = None
                    for tam in glc['fields'][1:-1]:
                        if row_cor[tam] > 0:
                            tam_ult = tam
                            acrescenta = round(
                                lm_cor_acresc / tam_tot * row_cor[tam],
                                0)
                            row_tot[field_tot] += acrescenta
                            row_tot[tam] += acrescenta
                            row_cor[field_tot] += acrescenta
                            row_cor[tam] += acrescenta
                            row_cor['{}|STYLE'.format(tam)] = \
                                'font-weight: bold; color: red'
                            tam_tot_final += row_cor[tam]

                    # em caso de distribuição do lm_cor por mais de uma cor
                    # o arredondamento pode não formar lm_cor. Se total ainda
                    # estiver abaico do lm_cor, acrescentar ao último tamanho
                    # alterado
                    lm_cor_acresc = lm_cor - tam_tot_final
                    if lm_cor_acresc > 0:
                        if tam_ult is not None:
                            row_tot[field_tot] += lm_cor_acresc
                            row_tot[tam] += lm_cor_acresc
                            row_cor[field_tot] += lm_cor_acresc
                            row_cor[tam] += lm_cor_acresc

            if glc != glm:
                self.context.update({
                    'glc': glc,
                })
<|endoftext|>


<commit_before>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import fields, models, api, _
from odoo.exceptions import ValidationError
import odoo.addons.decimal_precision as dp
from odoo.tools import float_compare

class SaleOrderLine(models.Model):

    _inherit = "sale.order.line"

    discount1 = fields.Float(
        'Discount 1 (%)',
        digits=dp.get_precision('Discount'),
    )
    discount2 = fields.Float(
        'Discount 2 (%)',
        digits=dp.get_precision('Discount'),
    )
    discount3 = fields.Float(
        'Discount 3 (%)',
        digits=dp.get_precision('Discount'),
    )
    # TODO do like in invoice line? Make normal field with constraint and
    # oncahnge?
    discount = fields.Float(
        compute='_compute_discount',
        store=True,
        readonly=True,
        # agregamos states vacio porque lo hereda de la definicion anterior
        states={},
    )

    @api.constrains('discount1', 'discount2', 'discount3')
    def check_discount_validity(self):
        for rec in self:
            error = []
            if rec.discount1 > 100:
                error.append('Discount 1')
            if rec.discount2 > 100:
                error.append('Discount 2')
            if rec.discount3 > 100:
                error.append('Discount 3')
            if error:
                raise ValidationError(_(
                    ",".join(error) + " must be less or equal than 100"
                ))

    @api.model
    def create(self, vals):
        self.inverse_vals(vals)
        return super().create(vals)

    @api.multi
    def write(self, vals):
        self.inverse_vals(vals)
        return super().write(vals)

    def inverse_vals(self, vals):
        """ No usamos metodo inverse porque en el create odoo termina llamando
        a inverse y unificando los descuentos en la primer linea.
        Además, solo actualizamos con el inverse el primer descuento
        principalmente por compatibilidad con listas que discriminen descuento
        y consideramos que las columnas 2 y 3 son descuentos adicionales y no
        las pisamos
        """
        precision = self.env['decimal.precision'].precision_get('Discount')
        if 'discount' in vals \
                and float_compare(vals.get('discount'), self.discount, precision_digits=precision) != 0 \
                and not vals.get('discount1') and not vals.get('discount2') and not vals.get('discount3'):
            vals.update({
                'discount1': vals.get('discount'),
            })

    @api.multi
    @api.depends('discount1', 'discount2', 'discount3')
    def _compute_discount(self):
        for rec in self:
            discount_factor = 1.0
            for discount in [rec.discount1, rec.discount2, rec.discount3]:
                discount_factor = discount_factor * (
                    (100.0 - discount) / 100.0)
            rec.discount = 100.0 - (discount_factor * 100.0)

    @api.multi
    def _prepare_invoice_line(self, qty):
        res = super(SaleOrderLine, self)._prepare_invoice_line(qty)
        res.update({
            'discount1': self.discount1,
            'discount2': self.discount2,
            'discount3': self.discount3
        })
        return res
<commit_msg>[12.0] [PORT] from 11.0 (#305)<commit_after>##############################################################################
# For copyright and license notices, see __manifest__.py file in module root
# directory
##############################################################################
from odoo import fields, models, api, _
from odoo.exceptions import ValidationError
import odoo.addons.decimal_precision as dp
from odoo.tools import float_compare

class SaleOrderLine(models.Model):

    _inherit = "sale.order.line"

    discount1 = fields.Float(
        'Discount 1 (%)',
        digits=dp.get_precision('Discount'),
    )
    discount2 = fields.Float(
        'Discount 2 (%)',
        digits=dp.get_precision('Discount'),
    )
    discount3 = fields.Float(
        'Discount 3 (%)',
        digits=dp.get_precision('Discount'),
    )
    # TODO do like in invoice line? Make normal field with constraint and
    # oncahnge?
    discount = fields.Float(
        compute='_compute_discount',
        store=True,
        readonly=True,
        # agregamos states vacio porque lo hereda de la definicion anterior
        states={},
    )

    @api.constrains('discount1', 'discount2', 'discount3')
    def check_discount_validity(self):
        for rec in self:
            error = []
            if rec.discount1 > 100:
                error.append('Discount 1')
            if rec.discount2 > 100:
                error.append('Discount 2')
            if rec.discount3 > 100:
                error.append('Discount 3')
            if error:
                raise ValidationError(_(
                    ",".join(error) + " must be less or equal than 100"
                ))

    @api.model
    def create(self, vals):
        self.inverse_vals(vals)
        return super().create(vals)

    @api.multi
    def write(self, vals):
        self.inverse_vals(vals)
        return super().write(vals)

    def inverse_vals(self, vals):
        """ No usamos metodo inverse porque en el create odoo termina llamando
        a inverse y unificando los descuentos en la primer linea.
        Además, solo actualizamos con el inverse el primer descuento
        principalmente por compatibilidad con listas que discriminen descuento
        y consideramos que las columnas 2 y 3 son descuentos adicionales y no
        las pisamos
        """
        precision = self.env['decimal.precision'].precision_get('Discount')
        if 'discount' in vals \
                and float_compare(vals.get('discount'), self.discount, precision_digits=precision) != 0 \
                and not {'discount1', 'discount2', 'discount3'} & set(vals.keys()):
            vals.update({
                'discount1': vals.get('discount'),
            })

    @api.multi
    @api.depends('discount1', 'discount2', 'discount3')
    def _compute_discount(self):
        for rec in self:
            discount_factor = 1.0
            for discount in [rec.discount1, rec.discount2, rec.discount3]:
                discount_factor = discount_factor * (
                    (100.0 - discount) / 100.0)
            rec.discount = 100.0 - (discount_factor * 100.0)

    @api.multi
    def _prepare_invoice_line(self, qty):
        res = super(SaleOrderLine, self)._prepare_invoice_line(qty)
        res.update({
            'discount1': self.discount1,
            'discount2': self.discount2,
            'discount3': self.discount3
        })
        return res
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

import datetime
import email
from email.mime.text import MIMEText
import fileinput
import logging
from logging.handlers import SysLogHandler
from optparse import make_option
import StringIO
import smtplib
import sys

from django.core.management.base import BaseCommand, CommandError
from django.utils import timezone

from modoboa.core.management.commands import CloseConnectionMixin
from modoboa.lib import parameters
from modoboa.lib.email_utils import split_mailbox, set_email_headers

from modoboa_admin.models import Mailbox

from ...models import ARmessage, ARhistoric
from ...modo_extension import PostfixAutoreply

logger = logging.getLogger()
logger.addHandler(SysLogHandler(address="/dev/log"))
logger.setLevel(logging.ERROR)

def send_autoreply(sender, mailbox, armessage, original_msg):
    """Send an autoreply message."""
    if armessage.fromdate > timezone.now():
        return

    if armessage.untildate is not None \
            and armessage.untildate < timezone.now():
        armessage.enabled = False
        armessage.save()
        return

    try:
        lastar = ARhistoric.objects.get(armessage=armessage.id, sender=sender)
        timeout = parameters.get_admin("AUTOREPLIES_TIMEOUT",
                                       app="modoboa_postfix_autoreply")
        delta = datetime.timedelta(seconds=int(timeout))
        now = timezone.make_aware(datetime.datetime.now(),
                                  timezone.get_default_timezone())
        if lastar.last_sent + delta > now:
            logger.debug(
                "no autoreply message sent because delta (%s) < timetout (%s)",
                delta, timeout
            )
            sys.exit(0)

    except ARhistoric.DoesNotExist:
        lastar = ARhistoric()
        lastar.armessage = armessage
        lastar.sender = sender

    msg = MIMEText(armessage.content.encode("utf-8"), _charset="utf-8")
    set_email_headers(
        msg, "Auto: {}".format(armessage.subject),
        mailbox.user.encoded_address, sender
    )
    msg["Auto-Submitted"] = "auto-replied"
    message_id = original_msg.get("Message-ID")
    if message_id:
        msg["In-Reply-To"] = message_id
        msg["References"] = message_id

    try:
        s = smtplib.SMTP()
        s.sendmail(mailbox.user.encoded_address, [sender], msg.as_string())
        s.quit()
    except smtplib.SMTPException as exp:
        logger.error("Failed to send autoreply message: %s", exp)
        sys.exit(1)

    logger.debug(
        "autoreply message sent to %s" % mailbox.user.encoded_address)

    lastar.last_sent = datetime.datetime.now()
    lastar.save()

class Command(BaseCommand, CloseConnectionMixin):
    args = "<sender> <recipient ...>"
    help = "Send autoreply emails"

    option_list = BaseCommand.option_list + (
        make_option(
            "--debug", "-d", action="store_true", dest="debug", default=False
        ),
    )

    def handle(self, *args, **options):
        if options["debug"]:
            logger.setLevel(logging.DEBUG)

        if len(args) < 2:
            logger.debug("autoreply %s", " ".join(args))

            raise CommandError(
                "usage: ./manage.py autoreply <sender> <recipient ...>")

        logger.debug(
            "autoreply sender=%s recipient=%s", args[0], ",".join(args[1:])
        )

        # Mailing list filter based on
        # https://tools.ietf.org/html/rfc5230#page-7
        sender = args[0]

        sender_localpart = split_mailbox(sender.lower())[0]
        if (
            (sender_localpart in ('mailer-daemon', 'listserv', 'majordomo')) or
            (sender_localpart.startswith('owner-')) or
            (sender_localpart.endswith('-request'))
        ):
            logger.debug("Skip auto reply, this mail comes from mailing list")
            sys.exit(0)

        content = StringIO.StringIO()
        for line in fileinput.input([]):
            content.write(line)
        content.seek(0)

        original_msg = email.message_from_file(content)
        ml_known_headers = [
            "List-Id", "List-Help", "List-Subscribe", "List-Unsubscribe",
            "List-Post", "List-Owner", "List-Archive"
        ]
        from_ml = False
        for header in ml_known_headers:
            if header in original_msg:
                from_ml = True
                break
        conditions = (
            original_msg.get("Precedence") == "bulk",
            original_msg.get("X-Mailer") == "PHPMailer",
            from_ml
        )
        if any(conditions):
            logger.debug(
                "Skip auto reply, this mail comes from mailing list")
            sys.exit(0)

        PostfixAutoreply().load()
        for fulladdress in args[1:]:
            address, domain = split_mailbox(fulladdress)
            try:
                mbox = Mailbox.objects.get(
                    address=address, domain__name=domain)
            except Mailbox.DoesNotExist:
                msg = "Unknown recipient %s" % (fulladdress)
                logger.debug("autoreply %s", msg)
                continue
            try:
                armessage = ARmessage.objects.get(mbox=mbox.id, enabled=True)
                logger.debug("autoreply message found")
            except ARmessage.DoesNotExist:
                logger.debug("autoreply message not found")
                continue

            send_autoreply(sender, mbox, armessage, original_msg)
<commit_msg>Force server to localhost.<commit_after>#!/usr/bin/env python
# coding: utf-8

import datetime
import email
from email.mime.text import MIMEText
import fileinput
import logging
from logging.handlers import SysLogHandler
from optparse import make_option
import StringIO
import smtplib
import sys

from django.core.management.base import BaseCommand, CommandError
from django.utils import timezone

from modoboa.core.management.commands import CloseConnectionMixin
from modoboa.lib import parameters
from modoboa.lib.email_utils import split_mailbox, set_email_headers

from modoboa_admin.models import Mailbox

from ...models import ARmessage, ARhistoric
from ...modo_extension import PostfixAutoreply

logger = logging.getLogger()
logger.addHandler(SysLogHandler(address="/dev/log"))
logger.setLevel(logging.ERROR)

def send_autoreply(sender, mailbox, armessage, original_msg):
    """Send an autoreply message."""
    if armessage.fromdate > timezone.now():
        return

    if armessage.untildate is not None \
            and armessage.untildate < timezone.now():
        armessage.enabled = False
        armessage.save()
        return

    try:
        lastar = ARhistoric.objects.get(armessage=armessage.id, sender=sender)
        timeout = parameters.get_admin("AUTOREPLIES_TIMEOUT",
                                       app="modoboa_postfix_autoreply")
        delta = datetime.timedelta(seconds=int(timeout))
        now = timezone.make_aware(datetime.datetime.now(),
                                  timezone.get_default_timezone())
        if lastar.last_sent + delta > now:
            logger.debug(
                "no autoreply message sent because delta (%s) < timetout (%s)",
                delta, timeout
            )
            sys.exit(0)

    except ARhistoric.DoesNotExist:
        lastar = ARhistoric()
        lastar.armessage = armessage
        lastar.sender = sender

    msg = MIMEText(armessage.content.encode("utf-8"), _charset="utf-8")
    set_email_headers(
        msg, "Auto: {}".format(armessage.subject),
        mailbox.user.encoded_address, sender
    )
    msg["Auto-Submitted"] = "auto-replied"
    message_id = original_msg.get("Message-ID")
    if message_id:
        msg["In-Reply-To"] = message_id
        msg["References"] = message_id

    try:
        s = smtplib.SMTP("localhost")
        s.sendmail(mailbox.user.encoded_address, [sender], msg.as_string())
        s.quit()
    except smtplib.SMTPException as exp:
        logger.error("Failed to send autoreply message: %s", exp)
        sys.exit(1)

    logger.debug(
        "autoreply message sent to %s" % mailbox.user.encoded_address)

    lastar.last_sent = datetime.datetime.now()
    lastar.save()

class Command(BaseCommand, CloseConnectionMixin):
    args = "<sender> <recipient ...>"
    help = "Send autoreply emails"

    option_list = BaseCommand.option_list + (
        make_option(
            "--debug", "-d", action="store_true", dest="debug", default=False
        ),
    )

    def handle(self, *args, **options):
        if options["debug"]:
            logger.setLevel(logging.DEBUG)

        if len(args) < 2:
            logger.debug("autoreply %s", " ".join(args))

            raise CommandError(
                "usage: ./manage.py autoreply <sender> <recipient ...>")

        logger.debug(
            "autoreply sender=%s recipient=%s", args[0], ",".join(args[1:])
        )

        # Mailing list filter based on
        # https://tools.ietf.org/html/rfc5230#page-7
        sender = args[0]

        sender_localpart = split_mailbox(sender.lower())[0]
        if (
            (sender_localpart in ('mailer-daemon', 'listserv', 'majordomo')) or
            (sender_localpart.startswith('owner-')) or
            (sender_localpart.endswith('-request'))
        ):
            logger.debug("Skip auto reply, this mail comes from mailing list")
            sys.exit(0)

        content = StringIO.StringIO()
        for line in fileinput.input([]):
            content.write(line)
        content.seek(0)

        original_msg = email.message_from_file(content)
        ml_known_headers = [
            "List-Id", "List-Help", "List-Subscribe", "List-Unsubscribe",
            "List-Post", "List-Owner", "List-Archive"
        ]
        from_ml = False
        for header in ml_known_headers:
            if header in original_msg:
                from_ml = True
                break
        conditions = (
            original_msg.get("Precedence") == "bulk",
            original_msg.get("X-Mailer") == "PHPMailer",
            from_ml
        )
        if any(conditions):
            logger.debug(
                "Skip auto reply, this mail comes from mailing list")
            sys.exit(0)

        PostfixAutoreply().load()
        for fulladdress in args[1:]:
            address, domain = split_mailbox(fulladdress)
            try:
                mbox = Mailbox.objects.get(
                    address=address, domain__name=domain)
            except Mailbox.DoesNotExist:
                msg = "Unknown recipient %s" % (fulladdress)
                logger.debug("autoreply %s", msg)
                continue
            try:
                armessage = ARmessage.objects.get(mbox=mbox.id, enabled=True)
                logger.debug("autoreply message found")
            except ARmessage.DoesNotExist:
                logger.debug("autoreply message not found")
                continue

            send_autoreply(sender, mbox, armessage, original_msg)
<|endoftext|>


<commit_before>"""The code in this module is mostly copy/pasted out of the distutils2 source
code, as recommended by Tarek Ziade.  As such, it may be subject to some change
as distutils2 development continues, and will have to be kept up to date.

I didn't want to use it directly from distutils2 itself, since I do not want it
to be an installation dependency for our packages yet--it is still too unstable
(the latest version on PyPI doesn't even install).
"""

import os
import re
import sys

from collections import defaultdict

import distutils.ccompiler

from distutils import log
from distutils.errors import DistutilsOptionError, DistutilsModuleError
from setuptools.dist import Distribution
from setuptools.extension import Extension
try:
    from ConfigParser import RawConfigParser
except ImportError:
    from configparser import RawConfigParser

# A simplified RE for this; just checks that the line ends with version
# predicates in ()
_VERSION_SPEC_RE = re.compile(r'\s*(.*?)\s*\((.*)\)\s*$')

def resolve_name(name):
    """Resolve a name like ``module.object`` to an object and return it.

    Raise ImportError if the module or name is not found.
    """

    parts = name.split('.')
    cursor = len(parts)
    module_name = parts[:cursor]

    while cursor > 0:
        try:
            ret = __import__('.'.join(module_name))
            break
        except ImportError:
            if cursor == 0:
                raise
            cursor -= 1
            module_name = parts[:cursor]
            ret = ''

    for part in parts[1:]:
        try:
            ret = getattr(ret, part)
        except AttributeError:
            raise ImportError(name)

    return ret

# TODO: This is getting pretty hefty; might want to break it up a bit
def cfg_to_args(path='setup.cfg'):
    """ Distutils2 to distutils1 compatibility util.

        This method uses an existing setup.cfg to generate a dictionary of
        keywords that can be used by distutils.core.setup(kwargs**).

        :param file:
            The setup.cfg path.
        :raises DistutilsFileError:
            When the setup.cfg file is not found.

    """
    # We need to declare the following constants here so that it's easier to
    # generate the setup.py afterwards, using inspect.getsource.

    # XXX ** == needs testing
    D1_D2_SETUP_ARGS = {"name": ("metadata",),
                        "version": ("metadata",),
                        "author": ("metadata",),
                        "author_email": ("metadata",),
                        "maintainer": ("metadata",),
                        "maintainer_email": ("metadata",),
                        "url": ("metadata", "home_page"),
                        "description": ("metadata", "summary"),
                        "long_description": ("metadata", "description"),
                        "download_url": ("metadata",),
                        "classifiers": ("metadata", "classifier"),
                        "platforms": ("metadata", "platform"),  # **
                        "license": ("metadata",),
                        # Use setuptools install_requires, not
                        # broken distutils requires
                        "install_requires": ("metadata", "requires_dist"),
                        "provides": ("metadata", "provides_dist"),  # **
                        "obsoletes": ("metadata", "obsoletes_dist"),  # **
                        "package_dir": ("files", 'packages_root'),
                        "packages": ("files",),
                        "package_data": ("files",),
                        "data_files": ("files",),
                        "scripts": ("files",),
                        "py_modules": ("files", "modules"),   # **
                        "cmdclass": ("global", "commands"),
                        # Not supported in distutils2, but provided for
                        # backwards compatibility with setuptools
                        "use_2to3": ("backwards_compat", "use_2to3"),
                        "zip_safe": ("backwards_compat", "zip_safe")
                        }

    MULTI_FIELDS = ("classifiers",
                    "platforms",
                    "install_requires",
                    "provides",
                    "obsoletes",
                    "packages",
                    "package_data",
                    "data_files",
                    "scripts",
                    "py_modules",
                    "cmdclass")

    BOOL_FIELDS = ("use_2to3", "zip_safe")

    # The method source code really starts here.
    parser = RawConfigParser()
    if not os.path.exists(path):
        raise DistutilsFileError("file '%s' does not exist" %
                                 os.path.abspath(path))
    parser.read(path)
    config = {}
    for section in parser.sections():
        config[section] = dict(parser.items(section))

    # Run setup_hooks, if configured
    # TODO: We need a better way of displaying errors that occur in setup_hook;
    # right now they only show up as errors 'parsing' the cfg file.  A
    # traceback and other info would be nice...
    setup_hooks = has_get_option(config, 'global', 'setup_hooks')
    if setup_hooks:
        setup_hooks = split_multiline(setup_hooks)
        for hook in setup_hooks:
            hook = resolve_name(hook)
            hook(config)

    register_custom_compilers(config)

    kwargs = {}

    for arg in D1_D2_SETUP_ARGS:
        if len(D1_D2_SETUP_ARGS[arg]) == 2:
            # The distutils field name is different than distutils2's.
            section, option = D1_D2_SETUP_ARGS[arg]

        elif len(D1_D2_SETUP_ARGS[arg]) == 1:
            # The distutils field name is the same thant distutils2's.
            section = D1_D2_SETUP_ARGS[arg][0]
            option = arg

        in_cfg_value = has_get_option(config, section, option)
        if not in_cfg_value:
            # There is no such option in the setup.cfg
            if arg == "long_description":
                in_cfg_value = has_get_option(config, section,
                                              "description_file")
                if in_cfg_value:
                    in_cfg_value = split_multiline(in_cfg_value)
                    value = ''
                    for filename in in_cfg_value:
                        description_file = open(filename)
                        try:
                            value += description_file.read().strip() + '\n\n'
                        finally:
                            description_file.close()
                    in_cfg_value = value
            else:
                continue

        if arg in MULTI_FIELDS:
            in_cfg_value = split_multiline(in_cfg_value)
        elif arg in BOOL_FIELDS:
            # Provide some flexibility here...
            if in_cfg_value.lower() in ('true', 't', '1', 'yes', 'y'):
                in_cfg_value = True
            else:
                in_cfg_value = False

        if in_cfg_value:
            if arg == 'install_requires':
                # Replaces PEP345-style version specs with the sort expected by
                # setuptools
                in_cfg_value = [_VERSION_SPEC_RE.sub(r'\1\2', pred)
                                for pred in in_cfg_value]
            elif arg == 'package_dir':
                in_cfg_value = {'': in_cfg_value}
            elif arg in ('package_data', 'data_files'):
                data_files = {}
                firstline = True
                prev = None
                for line in in_cfg_value:
                    if '=' in line:
                        key, value = line.split('=', 1)
                        key, value = (key.strip(), value.strip())
                        if key in data_files:
                            # Multiple duplicates of the same package name;
                            # this is for backwards compatibility of the old
                            # format prior to d2to1 0.2.6.
                            prev = data_files[key]
                            prev.extend(value.split())
                        else:
                            prev = data_files[key.strip()] = value.split()
                    elif firstline:
                        raise DistutilsOptionError(
                            'malformed package_data first line %r (misses '
                            '"=")' % line)
                    else:
                        prev.extend(line.strip().split())
                    firstline = False
                if arg == 'data_files':
                    # the data_files value is a pointlessly different structure
                    # from the package_data value
                    data_files = data_files.items()
                in_cfg_value = data_files
            elif arg == 'cmdclass':
                cmdclass = {}
                dist = Distribution()
                for cls in in_cfg_value:
                    cls = resolve_name(cls)
                    cmd = cls(dist)
                    cmdclass[cmd.get_command_name()] = cls
                in_cfg_value = cmdclass

        kwargs[arg] = in_cfg_value

    ext_modules = get_extension_modules(config)
    if ext_modules:
        kwargs['ext_modules'] = ext_modules

    entry_points = get_entry_points(config)
    if entry_points:
        kwargs['entry_points'] = entry_points

    wrap_commands(kwargs)
    return kwargs

def register_custom_compilers(config):
    """Handle custom compilers; this has no real equivalent in distutils, where
    additional compilers could only be added programmatically, so we have to
    hack it in somehow.
    """

    compilers = has_get_option(config, 'global', 'compilers')
    if compilers:
        compilers = split_multiline(compilers)
        for compiler in compilers:
            compiler = resolve_name(compiler)

            # In distutils2 compilers these class attributes exist; for
            # distutils1 we just have to make something up
            if hasattr(compiler, 'name'):
                name = compiler.name
            else:
                name = compiler.__name__
            if hasattr(compiler, 'description'):
                desc = compiler.description
            else:
                desc = 'custom compiler %s' % name

            module_name = compiler.__module__
            # Note; this *will* override built in compilers with the same name
            # TODO: Maybe display a warning about this?
            cc = distutils.ccompiler.compiler_class
            cc[name] = (module_name, compiler.__name__, desc)

            # HACK!!!!  Distutils assumes all compiler modules are in the
            # distutils package
            sys.modules['distutils.' + module_name] = sys.modules[module_name]

def get_extension_modules(config):
    """Handle extension modules"""

    EXTENSION_FIELDS = ("sources",
                        "include_dirs",
                        "define_macros",
                        "undef_macros",
                        "library_dirs",
                        "libraries",
                        "runtime_library_dirs",
                        "extra_objects",
                        "extra_compile_args",
                        "extra_link_args",
                        "export_symbols",
                        "swig_opts",
                        "depends")

    ext_modules = []
    for section in config:
        if ':' in section:
            labels = section.split(':', 1)
        else:
            # Backwards compatibility for old syntax; don't use this though
            labels = section.split('=', 1)
        labels = [l.strip() for l in labels]
        if (len(labels) == 2) and (labels[0] == 'extension'):
            ext_args = {}
            for field in EXTENSION_FIELDS:
                value = has_get_option(config, section, field)
                # All extension module options besides name can have multiple
                # values
                if not value:
                    continue
                value = split_multiline(value)
                if field == 'define_macros':
                    macros = []
                    for macro in value:
                        macro = macro.split('=', 1)
                        if len(macro) == 1:
                            macro = (macro[0].strip(), None)
                        else:
                            macro = (macro[0].strip(), macro[1].strip())
                        macros.append(macro)
                    value = macros
                ext_args[field] = value
            if ext_args:
                if 'name' not in ext_args:
                    ext_args['name'] = labels[1]
                ext_modules.append(Extension(ext_args.pop('name'),
                                             **ext_args))
    return ext_modules

def get_entry_points(config):
    """Process the [entry_points] section of setup.cfg to handle setuptools
    entry points.  This is, of course, not a standard feature of
    distutils2/packaging, but as there is not currently a standard alternative
    in packaging, we provide support for them.
    """

    if not 'entry_points' in config:
        return {}

    return dict((option, split_multiline(value))
                for option, value in config['entry_points'].items())

def wrap_commands(kwargs):
    dist = Distribution()

    # This should suffice to get the same config values and command classes
    # that the actual Distribution will see (not counting cmdclass, which is
    # handled below)
    dist.parse_config_files()

    for cmd, _ in dist.get_command_list():
        hooks = {}
        for opt, val in dist.get_option_dict(cmd).items():
            val = val[1]
            if opt.startswith('pre_hook.') or opt.startswith('post_hook.'):
                hook_type, alias = opt.split('.', 1)
                hook_dict = hooks.setdefault(hook_type, {})
                hook_dict[alias] = val
        if not hooks:
            continue

        if 'cmdclass' in kwargs and cmd in kwargs['cmdclass']:
            cmdclass = kwargs['cmdclass'][cmd]
        else:
            cmdclass = dist.get_command_class(cmd)

        new_cmdclass = wrap_command(cmd, cmdclass, hooks)
        kwargs.setdefault('cmdclass', {})[cmd] = new_cmdclass

def wrap_command(cmd, cmdclass, hooks):
    def run(self, cmdclass=cmdclass):
        self.run_command_hooks('pre_hook')
        cmdclass.run(self)
        self.run_command_hooks('post_hook')

    return type(cmd, (cmdclass, object),
                {'run': run, 'run_command_hooks': run_command_hooks,
                 'pre_hook': hooks.get('pre_hook'),
                 'post_hook': hooks.get('post_hook')})

def run_command_hooks(cmd_obj, hook_kind):
    """Run hooks registered for that command and phase.

    *cmd_obj* is a finalized command object; *hook_kind* is either
    'pre_hook' or 'post_hook'.
    """

    if hook_kind not in ('pre_hook', 'post_hook'):
        raise ValueError('invalid hook kind: %r' % hook_kind)

    hooks = getattr(cmd_obj, hook_kind, None)

    if hooks is None:
        return

    for hook in hooks.values():
        if isinstance(hook, str):
            try:
                hook_obj = resolve_name(hook)
            except ImportError:
                err = sys.exc_info()[1] # For py3k
                raise DistutilsModuleError(err)
        else:
            hook_obj = hook

        if not hasattr(hook_obj, '__call__'):
            raise DistutilsOptionError('hook %r is not callable' % hook)

        log.info('running %s %s for command %s',
                 hook_kind, hook, cmd_obj.get_command_name())
        hook_obj(cmd_obj)

def has_get_option(config, section, option):
    if section in config and option in config[section]:
        return config[section][option]
    elif section in config and option.replace('_', '-') in config[section]:
        return config[section][option.replace('_', '-')]
    else:
        return False

def split_multiline(value):
    """Special behaviour when we have a multi line options"""

    value = [element for element in
             (line.strip() for line in value.split('\n'))
             if element]
    return value

# The following classes are used to hack Distribution.command_options a bit
class DefaultGetDict(defaultdict):
    """Like defaultdict, but the get() method also sets and returns the default
    value.
    """

    def get(self, key, default=None):
        if default is None:
            default = self.default_factory()
        return super(DefaultGetDict, self).setdefault(key, default)

class IgnoreDict(dict):
    """A dictionary that ignores any insertions in which the key is a string
    matching any string in `ignore`.  The ignore list can also contain wildcard
    patterns using '*'.
    """

    def __init__(self, ignore):
        self.__ignore = re.compile(r'(%s)' % ('|'.join(
                                   [pat.replace('*', '.*')
                                    for pat in ignore])))

    def __setitem__(self, key, val):
        if self.__ignore.match(key):
            return
        super(IgnoreDict, self).__setitem__(key, val)

<commit_msg>add support for 'keywords' field<commit_after>"""The code in this module is mostly copy/pasted out of the distutils2 source
code, as recommended by Tarek Ziade.  As such, it may be subject to some change
as distutils2 development continues, and will have to be kept up to date.

I didn't want to use it directly from distutils2 itself, since I do not want it
to be an installation dependency for our packages yet--it is still too unstable
(the latest version on PyPI doesn't even install).
"""

import os
import re
import sys

from collections import defaultdict

import distutils.ccompiler

from distutils import log
from distutils.errors import DistutilsOptionError, DistutilsModuleError
from setuptools.dist import Distribution
from setuptools.extension import Extension
try:
    from ConfigParser import RawConfigParser
except ImportError:
    from configparser import RawConfigParser

# A simplified RE for this; just checks that the line ends with version
# predicates in ()
_VERSION_SPEC_RE = re.compile(r'\s*(.*?)\s*\((.*)\)\s*$')

def resolve_name(name):
    """Resolve a name like ``module.object`` to an object and return it.

    Raise ImportError if the module or name is not found.
    """

    parts = name.split('.')
    cursor = len(parts)
    module_name = parts[:cursor]

    while cursor > 0:
        try:
            ret = __import__('.'.join(module_name))
            break
        except ImportError:
            if cursor == 0:
                raise
            cursor -= 1
            module_name = parts[:cursor]
            ret = ''

    for part in parts[1:]:
        try:
            ret = getattr(ret, part)
        except AttributeError:
            raise ImportError(name)

    return ret

# TODO: This is getting pretty hefty; might want to break it up a bit
def cfg_to_args(path='setup.cfg'):
    """ Distutils2 to distutils1 compatibility util.

        This method uses an existing setup.cfg to generate a dictionary of
        keywords that can be used by distutils.core.setup(kwargs**).

        :param file:
            The setup.cfg path.
        :raises DistutilsFileError:
            When the setup.cfg file is not found.

    """
    # We need to declare the following constants here so that it's easier to
    # generate the setup.py afterwards, using inspect.getsource.

    # XXX ** == needs testing
    D1_D2_SETUP_ARGS = {"name": ("metadata",),
                        "version": ("metadata",),
                        "author": ("metadata",),
                        "author_email": ("metadata",),
                        "maintainer": ("metadata",),
                        "maintainer_email": ("metadata",),
                        "url": ("metadata", "home_page"),
                        "description": ("metadata", "summary"),
                        "keywords": ("metadata",),
                        "long_description": ("metadata", "description"),
                        "download_url": ("metadata",),
                        "classifiers": ("metadata", "classifier"),
                        "platforms": ("metadata", "platform"),  # **
                        "license": ("metadata",),
                        # Use setuptools install_requires, not
                        # broken distutils requires
                        "install_requires": ("metadata", "requires_dist"),
                        "provides": ("metadata", "provides_dist"),  # **
                        "obsoletes": ("metadata", "obsoletes_dist"),  # **
                        "package_dir": ("files", 'packages_root'),
                        "packages": ("files",),
                        "package_data": ("files",),
                        "data_files": ("files",),
                        "scripts": ("files",),
                        "py_modules": ("files", "modules"),   # **
                        "cmdclass": ("global", "commands"),
                        # Not supported in distutils2, but provided for
                        # backwards compatibility with setuptools
                        "use_2to3": ("backwards_compat", "use_2to3"),
                        "zip_safe": ("backwards_compat", "zip_safe")
                        }

    MULTI_FIELDS = ("classifiers",
                    "platforms",
                    "install_requires",
                    "provides",
                    "obsoletes",
                    "packages",
                    "package_data",
                    "data_files",
                    "scripts",
                    "py_modules",
                    "cmdclass")

    BOOL_FIELDS = ("use_2to3", "zip_safe")

    # The method source code really starts here.
    parser = RawConfigParser()
    if not os.path.exists(path):
        raise DistutilsFileError("file '%s' does not exist" %
                                 os.path.abspath(path))
    parser.read(path)
    config = {}
    for section in parser.sections():
        config[section] = dict(parser.items(section))

    # Run setup_hooks, if configured
    # TODO: We need a better way of displaying errors that occur in setup_hook;
    # right now they only show up as errors 'parsing' the cfg file.  A
    # traceback and other info would be nice...
    setup_hooks = has_get_option(config, 'global', 'setup_hooks')
    if setup_hooks:
        setup_hooks = split_multiline(setup_hooks)
        for hook in setup_hooks:
            hook = resolve_name(hook)
            hook(config)

    register_custom_compilers(config)

    kwargs = {}

    for arg in D1_D2_SETUP_ARGS:
        if len(D1_D2_SETUP_ARGS[arg]) == 2:
            # The distutils field name is different than distutils2's.
            section, option = D1_D2_SETUP_ARGS[arg]

        elif len(D1_D2_SETUP_ARGS[arg]) == 1:
            # The distutils field name is the same thant distutils2's.
            section = D1_D2_SETUP_ARGS[arg][0]
            option = arg

        in_cfg_value = has_get_option(config, section, option)
        if not in_cfg_value:
            # There is no such option in the setup.cfg
            if arg == "long_description":
                in_cfg_value = has_get_option(config, section,
                                              "description_file")
                if in_cfg_value:
                    in_cfg_value = split_multiline(in_cfg_value)
                    value = ''
                    for filename in in_cfg_value:
                        description_file = open(filename)
                        try:
                            value += description_file.read().strip() + '\n\n'
                        finally:
                            description_file.close()
                    in_cfg_value = value
            else:
                continue

        if arg in MULTI_FIELDS:
            in_cfg_value = split_multiline(in_cfg_value)
        elif arg in BOOL_FIELDS:
            # Provide some flexibility here...
            if in_cfg_value.lower() in ('true', 't', '1', 'yes', 'y'):
                in_cfg_value = True
            else:
                in_cfg_value = False

        if in_cfg_value:
            if arg == 'install_requires':
                # Replaces PEP345-style version specs with the sort expected by
                # setuptools
                in_cfg_value = [_VERSION_SPEC_RE.sub(r'\1\2', pred)
                                for pred in in_cfg_value]
            elif arg == 'package_dir':
                in_cfg_value = {'': in_cfg_value}
            elif arg in ('package_data', 'data_files'):
                data_files = {}
                firstline = True
                prev = None
                for line in in_cfg_value:
                    if '=' in line:
                        key, value = line.split('=', 1)
                        key, value = (key.strip(), value.strip())
                        if key in data_files:
                            # Multiple duplicates of the same package name;
                            # this is for backwards compatibility of the old
                            # format prior to d2to1 0.2.6.
                            prev = data_files[key]
                            prev.extend(value.split())
                        else:
                            prev = data_files[key.strip()] = value.split()
                    elif firstline:
                        raise DistutilsOptionError(
                            'malformed package_data first line %r (misses '
                            '"=")' % line)
                    else:
                        prev.extend(line.strip().split())
                    firstline = False
                if arg == 'data_files':
                    # the data_files value is a pointlessly different structure
                    # from the package_data value
                    data_files = data_files.items()
                in_cfg_value = data_files
            elif arg == 'cmdclass':
                cmdclass = {}
                dist = Distribution()
                for cls in in_cfg_value:
                    cls = resolve_name(cls)
                    cmd = cls(dist)
                    cmdclass[cmd.get_command_name()] = cls
                in_cfg_value = cmdclass

        kwargs[arg] = in_cfg_value

    ext_modules = get_extension_modules(config)
    if ext_modules:
        kwargs['ext_modules'] = ext_modules

    entry_points = get_entry_points(config)
    if entry_points:
        kwargs['entry_points'] = entry_points

    wrap_commands(kwargs)
    return kwargs

def register_custom_compilers(config):
    """Handle custom compilers; this has no real equivalent in distutils, where
    additional compilers could only be added programmatically, so we have to
    hack it in somehow.
    """

    compilers = has_get_option(config, 'global', 'compilers')
    if compilers:
        compilers = split_multiline(compilers)
        for compiler in compilers:
            compiler = resolve_name(compiler)

            # In distutils2 compilers these class attributes exist; for
            # distutils1 we just have to make something up
            if hasattr(compiler, 'name'):
                name = compiler.name
            else:
                name = compiler.__name__
            if hasattr(compiler, 'description'):
                desc = compiler.description
            else:
                desc = 'custom compiler %s' % name

            module_name = compiler.__module__
            # Note; this *will* override built in compilers with the same name
            # TODO: Maybe display a warning about this?
            cc = distutils.ccompiler.compiler_class
            cc[name] = (module_name, compiler.__name__, desc)

            # HACK!!!!  Distutils assumes all compiler modules are in the
            # distutils package
            sys.modules['distutils.' + module_name] = sys.modules[module_name]

def get_extension_modules(config):
    """Handle extension modules"""

    EXTENSION_FIELDS = ("sources",
                        "include_dirs",
                        "define_macros",
                        "undef_macros",
                        "library_dirs",
                        "libraries",
                        "runtime_library_dirs",
                        "extra_objects",
                        "extra_compile_args",
                        "extra_link_args",
                        "export_symbols",
                        "swig_opts",
                        "depends")

    ext_modules = []
    for section in config:
        if ':' in section:
            labels = section.split(':', 1)
        else:
            # Backwards compatibility for old syntax; don't use this though
            labels = section.split('=', 1)
        labels = [l.strip() for l in labels]
        if (len(labels) == 2) and (labels[0] == 'extension'):
            ext_args = {}
            for field in EXTENSION_FIELDS:
                value = has_get_option(config, section, field)
                # All extension module options besides name can have multiple
                # values
                if not value:
                    continue
                value = split_multiline(value)
                if field == 'define_macros':
                    macros = []
                    for macro in value:
                        macro = macro.split('=', 1)
                        if len(macro) == 1:
                            macro = (macro[0].strip(), None)
                        else:
                            macro = (macro[0].strip(), macro[1].strip())
                        macros.append(macro)
                    value = macros
                ext_args[field] = value
            if ext_args:
                if 'name' not in ext_args:
                    ext_args['name'] = labels[1]
                ext_modules.append(Extension(ext_args.pop('name'),
                                             **ext_args))
    return ext_modules

def get_entry_points(config):
    """Process the [entry_points] section of setup.cfg to handle setuptools
    entry points.  This is, of course, not a standard feature of
    distutils2/packaging, but as there is not currently a standard alternative
    in packaging, we provide support for them.
    """

    if not 'entry_points' in config:
        return {}

    return dict((option, split_multiline(value))
                for option, value in config['entry_points'].items())

def wrap_commands(kwargs):
    dist = Distribution()

    # This should suffice to get the same config values and command classes
    # that the actual Distribution will see (not counting cmdclass, which is
    # handled below)
    dist.parse_config_files()

    for cmd, _ in dist.get_command_list():
        hooks = {}
        for opt, val in dist.get_option_dict(cmd).items():
            val = val[1]
            if opt.startswith('pre_hook.') or opt.startswith('post_hook.'):
                hook_type, alias = opt.split('.', 1)
                hook_dict = hooks.setdefault(hook_type, {})
                hook_dict[alias] = val
        if not hooks:
            continue

        if 'cmdclass' in kwargs and cmd in kwargs['cmdclass']:
            cmdclass = kwargs['cmdclass'][cmd]
        else:
            cmdclass = dist.get_command_class(cmd)

        new_cmdclass = wrap_command(cmd, cmdclass, hooks)
        kwargs.setdefault('cmdclass', {})[cmd] = new_cmdclass

def wrap_command(cmd, cmdclass, hooks):
    def run(self, cmdclass=cmdclass):
        self.run_command_hooks('pre_hook')
        cmdclass.run(self)
        self.run_command_hooks('post_hook')

    return type(cmd, (cmdclass, object),
                {'run': run, 'run_command_hooks': run_command_hooks,
                 'pre_hook': hooks.get('pre_hook'),
                 'post_hook': hooks.get('post_hook')})

def run_command_hooks(cmd_obj, hook_kind):
    """Run hooks registered for that command and phase.

    *cmd_obj* is a finalized command object; *hook_kind* is either
    'pre_hook' or 'post_hook'.
    """

    if hook_kind not in ('pre_hook', 'post_hook'):
        raise ValueError('invalid hook kind: %r' % hook_kind)

    hooks = getattr(cmd_obj, hook_kind, None)

    if hooks is None:
        return

    for hook in hooks.values():
        if isinstance(hook, str):
            try:
                hook_obj = resolve_name(hook)
            except ImportError:
                err = sys.exc_info()[1] # For py3k
                raise DistutilsModuleError(err)
        else:
            hook_obj = hook

        if not hasattr(hook_obj, '__call__'):
            raise DistutilsOptionError('hook %r is not callable' % hook)

        log.info('running %s %s for command %s',
                 hook_kind, hook, cmd_obj.get_command_name())
        hook_obj(cmd_obj)

def has_get_option(config, section, option):
    if section in config and option in config[section]:
        return config[section][option]
    elif section in config and option.replace('_', '-') in config[section]:
        return config[section][option.replace('_', '-')]
    else:
        return False

def split_multiline(value):
    """Special behaviour when we have a multi line options"""

    value = [element for element in
             (line.strip() for line in value.split('\n'))
             if element]
    return value

# The following classes are used to hack Distribution.command_options a bit
class DefaultGetDict(defaultdict):
    """Like defaultdict, but the get() method also sets and returns the default
    value.
    """

    def get(self, key, default=None):
        if default is None:
            default = self.default_factory()
        return super(DefaultGetDict, self).setdefault(key, default)

class IgnoreDict(dict):
    """A dictionary that ignores any insertions in which the key is a string
    matching any string in `ignore`.  The ignore list can also contain wildcard
    patterns using '*'.
    """

    def __init__(self, ignore):
        self.__ignore = re.compile(r'(%s)' % ('|'.join(
                                   [pat.replace('*', '.*')
                                    for pat in ignore])))

    def __setitem__(self, key, val):
        if self.__ignore.match(key):
            return
        super(IgnoreDict, self).__setitem__(key, val)

<|endoftext|>


<commit_before>#!python
from __future__ import print_function
import sys
import re
import inspect
import optparse
import shutil

def get_options():
	parser = optparse.OptionParser()
	options, args = parser.parse_args()
	try:
		options.filename = args.pop()
	except IndexError:
		parser.error("Filename required")
	return options

# each of the replacement functions should have a docstring
#  which is a regular expression to be matched.

def replace_external_link(matcher):
	r"\[(?P<href>(?P<scheme>\w+)\://.+?) (?P<name>.+?)\]"
	return '`{name} <{href}>`_'.format(**matcher.groupdict())

def replace_wiki_link(matcher):
	r"\[wiki\:(?P<ref>.+?) (?P<name>.+?)\]"
	return '`{name} <TODO-fix wiki target {ref}>`_'.format(**matcher.groupdict())

# character array indexed by level for characters
heading_characters = [None, '*', '=', '-', '^']

def replace_headings(matcher):
	r"^(?P<level>=+) (?P<name>.*) (?P=level)$"
	level = len(matcher.groupdict()['level'])
	char = heading_characters[level]
	name = matcher.groupdict()['name']
	return '\n'.join([name, char*len(name)])

replacements = [
	replace_external_link,
	replace_wiki_link,
	replace_headings,
	]

def convert_file(filename):
	shutil.copy(filename, filename+'.bak')
	text = open(filename).read()
	new_text = text
	for repl in replacements:
		pattern = re.compile(inspect.getdoc(repl), re.MULTILINE)
		new_text = pattern.sub(repl, new_text)

	open(filename, 'w').write(new_text)
	print("done")

def handle_command_line():
	options = get_options()
	convert_file(options.filename)

if __name__ == '__main__':
	handle_command_line()
<commit_msg>Made the replacement list introspective<commit_after>#!python
from __future__ import print_function
import sys
import re
import inspect
import optparse
import shutil

def get_options():
	parser = optparse.OptionParser()
	options, args = parser.parse_args()
	try:
		options.filename = args.pop()
	except IndexError:
		parser.error("Filename required")
	return options

# each of the replacement functions should have a docstring
#  which is a regular expression to be matched.

def replace_external_link(matcher):
	r"\[(?P<href>(?P<scheme>\w+)\://.+?) (?P<name>.+?)\]"
	return '`{name} <{href}>`_'.format(**matcher.groupdict())

def replace_wiki_link(matcher):
	r"\[wiki\:(?P<ref>.+?) (?P<name>.+?)\]"
	return '`{name} <TODO-fix wiki target {ref}>`_'.format(**matcher.groupdict())

# character array indexed by level for characters
heading_characters = [None, '*', '=', '-', '^']

def replace_headings(matcher):
	r"^(?P<level>=+) (?P<name>.*) (?P=level)$"
	level = len(matcher.groupdict()['level'])
	char = heading_characters[level]
	name = matcher.groupdict()['name']
	return '\n'.join([name, char*len(name)])

replacements = [func for name, func in globals().items() if name.startswith('replace')]

def convert_file(filename):
	shutil.copy(filename, filename+'.bak')
	text = open(filename).read()
	new_text = text
	for repl in replacements:
		pattern = re.compile(inspect.getdoc(repl), re.MULTILINE)
		new_text = pattern.sub(repl, new_text)

	open(filename, 'w').write(new_text)
	print("done")

def handle_command_line():
	options = get_options()
	convert_file(options.filename)

if __name__ == '__main__':
	handle_command_line()
<|endoftext|>


<commit_before>import contextlib
import itertools
import time
import unittest.mock
from datetime import datetime
from os import path
from urllib.error import HTTPError
from urllib.parse import urlparse
from urllib.request import urlopen, Request

import pytest

from torchvision import datasets
from torchvision.datasets.utils import download_url, check_integrity

from common_utils import get_tmp_dir
from fakedata_generation import places365_root

def limit_requests_per_time(min_secs_between_requests=2.0):
    last_requests = {}

    def outer_wrapper(fn):
        def inner_wrapper(request, *args, **kwargs):
            url = request.full_url if isinstance(request, Request) else request

            netloc = urlparse(url).netloc
            last_request = last_requests.get(netloc)
            if last_request is not None:
                elapsed_secs = (datetime.now() - last_request).total_seconds()
                delta = min_secs_between_requests - elapsed_secs
                if delta > 0:
                    time.sleep(delta)

            response = fn(request, *args, **kwargs)
            last_requests[netloc] = datetime.now()

            return response

        return inner_wrapper

    return outer_wrapper

urlopen = limit_requests_per_time()(urlopen)

@contextlib.contextmanager
def log_download_attempts(
    urls_and_md5s=None,
    patch=True,
    download_url_location=".utils",
    patch_auxiliaries=None,
):
    if urls_and_md5s is None:
        urls_and_md5s = set()
    if download_url_location.startswith("."):
        download_url_location = f"torchvision.datasets{download_url_location}"
    if patch_auxiliaries is None:
        patch_auxiliaries = patch

    with contextlib.ExitStack() as stack:
        download_url_mock = stack.enter_context(
            unittest.mock.patch(
                f"{download_url_location}.download_url",
                wraps=None if patch else download_url,
            )
        )
        if patch_auxiliaries:
            # download_and_extract_archive
            stack.enter_context(unittest.mock.patch("torchvision.datasets.utils.extract_archive"))
        try:
            yield urls_and_md5s
        finally:
            for args, kwargs in download_url_mock.call_args_list:
                url = args[0]
                md5 = args[-1] if len(args) == 4 else kwargs.get("md5")
                urls_and_md5s.add((url, md5))

def retry(fn, times=1, wait=5.0):
    msgs = []
    for _ in range(times + 1):
        try:
            return fn()
        except AssertionError as error:
            msgs.append(str(error))
            time.sleep(wait)
    else:
        raise AssertionError(
            "\n".join(
                (
                    f"Assertion failed {times + 1} times with {wait:.1f} seconds intermediate wait time.\n",
                    *(f"{idx}: {error}" for idx, error in enumerate(msgs, 1)),
                )
            )
        )

@contextlib.contextmanager
def assert_server_response_ok():
    try:
        yield
    except HTTPError as error:
        raise AssertionError(f"The server returned {error.code}: {error.reason}.") from error

def assert_url_is_accessible(url):
    request = Request(url, headers=dict(method="HEAD"))
    with assert_server_response_ok():
        urlopen(request)

def assert_file_downloads_correctly(url, md5):
    with get_tmp_dir() as root:
        file = path.join(root, path.basename(url))
        with assert_server_response_ok():
            with urlopen(url) as response, open(file, "wb") as fh:
                fh.write(response.read())

        assert check_integrity(file, md5=md5), "The MD5 checksums mismatch"

class DownloadConfig:
    def __init__(self, url, md5=None, id=None):
        self.url = url
        self.md5 = md5
        self.id = id or url

    def __repr__(self):
        return self.id

def make_download_configs(urls_and_md5s, name=None):
    return [
        DownloadConfig(url, md5=md5, id=f"{name}, {url}" if name is not None else None) for url, md5 in urls_and_md5s
    ]

def collect_download_configs(dataset_loader, name=None, **kwargs):
    urls_and_md5s = set()
    try:
        with log_download_attempts(urls_and_md5s=urls_and_md5s, **kwargs):
            dataset = dataset_loader()
    except Exception:
        dataset = None

    if name is None and dataset is not None:
        name = type(dataset).__name__

    return make_download_configs(urls_and_md5s, name)

def places365():
    with log_download_attempts(patch=False) as urls_and_md5s:
        for split, small in itertools.product(("train-standard", "train-challenge", "val"), (False, True)):
            with places365_root(split=split, small=small) as places365:
                root, data = places365

                datasets.Places365(root, split=split, small=small, download=True)

    return make_download_configs(urls_and_md5s, name="Places365")

def caltech101():
    return collect_download_configs(lambda: datasets.Caltech101(".", download=True), name="Caltech101")

def caltech256():
    return collect_download_configs(lambda: datasets.Caltech256(".", download=True), name="Caltech256")

def cifar10():
    return collect_download_configs(lambda: datasets.CIFAR10(".", download=True), name="CIFAR10")

def cifar100():
    return collect_download_configs(lambda: datasets.CIFAR10(".", download=True), name="CIFAR100")

def voc():
    return itertools.chain(
        *[
            collect_download_configs(
                lambda: datasets.VOCSegmentation(".", year=year, download=True),
                name=f"VOC, {year}",
                download_url_location=".voc",
            )
            for year in ("2007", "2007-test", "2008", "2009", "2010", "2011", "2012")
        ]
    )

def mnist():
    return collect_download_configs(lambda: datasets.MNIST(".", download=True), name="MNIST")

def make_parametrize_kwargs(download_configs):
    argvalues = []
    ids = []
    for config in download_configs:
        argvalues.append((config.url, config.md5))
        ids.append(config.id)

    return dict(argnames=("url", "md5"), argvalues=argvalues, ids=ids)

@pytest.mark.parametrize(
    **make_parametrize_kwargs(
        itertools.chain(
            places365(),
            caltech101(),
            caltech256(),
            cifar10(),
            cifar100(),
            # The VOC download server is unstable. See https://github.com/pytorch/vision/issues/2953 for details.
            # voc(),
            mnist(),
        )
    )
)
def test_url_is_accessible(url, md5):
    retry(lambda: assert_url_is_accessible(url))

@pytest.mark.parametrize(**make_parametrize_kwargs(itertools.chain()))
def test_file_downloads_correctly(url, md5):
    retry(lambda: assert_file_downloads_correctly(url, md5))
<commit_msg>fashion_mnist (#3337)<commit_after>import contextlib
import itertools
import time
import unittest.mock
from datetime import datetime
from os import path
from urllib.error import HTTPError
from urllib.parse import urlparse
from urllib.request import urlopen, Request

import pytest

from torchvision import datasets
from torchvision.datasets.utils import download_url, check_integrity

from common_utils import get_tmp_dir
from fakedata_generation import places365_root

def limit_requests_per_time(min_secs_between_requests=2.0):
    last_requests = {}

    def outer_wrapper(fn):
        def inner_wrapper(request, *args, **kwargs):
            url = request.full_url if isinstance(request, Request) else request

            netloc = urlparse(url).netloc
            last_request = last_requests.get(netloc)
            if last_request is not None:
                elapsed_secs = (datetime.now() - last_request).total_seconds()
                delta = min_secs_between_requests - elapsed_secs
                if delta > 0:
                    time.sleep(delta)

            response = fn(request, *args, **kwargs)
            last_requests[netloc] = datetime.now()

            return response

        return inner_wrapper

    return outer_wrapper

urlopen = limit_requests_per_time()(urlopen)

@contextlib.contextmanager
def log_download_attempts(
    urls_and_md5s=None,
    patch=True,
    download_url_location=".utils",
    patch_auxiliaries=None,
):
    if urls_and_md5s is None:
        urls_and_md5s = set()
    if download_url_location.startswith("."):
        download_url_location = f"torchvision.datasets{download_url_location}"
    if patch_auxiliaries is None:
        patch_auxiliaries = patch

    with contextlib.ExitStack() as stack:
        download_url_mock = stack.enter_context(
            unittest.mock.patch(
                f"{download_url_location}.download_url",
                wraps=None if patch else download_url,
            )
        )
        if patch_auxiliaries:
            # download_and_extract_archive
            stack.enter_context(unittest.mock.patch("torchvision.datasets.utils.extract_archive"))
        try:
            yield urls_and_md5s
        finally:
            for args, kwargs in download_url_mock.call_args_list:
                url = args[0]
                md5 = args[-1] if len(args) == 4 else kwargs.get("md5")
                urls_and_md5s.add((url, md5))

def retry(fn, times=1, wait=5.0):
    msgs = []
    for _ in range(times + 1):
        try:
            return fn()
        except AssertionError as error:
            msgs.append(str(error))
            time.sleep(wait)
    else:
        raise AssertionError(
            "\n".join(
                (
                    f"Assertion failed {times + 1} times with {wait:.1f} seconds intermediate wait time.\n",
                    *(f"{idx}: {error}" for idx, error in enumerate(msgs, 1)),
                )
            )
        )

@contextlib.contextmanager
def assert_server_response_ok():
    try:
        yield
    except HTTPError as error:
        raise AssertionError(f"The server returned {error.code}: {error.reason}.") from error

def assert_url_is_accessible(url):
    request = Request(url, headers=dict(method="HEAD"))
    with assert_server_response_ok():
        urlopen(request)

def assert_file_downloads_correctly(url, md5):
    with get_tmp_dir() as root:
        file = path.join(root, path.basename(url))
        with assert_server_response_ok():
            with urlopen(url) as response, open(file, "wb") as fh:
                fh.write(response.read())

        assert check_integrity(file, md5=md5), "The MD5 checksums mismatch"

class DownloadConfig:
    def __init__(self, url, md5=None, id=None):
        self.url = url
        self.md5 = md5
        self.id = id or url

    def __repr__(self):
        return self.id

def make_download_configs(urls_and_md5s, name=None):
    return [
        DownloadConfig(url, md5=md5, id=f"{name}, {url}" if name is not None else None) for url, md5 in urls_and_md5s
    ]

def collect_download_configs(dataset_loader, name=None, **kwargs):
    urls_and_md5s = set()
    try:
        with log_download_attempts(urls_and_md5s=urls_and_md5s, **kwargs):
            dataset = dataset_loader()
    except Exception:
        dataset = None

    if name is None and dataset is not None:
        name = type(dataset).__name__

    return make_download_configs(urls_and_md5s, name)

def places365():
    with log_download_attempts(patch=False) as urls_and_md5s:
        for split, small in itertools.product(("train-standard", "train-challenge", "val"), (False, True)):
            with places365_root(split=split, small=small) as places365:
                root, data = places365

                datasets.Places365(root, split=split, small=small, download=True)

    return make_download_configs(urls_and_md5s, name="Places365")

def caltech101():
    return collect_download_configs(lambda: datasets.Caltech101(".", download=True), name="Caltech101")

def caltech256():
    return collect_download_configs(lambda: datasets.Caltech256(".", download=True), name="Caltech256")

def cifar10():
    return collect_download_configs(lambda: datasets.CIFAR10(".", download=True), name="CIFAR10")

def cifar100():
    return collect_download_configs(lambda: datasets.CIFAR10(".", download=True), name="CIFAR100")

def voc():
    return itertools.chain(
        *[
            collect_download_configs(
                lambda: datasets.VOCSegmentation(".", year=year, download=True),
                name=f"VOC, {year}",
                download_url_location=".voc",
            )
            for year in ("2007", "2007-test", "2008", "2009", "2010", "2011", "2012")
        ]
    )

def mnist():
    return collect_download_configs(lambda: datasets.MNIST(".", download=True), name="MNIST")

def fashion_mnist():
    return collect_download_configs(lambda: datasets.FashionMNIST(".", download=True), name="FashionMNIST")

def make_parametrize_kwargs(download_configs):
    argvalues = []
    ids = []
    for config in download_configs:
        argvalues.append((config.url, config.md5))
        ids.append(config.id)

    return dict(argnames=("url", "md5"), argvalues=argvalues, ids=ids)

@pytest.mark.parametrize(
    **make_parametrize_kwargs(
        itertools.chain(
            places365(),
            caltech101(),
            caltech256(),
            cifar10(),
            cifar100(),
            # The VOC download server is unstable. See https://github.com/pytorch/vision/issues/2953 for details.
            # voc(),
            mnist(),
            fashion_mnist(),
        )
    )
)
def test_url_is_accessible(url, md5):
    retry(lambda: assert_url_is_accessible(url))

@pytest.mark.parametrize(**make_parametrize_kwargs(itertools.chain()))
def test_file_downloads_correctly(url, md5):
    retry(lambda: assert_file_downloads_correctly(url, md5))
<|endoftext|>