<commit_before>from django.contrib import admin
from .models import Vehicle, Flight, Category, SpotTime, Lead, DentalOffice

@admin.register(Vehicle)
class VehicleAdmin(admin.ModelAdmin):
    list_display = ('name', )

@admin.register(DentalOffice)
class DentalOfficeAdmin(admin.ModelAdmin):
    list_display = ('name', )

@admin.register(Flight)
class FlightAdmin(admin.ModelAdmin):
    list_display = ('year', 'first_day', 'last_day')

    readonly_fields = ('year', )
    list_filter = ('year', )

@admin.register(Category)
class CategoryAdmin(admin.ModelAdmin):
    list_display = ('name', )

@admin.register(SpotTime)
class SpotTimeAdmin(admin.ModelAdmin):
    list_display = ('vehicle', 'scheduled', 'air_time')
    list_filter = ('vehicle', )

@admin.register(Lead)
class LeadAdmin(admin.ModelAdmin):
    list_display = ('vehicle', 'phone_number', 'name', 'birth_date', 'age', 'gender', 'category', 'lm',
                    'appointment_date', 'office',
                    'entered_date', 'entered_by', 'short_comments', 'email')

    list_filter = ('vehicle', 'lm', 'gender', 'category', 'appointment_date', 'office', 'entered_by')
    search_fields = ('phone_number', 'name', 'email')

    def save_model(self, request, obj, form, change):
        if getattr(obj, 'entered_by', None) is None:
            obj.entered_by = request.user
        obj.save()<commit_msg>defaulting vehicle when creating a lead to the closes air time spot<commit_after>from django.utils import timezone

from django.contrib import admin
from .models import Vehicle, Flight, Category, SpotTime, Lead, DentalOffice

@admin.register(Vehicle)
class VehicleAdmin(admin.ModelAdmin):
    list_display = ('name', )

@admin.register(DentalOffice)
class DentalOfficeAdmin(admin.ModelAdmin):
    list_display = ('name', )

@admin.register(Flight)
class FlightAdmin(admin.ModelAdmin):
    list_display = ('year', 'first_day', 'last_day')

    readonly_fields = ('year', )
    list_filter = ('year', )

@admin.register(Category)
class CategoryAdmin(admin.ModelAdmin):
    list_display = ('name', )

@admin.register(SpotTime)
class SpotTimeAdmin(admin.ModelAdmin):
    list_display = ('vehicle', 'scheduled', 'air_time')
    list_filter = ('vehicle', )

@admin.register(Lead)
class LeadAdmin(admin.ModelAdmin):
    list_display = ('vehicle', 'phone_number', 'name', 'birth_date', 'age', 'gender', 'category', 'lm',
                    'appointment_date', 'office',
                    'entered_date', 'entered_by', 'short_comments', 'email')

    list_filter = ('vehicle', 'lm', 'gender', 'category', 'appointment_date', 'office', 'entered_by')
    search_fields = ('phone_number', 'name', 'email')

    def save_model(self, request, obj, form, change):
        if getattr(obj, 'entered_by', None) is None:
            obj.entered_by = request.user
        obj.save()

    @staticmethod
    def guess_vehicle():
        now = timezone.now()
        spot_times = SpotTime.objects.filter(air_time__lte=now).order_by('-air_time')
        print spot_times
        try:
            spot_time = spot_times[0]
            return spot_time.vehicle
        except IndexError:
            return None

    def get_form(self, request, obj=None, **kwargs):
        form = super(LeadAdmin, self).get_form(request, obj, **kwargs)
        vehicle = self.guess_vehicle()
        if vehicle:
            form.base_fields['vehicle'].initial = vehicle
        return form<|endoftext|>


<commit_before>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import array

def _check_forward(e1, e2, f, y_expect):
    e1 = chainer.Variable(e1)
    e2 = chainer.Variable(e2)
    y = f(e1, e2)
    gradient_check.assert_allclose(y_expect, y.data)

def _check_backward(e1, e2, y_grad, f, bias):
    e1 = chainer.Variable(e1)
    e2 = chainer.Variable(e2)
    y = f(e1, e2)
    y.grad = y_grad
    y.backward()

    func = y.creator
    f = lambda: func.forward((e1.data, e2.data))

    ge1, ge2, gW = gradient_check.numerical_grad(
        f, (e1.data, e2.data, func.W), (y.grad,), eps=1e-2)

    gradient_check.assert_allclose(ge1, e1.grad, rtol=1e-3)
    gradient_check.assert_allclose(ge2, e2.grad, rtol=1e-3)
    gradient_check.assert_allclose(gW, func.gW, rtol=1e-3)

    if bias:
        gV1, gV2, gb = gradient_check.numerical_grad(
            f, (func.V1, func.V2, func.b),
            (y.grad,), eps=1e-2)
        gradient_check.assert_allclose(gV1, func.gV1, rtol=1e-3)
        gradient_check.assert_allclose(gV2, func.gV2, rtol=1e-3)
        gradient_check.assert_allclose(gb, func.gb, rtol=1e-3)

def _batch_to_gpu(*xs):
    return tuple(cuda.to_gpu(x) for x in xs)

def _uniform(*shape):
    return numpy.random.uniform(-1, 1, shape).astype(numpy.float32)

class TestBilinear(unittest.TestCase):

    in_shape = (3, 4)
    out_size = 4
    batch_size = 10

    def setUp(self):
        self.f = functions.Bilinear(
            self.in_shape[0], self.in_shape[1], self.out_size)
        self.f.W = _uniform(*self.f.W.shape)
        self.f.V1 = _uniform(*self.f.V1.shape)
        self.f.V2 = _uniform(*self.f.V2.shape)
        self.f.b = _uniform(*self.f.b.shape)
        self.f.zero_grads()

        self.W = self.f.W.copy()
        self.V1 = self.f.V1.copy()
        self.V2 = self.f.V2.copy()
        self.b = self.f.b.copy()

        self.e1 = _uniform(self.batch_size, self.in_shape[0])
        self.e2 = _uniform(self.batch_size, self.in_shape[1])
        self.gy = _uniform(self.batch_size, self.out_size)

        self.y = (
            numpy.einsum('ij,ik,jkl->il', self.e1, self.e2, self.W) +
            self.e1.dot(self.V1) + self.e2.dot(self.V2) + self.b)

    @condition.retry(3)
    def test_forward_cpu(self):
        _check_forward(self.e1, self.e2, self.f, self.y)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.f.to_gpu()
        _check_forward(cuda.to_gpu(self.e1),
                       cuda.to_gpu(self.e2),
                       self.f, self.y)

    @condition.retry(3)
    def test_backward_cpu(self):
        _check_backward(self.e1, self.e2, self.gy, self.f, True)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.f.to_gpu()
        _check_backward(cuda.to_gpu(self.e1),
                        cuda.to_gpu(self.e2),
                        cuda.to_gpu(self.gy),
                        self.f, True)

class TestBilinear2(TestBilinear):

    def setUp(self):
        super(TestBilinear2, self).setUp()

        assert self.in_shape[1] % 2 == 0
        self.e1 = _uniform(self.batch_size, 1, self.in_shape[0])
        self.e2 = _uniform(self.batch_size, self.in_shape[1] // 2, 2)
        self.gy = _uniform(self.batch_size, self.out_size)

        e1 = array.as_mat(self.e1)
        e2 = array.as_mat(self.e2)

        self.y = (
            numpy.einsum('ij,ik,jkl->il', e1, e2, self.W) +
            e1.dot(self.V1) + e2.dot(self.V2) + self.b)

class TestBilinear3(TestBilinear):

    out_size = 1

class TestBilinearWOBias(TestBilinear):

    def setUp(self):
        self.f = functions.Bilinear(
            self.in_shape[0], self.in_shape[1], self.out_size, True)
        self.f.W = numpy.random.uniform(
            -1, 1, self.f.W.shape).astype(numpy.float32)
        self.f.zero_grads()

        self.W = self.f.W.copy()

        self.e1 = _uniform(self.batch_size, self.in_shape[0])
        self.e2 = _uniform(self.batch_size, self.in_shape[1])
        self.gy = _uniform(self.batch_size, self.out_size)

        self.y = numpy.einsum('ij,ik,jkl->il', self.e1, self.e2, self.W)

    @condition.retry(3)
    def test_backward_cpu(self):
        _check_backward(self.e1, self.e2, self.gy, self.f, False)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.f.to_gpu()
        _check_backward(cuda.to_gpu(self.e1), cuda.to_gpu(self.e2),
                        cuda.to_gpu(self.gy), self.f, False)

class TestBilinearWOBias2(TestBilinearWOBias):

    def setUp(self):
        super(TestBilinearWOBias2, self).setUp()

        assert self.in_shape[1] % 2 == 0
        self.e1 = _uniform(self.batch_size, 1, self.in_shape[0])
        self.e2 = _uniform(self.batch_size, 2, self.in_shape[1] // 2)
        self.gy = _uniform(self.batch_size, self.out_size)

        e1 = array.as_mat(self.e1)
        e2 = array.as_mat(self.e2)

        self.y = numpy.einsum('ij,ik,jkl->il', e1, e2, self.W)

class TestBilinearWOBias3(TestBilinearWOBias):

    out_size = 1

class InitByInitialParameter(unittest.TestCase):

    in_shape = (2, 3)
    out_size = 4
    batch_size = 10

    def setUp(self):
        self.W = _uniform(self.in_shape[0], self.in_shape[1], self.out_size)
        self.V1 = _uniform(self.in_shape[0], self.out_size)
        self.V2 = _uniform(self.in_shape[1], self.out_size)
        self.b = _uniform(self.out_size,)

class NormalInitialParameter(InitByInitialParameter):

    def check_normal(self, initialW, initial_bias, nobias):
        functions.Bilinear(
            self.in_shape[0], self.in_shape[1], self.out_size, nobias,
            initialW, initial_bias)

    def test_normal_cpu_bias(self):
        self.check_normal(self.W, (self.V1, self.V2, self.b), False)

    def test_normal_cpu_nobias(self):
        self.check_normal(self.W, None, False)

    @attr.gpu
    def test_normal_gpu_bias(self):
        self.check_normal(cuda.to_gpu(self.W),
                          _batch_to_gpu(self.V1, self.V2, self.b), False)

    @attr.gpu
    def test_normal_gpu_nobias(self):
        self.check_normal(cuda.to_gpu(self.W), None, False)

class InvalidInitialParameter(InitByInitialParameter):

    def setUp(self):
        super(InvalidInitialParameter, self).setUp()
        self.invalidW = _uniform(self.in_shape[0] + 1, self.in_shape[1],
                                 self.out_size)
        self.invalidV1 = _uniform(self.in_shape[0] + 1, self.out_size)
        self.invalidV2 = _uniform(self.in_shape[1] + 1, self.out_size)
        self.invalidb = _uniform(self.out_size + 1,)

    def check_invalid(self, initialW, initial_bias, nobias):
        with self.assertRaises(AssertionError):
            functions.Bilinear(
                self.in_shape[0], self.in_shape[1], self.out_size, nobias,
                initialW, initial_bias)

    def test_invalidW_cpu(self):
        self.check_invalid(self.invalidW, (self.V1, self.V2, self.b), False)
        self.check_invalid(self.invalidW, None, True)

    @attr.gpu
    def test_invalidW_gpu(self):
        invalidW = cuda.to_gpu(self.invalidW)
        self.check_invalid(invalidW,
                           _batch_to_gpu(self.V1, self.V2, self.b), False)
        self.check_invalid(invalidW, None, True)

    def test_invalidV1_cpu(self):
        self.check_invalid(self.W, (self.invalidV1, self.V2, self.b), False)

    @attr.gpu
    def test_invaliV1_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.invalidV1, self.V2, self.b),
                           False)

    def test_invalidV2_cpu(self):
        self.check_invalid(self.W, (self.V1, self.invalidV2, self.b), False)

    @attr.gpu
    def test_invaliV2_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.V1, self.invalidV2, self.b),
                           False)

    def test_invalidb_cpu(self):
        self.check_invalid(self.W, (self.V1, self.V2, self.invalidb), False)

    @attr.gpu
    def test_invalib_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.V1, self.V2, self.invalidb),
                           False)

testing.run_module(__name__, __file__)
<commit_msg>Add more unitcases for Bilinear<commit_after>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition
from chainer.utils import array

def _check_forward(e1, e2, f, y_expect):
    e1 = chainer.Variable(e1)
    e2 = chainer.Variable(e2)
    y = f(e1, e2)
    gradient_check.assert_allclose(y_expect, y.data)

def _check_backward(e1, e2, y_grad, f, bias):
    e1 = chainer.Variable(e1)
    e2 = chainer.Variable(e2)
    y = f(e1, e2)
    y.grad = y_grad
    y.backward()

    func = y.creator
    f = lambda: func.forward((e1.data, e2.data))

    ge1, ge2, gW = gradient_check.numerical_grad(
        f, (e1.data, e2.data, func.W), (y.grad,), eps=1e-2)

    gradient_check.assert_allclose(ge1, e1.grad, rtol=1e-3)
    gradient_check.assert_allclose(ge2, e2.grad, rtol=1e-3)
    gradient_check.assert_allclose(gW, func.gW, rtol=1e-3)

    if bias:
        gV1, gV2, gb = gradient_check.numerical_grad(
            f, (func.V1, func.V2, func.b),
            (y.grad,), eps=1e-2)
        gradient_check.assert_allclose(gV1, func.gV1, rtol=1e-3)
        gradient_check.assert_allclose(gV2, func.gV2, rtol=1e-3)
        gradient_check.assert_allclose(gb, func.gb, rtol=1e-3)

def _batch_to_gpu(*xs):
    return tuple(cuda.to_gpu(x) for x in xs)

def _uniform(*shape):
    return numpy.random.uniform(-1, 1, shape).astype(numpy.float32)

class TestBilinear(unittest.TestCase):

    in_shape = (3, 4)
    out_size = 4
    batch_size = 10

    def setUp(self):
        self.f = functions.Bilinear(
            self.in_shape[0], self.in_shape[1], self.out_size)
        self.f.W = _uniform(*self.f.W.shape)
        self.f.V1 = _uniform(*self.f.V1.shape)
        self.f.V2 = _uniform(*self.f.V2.shape)
        self.f.b = _uniform(*self.f.b.shape)
        self.f.zero_grads()

        self.W = self.f.W.copy()
        self.V1 = self.f.V1.copy()
        self.V2 = self.f.V2.copy()
        self.b = self.f.b.copy()

        self.e1 = _uniform(self.batch_size, self.in_shape[0])
        self.e2 = _uniform(self.batch_size, self.in_shape[1])
        self.gy = _uniform(self.batch_size, self.out_size)

        self.y = (
            numpy.einsum('ij,ik,jkl->il', self.e1, self.e2, self.W) +
            self.e1.dot(self.V1) + self.e2.dot(self.V2) + self.b)

    @condition.retry(3)
    def test_forward_cpu(self):
        _check_forward(self.e1, self.e2, self.f, self.y)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.f.to_gpu()
        _check_forward(cuda.to_gpu(self.e1),
                       cuda.to_gpu(self.e2),
                       self.f, self.y)

    @condition.retry(3)
    def test_backward_cpu(self):
        _check_backward(self.e1, self.e2, self.gy, self.f, True)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.f.to_gpu()
        _check_backward(cuda.to_gpu(self.e1),
                        cuda.to_gpu(self.e2),
                        cuda.to_gpu(self.gy),
                        self.f, True)

class TestBilinear2(TestBilinear):

    def setUp(self):
        super(TestBilinear2, self).setUp()

        assert self.in_shape[1] % 2 == 0
        self.e1 = _uniform(self.batch_size, 1, self.in_shape[0])
        self.e2 = _uniform(self.batch_size, self.in_shape[1] // 2, 2)
        self.gy = _uniform(self.batch_size, self.out_size)

        e1 = array.as_mat(self.e1)
        e2 = array.as_mat(self.e2)

        self.y = (
            numpy.einsum('ij,ik,jkl->il', e1, e2, self.W) +
            e1.dot(self.V1) + e2.dot(self.V2) + self.b)

class TestBilinear3(TestBilinear):

    out_size = 1

class TestBilinear4(TestBilinear):

    in_shape = (1, 2)

class TestBilinear5(TestBilinear):

    in_shape = (2, 1)

class TestBilinear6(TestBilinear):

    in_shape = (1, 1)

class TestBilinear7(TestBilinear):

    in_shape = (1, 2)
    out_size = 1

class TestBilinear8(TestBilinear):

    in_shape = (2, 1)
    out_size = 1

class TestBilinear9(TestBilinear):

    in_shape = (1, 1)
    out_size = 1

class TestBilinearWOBias(TestBilinear):

    def setUp(self):
        self.f = functions.Bilinear(
            self.in_shape[0], self.in_shape[1], self.out_size, True)
        self.f.W = numpy.random.uniform(
            -1, 1, self.f.W.shape).astype(numpy.float32)
        self.f.zero_grads()

        self.W = self.f.W.copy()

        self.e1 = _uniform(self.batch_size, self.in_shape[0])
        self.e2 = _uniform(self.batch_size, self.in_shape[1])
        self.gy = _uniform(self.batch_size, self.out_size)

        self.y = numpy.einsum('ij,ik,jkl->il', self.e1, self.e2, self.W)

    @condition.retry(3)
    def test_backward_cpu(self):
        _check_backward(self.e1, self.e2, self.gy, self.f, False)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.f.to_gpu()
        _check_backward(cuda.to_gpu(self.e1), cuda.to_gpu(self.e2),
                        cuda.to_gpu(self.gy), self.f, False)

class TestBilinearWOBias2(TestBilinearWOBias):

    def setUp(self):
        super(TestBilinearWOBias2, self).setUp()

        assert self.in_shape[1] % 2 == 0
        self.e1 = _uniform(self.batch_size, 1, self.in_shape[0])
        self.e2 = _uniform(self.batch_size, 2, self.in_shape[1] // 2)
        self.gy = _uniform(self.batch_size, self.out_size)

        e1 = array.as_mat(self.e1)
        e2 = array.as_mat(self.e2)

        self.y = numpy.einsum('ij,ik,jkl->il', e1, e2, self.W)

class TestBilinearWOBias3(TestBilinearWOBias):

    out_size = 1

class TestBilinearWOBias4(TestBilinearWOBias):

    in_shape = (1, 2)

class TestBilinearWOBias5(TestBilinearWOBias):

    in_shape = (2, 1)

class TestBilinearWOBias6(TestBilinearWOBias):

    in_shape = (1, 1)

class TestBilinearWOBias7(TestBilinearWOBias):

    in_shape = (1, 2)
    out_size = 1

class TestBilinearWOBias8(TestBilinearWOBias):

    in_shape = (2, 1)
    out_size = 1

class TestBilinearWOBias9(TestBilinearWOBias):

    in_shape = (1, 1)
    out_size = 1

class InitByInitialParameter(unittest.TestCase):

    in_shape = (2, 3)
    out_size = 4
    batch_size = 10

    def setUp(self):
        self.W = _uniform(self.in_shape[0], self.in_shape[1], self.out_size)
        self.V1 = _uniform(self.in_shape[0], self.out_size)
        self.V2 = _uniform(self.in_shape[1], self.out_size)
        self.b = _uniform(self.out_size,)

class NormalInitialParameter(InitByInitialParameter):

    def check_normal(self, initialW, initial_bias, nobias):
        functions.Bilinear(
            self.in_shape[0], self.in_shape[1], self.out_size, nobias,
            initialW, initial_bias)

    def test_normal_cpu_bias(self):
        self.check_normal(self.W, (self.V1, self.V2, self.b), False)

    def test_normal_cpu_nobias(self):
        self.check_normal(self.W, None, False)

    @attr.gpu
    def test_normal_gpu_bias(self):
        self.check_normal(cuda.to_gpu(self.W),
                          _batch_to_gpu(self.V1, self.V2, self.b), False)

    @attr.gpu
    def test_normal_gpu_nobias(self):
        self.check_normal(cuda.to_gpu(self.W), None, False)

class InvalidInitialParameter(InitByInitialParameter):

    def setUp(self):
        super(InvalidInitialParameter, self).setUp()
        self.invalidW = _uniform(self.in_shape[0] + 1, self.in_shape[1],
                                 self.out_size)
        self.invalidV1 = _uniform(self.in_shape[0] + 1, self.out_size)
        self.invalidV2 = _uniform(self.in_shape[1] + 1, self.out_size)
        self.invalidb = _uniform(self.out_size + 1,)

    def check_invalid(self, initialW, initial_bias, nobias):
        with self.assertRaises(AssertionError):
            functions.Bilinear(
                self.in_shape[0], self.in_shape[1], self.out_size, nobias,
                initialW, initial_bias)

    def test_invalidW_cpu(self):
        self.check_invalid(self.invalidW, (self.V1, self.V2, self.b), False)
        self.check_invalid(self.invalidW, None, True)

    @attr.gpu
    def test_invalidW_gpu(self):
        invalidW = cuda.to_gpu(self.invalidW)
        self.check_invalid(invalidW,
                           _batch_to_gpu(self.V1, self.V2, self.b), False)
        self.check_invalid(invalidW, None, True)

    def test_invalidV1_cpu(self):
        self.check_invalid(self.W, (self.invalidV1, self.V2, self.b), False)

    @attr.gpu
    def test_invaliV1_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.invalidV1, self.V2, self.b),
                           False)

    def test_invalidV2_cpu(self):
        self.check_invalid(self.W, (self.V1, self.invalidV2, self.b), False)

    @attr.gpu
    def test_invaliV2_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.V1, self.invalidV2, self.b),
                           False)

    def test_invalidb_cpu(self):
        self.check_invalid(self.W, (self.V1, self.V2, self.invalidb), False)

    @attr.gpu
    def test_invalib_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.V1, self.V2, self.invalidb),
                           False)

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import re
import numpy as np
import six
from os.path import join
from six.moves import zip

from absl import flags

import tensorflow as tf

def configure_tpu(FLAGS):
  if FLAGS.use_tpu:
    tpu_cluster = tf.contrib.cluster_resolver.TPUClusterResolver(
        FLAGS.tpu, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
    master = tpu_cluster.get_master()
  else:
    tpu_cluster = None
    master = FLAGS.master

  session_config = tf.ConfigProto(allow_soft_placement=True)
  # Uncomment the following line if you hope to monitor GPU RAM growth
  # session_config.gpu_options.allow_growth = True

  if FLAGS.use_tpu:
    strategy = None
    tf.logging.info('Use TPU without distribute strategy.')
  elif FLAGS.num_core_per_host == 1:
    strategy = None
    tf.logging.info('Single device mode.')
  else:
    strategy = tf.contrib.distribute.MirroredStrategy(
        num_gpus=FLAGS.num_core_per_host)
    tf.logging.info('Use MirroredStrategy with %d devices.',
                    strategy.num_replicas_in_sync)

  per_host_input = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
  run_config = tf.contrib.tpu.RunConfig(
      master=master,
      model_dir=FLAGS.model_dir,
      session_config=session_config,
      tpu_config=tf.contrib.tpu.TPUConfig(
          iterations_per_loop=FLAGS.iterations,
          num_shards=FLAGS.num_hosts * FLAGS.num_core_per_host,
          per_host_input_for_training=per_host_input),
      keep_checkpoint_max=FLAGS.max_save,
      save_checkpoints_secs=None,
      save_checkpoints_steps=FLAGS.save_steps,
      train_distribute=strategy
  )
  return run_config

def init_from_checkpoint(FLAGS, global_vars=False):
  tvars = tf.global_variables() if global_vars else tf.trainable_variables()
  initialized_variable_names = {}
  scaffold_fn = None
  if FLAGS.init_checkpoint is not None:
    if FLAGS.init_checkpoint.endswith("latest"):
      ckpt_dir = os.path.dirname(FLAGS.init_checkpoint)
      init_checkpoint = tf.train.latest_checkpoint(ckpt_dir)
    else:
      init_checkpoint = FLAGS.init_checkpoint

    tf.logging.info("Initialize from the ckpt {}".format(init_checkpoint))

    (assignment_map, initialized_variable_names
    ) = get_assignment_map_from_checkpoint(tvars, init_checkpoint)
    if FLAGS.use_tpu:
      def tpu_scaffold():
        tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
        return tf.train.Scaffold()

      scaffold_fn = tpu_scaffold
    else:
      tf.train.init_from_checkpoint(init_checkpoint, assignment_map)

    # Log customized initialization
    tf.logging.info("**** Global Variables ****")
    for var in tvars:
      init_string = ""
      if var.name in initialized_variable_names:
        init_string = ", *INIT_FROM_CKPT*"
      tf.logging.info("  name = %s, shape = %s%s", var.name, var.shape,
                      init_string)
  return scaffold_fn

def get_train_op(FLAGS, total_loss, grads_and_vars=None):
  global_step = tf.train.get_or_create_global_step()

  # increase the learning rate linearly
  if FLAGS.warmup_steps > 0:
    warmup_lr = (tf.cast(global_step, tf.float32)
                 / tf.cast(FLAGS.warmup_steps, tf.float32)
                 * FLAGS.learning_rate)
  else:
    warmup_lr = 0.0

  # decay the learning rate
  if FLAGS.decay_method == "poly":
    decay_lr = tf.train.polynomial_decay(
        FLAGS.learning_rate,
        global_step=global_step - FLAGS.warmup_steps,
        decay_steps=FLAGS.train_steps - FLAGS.warmup_steps,
        end_learning_rate=FLAGS.learning_rate * FLAGS.min_lr_ratio)
  elif FLAGS.decay_method == "cos":
    decay_lr = tf.train.cosine_decay(
        FLAGS.learning_rate,
        global_step=global_step - FLAGS.warmup_steps,
        decay_steps=FLAGS.train_steps - FLAGS.warmup_steps,
        alpha=FLAGS.min_lr_ratio)
  else:
    raise ValueError(FLAGS.decay_method)

  learning_rate = tf.where(global_step < FLAGS.warmup_steps,
                           warmup_lr, decay_lr)

  if FLAGS.weight_decay == 0:
    optimizer = tf.train.AdamOptimizer(
        learning_rate=learning_rate,
        epsilon=FLAGS.adam_epsilon)
  elif FLAGS.weight_decay > 0 and FLAGS.num_core_per_host == 1:
    optimizer = AdamWeightDecayOptimizer(
        learning_rate=learning_rate,
        epsilon=FLAGS.adam_epsilon,
        exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"],
        weight_decay_rate=FLAGS.weight_decay)
  else:
    raise ValueError("Do not support `weight_decay > 0` with multi-gpu "
                     "training so far.")

  if FLAGS.use_tpu:
    optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)

  if grads_and_vars is None:
    grads_and_vars = optimizer.compute_gradients(total_loss)
  gradients, variables = zip(*grads_and_vars)
  clipped, gnorm = tf.clip_by_global_norm(gradients, FLAGS.clip)
  train_op = optimizer.apply_gradients(
      zip(clipped, variables), global_step=global_step)

  # Manually increment `global_step` for AdamWeightDecayOptimizer
  if isinstance(optimizer, AdamWeightDecayOptimizer):
    new_global_step = global_step + 1
    train_op = tf.group(train_op, [global_step.assign(new_global_step)])

  return train_op, learning_rate, gnorm

def clean_ckpt(_):
  input_ckpt = FLAGS.clean_input_ckpt
  output_model_dir = FLAGS.clean_output_model_dir

  tf.reset_default_graph()

  var_list = tf.contrib.framework.list_variables(input_ckpt)
  var_values, var_dtypes = {}, {}
  for (name, shape) in var_list:
    if not name.startswith("global_step") and "adam" not in name.lower():
      var_values[name] = None
      tf.logging.info("Include {}".format(name))
    else:
      tf.logging.info("Exclude {}".format(name))

  tf.logging.info("Loading from {}".format(input_ckpt))
  reader = tf.contrib.framework.load_checkpoint(input_ckpt)
  for name in var_values:
    tensor = reader.get_tensor(name)
    var_dtypes[name] = tensor.dtype
    var_values[name] = tensor

  with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
    tf_vars = [
        tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
        for v in var_values
    ]
  placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
  assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
  global_step = tf.Variable(
      0, name="global_step", trainable=False, dtype=tf.int64)
  saver = tf.train.Saver(tf.all_variables())

  if not tf.gfile.Exists(output_model_dir):
    tf.gfile.MakeDirs(output_model_dir)

  # Build a model consisting only of variables, set them to the average values.
  with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                           six.iteritems(var_values)):
      sess.run(assign_op, {p: value})

    # Use the built saver to save the averaged checkpoint.
    saver.save(sess, join(output_model_dir, "model.ckpt"),
               global_step=global_step)

def avg_checkpoints(model_dir, output_model_dir, last_k):
  tf.reset_default_graph()

  checkpoint_state = tf.train.get_checkpoint_state(model_dir)
  checkpoints = checkpoint_state.all_model_checkpoint_paths[- last_k:]
  var_list = tf.contrib.framework.list_variables(checkpoints[0])
  var_values, var_dtypes = {}, {}
  for (name, shape) in var_list:
    if not name.startswith("global_step"):
      var_values[name] = np.zeros(shape)
  for checkpoint in checkpoints:
    reader = tf.contrib.framework.load_checkpoint(checkpoint)
    for name in var_values:
      tensor = reader.get_tensor(name)
      var_dtypes[name] = tensor.dtype
      var_values[name] += tensor
    tf.logging.info("Read from checkpoint %s", checkpoint)
  for name in var_values:  # Average.
    var_values[name] /= len(checkpoints)

  with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
    tf_vars = [
        tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
        for v in var_values
    ]
  placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
  assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
  global_step = tf.Variable(
      0, name="global_step", trainable=False, dtype=tf.int64)
  saver = tf.train.Saver(tf.all_variables())

  # Build a model consisting only of variables, set them to the average values.
  with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                           six.iteritems(var_values)):
      sess.run(assign_op, {p: value})
    # Use the built saver to save the averaged checkpoint.
    saver.save(sess, join(output_model_dir, "model.ckpt"),
        global_step=global_step)

def get_assignment_map_from_checkpoint(tvars, init_checkpoint):
  """Compute the union of the current variables and checkpoint variables."""
  assignment_map = {}
  initialized_variable_names = {}

  name_to_variable = collections.OrderedDict()
  for var in tvars:
    name = var.name
    m = re.match("^(.*):\\d+$", name)
    if m is not None:
      name = m.group(1)
    name_to_variable[name] = var

  init_vars = tf.train.list_variables(init_checkpoint)

  assignment_map = collections.OrderedDict()
  for x in init_vars:
    (name, var) = (x[0], x[1])
    # tf.logging.info('original name: %s', name)
    if name not in name_to_variable:
      continue
    # assignment_map[name] = name
    assignment_map[name] = name_to_variable[name]
    initialized_variable_names[name] = 1
    initialized_variable_names[name + ":0"] = 1

  return (assignment_map, initialized_variable_names)

class AdamWeightDecayOptimizer(tf.train.Optimizer):
  """A basic Adam optimizer that includes "correct" L2 weight decay."""

  def __init__(self,
               learning_rate,
               weight_decay_rate=0.0,
               beta_1=0.9,
               beta_2=0.999,
               epsilon=1e-6,
               exclude_from_weight_decay=None,
               include_in_weight_decay=["r_s_bias", "r_r_bias", "r_w_bias"],
               name="AdamWeightDecayOptimizer"):
    """Constructs a AdamWeightDecayOptimizer."""
    super(AdamWeightDecayOptimizer, self).__init__(False, name)

    self.learning_rate = learning_rate
    self.weight_decay_rate = weight_decay_rate
    self.beta_1 = beta_1
    self.beta_2 = beta_2
    self.epsilon = epsilon
    self.exclude_from_weight_decay = exclude_from_weight_decay
    self.include_in_weight_decay = include_in_weight_decay

  def apply_gradients(self, grads_and_vars, global_step=None, name=None):
    """See base class."""
    assignments = []
    for (grad, param) in grads_and_vars:
      if grad is None or param is None:
        continue

      param_name = self._get_variable_name(param.name)

      m = tf.get_variable(
          name=param_name + "/adam_m",
          shape=param.shape.as_list(),
          dtype=tf.float32,
          trainable=False,
          initializer=tf.zeros_initializer())
      v = tf.get_variable(
          name=param_name + "/adam_v",
          shape=param.shape.as_list(),
          dtype=tf.float32,
          trainable=False,
          initializer=tf.zeros_initializer())

      # Standard Adam update.
      next_m = (
          tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad))
      next_v = (
          tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2,
                                                    tf.square(grad)))

      update = next_m / (tf.sqrt(next_v) + self.epsilon)

      # Just adding the square of the weights to the loss function is *not*
      # the correct way of using L2 regularization/weight decay with Adam,
      # since that will interact with the m and v parameters in strange ways.
      #
      # Instead we want ot decay the weights in a manner that doesn't interact
      # with the m/v parameters. This is equivalent to adding the square
      # of the weights to the loss with plain (non-momentum) SGD.
      if self._do_use_weight_decay(param_name):
        update += self.weight_decay_rate * param

      update_with_lr = self.learning_rate * update

      next_param = param - update_with_lr

      assignments.extend(
          [param.assign(next_param),
           m.assign(next_m),
           v.assign(next_v)])

    return tf.group(*assignments, name=name)

  def _do_use_weight_decay(self, param_name):
    """Whether to use L2 weight decay for `param_name`."""
    if not self.weight_decay_rate:
      return False
    for r in self.include_in_weight_decay:
      if re.search(r, param_name) is not None:
        return True

    if self.exclude_from_weight_decay:
      for r in self.exclude_from_weight_decay:
        if re.search(r, param_name) is not None:
          tf.logging.info('Adam WD excludes {}'.format(param_name))
          return False
    return True

  def _get_variable_name(self, param_name):
    """Get the variable name from the tensor name."""
    m = re.match("^(.*):\\d+$", param_name)
    if m is not None:
      param_name = m.group(1)
    return param_name

if __name__ == "__main__":
  flags.DEFINE_string("clean_input_ckpt", "", "input ckpt for cleaning")
  flags.DEFINE_string("clean_output_model_dir", "", "output dir for cleaned ckpt")

  FLAGS = flags.FLAGS

  tf.app.run(clean_ckpt)
<commit_msg>add lr layer decay<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import re
import numpy as np
import six
from os.path import join
from six.moves import zip

from absl import flags

import tensorflow as tf

def configure_tpu(FLAGS):
  if FLAGS.use_tpu:
    tpu_cluster = tf.contrib.cluster_resolver.TPUClusterResolver(
        FLAGS.tpu, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
    master = tpu_cluster.get_master()
  else:
    tpu_cluster = None
    master = FLAGS.master

  session_config = tf.ConfigProto(allow_soft_placement=True)
  # Uncomment the following line if you hope to monitor GPU RAM growth
  # session_config.gpu_options.allow_growth = True

  if FLAGS.use_tpu:
    strategy = None
    tf.logging.info('Use TPU without distribute strategy.')
  elif FLAGS.num_core_per_host == 1:
    strategy = None
    tf.logging.info('Single device mode.')
  else:
    strategy = tf.contrib.distribute.MirroredStrategy(
        num_gpus=FLAGS.num_core_per_host)
    tf.logging.info('Use MirroredStrategy with %d devices.',
                    strategy.num_replicas_in_sync)

  per_host_input = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
  run_config = tf.contrib.tpu.RunConfig(
      master=master,
      model_dir=FLAGS.model_dir,
      session_config=session_config,
      tpu_config=tf.contrib.tpu.TPUConfig(
          iterations_per_loop=FLAGS.iterations,
          num_shards=FLAGS.num_hosts * FLAGS.num_core_per_host,
          per_host_input_for_training=per_host_input),
      keep_checkpoint_max=FLAGS.max_save,
      save_checkpoints_secs=None,
      save_checkpoints_steps=FLAGS.save_steps,
      train_distribute=strategy
  )
  return run_config

def init_from_checkpoint(FLAGS, global_vars=False):
  tvars = tf.global_variables() if global_vars else tf.trainable_variables()
  initialized_variable_names = {}
  scaffold_fn = None
  if FLAGS.init_checkpoint is not None:
    if FLAGS.init_checkpoint.endswith("latest"):
      ckpt_dir = os.path.dirname(FLAGS.init_checkpoint)
      init_checkpoint = tf.train.latest_checkpoint(ckpt_dir)
    else:
      init_checkpoint = FLAGS.init_checkpoint

    tf.logging.info("Initialize from the ckpt {}".format(init_checkpoint))

    (assignment_map, initialized_variable_names
    ) = get_assignment_map_from_checkpoint(tvars, init_checkpoint)
    if FLAGS.use_tpu:
      def tpu_scaffold():
        tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
        return tf.train.Scaffold()

      scaffold_fn = tpu_scaffold
    else:
      tf.train.init_from_checkpoint(init_checkpoint, assignment_map)

    # Log customized initialization
    tf.logging.info("**** Global Variables ****")
    for var in tvars:
      init_string = ""
      if var.name in initialized_variable_names:
        init_string = ", *INIT_FROM_CKPT*"
      tf.logging.info("  name = %s, shape = %s%s", var.name, var.shape,
                      init_string)
  return scaffold_fn

def get_train_op(FLAGS, total_loss, grads_and_vars=None):
  global_step = tf.train.get_or_create_global_step()

  # increase the learning rate linearly
  if FLAGS.warmup_steps > 0:
    warmup_lr = (tf.cast(global_step, tf.float32)
                 / tf.cast(FLAGS.warmup_steps, tf.float32)
                 * FLAGS.learning_rate)
  else:
    warmup_lr = 0.0

  # decay the learning rate
  if FLAGS.decay_method == "poly":
    decay_lr = tf.train.polynomial_decay(
        FLAGS.learning_rate,
        global_step=global_step - FLAGS.warmup_steps,
        decay_steps=FLAGS.train_steps - FLAGS.warmup_steps,
        end_learning_rate=FLAGS.learning_rate * FLAGS.min_lr_ratio)
  elif FLAGS.decay_method == "cos":
    decay_lr = tf.train.cosine_decay(
        FLAGS.learning_rate,
        global_step=global_step - FLAGS.warmup_steps,
        decay_steps=FLAGS.train_steps - FLAGS.warmup_steps,
        alpha=FLAGS.min_lr_ratio)
  else:
    raise ValueError(FLAGS.decay_method)

  learning_rate = tf.where(global_step < FLAGS.warmup_steps,
                           warmup_lr, decay_lr)

  if FLAGS.weight_decay == 0:
    optimizer = tf.train.AdamOptimizer(
        learning_rate=learning_rate,
        epsilon=FLAGS.adam_epsilon)
  elif FLAGS.weight_decay > 0 and FLAGS.num_core_per_host == 1:
    optimizer = AdamWeightDecayOptimizer(
        learning_rate=learning_rate,
        epsilon=FLAGS.adam_epsilon,
        exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"],
        weight_decay_rate=FLAGS.weight_decay)
  else:
    raise ValueError("Do not support `weight_decay > 0` with multi-gpu "
                     "training so far.")

  if FLAGS.use_tpu:
    optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)

  if grads_and_vars is None:
    grads_and_vars = optimizer.compute_gradients(total_loss)
  gradients, variables = zip(*grads_and_vars)
  clipped, gnorm = tf.clip_by_global_norm(gradients, FLAGS.clip)

  if FLAGS.lr_layer_decay_rate != 1.0:
    n_layer = 0
    for i in range(len(clipped)):
      m = re.search(r"model/transformer/layer_(\d+?)/", variables[i].name)
      if not m: continue
      n_layer = max(n_layer, int(m.group(1)) + 1)

    for i in range(len(clipped)):
      for l in range(n_layer):
        if "model/transformer/layer_{}/".format(l) in variables[i].name:
          abs_rate = FLAGS.lr_layer_decay_rate ** (n_layer - 1 - l)
          clipped[i] *= abs_rate
          tf.logging.info("Apply mult {:.4f} to layer-{} grad of {}".format(
              abs_rate, l, variables[i].name))
          break

  train_op = optimizer.apply_gradients(
      zip(clipped, variables), global_step=global_step)

  # Manually increment `global_step` for AdamWeightDecayOptimizer
  if isinstance(optimizer, AdamWeightDecayOptimizer):
    new_global_step = global_step + 1
    train_op = tf.group(train_op, [global_step.assign(new_global_step)])

  return train_op, learning_rate, gnorm

def clean_ckpt(_):
  input_ckpt = FLAGS.clean_input_ckpt
  output_model_dir = FLAGS.clean_output_model_dir

  tf.reset_default_graph()

  var_list = tf.contrib.framework.list_variables(input_ckpt)
  var_values, var_dtypes = {}, {}
  for (name, shape) in var_list:
    if not name.startswith("global_step") and "adam" not in name.lower():
      var_values[name] = None
      tf.logging.info("Include {}".format(name))
    else:
      tf.logging.info("Exclude {}".format(name))

  tf.logging.info("Loading from {}".format(input_ckpt))
  reader = tf.contrib.framework.load_checkpoint(input_ckpt)
  for name in var_values:
    tensor = reader.get_tensor(name)
    var_dtypes[name] = tensor.dtype
    var_values[name] = tensor

  with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
    tf_vars = [
        tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
        for v in var_values
    ]
  placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
  assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
  global_step = tf.Variable(
      0, name="global_step", trainable=False, dtype=tf.int64)
  saver = tf.train.Saver(tf.all_variables())

  if not tf.gfile.Exists(output_model_dir):
    tf.gfile.MakeDirs(output_model_dir)

  # Build a model consisting only of variables, set them to the average values.
  with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                           six.iteritems(var_values)):
      sess.run(assign_op, {p: value})

    # Use the built saver to save the averaged checkpoint.
    saver.save(sess, join(output_model_dir, "model.ckpt"),
               global_step=global_step)

def avg_checkpoints(model_dir, output_model_dir, last_k):
  tf.reset_default_graph()

  checkpoint_state = tf.train.get_checkpoint_state(model_dir)
  checkpoints = checkpoint_state.all_model_checkpoint_paths[- last_k:]
  var_list = tf.contrib.framework.list_variables(checkpoints[0])
  var_values, var_dtypes = {}, {}
  for (name, shape) in var_list:
    if not name.startswith("global_step"):
      var_values[name] = np.zeros(shape)
  for checkpoint in checkpoints:
    reader = tf.contrib.framework.load_checkpoint(checkpoint)
    for name in var_values:
      tensor = reader.get_tensor(name)
      var_dtypes[name] = tensor.dtype
      var_values[name] += tensor
    tf.logging.info("Read from checkpoint %s", checkpoint)
  for name in var_values:  # Average.
    var_values[name] /= len(checkpoints)

  with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
    tf_vars = [
        tf.get_variable(v, shape=var_values[v].shape, dtype=var_dtypes[v])
        for v in var_values
    ]
  placeholders = [tf.placeholder(v.dtype, shape=v.shape) for v in tf_vars]
  assign_ops = [tf.assign(v, p) for (v, p) in zip(tf_vars, placeholders)]
  global_step = tf.Variable(
      0, name="global_step", trainable=False, dtype=tf.int64)
  saver = tf.train.Saver(tf.all_variables())

  # Build a model consisting only of variables, set them to the average values.
  with tf.Session() as sess:
    sess.run(tf.initialize_all_variables())
    for p, assign_op, (name, value) in zip(placeholders, assign_ops,
                                           six.iteritems(var_values)):
      sess.run(assign_op, {p: value})
    # Use the built saver to save the averaged checkpoint.
    saver.save(sess, join(output_model_dir, "model.ckpt"),
        global_step=global_step)

def get_assignment_map_from_checkpoint(tvars, init_checkpoint):
  """Compute the union of the current variables and checkpoint variables."""
  assignment_map = {}
  initialized_variable_names = {}

  name_to_variable = collections.OrderedDict()
  for var in tvars:
    name = var.name
    m = re.match("^(.*):\\d+$", name)
    if m is not None:
      name = m.group(1)
    name_to_variable[name] = var

  init_vars = tf.train.list_variables(init_checkpoint)

  assignment_map = collections.OrderedDict()
  for x in init_vars:
    (name, var) = (x[0], x[1])
    # tf.logging.info('original name: %s', name)
    if name not in name_to_variable:
      continue
    # assignment_map[name] = name
    assignment_map[name] = name_to_variable[name]
    initialized_variable_names[name] = 1
    initialized_variable_names[name + ":0"] = 1

  return (assignment_map, initialized_variable_names)

class AdamWeightDecayOptimizer(tf.train.Optimizer):
  """A basic Adam optimizer that includes "correct" L2 weight decay."""

  def __init__(self,
               learning_rate,
               weight_decay_rate=0.0,
               beta_1=0.9,
               beta_2=0.999,
               epsilon=1e-6,
               exclude_from_weight_decay=None,
               include_in_weight_decay=["r_s_bias", "r_r_bias", "r_w_bias"],
               name="AdamWeightDecayOptimizer"):
    """Constructs a AdamWeightDecayOptimizer."""
    super(AdamWeightDecayOptimizer, self).__init__(False, name)

    self.learning_rate = learning_rate
    self.weight_decay_rate = weight_decay_rate
    self.beta_1 = beta_1
    self.beta_2 = beta_2
    self.epsilon = epsilon
    self.exclude_from_weight_decay = exclude_from_weight_decay
    self.include_in_weight_decay = include_in_weight_decay

  def apply_gradients(self, grads_and_vars, global_step=None, name=None):
    """See base class."""
    assignments = []
    for (grad, param) in grads_and_vars:
      if grad is None or param is None:
        continue

      param_name = self._get_variable_name(param.name)

      m = tf.get_variable(
          name=param_name + "/adam_m",
          shape=param.shape.as_list(),
          dtype=tf.float32,
          trainable=False,
          initializer=tf.zeros_initializer())
      v = tf.get_variable(
          name=param_name + "/adam_v",
          shape=param.shape.as_list(),
          dtype=tf.float32,
          trainable=False,
          initializer=tf.zeros_initializer())

      # Standard Adam update.
      next_m = (
          tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad))
      next_v = (
          tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2,
                                                    tf.square(grad)))

      update = next_m / (tf.sqrt(next_v) + self.epsilon)

      # Just adding the square of the weights to the loss function is *not*
      # the correct way of using L2 regularization/weight decay with Adam,
      # since that will interact with the m and v parameters in strange ways.
      #
      # Instead we want ot decay the weights in a manner that doesn't interact
      # with the m/v parameters. This is equivalent to adding the square
      # of the weights to the loss with plain (non-momentum) SGD.
      if self._do_use_weight_decay(param_name):
        update += self.weight_decay_rate * param

      update_with_lr = self.learning_rate * update

      next_param = param - update_with_lr

      assignments.extend(
          [param.assign(next_param),
           m.assign(next_m),
           v.assign(next_v)])

    return tf.group(*assignments, name=name)

  def _do_use_weight_decay(self, param_name):
    """Whether to use L2 weight decay for `param_name`."""
    if not self.weight_decay_rate:
      return False
    for r in self.include_in_weight_decay:
      if re.search(r, param_name) is not None:
        return True

    if self.exclude_from_weight_decay:
      for r in self.exclude_from_weight_decay:
        if re.search(r, param_name) is not None:
          tf.logging.info('Adam WD excludes {}'.format(param_name))
          return False
    return True

  def _get_variable_name(self, param_name):
    """Get the variable name from the tensor name."""
    m = re.match("^(.*):\\d+$", param_name)
    if m is not None:
      param_name = m.group(1)
    return param_name

if __name__ == "__main__":
  flags.DEFINE_string("clean_input_ckpt", "", "input ckpt for cleaning")
  flags.DEFINE_string("clean_output_model_dir", "", "output dir for cleaned ckpt")

  FLAGS = flags.FLAGS

  tf.app.run(clean_ckpt)
<|endoftext|>


<commit_before>
import os
import logging

from flask import Flask

app = Flask(__name__)
app.config.from_object('alerta.settings')
app.config.from_pyfile('/etc/alertad.conf', silent=True)
app.config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True)

if 'SECRET_KEY' in os.environ:
    app.config['SECRET_KEY'] = os.environ['SECRET_KEY']

if not app.debug:
    app.logger.addHandler(logging.StreamHandler())
    app.logger.setLevel(logging.INFO)

from alerta.app.database import Mongo
db = Mongo()

import views
import management.views

def main():
    app.run(host='0.0.0.0', port=8080, debug=True)<commit_msg>flask should respect global debug setting<commit_after>
import os
import logging

from flask import Flask

app = Flask(__name__)

app.config.from_object('alerta.settings')
app.config.from_pyfile('/etc/alertad.conf', silent=True)
app.config.from_envvar('ALERTA_SVR_CONF_FILE', silent=True)

if 'SECRET_KEY' in os.environ:
    app.config['SECRET_KEY'] = os.environ['SECRET_KEY']

if not app.debug:
    app.logger.addHandler(logging.StreamHandler())
    app.logger.setLevel(logging.INFO)

from alerta.app.database import Mongo
db = Mongo()

import views
import management.views

def main():
    app.run(host='0.0.0.0', port=8080)

<|endoftext|>


<commit_before># encoding: utf8

# Temporary script to update the users browser listeners and played info

import cinje
from web.core import Application
from marrow.package.loader import load
from web.ext.djdb import DJDatabaseExtension
from web.ext.dj import DJExtension
from web.app.djrq.model.lastplay import DJs
from web.app.djrq.send_update import send_update
from web.app.djrq.templates.lastplayed import lastplayed_row
from sqlalchemy.sql import func
from time import sleep
import yaml

class FakeContext:
    """ A fake session, just used to query the database """
    def __init__(self, db={}):
        self.db = db

    @property
    def db(self):
        return self.__db

    @db.setter
    def db(self, db):
        self.__db=db

with open('../web/app/djrq/config.yaml') as f:
    config= yaml.safe_load(f)

if __name__ == '__main__':
    databases = DJDatabaseExtension(config=config)
    lp = databases.engines['lastplay']
    context = FakeContext()
    lp.start(context)
    l = lp.Session.query(DJs).filter(DJs.hide_from_menu == 0)
    djs = {}
    for djrow in l:
        package = 'web.app.djrq.model.'+djrow.databasetype
        djname = djrow.dj.lower()
        djs[djname] = {'databasetype': djrow.databasetype,
                    'package': 'web.app.djrq.model.'+djrow.databasetype,
                    'lp_id': None,
                    'current_listeners': 0,
                    'current_max_listeners': 0,
                    'queries': load(package + '.queries:Queries'),
                    'listeners': load(package + '.listeners:Listeners'),
                    'websocket': 'http://{}.rockitradio.info/pub?id={}'.format(djname, djname),
                    }
        #djs[djname]['websocket'] = 'http://dj-{}.gelth.local/pub?id=dj-{}'.format(djname, djname)

    lp.Session.close()
    lp.stop(context)

    while 1:
        # Get the latest played for each DJ
        for dj in databases.engines:
            elements_to_update = {}
            if dj == 'lastplay': continue

            db = databases.engines[dj]
            context = FakeContext()
            db.start(context)
            queries = djs[dj]['queries'](db=db.Session)

            try:
                lp = queries.get_last_played(count=1)
            except:
                print("No lastplay for {}".format(dj))
            else:
                try:
                    r = lp[0]
                except:
                    #print('Bad row', dj)
                    continue

                if r[0] == 0: continue

                if djs[dj]['lp_id'] is None:
                    print("Must be first run, setting id", dj, r.Played.played_id, r.Played.song.title)
                    djs[dj]['lp_id'] = r.Played.played_id
                elif r.Played.played_id == djs[dj]['lp_id']:
                    pass # Skip if no change
                else:
                    new_row = cinje.flatten(lastplayed_row(None, r, ma=True, played=True))
                    print('Sending update', new_row)
                    elements_to_update['lastplay'] = new_row
                    djs[dj]['lp_id'] = r.Played.played_id
            try:
                listeners_row = db.Session.query(djs[dj]['listeners']).one()
            except:
                pass # Skip because no listeners info
            else:
                if listeners_row.current != djs[dj]['current_listeners']:
                    djs[dj]['current_listeners'] = listeners_row.current
                    elements_to_update['listeners'] = listeners_row.current
                if listeners_row.max != djs[dj]['current_max_listeners']:
                    djs[dj]['current_max_listeners'] = listeners_row.max
                    elements_to_update['maxlisteners'] = listeners_row.max

            if elements_to_update != {}:
                print('Sending update to', djs[dj]['websocket'], elements_to_update)
                send_update(djs[dj]['websocket'], **elements_to_update)
            db.Session.close()
            db.stop(context)
        sleep(30)
<commit_msg>Changed update_browser_info to check for new info every 10 seconds.<commit_after># encoding: utf8

# Temporary script to update the users browser listeners and played info

import cinje
from web.core import Application
from marrow.package.loader import load
from web.ext.djdb import DJDatabaseExtension
from web.ext.dj import DJExtension
from web.app.djrq.model.lastplay import DJs
from web.app.djrq.send_update import send_update
from web.app.djrq.templates.lastplayed import lastplayed_row
from sqlalchemy.sql import func
from time import sleep
import yaml

class FakeContext:
    """ A fake session, just used to query the database """
    def __init__(self, db={}):
        self.db = db

    @property
    def db(self):
        return self.__db

    @db.setter
    def db(self, db):
        self.__db=db

with open('../web/app/djrq/config.yaml') as f:
    config= yaml.safe_load(f)

if __name__ == '__main__':
    databases = DJDatabaseExtension(config=config)
    lp = databases.engines['lastplay']
    context = FakeContext()
    lp.start(context)
    l = lp.Session.query(DJs).filter(DJs.hide_from_menu == 0)
    djs = {}
    for djrow in l:
        package = 'web.app.djrq.model.'+djrow.databasetype
        djname = djrow.dj.lower()
        djs[djname] = {'databasetype': djrow.databasetype,
                    'package': 'web.app.djrq.model.'+djrow.databasetype,
                    'lp_id': None,
                    'current_listeners': 0,
                    'current_max_listeners': 0,
                    'queries': load(package + '.queries:Queries'),
                    'listeners': load(package + '.listeners:Listeners'),
                    'websocket': 'http://{}.rockitradio.info/pub?id={}'.format(djname, djname),
                    }
        #djs[djname]['websocket'] = 'http://dj-{}.gelth.local/pub?id=dj-{}'.format(djname, djname)

    lp.Session.close()
    lp.stop(context)

    while 1:
        # Get the latest played for each DJ
        for dj in databases.engines:
            elements_to_update = {}
            if dj == 'lastplay': continue

            db = databases.engines[dj]
            context = FakeContext()
            db.start(context)
            queries = djs[dj]['queries'](db=db.Session)

            try:
                lp = queries.get_last_played(count=1)
            except:
                print("No lastplay for {}".format(dj))
            else:
                try:
                    r = lp[0]
                except:
                    #print('Bad row', dj)
                    continue

                if r[0] == 0: continue

                if djs[dj]['lp_id'] is None:
                    print("Must be first run, setting id", dj, r.Played.played_id, r.Played.song.title)
                    djs[dj]['lp_id'] = r.Played.played_id
                elif r.Played.played_id == djs[dj]['lp_id']:
                    pass # Skip if no change
                else:
                    new_row = cinje.flatten(lastplayed_row(None, r, ma=True, played=True))
                    print('Sending update', new_row)
                    elements_to_update['lastplay'] = new_row
                    djs[dj]['lp_id'] = r.Played.played_id
            try:
                listeners_row = db.Session.query(djs[dj]['listeners']).one()
            except:
                pass # Skip because no listeners info
            else:
                if listeners_row.current != djs[dj]['current_listeners']:
                    djs[dj]['current_listeners'] = listeners_row.current
                    elements_to_update['listeners'] = listeners_row.current
                if listeners_row.max != djs[dj]['current_max_listeners']:
                    djs[dj]['current_max_listeners'] = listeners_row.max
                    elements_to_update['maxlisteners'] = listeners_row.max

            if elements_to_update != {}:
                print('Sending update to', djs[dj]['websocket'], elements_to_update)
                send_update(djs[dj]['websocket'], **elements_to_update)
            db.Session.close()
            db.stop(context)
        sleep(10)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
#       simple.py
#
# Copyright (C) 2011 Thibault Saunier <thibault.saunier@collabora.co.uk>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3, or (at your option)
# any later version.

from gst import ges

import sys
import getopt
import gst
import glib

class Demo:
    def __init__(self, files):
        self.tl_objs = []
        self.set_pipeline (files)

        for f in files:
            self.add_tl_object(f)
            self.mainloop = glib.MainLoop()

    def add_tl_object(self, file):
        self.tl_objs.append(ges.TimelineFileSource (file))
        self.layer.add_object(self.tl_objs[-1], -1)
        transition = ges.TimelineStandardTransition("crossfade")
        transition.duration = gst.SECOND * 2
        self.layer.add_object(transition, -1)

    def bus_handler(self, unused_bus, message):
        if message.type == gst.MESSAGE_ERROR:
            print "ERROR"
            self.mainloop.quit()
        elif message.type == gst.MESSAGE_EOS:
            print "Done"
            self.mainloop.quit()

        return gst.BUS_PASS

    def set_pipeline (self, files):
        try:
            opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
        except getopt.error, msg:
            print msg
            print "for help use --help"

        ges.init()

        tl = ges.timeline_new_audio_video()
        layer = ges.SimpleTimelineLayer()
        tl.add_layer(layer)

        pipeline = ges.TimelinePipeline()
        pipeline.add_timeline (tl)
        bus = pipeline.get_bus()
        bus.set_sync_handler(self.bus_handler)

        self.pipeline = pipeline
        self.layer = layer

    def run(self):
        if (self.pipeline.set_state(gst.STATE_PLAYING) == gst.STATE_CHANGE_FAILURE):
            print "Couldn't start pipeline"

        self.mainloop.run()

def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "h", ["help"])
    except getopt.error, msg:
        print msg
        print "for help use --help"
        sys.exit(2)
        # process options
    for o, a in opts:
        if o in ("-h", "--help"):
            print __doc__
            sys.exit(0)

    demo = Demo(args)
    demo.run()

if __name__ == "__main__":
    main()
<commit_msg>pyges: fix and clean examples/simple.py<commit_after>#!/usr/bin/env python
#
#       simple.py
#
# Copyright (C) 2011 Thibault Saunier <thibault.saunier@collabora.co.uk>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3, or (at your option)
# any later version.

import sys
import optparse

import glib
import gobject
gobject.threads_init()

import gst
import ges

class Simple:
    def __init__(self, uris):
        # init ges to have debug logs
        ges.init()
        self.mainloop = glib.MainLoop()

        timeline = ges.timeline_new_audio_video()
        self.layer = ges.SimpleTimelineLayer()
        timeline.add_layer(self.layer)

        self.pipeline = ges.TimelinePipeline()
        self.pipeline.add_timeline(timeline)
        bus = self.pipeline.get_bus()
        bus.set_sync_handler(self.bus_handler)

        # all timeline objects except the last will have a transition at the end
        for n in uris[:-1]:
            self.add_timeline_object(n, True)
        self.add_timeline_object(uris[-1], False)

    def add_timeline_object(self, uri, do_transition):
        filesource = ges.TimelineFileSource (uri)
        filesource.set_duration(long (gst.SECOND * 5))
        self.layer.add_object(filesource, -1)

        if do_transition:
            transition = ges.TimelineStandardTransition("crossfade")
            transition.duration = gst.SECOND * 2
            self.layer.add_object(transition, -1)

    def bus_handler(self, unused_bus, message):
        if message.type == gst.MESSAGE_ERROR:
            print "ERROR"
            self.mainloop.quit()
        elif message.type == gst.MESSAGE_EOS:
            print "Done"
            self.mainloop.quit()

        return gst.BUS_PASS

    def run(self):
        if (self.pipeline.set_state(gst.STATE_PLAYING) == \
                gst.STATE_CHANGE_FAILURE):
            print "Couldn't start pipeline"

        self.mainloop.run()

def main(args):
    usage = "usage: %s URI-OF-VIDEO-1 ... URI-OF-VIDEO-n\n" % args[0]

    if len(args) < 2:
        sys.stderr.write(usage)
        sys.exit(1)

    parser = optparse.OptionParser (usage=usage)
    (options, args) = parser.parse_args ()

    simple = Simple(args)
    simple.run()

if __name__ == "__main__":
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before># https://github.com/freelawproject/courtlistener/issues/636
import logging
import os

from django.core.management import BaseCommand
from django.db import IntegrityError

from cl.corpus_importer.tasks import download_recap_item
from cl.lib.pacer import PacerXMLParser
from cl.lib.recap_utils import get_ia_document_url_from_path, \
    get_local_document_url_from_path
from cl.search.models import RECAPDocument, Docket
from cl.scrapers.tasks import get_page_count

logger = logging.getLogger(__name__)

class CleanupPacerXMLParser(PacerXMLParser):
    """Overrides the normal parser with one that does cleanup properly."""

    def make_recap_document(self, doc_node, docket_entry, entry_number,
                            attachment_number, document_type, debug):
        """Do nothing for items that don't start with zero. For ones that do,
        find the stripped version, fix it, download the correct item, extract
        it and finally save it to Solr.
        """

        if not entry_number.startswith('0'):
            # Only touch things where the new value leads with a zero.
            return None
        else:
            logger.info("  Doing docket_entry: %s, document_number, "
                        "%s and attachment number: %s" %
                        (docket_entry, entry_number, attachment_number))
        old_entry_number = int(entry_number)

        try:
            d = RECAPDocument.objects.get(
                docket_entry=docket_entry,
                document_number=old_entry_number,
                attachment_number=attachment_number or None,
            )
            logger.info("    Found item.")
        except RECAPDocument.DoesNotExist:
            logger.info("    Failed to find item.")
            return None

        d.document_number = entry_number
        if d.is_available:
            new_ia = get_ia_document_url_from_path(
                self.path, entry_number, attachment_number)
            logger.info("    Updating IA URL from %s to %s" %
                        (d.filepath_ia, new_ia))
            d.filepath_ia = new_ia

            if not os.path.isfile(d.filepath_local.path):
                # Set the value correctly and get the file from IA if we don't
                # already have it.
                new_local_path = os.path.join(
                    'recap',
                    get_local_document_url_from_path(self.path, entry_number,
                                                     attachment_number),
                )
                logger.info("    Updating local path from %s to %s" %
                            (d.filepath_local, new_local_path))
                d.filepath_local = new_local_path
                filename = d.filepath_ia.rsplit('/', 1)[-1]
                logger.info("    Downloading item with filename %s" % filename)
                if not debug:
                    download_recap_item(d.filepath_ia, filename)
            else:
                logger.info("    File already on disk. Punting.")

            if d.page_count is None:
                logger.info("    Getting page count.")
                extension = d.filepath_local.path.split('.')[-1]
                d.page_count = get_page_count(d.filepath_local.path, extension)
        else:
            logger.info("    Item not available in RECAP. Punting.")
            return None

        if not debug:
            try:
                d.save(do_extraction=True, index=True)
                logger.info("    Item saved at https://www.courtlistener.com%s"
                            % d.get_absolute_url())
            except IntegrityError:
                logger.info("    Integrity error while saving.")
                return None
        else:
            logger.info("    No save requested in debug mode.")

        return d

class Command(BaseCommand):
    help = "Fix issues identified in 636."

    def add_arguments(self, parser):
        parser.add_argument('--debug', dest='debug', action='store_true')
        parser.set_defaults(debug=False)

    def handle(self, *args, **options):
        dockets = Docket.objects.filter(
            docket_entries__recap_documents__ocr_status=RECAPDocument.OCR_FAILED)
        for docket in dockets:
            docket_path = docket.filepath_local.path
            try:
                logger.info("Doing docket at: %s" % docket_path)
                pacer_doc = CleanupPacerXMLParser(docket_path)
            except IOError:
                logger.info("Couldn't find docket at: %s" % docket_path)
            else:
                _ = pacer_doc.make_documents(
                    docket,
                    debug=options['debug'],
                )
<commit_msg>Moves extraction out of the save routine.<commit_after># https://github.com/freelawproject/courtlistener/issues/636
import logging
import os

from django.core.management import BaseCommand
from django.db import IntegrityError

from cl.corpus_importer.tasks import download_recap_item
from cl.lib.pacer import PacerXMLParser
from cl.lib.recap_utils import get_ia_document_url_from_path, \
    get_local_document_url_from_path
from cl.search.models import RECAPDocument, Docket
from cl.scrapers.tasks import get_page_count, extract_recap_pdf

logger = logging.getLogger(__name__)

class CleanupPacerXMLParser(PacerXMLParser):
    """Overrides the normal parser with one that does cleanup properly."""

    def make_recap_document(self, doc_node, docket_entry, entry_number,
                            attachment_number, document_type, debug):
        """Do nothing for items that don't start with zero. For ones that do,
        find the stripped version, fix it, download the correct item, extract
        it and finally save it to Solr.
        """

        if not entry_number.startswith('0'):
            # Only touch things where the new value leads with a zero.
            return None
        else:
            logger.info("  Doing docket_entry: %s, document_number, "
                        "%s and attachment number: %s" %
                        (docket_entry, entry_number, attachment_number))
        old_entry_number = int(entry_number)

        try:
            d = RECAPDocument.objects.get(
                docket_entry=docket_entry,
                document_number=old_entry_number,
                attachment_number=attachment_number or None,
            )
            logger.info("    Found item.")
        except RECAPDocument.DoesNotExist:
            logger.info("    Failed to find item.")
            return None

        d.document_number = entry_number
        if d.is_available:
            new_ia = get_ia_document_url_from_path(
                self.path, entry_number, attachment_number)
            logger.info("    Updating IA URL from %s to %s" %
                        (d.filepath_ia, new_ia))
            d.filepath_ia = new_ia

            if not os.path.isfile(d.filepath_local.path):
                # Set the value correctly and get the file from IA if we don't
                # already have it.
                new_local_path = os.path.join(
                    'recap',
                    get_local_document_url_from_path(self.path, entry_number,
                                                     attachment_number),
                )
                logger.info("    Updating local path from %s to %s" %
                            (d.filepath_local, new_local_path))
                d.filepath_local = new_local_path
                filename = d.filepath_ia.rsplit('/', 1)[-1]
                logger.info("    Downloading item with filename %s" % filename)
                if not debug:
                    download_recap_item(d.filepath_ia, filename)
            else:
                logger.info("    File already on disk. Punting.")

            if d.page_count is None:
                logger.info("    Getting page count.")
                extension = d.filepath_local.path.split('.')[-1]
                d.page_count = get_page_count(d.filepath_local.path, extension)
        else:
            logger.info("    Item not available in RECAP. Punting.")
            return None

        if not debug:
            try:
                extract_recap_pdf(d.pk)
                d.save(do_extraction=False, index=True)
                logger.info("    Item saved at https://www.courtlistener.com%s"
                            % d.get_absolute_url())
            except IntegrityError:
                logger.info("    Integrity error while saving.")
                return None
        else:
            logger.info("    No save requested in debug mode.")

        return d

class Command(BaseCommand):
    help = "Fix issues identified in 636."

    def add_arguments(self, parser):
        parser.add_argument('--debug', dest='debug', action='store_true')
        parser.set_defaults(debug=False)

    def handle(self, *args, **options):
        dockets = Docket.objects.filter(
            docket_entries__recap_documents__ocr_status=RECAPDocument.OCR_FAILED)
        for docket in dockets:
            docket_path = docket.filepath_local.path
            try:
                logger.info("Doing docket at: %s" % docket_path)
                pacer_doc = CleanupPacerXMLParser(docket_path)
            except IOError:
                logger.info("Couldn't find docket at: %s" % docket_path)
            else:
                _ = pacer_doc.make_documents(
                    docket,
                    debug=options['debug'],
                )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in module root
# directory
##############################################################################
from openerp import models, api, fields

class AccountPaymentGroup(models.Model):

    _inherit = "account.payment.group"

    # @api.model
    # def _get_regimen_ganancias(self):
    #     result = []
    #     for line in self.
    #     return

    retencion_ganancias = fields.Selection([
        # _get_regimen_ganancias,
        ('imposibilidad_retencion', 'Imposibilidad de Retención'),
        ('no_aplica', 'No Aplica'),
        ('nro_regimen', 'Nro Regimen'),
    ],
        'Retención Ganancias',
        readonly=True,
        states={'draft': [('readonly', False)],
                'confirmed': [('readonly', False)]}
    )
    regimen_ganancias_id = fields.Many2one(
        'afip.tabla_ganancias.alicuotasymontos',
        'Regimen Ganancias',
        readonly=True,
        ondelete='restrict',
        states={'draft': [('readonly', False)],
                'confirmed': [('readonly', False)]}
    )
    company_regimenes_ganancias_ids = fields.Many2many(
        'afip.tabla_ganancias.alicuotasymontos',
        compute='_company_regimenes_ganancias',
    )

    @api.multi
    @api.depends('company_id.regimenes_ganancias_ids')
    def _company_regimenes_ganancias(self):
        """
        Lo hacemos con campo computado y no related para que solo se setee
        y se exija si es pago de o a proveedor
        """
        for rec in self.filtered(lambda x: x.partner_type == 'supplier'):
            rec.company_regimenes_ganancias_ids = (
                rec.company_id.regimenes_ganancias_ids)

    @api.onchange('retencion_ganancias', 'commercial_partner_id')
    def change_retencion_ganancias(self):
        def_regimen = False
        if self.retencion_ganancias == 'nro_regimen':
            cia_regs = self.company_regimenes_ganancias_ids
            partner_regimen = (
                self.commercial_partner_id.default_regimen_ganancias_id)
            if partner_regimen and partner_regimen in cia_regs:
                def_regimen = partner_regimen
            elif cia_regs:
                def_regimen = cia_regs[0]
        self.regimen_ganancias_id = def_regimen

    @api.onchange('company_regimenes_ganancias_ids')
    def change_company_regimenes_ganancias(self):
        # partner_type == 'supplier' ya lo filtra el company_regimenes_ga...
        if self.company_regimenes_ganancias_ids:
            self.retencion_ganancias = 'nro_regimen'

    # sacamos esto por ahora ya que no es muy prolijo y nos se esta usando, si
    # lo llegamos a activar entonces tener en cuenta que en sipreco no queremos
    # que en borrador se setee ninguna regimen de ganancias
    # @api.model
    # def create(self, vals):
    #     """
    #     para casos donde se paga desde algun otro lugar (por ej. liquidador
    #     de impuestos), seteamos no aplica si no hay nada seteado
    #     """
    #     payment_group = super(AccountPaymentGroup, self).create(vals)
    #     if (
    #             payment_group.company_regimenes_ganancias_ids and
    #             payment_group.partner_type == 'supplier' and
    #             not payment_group.retencion_ganancias and
    #             not payment_group.regimen_ganancias_id):
    #         payment_group.retencion_ganancias = 'no_aplica'
    #     return payment_group
<commit_msg>[9.0] Fix onchange of m2m computed fields used in domains<commit_after># -*- coding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in module root
# directory
##############################################################################
from openerp import models, api, fields

class AccountPaymentGroup(models.Model):

    _inherit = "account.payment.group"

    # @api.model
    # def _get_regimen_ganancias(self):
    #     result = []
    #     for line in self.
    #     return

    retencion_ganancias = fields.Selection([
        # _get_regimen_ganancias,
        ('imposibilidad_retencion', 'Imposibilidad de Retención'),
        ('no_aplica', 'No Aplica'),
        ('nro_regimen', 'Nro Regimen'),
    ],
        'Retención Ganancias',
        readonly=True,
        states={'draft': [('readonly', False)],
                'confirmed': [('readonly', False)]}
    )
    regimen_ganancias_id = fields.Many2one(
        'afip.tabla_ganancias.alicuotasymontos',
        'Regimen Ganancias',
        readonly=True,
        ondelete='restrict',
        states={'draft': [('readonly', False)],
                'confirmed': [('readonly', False)]}
    )
    company_regimenes_ganancias_ids = fields.Many2many(
        'afip.tabla_ganancias.alicuotasymontos',
        compute='_company_regimenes_ganancias',
    )

    @api.multi
    def onchange(self, values, field_name, field_onchange):
        """
        Idea obtenida de aca
        https://github.com/odoo/odoo/issues/16072#issuecomment-289833419
        por el cambio que se introdujo en esa mimsa conversación, TODO en v11
        no haría mas falta, simplemente domain="[('id', 'in', x2m_field)]"
        Otras posibilidades que probamos pero no resultaron del todo fue:
        * agregar onchange sobre campos calculados y que devuelvan un dict con
        domain. El tema es que si se entra a un registro guardado el onchange
        no se ejecuta
        * usae el modulo de web_domain_field que esta en un pr a la oca
        """
        for field in field_onchange.keys():
            if field.startswith('company_regimenes_ganancias_ids.'):
                del field_onchange[field]
        return super(AccountPaymentGroup, self).onchange(
            values, field_name, field_onchange)

    @api.multi
    @api.depends('company_id.regimenes_ganancias_ids')
    def _company_regimenes_ganancias(self):
        """
        Lo hacemos con campo computado y no related para que solo se setee
        y se exija si es pago de o a proveedor
        """
        for rec in self.filtered(lambda x: x.partner_type == 'supplier'):
            rec.company_regimenes_ganancias_ids = (
                rec.company_id.regimenes_ganancias_ids)

    @api.onchange('retencion_ganancias', 'commercial_partner_id')
    def change_retencion_ganancias(self):
        def_regimen = False
        if self.retencion_ganancias == 'nro_regimen':
            cia_regs = self.company_regimenes_ganancias_ids
            partner_regimen = (
                self.commercial_partner_id.default_regimen_ganancias_id)
            if partner_regimen and partner_regimen in cia_regs:
                def_regimen = partner_regimen
            elif cia_regs:
                def_regimen = cia_regs[0]
        self.regimen_ganancias_id = def_regimen

    @api.onchange('company_regimenes_ganancias_ids')
    def change_company_regimenes_ganancias(self):
        # partner_type == 'supplier' ya lo filtra el company_regimenes_ga...
        if self.company_regimenes_ganancias_ids:
            self.retencion_ganancias = 'nro_regimen'

    # sacamos esto por ahora ya que no es muy prolijo y nos se esta usando, si
    # lo llegamos a activar entonces tener en cuenta que en sipreco no queremos
    # que en borrador se setee ninguna regimen de ganancias
    # @api.model
    # def create(self, vals):
    #     """
    #     para casos donde se paga desde algun otro lugar (por ej. liquidador
    #     de impuestos), seteamos no aplica si no hay nada seteado
    #     """
    #     payment_group = super(AccountPaymentGroup, self).create(vals)
    #     if (
    #             payment_group.company_regimenes_ganancias_ids and
    #             payment_group.partner_type == 'supplier' and
    #             not payment_group.retencion_ganancias and
    #             not payment_group.regimen_ganancias_id):
    #         payment_group.retencion_ganancias = 'no_aplica'
    #     return payment_group
<|endoftext|>