<commit_before>from __future__ import absolute_import
from abc import ABCMeta, abstractmethod
from firedrake import Function, TestFunction, TrialFunction, \
    LinearVariationalProblem, LinearVariationalSolver, FacetNormal, \
    dx, dot, grad, div, jump, avg, dS, dS_v, dS_h, action, inner, outer

class Advection(object):
    """
    Base class for advection schemes for dcore.

    :arg state: :class:`.State` object.
    """
    __metaclass__ = ABCMeta

    def __init__(self, state):
        self.state = state
        self.ubar = Function(state.V[0])

    @abstractmethod
    def apply(self, x, x_out):
        """
        Function takes x as input, computes F(x) and returns x_out
        as output.

        :arg x: :class:`.Function` object, the input Function.
        :arg x_out: :class:`.Function` object, the output Function.
        """
        pass

class NoAdvection(Advection):
    """
    An non-advection scheme that does nothing.
    """

    def apply(self, x_in, x_out):

        x_out.assign(x_in)

class LinearAdvection_Vt(Advection):
    """
    An advection scheme that uses the linearised background state
    in evaluation of the advection term for the Vt temperature space.

    :arg state: :class:`.State` object.
    :arg qbar: :class:`.Function` object. The reference function that we
    are linearising around.
    :arg options: a PETSc options dictionary
    """

    def __init__(self, state, V, qbar, options=None):
        super(LinearAdvection_Vt, self).__init__(state)

        p = TestFunction(state.V[2])
        q = TrialFunction(state.V[2])

        self.dq = Function(state.V[2])

        a = p*q*dx
        k = state.parameters.k             # Upward pointing unit vector
        L = -p*dot(self.ubar,k)*dot(k,grad(qbar))*dx

        aProblem = LinearVariationalProblem(a,L,self.dq)
        if options is None:
            options = {'ksp_type':'cg',
                       'pc_type':'bjacobi',
                       'sub_pc_type':'ilu'}

        self.solver = LinearVariationalSolver(aProblem,
                                              solver_parameters=options)

    def apply(self, x_in, x_out):
        dt = self.state.timestepping.dt
        self.solver.solve()
        x_out.assign(x_in + dt*self.dq)

class LinearAdvection_V3(Advection):
    """
    An advection scheme that uses the linearised background state
    in evaluation of the advection term for the V3 DG space.

    :arg state: :class:`.State` object.
    :arg qbar: :class:`.Function` object. The reference function that we
    are linearising around.
    :arg options: a PETSc options dictionary
    """

    def __init__(self, state, qbar, options=None):
        super(LinearAdvection_V3, self).__init__(state)

        p = TestFunction(state.V[1])
        q = TrialFunction(state.V[1])

        self.dq = Function(state.V[1])

        n = FacetNormal(state.mesh)

        a = p*q*dx
        L = (dot(grad(p), self.ubar)*qbar*dx -
             jump(self.ubar*p, n)*avg(qbar)*(dS_v + dS_h))

        aProblem = LinearVariationalProblem(a,L,self.dq)
        if options is None:
            options = {'ksp_type':'cg',
                       'pc_type':'bjacobi',
                       'sub_pc_type':'ilu'}

        self.solver = LinearVariationalSolver(aProblem,
                                              solver_parameters=options)

    def apply(self, x_in, x_out):
        dt = self.state.timestepping.dt
        self.solver.solve()
        x_out.assign(x_in + dt*self.dq)

class DGAdvection(Advection):

    """
    DG 3 step SSPRK advection scheme that can be applied to a scalar
    or vector field

    :arg state: :class:`.State` object.
    :arg V: function space of advected field - should be DG
    :arg continuity: optional boolean
         if True, the advection equation is of the form:
         D_t + (u.\nabla)D = 0
         if False, the advection equations in of the form:
         D_t +\nabla .(uD) = 0
    """

    def __init__(self, state, V, continuity=False):

        super(DGAdvection, self).__init__(state)

        element = V.fiat_element
        assert element.entity_dofs() == element.entity_closure_dofs(), "Provided space is not discontinuous"
        dt = state.timestepping.dt

        phi = TestFunction(V)
        D = TrialFunction(V)
        self.D1 = Function(V)
        self.dD = Function(V)

        n = FacetNormal(state.mesh)
        # ( dot(v, n) + |dot(v, n)| )/2.0
        un = 0.5*(dot(self.ubar, n) + abs(dot(self.ubar, n)))

        a_mass = inner(phi,D)*dx

        if continuity:
            a_int = -inner(div(outer(phi,self.ubar)),D)*dx
        else:
            a_int = -inner(grad(phi), outer(D, self.ubar))*dx

        a_flux = (dot(jump(phi), un('+')*D('+') - un('-')*D('-')))*dS
        arhs = a_mass - dt*(a_int + a_flux)

        DGproblem = LinearVariationalProblem(a_mass, action(arhs,self.D1),
                                             self.dD)
        self.DGsolver = LinearVariationalSolver(DGproblem,
                                                solver_parameters={
                                                    'ksp_type':'preonly',
                                                    'pc_type':'bjacobi',
                                                    'sub_pc_type': 'ilu'})

    def apply(self, x_in, x_out):
        # SSPRK Stage 1
        self.D1.assign(x_in)
        self.DGsolver.solve()
        self.D1.assign(self.dD)

        # SSPRK Stage 2
        self.DGsolver.solve()
        self.D1.assign(0.75*x_in + 0.25*self.dD)

        # SSPRK Stage 3
        self.DGsolver.solve()

        x_out.assign((1.0/3.0)*x_in + (2.0/3.0)*self.dD)
<commit_msg>Gah lint!<commit_after>from __future__ import absolute_import
from abc import ABCMeta, abstractmethod
from firedrake import Function, TestFunction, TrialFunction, \
    LinearVariationalProblem, LinearVariationalSolver, FacetNormal, \
    dx, dot, grad, div, jump, avg, dS, dS_v, dS_h, action, inner, outer

class Advection(object):
    """
    Base class for advection schemes for dcore.

    :arg state: :class:`.State` object.
    """
    __metaclass__ = ABCMeta

    def __init__(self, state):
        self.state = state
        self.ubar = Function(state.V[0])

    @abstractmethod
    def apply(self, x, x_out):
        """
        Function takes x as input, computes F(x) and returns x_out
        as output.

        :arg x: :class:`.Function` object, the input Function.
        :arg x_out: :class:`.Function` object, the output Function.
        """
        pass

class NoAdvection(Advection):
    """
    An non-advection scheme that does nothing.
    """

    def apply(self, x_in, x_out):

        x_out.assign(x_in)

class LinearAdvection_Vt(Advection):
    """
    An advection scheme that uses the linearised background state
    in evaluation of the advection term for the Vt temperature space.

    :arg state: :class:`.State` object.
    :arg qbar: :class:`.Function` object. The reference function that we
    are linearising around.
    :arg options: a PETSc options dictionary
    """

    def __init__(self, state, V, qbar, options=None):
        super(LinearAdvection_Vt, self).__init__(state)

        p = TestFunction(state.V[2])
        q = TrialFunction(state.V[2])

        self.dq = Function(state.V[2])

        a = p*q*dx
        k = state.parameters.k             # Upward pointing unit vector
        L = -p*dot(self.ubar,k)*dot(k,grad(qbar))*dx

        aProblem = LinearVariationalProblem(a,L,self.dq)
        if options is None:
            options = {'ksp_type':'cg',
                       'pc_type':'bjacobi',
                       'sub_pc_type':'ilu'}

        self.solver = LinearVariationalSolver(aProblem,
                                              solver_parameters=options)

    def apply(self, x_in, x_out):
        dt = self.state.timestepping.dt
        self.solver.solve()
        x_out.assign(x_in + dt*self.dq)

class LinearAdvection_V3(Advection):
    """
    An advection scheme that uses the linearised background state
    in evaluation of the advection term for the V3 DG space.

    :arg state: :class:`.State` object.
    :arg qbar: :class:`.Function` object. The reference function that we
    are linearising around.
    :arg options: a PETSc options dictionary
    """

    def __init__(self, state, qbar, options=None):
        super(LinearAdvection_V3, self).__init__(state)

        p = TestFunction(state.V[1])
        q = TrialFunction(state.V[1])

        self.dq = Function(state.V[1])

        n = FacetNormal(state.mesh)

        a = p*q*dx
        L = (dot(grad(p), self.ubar)*qbar*dx -
             jump(self.ubar*p, n)*avg(qbar)*(dS_v + dS_h))

        aProblem = LinearVariationalProblem(a,L,self.dq)
        if options is None:
            options = {'ksp_type':'cg',
                       'pc_type':'bjacobi',
                       'sub_pc_type':'ilu'}

        self.solver = LinearVariationalSolver(aProblem,
                                              solver_parameters=options)

    def apply(self, x_in, x_out):
        dt = self.state.timestepping.dt
        self.solver.solve()
        x_out.assign(x_in + dt*self.dq)

class DGAdvection(Advection):

    """
    DG 3 step SSPRK advection scheme that can be applied to a scalar
    or vector field

    :arg state: :class:`.State` object.
    :arg V: function space of advected field - should be DG
    :arg continuity: optional boolean
         if True, the advection equation is of the form:
         D_t + (u.\nabla)D = 0
         if False, the advection equations in of the form:
         D_t +\nabla .(uD) = 0
    """

    def __init__(self, state, V, continuity=False):

        super(DGAdvection, self).__init__(state)

        element = V.fiat_element
        assert element.entity_dofs() == element.entity_closure_dofs(), "Provided space is not discontinuous"
        dt = state.timestepping.dt

        phi = TestFunction(V)
        D = TrialFunction(V)
        self.D1 = Function(V)
        self.dD = Function(V)

        n = FacetNormal(state.mesh)
        # ( dot(v, n) + |dot(v, n)| )/2.0
        un = 0.5*(dot(self.ubar, n) + abs(dot(self.ubar, n)))

        a_mass = inner(phi,D)*dx

        if continuity:
            a_int = -inner(div(outer(phi,self.ubar)),D)*dx
        else:
            a_int = -inner(grad(phi), outer(D, self.ubar))*dx

        a_flux = (dot(jump(phi), un('+')*D('+') - un('-')*D('-')))*dS
        arhs = a_mass - dt*(a_int + a_flux)

        DGproblem = LinearVariationalProblem(a_mass, action(arhs,self.D1),
                                             self.dD)
        self.DGsolver = LinearVariationalSolver(DGproblem,
                                                solver_parameters={
                                                    'ksp_type':'preonly',
                                                    'pc_type':'bjacobi',
                                                    'sub_pc_type': 'ilu'})

    def apply(self, x_in, x_out):
        # SSPRK Stage 1
        self.D1.assign(x_in)
        self.DGsolver.solve()
        self.D1.assign(self.dD)

        # SSPRK Stage 2
        self.DGsolver.solve()
        self.D1.assign(0.75*x_in + 0.25*self.dD)

        # SSPRK Stage 3
        self.DGsolver.solve()

        x_out.assign((1.0/3.0)*x_in + (2.0/3.0)*self.dD)
<|endoftext|>


<commit_before>'''
    Copyright (C) 2005-17 www.interpss.org

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
'''

'''
 Use NN-model to predict the bus voltage for a set of scale-factors
'''

from datetime import datetime
import numpy as np
import tensorflow as tf

import sys
sys.path.insert(0, '../..')

import lib.common_func as cf

train_points = 1000

#
# load the IEEE-14Bus case
#
filename = 'testdata/ieee14.ieee'
noBus, noBranch = cf.ipss_app.loadCase(filename, 'InterfacePowerRandomChangeTrainCaseBuilder')
print(filename, ' loaded,  no of Buses, Branches:', noBus, ', ', noBranch)

# define model size
size = noBus * 2
#print('size: ', size)

# define model variables
W1 = tf.Variable(tf.zeros([size*2,6]))
b1 = tf.Variable(tf.zeros([6]))

# define model

def nn_model(data):
    output = tf.matmul(data, W1) + b1
    return output

# define loss
x = tf.placeholder(tf.float32, [None, size*2])
y = tf.placeholder(tf.float32)

error = tf.square(nn_model(x) - y)
loss = tf.reduce_sum(error)

# define training optimization
optimizer = tf.train.AdadeltaOptimizer()
train = optimizer.minimize(loss)
init = tf.global_variables_initializer()
# run the computation graph
with tf.Session() as sess :
    sess.run(init)

    # run the training part
    # =====================

    print('Begin training: ', datetime.now())

    # retrieve training set
    trainSet = cf.ipss_app.getTrainSet(train_points)
    train_x, train_y = cf.transfer2PyArrays(trainSet)
    train_x,aver_x,ran_x = cf.normalization(train_x);

    train_y,aver_y,ran_y = cf.normalization(train_y);
    # run the training part
    for i in range(1000):
        if (i % 100 == 0) : print('Training step: ', i)
        sess.run(train, {x:train_x, y:train_y})

    print('End training: ', datetime.now())

    '''
    print('W1: ', sess.run(W1))
    print('b1: ', sess.run(b1))
    '''

    # run the verification part
    # =========================
    testSize=100
    misSet = np.zeros((testSize,6))
    # retrieve a test case
    for i in range(testSize) :
    #for factor in [0.45, 1.0, 1.55] :
        testCase = cf.ipss_app.getTestCase()
        test_x, test_y = cf.transfer2PyArrays(testCase)
        test_x =  np.divide(np.subtract(test_x,aver_x),ran_x)
        # compute model output (network voltage)
        model_y = sess.run(nn_model(x), {x:test_x})
        #printArray(model_y, 'model_y')
        misSet[i] =  np.abs(model_y[0]*ran_y+aver_y-test_y[0])

#         netVoltage = cf.transfer2JavaDblAry(model_y[0]*ran_y+aver_y, size)
#         mismatchSet[i] = np.array([cf.ipss_app.getMismatch(netVoltage)[0],cf.ipss_app.getMismatch(netVoltage)[1]])
#     train_mm,aver_mm,ran_mm = cf.normalization(mismatchSet);
    train_m,aver_m,ran_m = cf.normalization(misSet);
#     print('model out mismatch(aver): ', aver_mm)
#     print('model out mismatch(range): ', ran_mm)
#     print('aver case max error: ', aver_m)
#     print('max case max error : ', ran_m )
    print('max case max error : ', np.max(ran_m) )
    print('aver case max error : ', np.average(np.max(misSet, axis =1)) )
<commit_msg>no logic changed<commit_after>'''
    Copyright (C) 2005-17 www.interpss.org

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
'''

'''
 Use NN-model to predict the bus voltage for a set of scale-factors
'''

from datetime import datetime
import numpy as np
import tensorflow as tf

import sys
sys.path.insert(0, '../..')

import lib.common_func as cf

train_points = 1000

#
# load the IEEE-14Bus case
#
filename = 'testdata/ieee14.ieee'
noBus, noBranch = cf.ipss_app.loadCase(filename, 'InterfacePowerRandomChangeTrainCaseBuilder')
print(filename, ' loaded,  no of Buses, Branches:', noBus, ', ', noBranch)

# define model size
size = noBus * 4
#print('size: ', size)

# define model variables
W1 = tf.Variable(tf.zeros([size,6]))
b1 = tf.Variable(tf.zeros([6]))

# define model

def nn_model(data):
    output = tf.matmul(data, W1) + b1
    return output

# define loss
x = tf.placeholder(tf.float32, [None, size*2])
y = tf.placeholder(tf.float32)

error = tf.square(nn_model(x) - y)
loss = tf.reduce_sum(error)

# define training optimization
optimizer = tf.train.AdadeltaOptimizer()
train = optimizer.minimize(loss)
init = tf.global_variables_initializer()
# run the computation graph
with tf.Session() as sess :
    sess.run(init)

    # run the training part
    # =====================

    print('Begin training: ', datetime.now())

    # retrieve training set
    trainSet = cf.ipss_app.getTrainSet(train_points)
    train_x, train_y = cf.transfer2PyArrays(trainSet)
    train_x,aver_x,ran_x = cf.normalization(train_x);

    train_y,aver_y,ran_y = cf.normalization(train_y);
    # run the training part
    for i in range(1000):
        if (i % 100 == 0) : print('Training step: ', i)
        sess.run(train, {x:train_x, y:train_y})

    print('End training: ', datetime.now())

    '''
    print('W1: ', sess.run(W1))
    print('b1: ', sess.run(b1))
    '''

    # run the verification part
    # =========================
    testSize=100
    misSet = np.zeros((testSize,6))
    # retrieve a test case
    for i in range(testSize) :
    #for factor in [0.45, 1.0, 1.55] :
        testCase = cf.ipss_app.getTestCase()
        test_x, test_y = cf.transfer2PyArrays(testCase)
        test_x =  np.divide(np.subtract(test_x,aver_x),ran_x)
        # compute model output (network voltage)
        model_y = sess.run(nn_model(x), {x:test_x})
        #printArray(model_y, 'model_y')
        misSet[i] =  np.abs(model_y[0]*ran_y+aver_y-test_y[0])

#         netVoltage = cf.transfer2JavaDblAry(model_y[0]*ran_y+aver_y, size)
#         mismatchSet[i] = np.array([cf.ipss_app.getMismatch(netVoltage)[0],cf.ipss_app.getMismatch(netVoltage)[1]])
#     train_mm,aver_mm,ran_mm = cf.normalization(mismatchSet);
    train_m,aver_m,ran_m = cf.normalization(misSet);
#     print('model out mismatch(aver): ', aver_mm)
#     print('model out mismatch(range): ', ran_mm)
#     print('aver case max error: ', aver_m)
#     print('max case max error : ', ran_m )
    print('max case max error : ', np.max(ran_m) )
    print('aver case max error : ', np.average(np.max(misSet, axis =1)) )
<|endoftext|>


<commit_before>import logging
import sys
from datetime import datetime, timedelta
from django.conf import settings
from django.db import models
from django.test import TestCase, override_settings
from django.utils import timezone
from dj_phab.queryset import DateGroupingQuerySet

if 'test' in sys.argv:
    # System will fallback to syncdb behavior -- which allows the sample model below
    # to have a DB table built
    settings.MIGRATION_MODULES = {
        'dj_phab': 'dj_phab.migrations_not_used_in_tests'
    }

class DGQSTestModel(models.Model):
    a_date = models.DateTimeField()
    value = models.PositiveIntegerField()

    objects = models.Manager.from_queryset(DateGroupingQuerySet)()

class TestDateGroupingQuerySet(TestCase):
    def setUp(self):
        self.tz = timezone.get_current_timezone()

    def make_models(self, count, interval):
        instances = []
        base_date = timezone.make_aware(datetime(2014, 1, 1), self.tz)
        for i in range(count):
            val = i * interval
            instances.append(DGQSTestModel(**{
                'a_date': base_date + timedelta(val),
                'value': val,
            }))
            logging.debug([i, val, base_date + timedelta(val)])
        DGQSTestModel.objects.bulk_create(instances)

    def test_smoke(self):
        pass

    def test_group_year(self):
        # 2 each in 2014, 2015, 2016, 2017
        # values 0, 200, 400, 600, 800, 1000, 1200, 1400
        # sums 200, 1000, 1800, 2600
        self.make_models(8, 200)
        sums = DGQSTestModel.objects.group_by_date('a_date', 'year')\
                            .annotate(sum=models.Sum('value'))
        tuples = [(row['a_date_year'], row['sum']) for row in sums]
        self.assertItemsEqual(tuples, [(2014, 200), (2015, 1000), (2016, 1800), (2017, 2600)])

    def test_group_month(self):
        # 2 each in 2014-01, 2014-02, 2014-03, 2014-04
        # values 0, 16, 32, 48, 64, 80, 96, 112
        # sums 16, 80, 144, 208
        self.make_models(8, 16)
        sums = DGQSTestModel.objects.group_by_date('a_date', 'month')\
                            .annotate(sum=models.Sum('value'))
        tuples = [(row['a_date_year'], row['a_date_month'], row['sum']) for row in sums]
        self.assertItemsEqual(tuples, [(2014, 1, 16),
                                       (2014, 2, 80),
                                       (2014, 3, 144),
                                       (2014, 4, 208)])

    def test_group_week(self):
        # 2 each in 2014-01 weeks 1, 2, 3, 4
        # values 0, 3, 7, 10, 14, 18, 21, 25
        # sums 3, 17, 32, 46
        self.make_models(8, 3.6)
        sums = DGQSTestModel.objects.group_by_date('a_date', 'week')\
                            .annotate(sum=models.Sum('value'))
        tuples = [(row['a_date_year'], row['a_date_week'], row['sum']) for row in sums]
        self.assertItemsEqual(tuples, [(2014, 1, 3),
                                       (2014, 2, 17),
                                       (2014, 3, 32),
                                       (2014, 4, 46)])

    def test_group_day(self):
        self.fail()

    def test_group_and_filter(self):
        self.fail()
<commit_msg>test days<commit_after>import logging
import sys
from datetime import datetime, timedelta
from django.conf import settings
from django.db import models
from django.test import TestCase, override_settings
from django.utils import timezone
from dj_phab.queryset import DateGroupingQuerySet

if 'test' in sys.argv:
    # System will fallback to syncdb behavior -- which allows the sample model below
    # to have a DB table built
    settings.MIGRATION_MODULES = {
        'dj_phab': 'dj_phab.migrations_not_used_in_tests'
    }

class DGQSTestModel(models.Model):
    a_date = models.DateTimeField()
    value = models.PositiveIntegerField()

    objects = models.Manager.from_queryset(DateGroupingQuerySet)()

class TestDateGroupingQuerySet(TestCase):
    def setUp(self):
        self.tz = timezone.get_current_timezone()

    def make_models(self, count, interval):
        instances = []
        base_date = timezone.make_aware(datetime(2014, 1, 1), self.tz)
        for i in range(count):
            val = i * interval
            instances.append(DGQSTestModel(**{
                'a_date': base_date + timedelta(val),
                'value': val,
            }))
            logging.debug([i, val, base_date + timedelta(val)])
        DGQSTestModel.objects.bulk_create(instances)

    def test_smoke(self):
        pass

    def test_group_year(self):
        # 2 each in 2014, 2015, 2016, 2017
        # values 0, 200, 400, 600, 800, 1000, 1200, 1400
        # sums 200, 1000, 1800, 2600
        self.make_models(8, 200)
        sums = DGQSTestModel.objects.group_by_date('a_date', 'year')\
                            .annotate(sum=models.Sum('value'))
        tuples = [(row['a_date_year'], row['sum']) for row in sums]
        self.assertItemsEqual(tuples, [(2014, 200), (2015, 1000), (2016, 1800), (2017, 2600)])

    def test_group_month(self):
        # 2 each in 2014-01, 2014-02, 2014-03, 2014-04
        # values 0, 16, 32, 48, 64, 80, 96, 112
        # sums 16, 80, 144, 208
        self.make_models(8, 16)
        sums = DGQSTestModel.objects.group_by_date('a_date', 'month')\
                            .annotate(sum=models.Sum('value'))
        tuples = [(row['a_date_year'], row['a_date_month'], row['sum']) for row in sums]
        self.assertItemsEqual(tuples, [(2014, 1, 16),
                                       (2014, 2, 80),
                                       (2014, 3, 144),
                                       (2014, 4, 208)])

    def test_group_week(self):
        # 2 each in 2014-01 weeks 1, 2, 3, 4
        # values (rounding down) 0, 3, 7, 10, 14, 18, 21, 25
        # sums 3, 17, 32, 46
        self.make_models(8, 3.6)
        sums = DGQSTestModel.objects.group_by_date('a_date', 'week')\
                            .annotate(sum=models.Sum('value'))
        tuples = [(row['a_date_year'], row['a_date_week'], row['sum']) for row in sums]
        self.assertItemsEqual(tuples, [(2014, 1, 3),
                                       (2014, 2, 17),
                                       (2014, 3, 32),
                                       (2014, 4, 46)])

    def test_group_day(self):
        # 2 each in 2014-01-01, 2014-01-02, 2014-01-03, 2014-01-04
        # values (rounding down) 0, 0, 1, 1, 2, 2, 3, 3
        # sums 0, 2, 4, 6
        self.make_models(8, 0.55)
        sums = DGQSTestModel.objects.group_by_date('a_date', 'day')\
                            .annotate(sum=models.Sum('value'))
        tuples = [(row['a_date_year'],
                   row['a_date_month'],
                   row['a_date_day'],
                   row['sum'])
                  for row in sums]
        self.assertItemsEqual(tuples, [(2014, 1, 1, 0),
                                       (2014, 1, 2, 2),
                                       (2014, 1, 3, 4),
                                       (2014, 1, 4, 6)])

    def test_group_and_filter(self):
        self.fail()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from math import pi

import os
import random
import torch
import unittest
import gpytorch
from torch import optim
from gpytorch.likelihoods import GaussianLikelihood
from gpytorch.models import AbstractVariationalGP
from gpytorch.variational import CholeskyVariationalDistribution
from gpytorch.variational import VariationalStrategy

def train_data(cuda=False):
    train_x = torch.linspace(0, 1, 260)
    train_y = torch.cos(train_x * (2 * pi))
    if cuda:
        return train_x.cuda(), train_y.cuda()
    else:
        return train_x, train_y

class SVGPRegressionModel(AbstractVariationalGP):
    def __init__(self, inducing_points):
        variational_distribution = CholeskyVariationalDistribution(inducing_points.size(-1))
        variational_strategy = VariationalStrategy(self,
                                                   inducing_points,
                                                   variational_distribution,
                                                   learn_inducing_locations=True)
        super(SVGPRegressionModel, self).__init__(variational_strategy)
        self.mean_module = gpytorch.means.ConstantMean()
        self.covar_module = gpytorch.kernels.ScaleKernel(
            gpytorch.kernels.RBFKernel(
                log_lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(0.001, 1., sigma=0.1, log_transform=True)
            )
        )

    def forward(self, x):
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        latent_pred = gpytorch.distributions.MultivariateNormal(mean_x, covar_x)
        return latent_pred

class TestSVGPClassification(unittest.TestCase):
    def setUp(self):
        if os.getenv("UNLOCK_SEED") is None or os.getenv("UNLOCK_SEED").lower() == "false":
            self.rng_state = torch.get_rng_state()
            torch.manual_seed(0)
            if torch.cuda.is_available():
                torch.cuda.manual_seed_all(0)
            random.seed(0)

    def tearDown(self):
        if hasattr(self, "rng_state"):
            torch.set_rng_state(self.rng_state)

    def test_classification_error(self):
        train_x, train_y = train_data()
        likelihood = GaussianLikelihood()
        model = SVGPRegressionModel(train_x[:25])
        mll = gpytorch.mlls.VariationalELBO(likelihood, model, num_data=len(train_y))

        # Find optimal model hyperparameters
        model.train()
        likelihood.train()
        optimizer = optim.Adam([{'params': model.parameters()}, {'params': likelihood.parameters()}], lr=0.1)
        optimizer.n_iter = 0
        for _ in range(200):
            optimizer.zero_grad()
            output = model(train_x)
            loss = -mll(output, train_y)
            loss.backward()
            optimizer.n_iter += 1
            optimizer.step()

        for param in model.parameters():
            self.assertTrue(param.grad is not None)
            self.assertGreater(param.grad.norm().item(), 0)
        for param in likelihood.parameters():
            self.assertTrue(param.grad is not None)
            self.assertGreater(param.grad.norm().item(), 0)

        # Set back to eval mode
        model.eval()
        likelihood.eval()
        test_preds = likelihood(model(train_x)).mean.squeeze()
        mean_abs_error = torch.mean(torch.abs(train_y - test_preds) / 2)
        assert mean_abs_error.item() < 1e-1

    def test_classification_error_cuda(self):
        if torch.cuda.is_available():
            train_x, train_y = train_data(cuda=True)
            likelihood = GaussianLikelihood().cuda()
            model = SVGPRegressionModel(train_x[:25]).cuda()
            mll = gpytorch.mlls.VariationalMarginalLogLikelihood(likelihood, model, num_data=len(train_y))

            # Find optimal model hyperparameters
            model.train()
            optimizer = optim.Adam([{'params': model.parameters()}, {'params': likelihood.parameters()}], lr=0.1)
            optimizer.n_iter = 0
            for _ in range(200):
                optimizer.zero_grad()
                output = model(train_x)
                loss = -mll(output, train_y)
                loss.backward()
                optimizer.n_iter += 1
                optimizer.step()

            for param in model.parameters():
                self.assertTrue(param.grad is not None)
                self.assertGreater(param.grad.norm().item(), 0)
            for param in likelihood.parameters():
                self.assertTrue(param.grad is not None)
                self.assertGreater(param.grad.norm().item(), 0)
            optimizer.step()

            # Set back to eval mode
            model.eval()
            test_preds = likelihood(model(train_x)).mean.squeeze()
            mean_abs_error = torch.mean(torch.abs(train_y - test_preds) / 2)
            self.assertLess(mean_abs_error.item(), 1e-1)

if __name__ == "__main__":
    unittest.main()
<commit_msg>Fix class / method names in unit test<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from math import pi

import os
import random
import torch
import unittest
import gpytorch
from torch import optim
from gpytorch.likelihoods import GaussianLikelihood
from gpytorch.models import AbstractVariationalGP
from gpytorch.variational import CholeskyVariationalDistribution
from gpytorch.variational import VariationalStrategy

def train_data(cuda=False):
    train_x = torch.linspace(0, 1, 260)
    train_y = torch.cos(train_x * (2 * pi))
    if cuda:
        return train_x.cuda(), train_y.cuda()
    else:
        return train_x, train_y

class SVGPRegressionModel(AbstractVariationalGP):
    def __init__(self, inducing_points):
        variational_distribution = CholeskyVariationalDistribution(inducing_points.size(-1))
        variational_strategy = VariationalStrategy(self,
                                                   inducing_points,
                                                   variational_distribution,
                                                   learn_inducing_locations=True)
        super(SVGPRegressionModel, self).__init__(variational_strategy)
        self.mean_module = gpytorch.means.ConstantMean()
        self.covar_module = gpytorch.kernels.ScaleKernel(
            gpytorch.kernels.RBFKernel(
                log_lengthscale_prior=gpytorch.priors.SmoothedBoxPrior(0.001, 1., sigma=0.1, log_transform=True)
            )
        )

    def forward(self, x):
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        latent_pred = gpytorch.distributions.MultivariateNormal(mean_x, covar_x)
        return latent_pred

class TestSVGPRegression(unittest.TestCase):
    def setUp(self):
        if os.getenv("UNLOCK_SEED") is None or os.getenv("UNLOCK_SEED").lower() == "false":
            self.rng_state = torch.get_rng_state()
            torch.manual_seed(0)
            if torch.cuda.is_available():
                torch.cuda.manual_seed_all(0)
            random.seed(0)

    def tearDown(self):
        if hasattr(self, "rng_state"):
            torch.set_rng_state(self.rng_state)

    def test_regression_error(self):
        train_x, train_y = train_data()
        likelihood = GaussianLikelihood()
        model = SVGPRegressionModel(train_x[:25])
        mll = gpytorch.mlls.VariationalELBO(likelihood, model, num_data=len(train_y))

        # Find optimal model hyperparameters
        model.train()
        likelihood.train()
        optimizer = optim.Adam([{'params': model.parameters()}, {'params': likelihood.parameters()}], lr=0.1)
        optimizer.n_iter = 0
        for _ in range(200):
            optimizer.zero_grad()
            output = model(train_x)
            loss = -mll(output, train_y)
            loss.backward()
            optimizer.n_iter += 1
            optimizer.step()

        for param in model.parameters():
            self.assertTrue(param.grad is not None)
            self.assertGreater(param.grad.norm().item(), 0)
        for param in likelihood.parameters():
            self.assertTrue(param.grad is not None)
            self.assertGreater(param.grad.norm().item(), 0)

        # Set back to eval mode
        model.eval()
        likelihood.eval()
        test_preds = likelihood(model(train_x)).mean.squeeze()
        mean_abs_error = torch.mean(torch.abs(train_y - test_preds) / 2)
        assert mean_abs_error.item() < 1e-1

    def test_regression_error_cuda(self):
        if torch.cuda.is_available():
            train_x, train_y = train_data(cuda=True)
            likelihood = GaussianLikelihood().cuda()
            model = SVGPRegressionModel(train_x[:25]).cuda()
            mll = gpytorch.mlls.VariationalMarginalLogLikelihood(likelihood, model, num_data=len(train_y))

            # Find optimal model hyperparameters
            model.train()
            optimizer = optim.Adam([{'params': model.parameters()}, {'params': likelihood.parameters()}], lr=0.1)
            optimizer.n_iter = 0
            for _ in range(200):
                optimizer.zero_grad()
                output = model(train_x)
                loss = -mll(output, train_y)
                loss.backward()
                optimizer.n_iter += 1
                optimizer.step()

            for param in model.parameters():
                self.assertTrue(param.grad is not None)
                self.assertGreater(param.grad.norm().item(), 0)
            for param in likelihood.parameters():
                self.assertTrue(param.grad is not None)
                self.assertGreater(param.grad.norm().item(), 0)
            optimizer.step()

            # Set back to eval mode
            model.eval()
            test_preds = likelihood(model(train_x)).mean.squeeze()
            mean_abs_error = torch.mean(torch.abs(train_y - test_preds) / 2)
            self.assertLess(mean_abs_error.item(), 1e-1)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>class List(list):
    """
    Same as pypeg2.List except that the constructor has been modified
    to work with Python 2.x.

    See http://goo.gl/VKPY3.
    """

    def __init__(self, *args, **kwargs):
        """Construct a List, and construct its attributes from keyword
        arguments.
        """
        if len(args) == 1:
            if isinstance(args[0], str):
                self.append(args[0])
            else:
                super(List, self).__init__(args[0])
        else:
            super(List, self).__init__(args)

        for k, v in kwargs.items():
            setattr(self, k, v)

    def __repr__(self):
        """x.__repr__() <==> repr(x)"""
        return ''.join((type(self).__name__, "(", super(List, self).__repr__(),
            ")"))

    def __eq__(self, other):
        return super(List, self).__eq__(list(other))
<commit_msg>Remove unused module.<commit_after><|endoftext|>


<commit_before>from django.conf.urls import patterns
from mapreduce.main import create_handlers_map
from djangae.utils import djangae_webapp

wrapped_urls = ((url_re.replace('.*/', '^', 1), djangae_webapp(func)) for url_re, func in create_handlers_map())
urlpatterns = patterns('', *wrapped_urls)
<commit_msg>import fix to make sure mapreduce is not a dep<commit_after>from django.conf.urls import patterns
from djangae.utils import djangae_webapp

try:
    from mapreduce.main import create_handlers_map
    wrapped_urls = ((url_re.replace('.*/', '^', 1), djangae_webapp(func)) for url_re, func in create_handlers_map())
except ImportError as e:
    wrapped_urls = []

urlpatterns = patterns('', *wrapped_urls)
<|endoftext|>


<commit_before>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Lesser General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Lesser General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
Copy articles to a different amcat server using the API

Limitations: does not copy UUID or parent/child relations
"""

import argparse
import requests
import logging
import itertools

from amcatclient import AmcatAPI

SET_ARGS = ["name", "provenance"]
ART_ARGS = ["metastring", "byline", "uuid", "author", "headline", "text",
            "section", "url", "length", "addressee", "externalid",
            "insertdate", "date", "pagenr", "medium", "parent"]

def create_set(src_api, src_project, src_set, trg_api, trg_project):
    s = src_api.get_set(src_project, src_set)
    s = {k: v for (k, v) in s.items() if k in SET_ARGS}

    provenance = "Copied from {src_api.host} project {src_project} set {src_set}".format(**locals())
    if s.get('provenance'):
        s['provenance'] = "\n".join([s['provenance'], provenance])
    else:
        s['provenance'] = provenance

    result = trg.create_set(trg_project, s)
    logging.info("Created set {id}:{name} in project {project}"
                 .format(**result))
    return result["id"]

def copy_articles(src_api, src_project, src_set,
                  trg_api, trg_project, trg_set=None,
                  batch_size=100, from_page=1):
    if trg_set is None:
        trg_set = create_set(src_api, src_project, src_set, trg_api, trg_project)
    articles = src_api.list_articles(src_project, src_set, page=from_page, page_size=batch_size, order_by="parent", text=True)
    for i in itertools.count(from_page):
        batch = list(itertools.islice(articles, batch_size))
        if not batch:
            logging.info("Done")
            break
        logging.info("Copying batch {i}: {n} articles"
                     .format(n=len(batch), **locals()))

        def convert(a):
            a = {k: v for (k, v) in a.items() if k in ART_ARGS and v}
            if not a.get('text'): a['text'] = "-"
            if not a.get('headline'): a['headline']="-"
            a['title'] = a.pop('headline')
            return a
        batch = [convert(a) for a in batch]

        trg_api.create_articles(trg_project, trg_set, batch)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(epilog=__doc__)
    parser.add_argument("source_url", help='URL of the source '
                        '(e.g. "http://amcat-dev.labs.vu.nl")')
    parser.add_argument("target_url", help='URL of the target '
                        '(e.g. "http://amcat.vu.nl")')
    parser.add_argument("source_project", help='Article set ID in the source',
                        type=int)
    parser.add_argument("source_set", help='Article set ID in the source',
                        type=int)
    parser.add_argument("target_project", help='Project ID in the target',
                        type=int)
    parser.add_argument("--target-set", "-s", help='Article set ID in the '
                        'target (if omitted, a new set will be created',
                        type=int)
    parser.add_argument("--batch-size", "-b", help='Batch size for copying',
                        type=int, default=100)
    parser.add_argument("--from-page", "-p", help='Start from page (batch)',
                        type=int, default=1)

    args = parser.parse_args()

    fmt = '[%(asctime)s %(levelname)s %(name)s] %(message)s'
    logging.basicConfig(format=fmt, level=logging.INFO)
    logging.getLogger("requests").setLevel(logging.WARNING)

    src = AmcatAPI(args.source_url)
    trg = AmcatAPI(args.target_url)

    copy_articles(src, args.source_project, args.source_set,
                  trg, args.target_project, args.target_set,
                  args.batch_size, args.from_page)
<commit_msg>make copy_articles version aware<commit_after>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Lesser General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Lesser General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
Copy articles to a different amcat server using the API

Limitations: does not copy UUID or parent/child relations
"""

import argparse
import requests
import logging
import itertools

from amcatclient import AmcatAPI

SET_ARGS = ["name", "provenance"]
ART_ARGS = ["metastring", "byline", "uuid", "author", "headline", "text",
            "section", "url", "length", "addressee", "externalid",
            "insertdate", "date", "pagenr", "medium", "parent"]

def create_set(src_api, src_project, src_set, trg_api, trg_project):
    s = src_api.get_set(src_project, src_set)
    s = {k: v for (k, v) in s.items() if k in SET_ARGS}

    provenance = "Copied from {src_api.host} project {src_project} set {src_set}".format(**locals())
    if s.get('provenance'):
        s['provenance'] = "\n".join([s['provenance'], provenance])
    else:
        s['provenance'] = provenance

    result = trg.create_set(trg_project, s)
    logging.info("Created set {id}:{name} in project {project}"
                 .format(**result))
    return result["id"]

def copy_articles(src_api, src_project, src_set,
                  trg_api, trg_project, trg_set=None,
                  batch_size=100, from_page=1):

    srcv = src_api.get_version()
    trgv = trg_api.get_version()

    if not (srcv.major == 3 and trgv.major == 3 and srcv.minor in (4,5) and trgv.minor in (4,5)):
        raise Exception("copy_articles only possible between versions 3.4 and 3.5")

    if trg_set is None:
        trg_set = create_set(src_api, src_project, src_set, trg_api, trg_project)
    if srcv.minor == 5:
        kargs = {}
    else:
        kargs = dict(order_by='parent')
    articles = src_api.list_articles(src_project, src_set, page=from_page, page_size=batch_size, text=True, **kargs)
    for i in itertools.count(from_page):
        batch = list(itertools.islice(articles, batch_size))
        if not batch:
            logging.info("Done")
            break
        logging.info("Copying batch {i}: {n} articles"
                     .format(n=len(batch), **locals()))

        def convert(a):
            a = {k: v for (k, v) in a.items() if k in ART_ARGS and v}
            if not a.get('text'): a['text'] = "-"
            # someone decided to rename headline to title in 3.5, so check and rename as needed
            title = a.pop('headline', '-') if srcv.minor == 4 else a.pop("title", '-')
            medium = a.pop('medium', None) if srcv.minor == 4 else a.pop("publisher", None)
            if trgv.minor == 5:
                a['title'] = title
                if medium:
                    a['publisher'] = medium
            if trgv.minor == 4:
                a['headline'] = title
                a['medium'] = medium or "-"
            return a
        batch = [convert(a) for a in batch]

        trg_api.create_articles(trg_project, trg_set, batch)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(epilog=__doc__)
    parser.add_argument("source_url", help='URL of the source '
                        '(e.g. "http://amcat-dev.labs.vu.nl")')
    parser.add_argument("target_url", help='URL of the target '
                        '(e.g. "http://amcat.vu.nl")')
    parser.add_argument("source_project", help='Article set ID in the source',
                        type=int)
    parser.add_argument("source_set", help='Article set ID in the source',
                        type=int)
    parser.add_argument("target_project", help='Project ID in the target',
                        type=int)
    parser.add_argument("--target-set", "-s", help='Article set ID in the '
                        'target (if omitted, a new set will be created',
                        type=int)
    parser.add_argument("--batch-size", "-b", help='Batch size for copying',
                        type=int, default=100)
    parser.add_argument("--from-page", "-p", help='Start from page (batch)',
                        type=int, default=1)

    args = parser.parse_args()

    fmt = '[%(asctime)s %(levelname)s %(name)s] %(message)s'
    logging.basicConfig(format=fmt, level=logging.INFO)
    logging.getLogger("requests").setLevel(logging.WARNING)

    src = AmcatAPI(args.source_url)
    trg = AmcatAPI(args.target_url)

    copy_articles(src, args.source_project, args.source_set,
                  trg, args.target_project, args.target_set,
                  args.batch_size, args.from_page)
<|endoftext|>


<commit_before>"""
Built-in, globally-available admin actions.
"""

from django import template
from django.core.exceptions import PermissionDenied
from django.contrib.admin import helpers
from django.contrib.admin.util import get_deleted_objects, model_ngettext
from django.shortcuts import render_to_response
from django.utils.encoding import force_unicode
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy, ugettext as _

def delete_selected(modeladmin, request, queryset):
    """
    Default action which deletes the selected objects.

    This action first displays a confirmation page whichs shows all the
    deleteable objects, or, if the user has no permission one of the related
    childs (foreignkeys), a "permission denied" message.

    Next, it delets all selected objects and redirects back to the change list.
    """
    opts = modeladmin.model._meta
    app_label = opts.app_label

    # Check that the user has delete permission for the actual model
    if not modeladmin.has_delete_permission(request):
        raise PermissionDenied

    # Populate deletable_objects, a data structure of all related objects that
    # will also be deleted.

    # deletable_objects must be a list if we want to use '|unordered_list' in the template
    deletable_objects = []
    perms_needed = set()
    i = 0
    for obj in queryset:
        deletable_objects.append([mark_safe(u'%s: %s' % (escape(force_unicode(capfirst(opts.verbose_name))), obj.pk, escape(obj))), []])
        get_deleted_objects(deletable_objects[i], perms_needed, request.user, obj, opts, 1, modeladmin.admin_site, levels_to_root=2)
        i=i+1

    # The user has already confirmed the deletion.
    # Do the deletion and return a None to display the change list view again.
    if request.POST.get('post'):
        if perms_needed:
            raise PermissionDenied
        n = queryset.count()
        if n:
            for obj in queryset:
                obj_display = force_unicode(obj)
                modeladmin.log_deletion(request, obj, obj_display)
            queryset.delete()
            modeladmin.message_user(request, _("Successfully deleted %(count)d %(items)s.") % {
                "count": n, "items": model_ngettext(modeladmin.opts, n)
            })
        # Return None to display the change list page again.
        return None

    context = {
        "title": _("Are you sure?"),
        "object_name": force_unicode(opts.verbose_name),
        "deletable_objects": deletable_objects,
        'queryset': queryset,
        "perms_lacking": perms_needed,
        "opts": opts,
        "root_path": modeladmin.admin_site.root_path,
        "app_label": app_label,
        'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME,
    }

    # Display the confirmation page
    return render_to_response(modeladmin.delete_confirmation_template or [
        "admin/%s/%s/delete_selected_confirmation.html" % (app_label, opts.object_name.lower()),
        "admin/%s/delete_selected_confirmation.html" % app_label,
        "admin/delete_selected_confirmation.html"
    ], context, context_instance=template.RequestContext(request))

delete_selected.short_description = ugettext_lazy("Delete selected %(verbose_name_plural)s")
<commit_msg>Fixed #10997: fixed a Python 2.4-ism in admin actions.<commit_after>"""
Built-in, globally-available admin actions.
"""

from django import template
from django.core.exceptions import PermissionDenied
from django.contrib.admin import helpers
from django.contrib.admin.util import get_deleted_objects, model_ngettext
from django.shortcuts import render_to_response
from django.utils.encoding import force_unicode
from django.utils.html import escape
from django.utils.safestring import mark_safe
from django.utils.text import capfirst
from django.utils.translation import ugettext_lazy, ugettext as _
try:
    set
except NameError:
    from sets import Set as set     # Python 2.3 fallback

def delete_selected(modeladmin, request, queryset):
    """
    Default action which deletes the selected objects.

    This action first displays a confirmation page whichs shows all the
    deleteable objects, or, if the user has no permission one of the related
    childs (foreignkeys), a "permission denied" message.

    Next, it delets all selected objects and redirects back to the change list.
    """
    opts = modeladmin.model._meta
    app_label = opts.app_label

    # Check that the user has delete permission for the actual model
    if not modeladmin.has_delete_permission(request):
        raise PermissionDenied

    # Populate deletable_objects, a data structure of all related objects that
    # will also be deleted.

    # deletable_objects must be a list if we want to use '|unordered_list' in the template
    deletable_objects = []
    perms_needed = set()
    i = 0
    for obj in queryset:
        deletable_objects.append([mark_safe(u'%s: %s' % (escape(force_unicode(capfirst(opts.verbose_name))), obj.pk, escape(obj))), []])
        get_deleted_objects(deletable_objects[i], perms_needed, request.user, obj, opts, 1, modeladmin.admin_site, levels_to_root=2)
        i=i+1

    # The user has already confirmed the deletion.
    # Do the deletion and return a None to display the change list view again.
    if request.POST.get('post'):
        if perms_needed:
            raise PermissionDenied
        n = queryset.count()
        if n:
            for obj in queryset:
                obj_display = force_unicode(obj)
                modeladmin.log_deletion(request, obj, obj_display)
            queryset.delete()
            modeladmin.message_user(request, _("Successfully deleted %(count)d %(items)s.") % {
                "count": n, "items": model_ngettext(modeladmin.opts, n)
            })
        # Return None to display the change list page again.
        return None

    context = {
        "title": _("Are you sure?"),
        "object_name": force_unicode(opts.verbose_name),
        "deletable_objects": deletable_objects,
        'queryset': queryset,
        "perms_lacking": perms_needed,
        "opts": opts,
        "root_path": modeladmin.admin_site.root_path,
        "app_label": app_label,
        'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME,
    }

    # Display the confirmation page
    return render_to_response(modeladmin.delete_confirmation_template or [
        "admin/%s/%s/delete_selected_confirmation.html" % (app_label, opts.object_name.lower()),
        "admin/%s/delete_selected_confirmation.html" % app_label,
        "admin/delete_selected_confirmation.html"
    ], context, context_instance=template.RequestContext(request))

delete_selected.short_description = ugettext_lazy("Delete selected %(verbose_name_plural)s")
<|endoftext|>


<commit_before>import os
import ssl

from . import errors
from .transport import SSLAdapter

class TLSConfig(object):
    """
    TLS configuration.

    Args:
        client_cert (tuple of str): Path to client cert, path to client key.
        ca_cert (str): Path to CA cert file.
        verify (bool or str): This can be ``False`` or a path to a CA cert
            file.
        ssl_version (int): A valid `SSL version`_.
        assert_hostname (bool): Verify the hostname of the server.

    .. _`SSL version`:
        https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1
    """
    cert = None
    ca_cert = None
    verify = None
    ssl_version = None

    def __init__(self, client_cert=None, ca_cert=None, verify=None,
                 ssl_version=None, assert_hostname=None,
                 assert_fingerprint=None):
        # Argument compatibility/mapping with
        # https://docs.docker.com/engine/articles/https/
        # This diverges from the Docker CLI in that users can specify 'tls'
        # here, but also disable any public/default CA pool verification by
        # leaving tls_verify=False

        self.assert_hostname = assert_hostname
        self.assert_fingerprint = assert_fingerprint

        # TLS v1.0 seems to be the safest default; SSLv23 fails in mysterious
        # ways: https://github.com/docker/docker-py/issues/963

        self.ssl_version = ssl_version or ssl.PROTOCOL_TLSv1

        # "tls" and "tls_verify" must have both or neither cert/key files
        # In either case, Alert the user when both are expected, but any are
        # missing.

        if client_cert:
            try:
                tls_cert, tls_key = client_cert
            except ValueError:
                raise errors.TLSParameterError(
                    'client_config must be a tuple of'
                    ' (client certificate, key file)'
                )

            if not (tls_cert and tls_key) or (not os.path.isfile(tls_cert) or
                                              not os.path.isfile(tls_key)):
                raise errors.TLSParameterError(
                    'Path to a certificate and key files must be provided'
                    ' through the client_config param'
                )
            self.cert = (tls_cert, tls_key)

        # If verify is set, make sure the cert exists
        self.verify = verify
        self.ca_cert = ca_cert
        if self.verify and self.ca_cert and not os.path.isfile(self.ca_cert):
            raise errors.TLSParameterError(
                'Invalid CA certificate provided for `tls_ca_cert`.'
            )

    def configure_client(self, client):
        """
        Configure a client with these TLS options.
        """
        client.ssl_version = self.ssl_version

        if self.verify and self.ca_cert:
            client.verify = self.ca_cert
        else:
            client.verify = self.verify

        if self.cert:
            client.cert = self.cert

        client.mount('https://', SSLAdapter(
            ssl_version=self.ssl_version,
            assert_hostname=self.assert_hostname,
            assert_fingerprint=self.assert_fingerprint,
        ))
<commit_msg>Change default TLS version<commit_after>import os
import ssl

from . import errors
from .transport import SSLAdapter

class TLSConfig(object):
    """
    TLS configuration.

    Args:
        client_cert (tuple of str): Path to client cert, path to client key.
        ca_cert (str): Path to CA cert file.
        verify (bool or str): This can be ``False`` or a path to a CA cert
            file.
        ssl_version (int): A valid `SSL version`_.
        assert_hostname (bool): Verify the hostname of the server.

    .. _`SSL version`:
        https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1
    """
    cert = None
    ca_cert = None
    verify = None
    ssl_version = None

    def __init__(self, client_cert=None, ca_cert=None, verify=None,
                 ssl_version=None, assert_hostname=None,
                 assert_fingerprint=None):
        # Argument compatibility/mapping with
        # https://docs.docker.com/engine/articles/https/
        # This diverges from the Docker CLI in that users can specify 'tls'
        # here, but also disable any public/default CA pool verification by
        # leaving tls_verify=False

        self.assert_hostname = assert_hostname
        self.assert_fingerprint = assert_fingerprint

        # TODO(dperny): according to the python docs, PROTOCOL_TLSvWhatever is
        # depcreated, and it's recommended to use OPT_NO_TLSvWhatever instead
        # to exclude versions. But I think that might require a bigger
        # architectural change, so I've opted not to pursue it at this time

        # If the user provides an SSL version, we should use their preference
        if ssl_version:
            self.ssl_version = ssl_version
        else:
            # If the user provides no ssl version, we should default to
            # TLSv1_2.  This option is the most secure, and will work for the
            # majority of users with reasonably up-to-date software. However,
            # before doing so, detect openssl version to ensure we can support
            # it.

            # ssl.OPENSSL_VERSION_INFO returns a tuple of 5 integers
            # representing version info. We want any OpenSSL version greater
            # than 1.0.1. Python compares tuples lexigraphically, which means
            # this comparison will work.
            if ssl.OPENSSL_VERSION_INFO > (1, 0, 1, 0, 0):
                # If this version is high enough to support TLSv1_2, then we
                # should use it.
                self.ssl_version = ssl.PROTOCOL_TLSv1_2
            else:
                # If we can't, use a differnent default. Before the commit
                # introducing this version detection, the comment read:
                # >>> TLS v1.0 seems to be the safest default; SSLv23 fails in
                # >>> mysterious ways:
                # >>> https://github.com/docker/docker-py/issues/963
                # Which is why we choose PROTOCOL_TLSv1
                self.ssl_version = ssl.PROTOCOL_TLSv1

        # "tls" and "tls_verify" must have both or neither cert/key files In
        # either case, Alert the user when both are expected, but any are
        # missing.

        if client_cert:
            try:
                tls_cert, tls_key = client_cert
            except ValueError:
                raise errors.TLSParameterError(
                    'client_config must be a tuple of'
                    ' (client certificate, key file)'
                )

            if not (tls_cert and tls_key) or (not os.path.isfile(tls_cert) or
                                              not os.path.isfile(tls_key)):
                raise errors.TLSParameterError(
                    'Path to a certificate and key files must be provided'
                    ' through the client_config param'
                )
            self.cert = (tls_cert, tls_key)

        # If verify is set, make sure the cert exists
        self.verify = verify
        self.ca_cert = ca_cert
        if self.verify and self.ca_cert and not os.path.isfile(self.ca_cert):
            raise errors.TLSParameterError(
                'Invalid CA certificate provided for `tls_ca_cert`.'
            )

    def configure_client(self, client):
        """
        Configure a client with these TLS options.
        """
        client.ssl_version = self.ssl_version

        if self.verify and self.ca_cert:
            client.verify = self.ca_cert
        else:
            client.verify = self.verify

        if self.cert:
            client.cert = self.cert

        client.mount('https://', SSLAdapter(
            ssl_version=self.ssl_version,
            assert_hostname=self.assert_hostname,
            assert_fingerprint=self.assert_fingerprint,
        ))
<|endoftext|>


<commit_before>"""
This encapsulates the logic for displaying filters in the Django admin.
Filters are specified in models with the "list_filter" option.

Each filter subclass knows how to display a filter for a field that passes a
certain test -- e.g. being a DateField or ForeignKey.
"""
import datetime

from django.contrib.admin.options import IncorrectLookupParameters
from django.contrib.admin.utils import (
    get_model_from_relation, prepare_lookup_value, reverse_field_path,
)
from django.core.exceptions import ImproperlyConfigured, ValidationError
from django.db import models
from django.utils import timezone
from django.utils.encoding import force_text, smart_text
from django.utils.translation import ugettext_lazy as _

class ListFilter(object):
    title = None  # Human-readable title to appear in the right sidebar.
    template = 'admin/filter.html'

    def __init__(self, request, params, model, model_admin):
        # This dictionary will eventually contain the request's query string
        # parameters actually used by this filter.
        self.used_parameters = {}
        if self.title is None:
            raise ImproperlyConfigured(
                "The list filter '%s' does not specify "
                "a 'title'." % self.__class__.__name__)

    def has_output(self):
        """
        Returns True if some choices would be output for this filter.
        """
        raise NotImplementedError('subclasses of ListFilter must provide a has_output() method')

    def choices(self, cl):
        """
        Returns choices ready to be output in the template.
        """
        raise NotImplementedError('subclasses of ListFilter must provide a choices() method')

    def queryset(self, request, queryset):
        """
        Returns the filtered queryset.
        """
        raise NotImplementedError('subclasses of ListFilter must provide a queryset() method')

    def expected_parameters(self):
        """
        Returns the list of parameter names that are expected from the
        request's query string and that will be used by this filter.
        """
        raise NotImplementedError('subclasses of ListFilter must provide an expected_parameters() method')

class SimpleListFilter(ListFilter):
    # The parameter that should be used in the query string for that filter.
    parameter_name = None

    def __init__(self, request, params, model, model_admin):
        super(SimpleListFilter, self).__init__(
            request, params, model, model_admin)
        if self.parameter_name is None:
            raise ImproperlyConfigured(
                "The list filter '%s' does not specify "
                "a 'parameter_name'." % self.__class__.__name__)
        if self.parameter_name in params:
            value = params.pop(self.parameter_name)
            self.used_parameters[self.parameter_name] = value
        lookup_choices = self.lookups(request, model_admin)
        if lookup_choices is None:
            lookup_choices = ()
        self.lookup_choices = list(lookup_choices)

    def has_output(self):
        return len(self.lookup_choices) > 0

    def value(self):
        """
        Returns the value (in string format) provided in the request's
        query string for this filter, if any. If the value wasn't provided then
        returns None.
        """
        return self.used_parameters.get(self.parameter_name)

    def lookups(self, request, model_admin):
        """
        Must be overridden to return a list of tuples (value, verbose value)
        """
        raise NotImplementedError(
            'The SimpleListFilter.lookups() method must be overridden to '
            'return a list of tuples (value, verbose value)')

    def expected_parameters(self):
        return [self.parameter_name]

    def choices(self, cl):
        yield {
            'selected': self.value() is None,
            'query_string': cl.get_query_string({}, [self.parameter_name]),
            'display': _('All'),
        }
        for lookup, title in self.lookup_choices:
            yield {
                'selected': self.value() == force_text(lookup),
                'query_string': cl.get_query_string({
                    self.parameter_name: lookup,
                }, []),
                'display': title,
            }

class FieldListFilter(ListFilter):
    _field_list_filters = []
    _take_priority_index = 0

    def __init__(self, field, request, params, model, model_admin, field_path):
        self.field = field
        self.field_path = field_path
        self.title = getattr(field, 'verbose_name', field_path)
        super(FieldListFilter, self).__init__(
            request, params, model, model_admin)
        for p in self.expected_parameters():
            if p in params:
                value = params.pop(p)
                self.used_parameters[p] = prepare_lookup_value(p, value)

    def has_output(self):
        return True

    def queryset(self, request, queryset):
        try:
            return queryset.filter(**self.used_parameters)
        except ValidationError as e:
            raise IncorrectLookupParameters(e)

    @classmethod
    def register(cls, test, list_filter_class, take_priority=False):
        if take_priority:
            # This is to allow overriding the default filters for certain types
            # of fields with some custom filters. The first found in the list
            # is used in priority.
            cls._field_list_filters.insert(
                cls._take_priority_index, (test, list_filter_class))
            cls._take_priority_index += 1
        else:
            cls._field_list_filters.append((test, list_filter_class))

    @classmethod
    def create(cls, field, request, params, model, model_admin, field_path):
        for test, list_filter_class in cls._field_list_filters:
            if not test(field):
                continue
            return list_filter_class(field, request, params,
                model, model_admin, field_path=field_path)

class RelatedFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        other_model = get_model_from_relation(field)
        self.lookup_kwarg = '%s__%s__exact' % (field_path, field.target_field.name)
        self.lookup_kwarg_isnull = '%s__isnull' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        self.lookup_val_isnull = request.GET.get(self.lookup_kwarg_isnull)
        super(RelatedFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)
        self.lookup_choices = self.field_choices(field, request, model_admin)
        if hasattr(field, 'verbose_name'):
            self.lookup_title = field.verbose_name
        else:
            self.lookup_title = other_model._meta.verbose_name
        self.title = self.lookup_title
        self.empty_value_display = model_admin.get_empty_value_display()

    @property
    def include_empty_choice(self):
        """
        Return True if a "(None)" choice should be included, which filters
        out everything except empty relationships.
        """
        return self.field.null or (self.field.is_relation and self.field.many_to_many)

    def has_output(self):
        if self.include_empty_choice:
            extra = 1
        else:
            extra = 0
        return len(self.lookup_choices) + extra > 1

    def expected_parameters(self):
        return [self.lookup_kwarg, self.lookup_kwarg_isnull]

    def field_choices(self, field, request, model_admin):
        return field.get_choices(include_blank=False)

    def choices(self, cl):
        yield {
            'selected': self.lookup_val is None and not self.lookup_val_isnull,
            'query_string': cl.get_query_string({},
                [self.lookup_kwarg, self.lookup_kwarg_isnull]),
            'display': _('All'),
        }
        for pk_val, val in self.lookup_choices:
            yield {
                'selected': self.lookup_val == smart_text(pk_val),
                'query_string': cl.get_query_string({
                    self.lookup_kwarg: pk_val,
                }, [self.lookup_kwarg_isnull]),
                'display': val,
            }
        if self.include_empty_choice:
            yield {
                'selected': bool(self.lookup_val_isnull),
                'query_string': cl.get_query_string({
                    self.lookup_kwarg_isnull: 'True',
                }, [self.lookup_kwarg]),
                'display': self.empty_value_display,
            }

FieldListFilter.register(lambda f: f.remote_field, RelatedFieldListFilter)

class BooleanFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.lookup_kwarg = '%s__exact' % field_path
        self.lookup_kwarg2 = '%s__isnull' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        self.lookup_val2 = request.GET.get(self.lookup_kwarg2)
        super(BooleanFieldListFilter, self).__init__(field,
            request, params, model, model_admin, field_path)

    def expected_parameters(self):
        return [self.lookup_kwarg, self.lookup_kwarg2]

    def choices(self, cl):
        for lookup, title in (
                (None, _('All')),
                ('1', _('Yes')),
                ('0', _('No'))):
            yield {
                'selected': self.lookup_val == lookup and not self.lookup_val2,
                'query_string': cl.get_query_string({
                    self.lookup_kwarg: lookup,
                }, [self.lookup_kwarg2]),
                'display': title,
            }
        if isinstance(self.field, models.NullBooleanField):
            yield {
                'selected': self.lookup_val2 == 'True',
                'query_string': cl.get_query_string({
                    self.lookup_kwarg2: 'True',
                }, [self.lookup_kwarg]),
                'display': _('Unknown'),
            }

FieldListFilter.register(lambda f: isinstance(f,
    (models.BooleanField, models.NullBooleanField)), BooleanFieldListFilter)

class ChoicesFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.lookup_kwarg = '%s__exact' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        super(ChoicesFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)

    def expected_parameters(self):
        return [self.lookup_kwarg]

    def choices(self, cl):
        yield {
            'selected': self.lookup_val is None,
            'query_string': cl.get_query_string({}, [self.lookup_kwarg]),
            'display': _('All')
        }
        for lookup, title in self.field.flatchoices:
            yield {
                'selected': smart_text(lookup) == self.lookup_val,
                'query_string': cl.get_query_string({
                    self.lookup_kwarg: lookup}),
                'display': title,
            }

FieldListFilter.register(lambda f: bool(f.choices), ChoicesFieldListFilter)

class DateFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.field_generic = '%s__' % field_path
        self.date_params = {k: v for k, v in params.items()
                            if k.startswith(self.field_generic)}

        now = timezone.now()
        # When time zone support is enabled, convert "now" to the user's time
        # zone so Django's definition of "Today" matches what the user expects.
        if timezone.is_aware(now):
            now = timezone.localtime(now)

        if isinstance(field, models.DateTimeField):
            today = now.replace(hour=0, minute=0, second=0, microsecond=0)
        else:       # field is a models.DateField
            today = now.date()
        tomorrow = today + datetime.timedelta(days=1)
        if today.month == 12:
            next_month = today.replace(year=today.year + 1, month=1, day=1)
        else:
            next_month = today.replace(month=today.month + 1, day=1)
        next_year = today.replace(year=today.year + 1, month=1, day=1)

        self.lookup_kwarg_since = '%s__gte' % field_path
        self.lookup_kwarg_until = '%s__lt' % field_path
        self.links = (
            (_('Any date'), {}),
            (_('Today'), {
                self.lookup_kwarg_since: str(today),
                self.lookup_kwarg_until: str(tomorrow),
            }),
            (_('Past 7 days'), {
                self.lookup_kwarg_since: str(today - datetime.timedelta(days=7)),
                self.lookup_kwarg_until: str(tomorrow),
            }),
            (_('This month'), {
                self.lookup_kwarg_since: str(today.replace(day=1)),
                self.lookup_kwarg_until: str(next_month),
            }),
            (_('This year'), {
                self.lookup_kwarg_since: str(today.replace(month=1, day=1)),
                self.lookup_kwarg_until: str(next_year),
            }),
        )
        super(DateFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)

    def expected_parameters(self):
        return [self.lookup_kwarg_since, self.lookup_kwarg_until]

    def choices(self, cl):
        for title, param_dict in self.links:
            yield {
                'selected': self.date_params == param_dict,
                'query_string': cl.get_query_string(
                    param_dict, [self.field_generic]),
                'display': title,
            }

FieldListFilter.register(
    lambda f: isinstance(f, models.DateField), DateFieldListFilter)

# This should be registered last, because it's a last resort. For example,
# if a field is eligible to use the BooleanFieldListFilter, that'd be much
# more appropriate, and the AllValuesFieldListFilter won't get used for it.
class AllValuesFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.lookup_kwarg = field_path
        self.lookup_kwarg_isnull = '%s__isnull' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        self.lookup_val_isnull = request.GET.get(self.lookup_kwarg_isnull)
        self.empty_value_display = model_admin.get_empty_value_display()
        parent_model, reverse_path = reverse_field_path(model, field_path)
        # Obey parent ModelAdmin queryset when deciding which options to show
        if model == parent_model:
            queryset = model_admin.get_queryset(request)
        else:
            queryset = parent_model._default_manager.all()
        self.lookup_choices = (queryset
                               .distinct()
                               .order_by(field.name)
                               .values_list(field.name, flat=True))
        super(AllValuesFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)

    def expected_parameters(self):
        return [self.lookup_kwarg, self.lookup_kwarg_isnull]

    def choices(self, cl):
        yield {
            'selected': (self.lookup_val is None
                and self.lookup_val_isnull is None),
            'query_string': cl.get_query_string({},
                [self.lookup_kwarg, self.lookup_kwarg_isnull]),
            'display': _('All'),
        }
        include_none = False
        for val in self.lookup_choices:
            if val is None:
                include_none = True
                continue
            val = smart_text(val)
            yield {
                'selected': self.lookup_val == val,
                'query_string': cl.get_query_string({
                    self.lookup_kwarg: val,
                }, [self.lookup_kwarg_isnull]),
                'display': val,
            }
        if include_none:
            yield {
                'selected': bool(self.lookup_val_isnull),
                'query_string': cl.get_query_string({
                    self.lookup_kwarg_isnull: 'True',
                }, [self.lookup_kwarg]),
                'display': self.empty_value_display,
            }

FieldListFilter.register(lambda f: True, AllValuesFieldListFilter)

class RelatedOnlyFieldListFilter(RelatedFieldListFilter):
    def field_choices(self, field, request, model_admin):
        pk_qs = model_admin.get_queryset(request).distinct().values_list('%s__pk' % self.field_path, flat=True)
        return field.get_choices(include_blank=False, limit_choices_to={'pk__in': pk_qs})
<commit_msg>Made ListFilter.choices() argument name more explicit.<commit_after>"""
This encapsulates the logic for displaying filters in the Django admin.
Filters are specified in models with the "list_filter" option.

Each filter subclass knows how to display a filter for a field that passes a
certain test -- e.g. being a DateField or ForeignKey.
"""
import datetime

from django.contrib.admin.options import IncorrectLookupParameters
from django.contrib.admin.utils import (
    get_model_from_relation, prepare_lookup_value, reverse_field_path,
)
from django.core.exceptions import ImproperlyConfigured, ValidationError
from django.db import models
from django.utils import timezone
from django.utils.encoding import force_text, smart_text
from django.utils.translation import ugettext_lazy as _

class ListFilter(object):
    title = None  # Human-readable title to appear in the right sidebar.
    template = 'admin/filter.html'

    def __init__(self, request, params, model, model_admin):
        # This dictionary will eventually contain the request's query string
        # parameters actually used by this filter.
        self.used_parameters = {}
        if self.title is None:
            raise ImproperlyConfigured(
                "The list filter '%s' does not specify "
                "a 'title'." % self.__class__.__name__)

    def has_output(self):
        """
        Returns True if some choices would be output for this filter.
        """
        raise NotImplementedError('subclasses of ListFilter must provide a has_output() method')

    def choices(self, changelist):
        """
        Returns choices ready to be output in the template.

        `changelist` is the ChangeList to be displayed.
        """
        raise NotImplementedError('subclasses of ListFilter must provide a choices() method')

    def queryset(self, request, queryset):
        """
        Returns the filtered queryset.
        """
        raise NotImplementedError('subclasses of ListFilter must provide a queryset() method')

    def expected_parameters(self):
        """
        Returns the list of parameter names that are expected from the
        request's query string and that will be used by this filter.
        """
        raise NotImplementedError('subclasses of ListFilter must provide an expected_parameters() method')

class SimpleListFilter(ListFilter):
    # The parameter that should be used in the query string for that filter.
    parameter_name = None

    def __init__(self, request, params, model, model_admin):
        super(SimpleListFilter, self).__init__(
            request, params, model, model_admin)
        if self.parameter_name is None:
            raise ImproperlyConfigured(
                "The list filter '%s' does not specify "
                "a 'parameter_name'." % self.__class__.__name__)
        if self.parameter_name in params:
            value = params.pop(self.parameter_name)
            self.used_parameters[self.parameter_name] = value
        lookup_choices = self.lookups(request, model_admin)
        if lookup_choices is None:
            lookup_choices = ()
        self.lookup_choices = list(lookup_choices)

    def has_output(self):
        return len(self.lookup_choices) > 0

    def value(self):
        """
        Returns the value (in string format) provided in the request's
        query string for this filter, if any. If the value wasn't provided then
        returns None.
        """
        return self.used_parameters.get(self.parameter_name)

    def lookups(self, request, model_admin):
        """
        Must be overridden to return a list of tuples (value, verbose value)
        """
        raise NotImplementedError(
            'The SimpleListFilter.lookups() method must be overridden to '
            'return a list of tuples (value, verbose value)')

    def expected_parameters(self):
        return [self.parameter_name]

    def choices(self, changelist):
        yield {
            'selected': self.value() is None,
            'query_string': changelist.get_query_string({}, [self.parameter_name]),
            'display': _('All'),
        }
        for lookup, title in self.lookup_choices:
            yield {
                'selected': self.value() == force_text(lookup),
                'query_string': changelist.get_query_string({self.parameter_name: lookup}, []),
                'display': title,
            }

class FieldListFilter(ListFilter):
    _field_list_filters = []
    _take_priority_index = 0

    def __init__(self, field, request, params, model, model_admin, field_path):
        self.field = field
        self.field_path = field_path
        self.title = getattr(field, 'verbose_name', field_path)
        super(FieldListFilter, self).__init__(
            request, params, model, model_admin)
        for p in self.expected_parameters():
            if p in params:
                value = params.pop(p)
                self.used_parameters[p] = prepare_lookup_value(p, value)

    def has_output(self):
        return True

    def queryset(self, request, queryset):
        try:
            return queryset.filter(**self.used_parameters)
        except ValidationError as e:
            raise IncorrectLookupParameters(e)

    @classmethod
    def register(cls, test, list_filter_class, take_priority=False):
        if take_priority:
            # This is to allow overriding the default filters for certain types
            # of fields with some custom filters. The first found in the list
            # is used in priority.
            cls._field_list_filters.insert(
                cls._take_priority_index, (test, list_filter_class))
            cls._take_priority_index += 1
        else:
            cls._field_list_filters.append((test, list_filter_class))

    @classmethod
    def create(cls, field, request, params, model, model_admin, field_path):
        for test, list_filter_class in cls._field_list_filters:
            if not test(field):
                continue
            return list_filter_class(field, request, params,
                model, model_admin, field_path=field_path)

class RelatedFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        other_model = get_model_from_relation(field)
        self.lookup_kwarg = '%s__%s__exact' % (field_path, field.target_field.name)
        self.lookup_kwarg_isnull = '%s__isnull' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        self.lookup_val_isnull = request.GET.get(self.lookup_kwarg_isnull)
        super(RelatedFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)
        self.lookup_choices = self.field_choices(field, request, model_admin)
        if hasattr(field, 'verbose_name'):
            self.lookup_title = field.verbose_name
        else:
            self.lookup_title = other_model._meta.verbose_name
        self.title = self.lookup_title
        self.empty_value_display = model_admin.get_empty_value_display()

    @property
    def include_empty_choice(self):
        """
        Return True if a "(None)" choice should be included, which filters
        out everything except empty relationships.
        """
        return self.field.null or (self.field.is_relation and self.field.many_to_many)

    def has_output(self):
        if self.include_empty_choice:
            extra = 1
        else:
            extra = 0
        return len(self.lookup_choices) + extra > 1

    def expected_parameters(self):
        return [self.lookup_kwarg, self.lookup_kwarg_isnull]

    def field_choices(self, field, request, model_admin):
        return field.get_choices(include_blank=False)

    def choices(self, changelist):
        yield {
            'selected': self.lookup_val is None and not self.lookup_val_isnull,
            'query_string': changelist.get_query_string({},
                [self.lookup_kwarg, self.lookup_kwarg_isnull]),
            'display': _('All'),
        }
        for pk_val, val in self.lookup_choices:
            yield {
                'selected': self.lookup_val == smart_text(pk_val),
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg: pk_val,
                }, [self.lookup_kwarg_isnull]),
                'display': val,
            }
        if self.include_empty_choice:
            yield {
                'selected': bool(self.lookup_val_isnull),
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg_isnull: 'True',
                }, [self.lookup_kwarg]),
                'display': self.empty_value_display,
            }

FieldListFilter.register(lambda f: f.remote_field, RelatedFieldListFilter)

class BooleanFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.lookup_kwarg = '%s__exact' % field_path
        self.lookup_kwarg2 = '%s__isnull' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        self.lookup_val2 = request.GET.get(self.lookup_kwarg2)
        super(BooleanFieldListFilter, self).__init__(field,
            request, params, model, model_admin, field_path)

    def expected_parameters(self):
        return [self.lookup_kwarg, self.lookup_kwarg2]

    def choices(self, changelist):
        for lookup, title in (
                (None, _('All')),
                ('1', _('Yes')),
                ('0', _('No'))):
            yield {
                'selected': self.lookup_val == lookup and not self.lookup_val2,
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg: lookup,
                }, [self.lookup_kwarg2]),
                'display': title,
            }
        if isinstance(self.field, models.NullBooleanField):
            yield {
                'selected': self.lookup_val2 == 'True',
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg2: 'True',
                }, [self.lookup_kwarg]),
                'display': _('Unknown'),
            }

FieldListFilter.register(lambda f: isinstance(f,
    (models.BooleanField, models.NullBooleanField)), BooleanFieldListFilter)

class ChoicesFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.lookup_kwarg = '%s__exact' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        super(ChoicesFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)

    def expected_parameters(self):
        return [self.lookup_kwarg]

    def choices(self, changelist):
        yield {
            'selected': self.lookup_val is None,
            'query_string': changelist.get_query_string({}, [self.lookup_kwarg]),
            'display': _('All')
        }
        for lookup, title in self.field.flatchoices:
            yield {
                'selected': smart_text(lookup) == self.lookup_val,
                'query_string': changelist.get_query_string({self.lookup_kwarg: lookup}),
                'display': title,
            }

FieldListFilter.register(lambda f: bool(f.choices), ChoicesFieldListFilter)

class DateFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.field_generic = '%s__' % field_path
        self.date_params = {k: v for k, v in params.items()
                            if k.startswith(self.field_generic)}

        now = timezone.now()
        # When time zone support is enabled, convert "now" to the user's time
        # zone so Django's definition of "Today" matches what the user expects.
        if timezone.is_aware(now):
            now = timezone.localtime(now)

        if isinstance(field, models.DateTimeField):
            today = now.replace(hour=0, minute=0, second=0, microsecond=0)
        else:       # field is a models.DateField
            today = now.date()
        tomorrow = today + datetime.timedelta(days=1)
        if today.month == 12:
            next_month = today.replace(year=today.year + 1, month=1, day=1)
        else:
            next_month = today.replace(month=today.month + 1, day=1)
        next_year = today.replace(year=today.year + 1, month=1, day=1)

        self.lookup_kwarg_since = '%s__gte' % field_path
        self.lookup_kwarg_until = '%s__lt' % field_path
        self.links = (
            (_('Any date'), {}),
            (_('Today'), {
                self.lookup_kwarg_since: str(today),
                self.lookup_kwarg_until: str(tomorrow),
            }),
            (_('Past 7 days'), {
                self.lookup_kwarg_since: str(today - datetime.timedelta(days=7)),
                self.lookup_kwarg_until: str(tomorrow),
            }),
            (_('This month'), {
                self.lookup_kwarg_since: str(today.replace(day=1)),
                self.lookup_kwarg_until: str(next_month),
            }),
            (_('This year'), {
                self.lookup_kwarg_since: str(today.replace(month=1, day=1)),
                self.lookup_kwarg_until: str(next_year),
            }),
        )
        super(DateFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)

    def expected_parameters(self):
        return [self.lookup_kwarg_since, self.lookup_kwarg_until]

    def choices(self, changelist):
        for title, param_dict in self.links:
            yield {
                'selected': self.date_params == param_dict,
                'query_string': changelist.get_query_string(param_dict, [self.field_generic]),
                'display': title,
            }

FieldListFilter.register(
    lambda f: isinstance(f, models.DateField), DateFieldListFilter)

# This should be registered last, because it's a last resort. For example,
# if a field is eligible to use the BooleanFieldListFilter, that'd be much
# more appropriate, and the AllValuesFieldListFilter won't get used for it.
class AllValuesFieldListFilter(FieldListFilter):
    def __init__(self, field, request, params, model, model_admin, field_path):
        self.lookup_kwarg = field_path
        self.lookup_kwarg_isnull = '%s__isnull' % field_path
        self.lookup_val = request.GET.get(self.lookup_kwarg)
        self.lookup_val_isnull = request.GET.get(self.lookup_kwarg_isnull)
        self.empty_value_display = model_admin.get_empty_value_display()
        parent_model, reverse_path = reverse_field_path(model, field_path)
        # Obey parent ModelAdmin queryset when deciding which options to show
        if model == parent_model:
            queryset = model_admin.get_queryset(request)
        else:
            queryset = parent_model._default_manager.all()
        self.lookup_choices = (queryset
                               .distinct()
                               .order_by(field.name)
                               .values_list(field.name, flat=True))
        super(AllValuesFieldListFilter, self).__init__(
            field, request, params, model, model_admin, field_path)

    def expected_parameters(self):
        return [self.lookup_kwarg, self.lookup_kwarg_isnull]

    def choices(self, changelist):
        yield {
            'selected': (self.lookup_val is None
                and self.lookup_val_isnull is None),
            'query_string': changelist.get_query_string({},
                [self.lookup_kwarg, self.lookup_kwarg_isnull]),
            'display': _('All'),
        }
        include_none = False
        for val in self.lookup_choices:
            if val is None:
                include_none = True
                continue
            val = smart_text(val)
            yield {
                'selected': self.lookup_val == val,
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg: val,
                }, [self.lookup_kwarg_isnull]),
                'display': val,
            }
        if include_none:
            yield {
                'selected': bool(self.lookup_val_isnull),
                'query_string': changelist.get_query_string({
                    self.lookup_kwarg_isnull: 'True',
                }, [self.lookup_kwarg]),
                'display': self.empty_value_display,
            }

FieldListFilter.register(lambda f: True, AllValuesFieldListFilter)

class RelatedOnlyFieldListFilter(RelatedFieldListFilter):
    def field_choices(self, field, request, model_admin):
        pk_qs = model_admin.get_queryset(request).distinct().values_list('%s__pk' % self.field_path, flat=True)
        return field.get_choices(include_blank=False, limit_choices_to={'pk__in': pk_qs})
<|endoftext|>