<commit_before># Copyright (c) 2001-2014, Canal TP and/or its affiliates. All rights reserved.
#
# This file is part of Navitia,
#     the software to build cool stuff with public transport.
#
# Hope you'll enjoy and contribute to this project,
#     powered by Canal TP (www.canaltp.fr).
# Help us simplify mobility and open public transport:
#     a non ending quest to the responsive locomotion way of traveling!
#
# LICENCE: This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Stay tuned using
# twitter @navitia
# IRC #navitia on freenode
# https://groups.google.com/d/forum/navitia
# www.navitia.io

import navitiacommon.type_pb2 as type_pb2
import navitiacommon.request_pb2 as request_pb2
import navitiacommon.response_pb2 as response_pb2
import itertools

pb_type = {
    'stop_area': type_pb2.STOP_AREA,
    'stop_point': type_pb2.STOP_POINT,
    'address': type_pb2.ADDRESS,
    'poi': type_pb2.POI,
    'administrative_region': type_pb2.ADMINISTRATIVE_REGION,
    'line': type_pb2.LINE
}

pt_object_type = {
    'network': type_pb2.NETWORK,
    'commercial_mode': type_pb2.COMMERCIAL_MODE,
    'line': type_pb2.LINE,
    'route': type_pb2.ROUTE,
    'stop_area': type_pb2.STOP_AREA
}

def compare(obj1, obj2, compare_generator):
    """
    To decide that 2 objects are equals, we loop through all values of the
    compare_generator and stop at the first non equals value

    Note: the fillvalue is the value used when a generator is consumed
    (if the 2 generators does not return the same number of elt).
    by setting it to object(), we ensure that it will be !=
    from any values returned by the other generator
    """
    return all(a == b for a, b in itertools.izip_longest(compare_generator(obj1),
                                                         compare_generator(obj2),
                                                         fillvalue=object()))

def are_equals(journey1, journey2):
    """
    To decide that 2 journeys are equals, we loop through all values of the
    compare_journey_generator and stop at the first non equals value
    """
    return compare(journey1, journey2, compare_journey_generator)

def compare_journey_generator(journey):
    """
    Generator used to compare journeys together

    2 journeys are equals if they share for all the sections the same :
     * departure time
     * arrival time
     * vj
     * type
     * departure location
     * arrival location
    """
    yield journey.departure_date_time
    yield journey.arrival_date_time
    yield journey.destination.uri if journey.destination else 'no_destination'
    for s in journey.sections:
        yield s.type
        yield s.begin_date_time
        yield s.end_date_time
        yield s.uris.vehicle_journey
        #NOTE: we want to ensure that we always yield the same number of elt
        yield s.origin.uri if s.origin else 'no_origin'
        yield s.destination.uri if s.destination else 'no_destination'

def count_typed_journeys(journeys):
        return sum(1 for journey in journeys if journey.type)

class JourneySorter(object):
    """
    abstract class for journey sorter
    """
    def __init__(self, clockwise):
        self.clockwise = clockwise

    def __call__(self, j1, j2):
        raise NotImplementedError()

    def sort_by_duration_and_transfert(self, j1, j2):
        """
        we compare the duration, hence it will indirectly compare
        the departure for clockwise, and the arrival for not clockwise
        """
        if j1.duration != j2.duration:
            return j2.duration - j1.duration

        if j1.nb_transfers != j2.nb_transfers:
            return j1.nb_transfers - j2.nb_transfers

        #afterward we compare the non pt duration
        non_pt_duration_j1 = non_pt_duration_j2 = None
        for journey in [j1, j2]:
            non_pt_duration = 0
            for section in journey.sections:
                if section.type != response_pb2.PUBLIC_TRANSPORT:
                    non_pt_duration += section.duration
            if non_pt_duration_j1 is None:
                non_pt_duration_j1 = non_pt_duration
            else:
                non_pt_duration_j2 = non_pt_duration
        return non_pt_duration_j1 - non_pt_duration_j2

class ArrivalJourneySorter(JourneySorter):
    """
    Journey comparator for sort, on clockwise the sort is done by arrival time

    the comparison is different if the query is for clockwise search or not
    """
    def __init__(self, clockwise):
        super(ArrivalJourneySorter, self).__init__(clockwise)

    def __call__(self, j1, j2):
        if self.clockwise:
            #for clockwise query, we want to sort first on the arrival time
            if j1.arrival_date_time != j2.arrival_date_time:
                return -1 if j1.arrival_date_time < j2.arrival_date_time else 1
        else:
            #for non clockwise the first sort is done on departure
            if j1.departure_date_time != j2.departure_date_time:
                return -1 if j1.departure_date_time > j2.departure_date_time else 1

        return self.sort_by_duration_and_transfert(j1, j2)

class DepartureJourneySorter(JourneySorter):
    """
    Journey comparator for sort, the sort is done by departure time

    """
    def __init__(self, clockwise):
        super(DepartureJourneySorter, self).__init__(clockwise)

    def __call__(self, j1, j2):

        if j1.departure_date_time != j2.departure_date_time:
            return -1 if j1.departure_date_time < j2.departure_date_time else 1

        return self.sort_by_duration_and_transfert(j1, j2)

def build_pagination(request, resp):
    pagination = resp.pagination
    if pagination.totalResult > 0:
        query_args = ""
        for key, value in request.iteritems():
            if key != "startPage":
                if isinstance(value, type([])):
                    for v in value:
                        query_args += key + "=" + unicode(v) + "&"
                else:
                    query_args += key + "=" + unicode(value) + "&"
        if pagination.startPage > 0:
            page = pagination.startPage - 1
            pagination.previousPage = query_args
            pagination.previousPage += "start_page=%i" % page
        last_id_page = (pagination.startPage + 1) * pagination.itemsPerPage
        if last_id_page < pagination.totalResult:
            page = pagination.startPage + 1
            pagination.nextPage = query_args + "start_page=%i" % page

journey_sorter = {
    'arrival_time': ArrivalJourneySorter,
    'departure_time': DepartureJourneySorter
}
<commit_msg>Revert "jormungandr: when sorting by departure, we always sort by departure"<commit_after># Copyright (c) 2001-2014, Canal TP and/or its affiliates. All rights reserved.
#
# This file is part of Navitia,
#     the software to build cool stuff with public transport.
#
# Hope you'll enjoy and contribute to this project,
#     powered by Canal TP (www.canaltp.fr).
# Help us simplify mobility and open public transport:
#     a non ending quest to the responsive locomotion way of traveling!
#
# LICENCE: This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# Stay tuned using
# twitter @navitia
# IRC #navitia on freenode
# https://groups.google.com/d/forum/navitia
# www.navitia.io

import navitiacommon.type_pb2 as type_pb2
import navitiacommon.request_pb2 as request_pb2
import navitiacommon.response_pb2 as response_pb2
import itertools

pb_type = {
    'stop_area': type_pb2.STOP_AREA,
    'stop_point': type_pb2.STOP_POINT,
    'address': type_pb2.ADDRESS,
    'poi': type_pb2.POI,
    'administrative_region': type_pb2.ADMINISTRATIVE_REGION,
    'line': type_pb2.LINE
}

pt_object_type = {
    'network': type_pb2.NETWORK,
    'commercial_mode': type_pb2.COMMERCIAL_MODE,
    'line': type_pb2.LINE,
    'route': type_pb2.ROUTE,
    'stop_area': type_pb2.STOP_AREA
}

def compare(obj1, obj2, compare_generator):
    """
    To decide that 2 objects are equals, we loop through all values of the
    compare_generator and stop at the first non equals value

    Note: the fillvalue is the value used when a generator is consumed
    (if the 2 generators does not return the same number of elt).
    by setting it to object(), we ensure that it will be !=
    from any values returned by the other generator
    """
    return all(a == b for a, b in itertools.izip_longest(compare_generator(obj1),
                                                         compare_generator(obj2),
                                                         fillvalue=object()))

def are_equals(journey1, journey2):
    """
    To decide that 2 journeys are equals, we loop through all values of the
    compare_journey_generator and stop at the first non equals value
    """
    return compare(journey1, journey2, compare_journey_generator)

def compare_journey_generator(journey):
    """
    Generator used to compare journeys together

    2 journeys are equals if they share for all the sections the same :
     * departure time
     * arrival time
     * vj
     * type
     * departure location
     * arrival location
    """
    yield journey.departure_date_time
    yield journey.arrival_date_time
    yield journey.destination.uri if journey.destination else 'no_destination'
    for s in journey.sections:
        yield s.type
        yield s.begin_date_time
        yield s.end_date_time
        yield s.uris.vehicle_journey
        #NOTE: we want to ensure that we always yield the same number of elt
        yield s.origin.uri if s.origin else 'no_origin'
        yield s.destination.uri if s.destination else 'no_destination'

def count_typed_journeys(journeys):
        return sum(1 for journey in journeys if journey.type)

class JourneySorter(object):
    """
    abstract class for journey sorter
    """
    def __init__(self, clockwise):
        self.clockwise = clockwise

    def __call__(self, j1, j2):
        raise NotImplementedError()

    def sort_by_duration_and_transfert(self, j1, j2):
        """
        we compare the duration, hence it will indirectly compare
        the departure for clockwise, and the arrival for not clockwise
        """
        if j1.duration != j2.duration:
            return j2.duration - j1.duration

        if j1.nb_transfers != j2.nb_transfers:
            return j1.nb_transfers - j2.nb_transfers

        #afterward we compare the non pt duration
        non_pt_duration_j1 = non_pt_duration_j2 = None
        for journey in [j1, j2]:
            non_pt_duration = 0
            for section in journey.sections:
                if section.type != response_pb2.PUBLIC_TRANSPORT:
                    non_pt_duration += section.duration
            if non_pt_duration_j1 is None:
                non_pt_duration_j1 = non_pt_duration
            else:
                non_pt_duration_j2 = non_pt_duration
        return non_pt_duration_j1 - non_pt_duration_j2

class ArrivalJourneySorter(JourneySorter):
    """
    Journey comparator for sort, on clockwise the sort is done by arrival time

    the comparison is different if the query is for clockwise search or not
    """
    def __init__(self, clockwise):
        super(ArrivalJourneySorter, self).__init__(clockwise)

    def __call__(self, j1, j2):
        if self.clockwise:
            #for clockwise query, we want to sort first on the arrival time
            if j1.arrival_date_time != j2.arrival_date_time:
                return -1 if j1.arrival_date_time < j2.arrival_date_time else 1
        else:
            #for non clockwise the first sort is done on departure
            if j1.departure_date_time != j2.departure_date_time:
                return -1 if j1.departure_date_time > j2.departure_date_time else 1

        return self.sort_by_duration_and_transfert(j1, j2)

class DepartureJourneySorter(JourneySorter):
    """
    Journey comparator for sort, on clockwise the sort is done by departure time

    the comparison is different if the query is for clockwise search or not
    """
    def __init__(self, clockwise):
        super(DepartureJourneySorter, self).__init__(clockwise)

    def __call__(self, j1, j2):

        if self.clockwise:
            #for clockwise query, we want to sort first on the departure time
            if j1.departure_date_time != j2.departure_date_time:
                return -1 if j1.departure_date_time < j2.departure_date_time else 1
        else:
            #for non clockwise the first sort is done on arrival
            if j1.arrival_date_time != j2.arrival_date_time:
                return -1 if j1.arrival_date_time > j2.arrival_date_time else 1

        return self.sort_by_duration_and_transfert(j1, j2)

def build_pagination(request, resp):
    pagination = resp.pagination
    if pagination.totalResult > 0:
        query_args = ""
        for key, value in request.iteritems():
            if key != "startPage":
                if isinstance(value, type([])):
                    for v in value:
                        query_args += key + "=" + unicode(v) + "&"
                else:
                    query_args += key + "=" + unicode(value) + "&"
        if pagination.startPage > 0:
            page = pagination.startPage - 1
            pagination.previousPage = query_args
            pagination.previousPage += "start_page=%i" % page
        last_id_page = (pagination.startPage + 1) * pagination.itemsPerPage
        if last_id_page < pagination.totalResult:
            page = pagination.startPage + 1
            pagination.nextPage = query_args + "start_page=%i" % page

journey_sorter = {
    'arrival_time': ArrivalJourneySorter,
    'departure_time': DepartureJourneySorter
}
<|endoftext|>


<commit_before>import GPflow
import tensorflow as tf
from GPflow.tf_hacks import eye
import numpy as np
import unittest
from IPython import embed
from reference import *

def referenceUnivariateLogMarginalLikelihood( y, K, noiseVariance ):
    return -0.5 * y * y / ( K + noiseVariance ) - 0.5 * np.log( K + noiseVariance ) - 0.5 * np.log( np.pi * 2. )

def referenceUnivariatePosterior( y, K, noiseVariance ):
    mean = K * y / ( K + noiseVariance )
    variance = K - K / ( K + noiseVariance )
    return mean, variance

def referenceUnivariatePriorKL( meanA, meanB, varA, varB ):
    #KL[ qA | qB ] = E_{qA} \log [qA / qB] where qA and qB are univariate normal distributions.
    return 0.5 * ( np.log( varB ) - np.log( varA) - 1. + varA/varB + (meanB-meanA) * (meanB - meanA) / varB )

def referenceMultivariatePriorKL( meanA, covA, meanB, covB ):
    #KL[ qA | qB ] = E_{qA} \log [qA / qB] where qA and aB are K dimensional multivariate normal distributions.
    #Analytically tractable and equal to...
    #... 0.5 * ( Tr( covB^{-1} covA) + (meanB - meanA)^T covB^{-1} (meanB - meanA) - K + log( det( covB ) ) - log ( det( covA ) ) )
    K = covA.shape[0]
    traceTerm = 0.5 * np.trace( np.linalg.solve( covB, covA ) )
    delta = meanB - meanA
    mahalanobisTerm = 0.5 * np.dot( delta.T, np.linalg.solve( covB, delta ) )
    constantTerm = -0.5 * K
    priorLogDeterminantTerm = 0.5*np.linalg.slogdet( covB )[1]
    variationalLogDeterminantTerm = -0.5 * np.linalg.slogdet( covA )[1]
    return traceTerm + mahalanobisTerm + constantTerm + priorLogDeterminantTerm + variationalLogDeterminantTerm

def kernel(kernelVariance=1,lengthScale=1.):
    kern = GPflow.kernels.RBF(1)
    kern.variance = kernelVariance
    kern.lengthscales = lengthScale
    return kern

class VariationalUnivariateTest(unittest.TestCase):
    def setUp(self):
        #def __init__(self, X, Y, kern, likelihood, Z, mean_function=Zero(), num_latent=None, q_diag=False, whiten=True):
        self.y_real = 2.
        self.K = 1.
        self.noiseVariance = 0.5
        self.univariate = 1
        self.oneLatentFunction = 1
        self.meanZero = 0.
        self.X = np.atleast_2d( np.array( [0.] ) )
        self.Y = np.atleast_2d( np.array( [self.y_real] ) )
        self.Z = self.X.copy()
        self.lik = GPflow.likelihoods.Gaussian()
        self.lik.variance = self.noiseVariance
        self.posteriorMean, self.posteriorVariance = referenceUnivariatePosterior( y=self.y_real, K = self.K, noiseVariance = self.noiseVariance )
        self.posteriorStd = np.sqrt( self.posteriorVariance )

    def get_model( self, is_diagonal, is_whitened ):
        model = GPflow.svgp.SVGP( X=self.X, Y=self.Y, kern=kernel(kernelVariance=self.K), likelihood=self.lik, Z=self.Z, q_diag = is_diagonal, whiten = is_whitened )
        if is_diagonal:
            model.q_sqrt = np.ones((self.univariate, self.oneLatentFunction))*self.posteriorStd
        else:
            model.q_sqrt = np.ones( (self.univariate, self.univariate, self.oneLatentFunction ) ) * self.posteriorStd
        model.q_mu =  np.ones((self.univariate, self.oneLatentFunction))* self.posteriorMean
        model.make_tf_array(model._free_vars)
        return model

    def test_prior_KL( self ):
        meanA = self.posteriorMean
        varA = self.posteriorVariance
        meanB = self.meanZero #Assumes a zero
        varB = self.K

        referenceKL = referenceUnivariatePriorKL( meanA, meanB, varA, varB )

        for is_diagonal in [True,False]:
            for is_whitened in [True,False]:
                model = self.get_model( is_diagonal, is_whitened )
                with model.tf_mode():
                    prior_KL_function = model.build_prior_KL()
                test_prior_KL = prior_KL_function.eval( session = model._session, feed_dict = {model._free_vars: model.get_free_state() } )
                self.failUnless( np.abs( referenceKL - test_prior_KL ) < 1e-4 )

    def test_build_likelihood(self):
        #reference marginal likelihood
        log_marginal_likelihood = referenceUnivariateLogMarginalLikelihood( y=self.y_real, K = self.K, noiseVariance = self.noiseVariance )

        for is_diagonal in [True,False]:
            for is_whitened in [True,False]:
                model = self.get_model( is_diagonal, is_whitened )
                model._compile()
                with model.tf_mode():
                    model_likelihood_function = model.build_likelihood()
                model_likelihood = model_likelihood_function.eval( session = model._session, feed_dict = {model._free_vars: model.get_free_state() } )
                self.failUnless( np.abs( model_likelihood - log_marginal_likelihood ) < 1e-4 )

    def testUnivariateConditionals(self):
        for is_diagonal in [True,False]:
            for is_whitened in [True,False]:
                model = self.get_model( is_diagonal, is_whitened )
                with model.tf_mode():
                    if is_whitened:
                        fmean_func, fvar_func = GPflow.conditionals.gaussian_gp_predict_whitened(self.X, self.Z, model.kern, model.q_mu, model.q_sqrt, self.oneLatentFunction )
                    else:
                        fmean_func, fvar_func = GPflow.conditionals.gaussian_gp_predict(self.X, self.Z, model.kern, model.q_mu, model.q_sqrt, self.oneLatentFunction)
                mean_value = fmean_func.eval( session = model._session, feed_dict = {model._free_vars: model.get_free_state() } )[0,0]
                var_value = fvar_func.eval( session = model._session, feed_dict = {model._free_vars: model.get_free_state() } )[0,0]
                self.failUnless( np.abs( mean_value - self.posteriorMean ) < 1e-4 )
                self.failUnless( np.abs( var_value - self.posteriorVariance ) < 1e-4 )

class VariationalMultivariateTest(unittest.TestCase):
    def setUp( self ):
        self.nDimensions = 3
        self.rng = np.random.RandomState(1)
        self.Y = self.rng.randn( self.nDimensions, 1 )
        self.X = self.rng.randn( self.nDimensions, 1 )
        self.Z = self.X.copy()
        self.noiseVariance = 0.5
        self.signalVariance = 1.5
        self.lengthScale = 1.7
        self.oneLatentFunction = 1
        self.lik = GPflow.likelihoods.Gaussian()
        self.lik.variance = self.noiseVariance
        self.q_mean = self.rng.randn( self.nDimensions, self.oneLatentFunction )
        self.q_sqrt_diag = self.rng.rand( self.nDimensions, self.oneLatentFunction )
        self.q_sqrt_full = np.tril( self.rng.rand( self.nDimensions, self.nDimensions ) )

    def getModel( self, is_diagonal, is_whitened  ):
        model = GPflow.svgp.SVGP( X=self.X, Y=self.Y, kern=kernel(kernelVariance=self.signalVariance,lengthScale=self.lengthScale), likelihood=self.lik, Z=self.Z, q_diag = is_diagonal, whiten = is_whitened )
        if is_diagonal:
            model.q_sqrt = self.q_sqrt_diag
        else:
            model.q_sqrt = self.q_sqrt_full[:,:,None]
        model.q_mu =  self.q_mean
        model.make_tf_array(model._free_vars)
        return model

    def test_refrence_implementation_consistency( self ):
        rng = np.random.RandomState(10)
        qMean = rng.randn()
        qCov = rng.rand()
        pMean = rng.rand()
        pCov = rng.rand()
        univariate_KL = referenceUnivariatePriorKL( qMean, pMean, qCov, pCov )
        multivariate_KL = referenceMultivariatePriorKL( np.array( [[qMean]] ), np.array( [[qCov]]), np.array( [[pMean]] ), np.array( [[pCov]] ) )
        self.failUnless( np.abs( univariate_KL - multivariate_KL ) < 1e-4 )

    def test_prior_KL_fullQ( self ):
        covQ = np.dot( self.q_sqrt_full, self.q_sqrt_full.T )
        mean_prior = np.zeros( ( self.nDimensions, 1 ) )

        for is_whitened in [True,False]:
            model = self.getModel( False, is_whitened )

            if is_whitened:
                cov_prior = np.eye( self.nDimensions )
            else:
                cov_prior = referenceRbfKernel( self.X, self.lengthScale, self.signalVariance )

            referenceKL = referenceMultivariatePriorKL( self.q_mean, covQ, mean_prior, cov_prior )
            #now get test KL.
            with model.tf_mode():
                prior_KL_function = model.build_prior_KL()
            test_prior_KL = prior_KL_function.eval( session = model._session, feed_dict = {model._free_vars: model.get_free_state() } )
            self.failUnless( np.abs( referenceKL - test_prior_KL ) < 1e-4 )

if __name__ == "__main__":
    unittest.main()

<commit_msg>removing IPython import<commit_after>import GPflow
import tensorflow as tf
from GPflow.tf_hacks import eye
import numpy as np
import unittest
from reference import *

def referenceUnivariateLogMarginalLikelihood( y, K, noiseVariance ):
    return -0.5 * y * y / ( K + noiseVariance ) - 0.5 * np.log( K + noiseVariance ) - 0.5 * np.log( np.pi * 2. )

def referenceUnivariatePosterior( y, K, noiseVariance ):
    mean = K * y / ( K + noiseVariance )
    variance = K - K / ( K + noiseVariance )
    return mean, variance

def referenceUnivariatePriorKL( meanA, meanB, varA, varB ):
    #KL[ qA | qB ] = E_{qA} \log [qA / qB] where qA and qB are univariate normal distributions.
    return 0.5 * ( np.log( varB ) - np.log( varA) - 1. + varA/varB + (meanB-meanA) * (meanB - meanA) / varB )

def referenceMultivariatePriorKL( meanA, covA, meanB, covB ):
    #KL[ qA | qB ] = E_{qA} \log [qA / qB] where qA and aB are K dimensional multivariate normal distributions.
    #Analytically tractable and equal to...
    #... 0.5 * ( Tr( covB^{-1} covA) + (meanB - meanA)^T covB^{-1} (meanB - meanA) - K + log( det( covB ) ) - log ( det( covA ) ) )
    K = covA.shape[0]
    traceTerm = 0.5 * np.trace( np.linalg.solve( covB, covA ) )
    delta = meanB - meanA
    mahalanobisTerm = 0.5 * np.dot( delta.T, np.linalg.solve( covB, delta ) )
    constantTerm = -0.5 * K
    priorLogDeterminantTerm = 0.5*np.linalg.slogdet( covB )[1]
    variationalLogDeterminantTerm = -0.5 * np.linalg.slogdet( covA )[1]
    return traceTerm + mahalanobisTerm + constantTerm + priorLogDeterminantTerm + variationalLogDeterminantTerm

def kernel(kernelVariance=1,lengthScale=1.):
    kern = GPflow.kernels.RBF(1)
    kern.variance = kernelVariance
    kern.lengthscales = lengthScale
    return kern

class VariationalUnivariateTest(unittest.TestCase):
    def setUp(self):
        #def __init__(self, X, Y, kern, likelihood, Z, mean_function=Zero(), num_latent=None, q_diag=False, whiten=True):
        self.y_real = 2.
        self.K = 1.
        self.noiseVariance = 0.5
        self.univariate = 1
        self.oneLatentFunction = 1
        self.meanZero = 0.
        self.X = np.atleast_2d( np.array( [0.] ) )
        self.Y = np.atleast_2d( np.array( [self.y_real] ) )
        self.Z = self.X.copy()
        self.lik = GPflow.likelihoods.Gaussian()
        self.lik.variance = self.noiseVariance
        self.posteriorMean, self.posteriorVariance = referenceUnivariatePosterior( y=self.y_real, K = self.K, noiseVariance = self.noiseVariance )
        self.posteriorStd = np.sqrt( self.posteriorVariance )

    def get_model( self, is_diagonal, is_whitened ):
        model = GPflow.svgp.SVGP( X=self.X, Y=self.Y, kern=kernel(kernelVariance=self.K), likelihood=self.lik, Z=self.Z, q_diag = is_diagonal, whiten = is_whitened )
        if is_diagonal:
            model.q_sqrt = np.ones((self.univariate, self.oneLatentFunction))*self.posteriorStd
        else:
            model.q_sqrt = np.ones( (self.univariate, self.univariate, self.oneLatentFunction ) ) * self.posteriorStd
        model.q_mu =  np.ones((self.univariate, self.oneLatentFunction))* self.posteriorMean
        model.make_tf_array(model._free_vars)
        return model

    def test_prior_KL( self ):
        meanA = self.posteriorMean
        varA = self.posteriorVariance
        meanB = self.meanZero #Assumes a zero
        varB = self.K

        referenceKL = referenceUnivariatePriorKL( meanA, meanB, varA, varB )

        for is_diagonal in [True,False]:
            for is_whitened in [True,False]:
                model = self.get_model( is_diagonal, is_whitened )
                with model.tf_mode():
                    prior_KL_function = model.build_prior_KL()
                test_prior_KL = prior_KL_function.eval( session = model._session, feed_dict = {model._free_vars: model.get_free_state() } )
                self.failUnless( np.abs( referenceKL - test_prior_KL ) < 1e-4 )

    def test_build_likelihood(self):
        #reference marginal likelihood
        log_marginal_likelihood = referenceUnivariateLogMarginalLikelihood( y=self.y_real, K = self.K, noiseVariance = self.noiseVariance )

        for is_diagonal in [True,False]:
            for is_whitened in [True,False]:
                model = self.get_model( is_diagonal, is_whitened )
                model._compile()
                with model.tf_mode():
                    model_likelihood_function = model.build_likelihood()
                model_likelihood = model_likelihood_function.eval( session = model._session, feed_dict = {model._free_vars: model.get_free_state() } )
                self.failUnless( np.abs( model_likelihood - log_marginal_likelihood ) < 1e-4 )

    def testUnivariateConditionals(self):
        for is_diagonal in [True,False]:
            for is_whitened in [True,False]:
                model = self.get_model( is_diagonal, is_whitened )
                with model.tf_mode():
                    if is_whitened:
                        fmean_func, fvar_func = GPflow.conditionals.gaussian_gp_predict_whitened(self.X, self.Z, model.kern, model.q_mu, model.q_sqrt, self.oneLatentFunction )
                    else:
                        fmean_func, fvar_func = GPflow.conditionals.gaussian_gp_predict(self.X, self.Z, model.kern, model.q_mu, model.q_sqrt, self.oneLatentFunction)
                mean_value = fmean_func.eval( session = model._session, feed_dict = {model._free_vars: model.get_free_state() } )[0,0]
                var_value = fvar_func.eval( session = model._session, feed_dict = {model._free_vars: model.get_free_state() } )[0,0]
                self.failUnless( np.abs( mean_value - self.posteriorMean ) < 1e-4 )
                self.failUnless( np.abs( var_value - self.posteriorVariance ) < 1e-4 )

class VariationalMultivariateTest(unittest.TestCase):
    def setUp( self ):
        self.nDimensions = 3
        self.rng = np.random.RandomState(1)
        self.Y = self.rng.randn( self.nDimensions, 1 )
        self.X = self.rng.randn( self.nDimensions, 1 )
        self.Z = self.X.copy()
        self.noiseVariance = 0.5
        self.signalVariance = 1.5
        self.lengthScale = 1.7
        self.oneLatentFunction = 1
        self.lik = GPflow.likelihoods.Gaussian()
        self.lik.variance = self.noiseVariance
        self.q_mean = self.rng.randn( self.nDimensions, self.oneLatentFunction )
        self.q_sqrt_diag = self.rng.rand( self.nDimensions, self.oneLatentFunction )
        self.q_sqrt_full = np.tril( self.rng.rand( self.nDimensions, self.nDimensions ) )

    def getModel( self, is_diagonal, is_whitened  ):
        model = GPflow.svgp.SVGP( X=self.X, Y=self.Y, kern=kernel(kernelVariance=self.signalVariance,lengthScale=self.lengthScale), likelihood=self.lik, Z=self.Z, q_diag = is_diagonal, whiten = is_whitened )
        if is_diagonal:
            model.q_sqrt = self.q_sqrt_diag
        else:
            model.q_sqrt = self.q_sqrt_full[:,:,None]
        model.q_mu =  self.q_mean
        model.make_tf_array(model._free_vars)
        return model

    def test_refrence_implementation_consistency( self ):
        rng = np.random.RandomState(10)
        qMean = rng.randn()
        qCov = rng.rand()
        pMean = rng.rand()
        pCov = rng.rand()
        univariate_KL = referenceUnivariatePriorKL( qMean, pMean, qCov, pCov )
        multivariate_KL = referenceMultivariatePriorKL( np.array( [[qMean]] ), np.array( [[qCov]]), np.array( [[pMean]] ), np.array( [[pCov]] ) )
        self.failUnless( np.abs( univariate_KL - multivariate_KL ) < 1e-4 )

    def test_prior_KL_fullQ( self ):
        covQ = np.dot( self.q_sqrt_full, self.q_sqrt_full.T )
        mean_prior = np.zeros( ( self.nDimensions, 1 ) )

        for is_whitened in [True,False]:
            model = self.getModel( False, is_whitened )

            if is_whitened:
                cov_prior = np.eye( self.nDimensions )
            else:
                cov_prior = referenceRbfKernel( self.X, self.lengthScale, self.signalVariance )

            referenceKL = referenceMultivariatePriorKL( self.q_mean, covQ, mean_prior, cov_prior )
            #now get test KL.
            with model.tf_mode():
                prior_KL_function = model.build_prior_KL()
            test_prior_KL = prior_KL_function.eval( session = model._session, feed_dict = {model._free_vars: model.get_free_state() } )
            self.failUnless( np.abs( referenceKL - test_prior_KL ) < 1e-4 )

if __name__ == "__main__":
    unittest.main()

<|endoftext|>


<commit_before>import sys
import ast
import os
import glob
import re
import stat

from setuptools.command.egg_info import egg_info, manifest_maker
from setuptools.dist import Distribution
from setuptools.extern.six.moves import map

import pytest

from . import environment
from .files import build_files
from .textwrap import DALS
from . import contexts

class Environment(str):
    pass

class TestEggInfo(object):

    setup_script = DALS("""
        from setuptools import setup

        setup(
            name='foo',
            py_modules=['hello'],
            entry_points={'console_scripts': ['hi = hello.run']},
            zip_safe=False,
        )
        """)

    def _create_project(self):
        build_files({
            'setup.py': self.setup_script,
            'hello.py': DALS("""
                def run():
                    print('hello')
                """)
        })

    @pytest.yield_fixture
    def env(self):
        with contexts.tempdir(prefix='setuptools-test.') as env_dir:
            env = Environment(env_dir)
            os.chmod(env_dir, stat.S_IRWXU)
            subs = 'home', 'lib', 'scripts', 'data', 'egg-base'
            env.paths = dict(
                (dirname, os.path.join(env_dir, dirname))
                for dirname in subs
            )
            list(map(os.mkdir, env.paths.values()))
            build_files({
                env.paths['home']: {
                    '.pydistutils.cfg': DALS("""
                    [egg_info]
                    egg-base = %(egg-base)s
                    """ % env.paths)
                }
            })
            yield env

    def test_egg_info_save_version_info_setup_empty(self, tmpdir_cwd, env):
        """
        When the egg_info section is empty or not present, running
        save_version_info should add the settings to the setup.cfg
        in a deterministic order, consistent with the ordering found
        on Python 2.7 with PYTHONHASHSEED=0.
        """
        setup_cfg = os.path.join(env.paths['home'], 'setup.cfg')
        dist = Distribution()
        ei = egg_info(dist)
        ei.initialize_options()
        ei.save_version_info(setup_cfg)

        with open(setup_cfg, 'r') as f:
            content = f.read()

        assert '[egg_info]' in content
        assert 'tag_build =' in content
        assert 'tag_date = 0' in content

        expected_order = 'tag_build', 'tag_date',

        self._validate_content_order(content, expected_order)

    @staticmethod
    def _validate_content_order(content, expected):
        """
        Assert that the strings in expected appear in content
        in order.
        """
        pattern = '.*'.join(expected)
        flags = re.MULTILINE | re.DOTALL
        assert re.search(pattern, content, flags)

    def test_egg_info_save_version_info_setup_defaults(self, tmpdir_cwd, env):
        """
        When running save_version_info on an existing setup.cfg
        with the 'default' values present from a previous run,
        the file should remain unchanged.
        """
        setup_cfg = os.path.join(env.paths['home'], 'setup.cfg')
        build_files({
            setup_cfg: DALS("""
            [egg_info]
            tag_build =
            tag_date = 0
            """),
        })
        dist = Distribution()
        ei = egg_info(dist)
        ei.initialize_options()
        ei.save_version_info(setup_cfg)

        with open(setup_cfg, 'r') as f:
            content = f.read()

        assert '[egg_info]' in content
        assert 'tag_build =' in content
        assert 'tag_date = 0' in content

        expected_order = 'tag_build', 'tag_date',

        self._validate_content_order(content, expected_order)

    def test_egg_base_installed_egg_info(self, tmpdir_cwd, env):
        self._create_project()

        self._run_install_command(tmpdir_cwd, env)
        actual = self._find_egg_info_files(env.paths['lib'])

        expected = [
            'PKG-INFO',
            'SOURCES.txt',
            'dependency_links.txt',
            'entry_points.txt',
            'not-zip-safe',
            'top_level.txt',
        ]
        assert sorted(actual) == expected

    def test_manifest_template_is_read(self, tmpdir_cwd, env):
        self._create_project()
        build_files({
            'MANIFEST.in': DALS("""
                recursive-include docs *.rst
            """),
            'docs': {
                'usage.rst': "Run 'hi'",
            }
        })
        self._run_install_command(tmpdir_cwd, env)
        egg_info_dir = self._find_egg_info_files(env.paths['lib']).base
        sources_txt = os.path.join(egg_info_dir, 'SOURCES.txt')
        with open(sources_txt) as f:
            assert 'docs/usage.rst' in f.read().split('\n')

    def _setup_script_with_requires(self, requires, use_setup_cfg=False):
        setup_script = DALS(
            '''
            from setuptools import setup

            setup(name='foo', zip_safe=False, %s)
            '''
        ) % ('' if use_setup_cfg else requires)
        setup_config = requires if use_setup_cfg else ''
        build_files({'setup.py': setup_script,
                     'setup.cfg': setup_config})

    mismatch_marker = "python_version<'{this_ver}'".format(
        this_ver=sys.version_info[0],
    )
    # Alternate equivalent syntax.
    mismatch_marker_alternate = 'python_version < "{this_ver}"'.format(
        this_ver=sys.version_info[0],
    )
    invalid_marker = "<=>++"

    class RequiresTestHelper(object):

        @staticmethod
        def parametrize(*test_list, **format_dict):
            idlist = []
            argvalues = []
            for test in test_list:
                test_params = test.lstrip().split('\n\n', 3)
                name_kwargs = test_params.pop(0).split('\n')
                if len(name_kwargs) > 1:
                    install_cmd_kwargs = ast.literal_eval(name_kwargs[1].strip())
                else:
                    install_cmd_kwargs = {}
                name = name_kwargs[0].strip()
                setup_py_requires, setup_cfg_requires, expected_requires = (
                    DALS(a).format(**format_dict) for a in test_params
                )
                for id_, requires, use_cfg in (
                    (name, setup_py_requires, False),
                    (name + '_in_setup_cfg', setup_cfg_requires, True),
                ):
                    idlist.append(id_)
                    marks = ()
                    if requires.startswith('@xfail\n'):
                        requires = requires[7:]
                        marks = pytest.mark.xfail
                    argvalues.append(pytest.param(requires, use_cfg,
                                                  expected_requires,
                                                  install_cmd_kwargs,
                                                  marks=marks))
            return pytest.mark.parametrize('requires,use_setup_cfg,'
                                           'expected_requires,install_cmd_kwargs',
                                           argvalues, ids=idlist)

    @RequiresTestHelper.parametrize(
        # Format of a test:
        #
        # id
        # install_cmd_kwargs [optional]
        #
        # requires block (when used in setup.py)
        #
        # requires block (when used in setup.cfg)
        #
        # expected contents of requires.txt

        '''
        install_requires_with_marker

        install_requires=["barbazquux;{mismatch_marker}"],

        [options]
        install_requires =
            barbazquux; {mismatch_marker}

        [:{mismatch_marker_alternate}]
        barbazquux
        ''',

        '''
        install_requires_with_extra
        {'cmd': ['egg_info']}

        install_requires=["barbazquux [test]"],

        [options]
        install_requires =
            barbazquux [test]

        barbazquux[test]
        ''',

        '''
        install_requires_with_extra_and_marker

        install_requires=["barbazquux [test]; {mismatch_marker}"],

        [options]
        install_requires =
            barbazquux [test]; {mismatch_marker}

        [:{mismatch_marker_alternate}]
        barbazquux[test]
        ''',

        '''
        setup_requires_with_markers

        setup_requires=["barbazquux;{mismatch_marker}"],

        [options]
        setup_requires =
            barbazquux; {mismatch_marker}

        ''',

        '''
        tests_require_with_markers
        {'cmd': ['test'], 'output': "Ran 0 tests in"}

        tests_require=["barbazquux;{mismatch_marker}"],

        [options]
        tests_require =
            barbazquux; {mismatch_marker}

        ''',

        '''
        extras_require_with_extra
        {'cmd': ['egg_info']}

        extras_require={{"extra": ["barbazquux [test]"]}},

        [options.extras_require]
        extra = barbazquux [test]

        [extra]
        barbazquux[test]
        ''',

        '''
        extras_require_with_extra_and_marker_in_req

        extras_require={{"extra": ["barbazquux [test]; {mismatch_marker}"]}},

        [options.extras_require]
        extra =
            barbazquux [test]; {mismatch_marker}

        [extra]

        [extra:{mismatch_marker_alternate}]
        barbazquux[test]
        ''',

        # FIXME: ConfigParser does not allow : in key names!
        '''
        extras_require_with_marker

        extras_require={{":{mismatch_marker}": ["barbazquux"]}},

        @xfail
        [options.extras_require]
        :{mismatch_marker} = barbazquux

        [:{mismatch_marker}]
        barbazquux
        ''',

        '''
        extras_require_with_marker_in_req

        extras_require={{"extra": ["barbazquux; {mismatch_marker}"]}},

        [options.extras_require]
        extra =
            barbazquux; {mismatch_marker}

        [extra]

        [extra:{mismatch_marker_alternate}]
        barbazquux
        ''',

        '''
        extras_require_with_empty_section

        extras_require={{"empty": []}},

        [options.extras_require]
        empty =

        [empty]
        ''',
        # Format arguments.
        invalid_marker=invalid_marker,
        mismatch_marker=mismatch_marker,
        mismatch_marker_alternate=mismatch_marker_alternate,
    )
    def test_requires(self, tmpdir_cwd, env,
                      requires, use_setup_cfg,
                      expected_requires, install_cmd_kwargs):
        self._setup_script_with_requires(requires, use_setup_cfg)
        self._run_install_command(tmpdir_cwd, env, **install_cmd_kwargs)
        egg_info_dir = os.path.join('.', 'foo.egg-info')
        requires_txt = os.path.join(egg_info_dir, 'requires.txt')
        if os.path.exists(requires_txt):
            with open(requires_txt) as fp:
                install_requires = fp.read()
        else:
            install_requires = ''
        assert install_requires.lstrip() == expected_requires
        assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == []

    def test_extras_require_with_invalid_marker(self, tmpdir_cwd, env):
        tmpl = 'extras_require={{":{marker}": ["barbazquux"]}},'
        req = tmpl.format(marker=self.invalid_marker)
        self._setup_script_with_requires(req)
        with pytest.raises(AssertionError):
            self._run_install_command(tmpdir_cwd, env)
        assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == []

    def test_extras_require_with_invalid_marker_in_req(self, tmpdir_cwd, env):
        tmpl = 'extras_require={{"extra": ["barbazquux; {marker}"]}},'
        req = tmpl.format(marker=self.invalid_marker)
        self._setup_script_with_requires(req)
        with pytest.raises(AssertionError):
            self._run_install_command(tmpdir_cwd, env)
        assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == []

    def test_long_description_content_type(self, tmpdir_cwd, env):
        # Test that specifying a `long_description_content_type` keyword arg to
        # the `setup` function results in writing a `Description-Content-Type`
        # line to the `PKG-INFO` file in the `<distribution>.egg-info`
        # directory.
        # `Description-Content-Type` is described at
        # https://github.com/pypa/python-packaging-user-guide/pull/258

        self._setup_script_with_requires(
            """long_description_content_type='text/markdown',""")
        environ = os.environ.copy().update(
            HOME=env.paths['home'],
        )
        code, data = environment.run_setup_py(
            cmd=['egg_info'],
            pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]),
            data_stream=1,
            env=environ,
        )
        egg_info_dir = os.path.join('.', 'foo.egg-info')
        with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file:
            pkg_info_lines = pkginfo_file.read().split('\n')
        expected_line = 'Description-Content-Type: text/markdown'
        assert expected_line in pkg_info_lines

    def test_python_requires_egg_info(self, tmpdir_cwd, env):
        self._setup_script_with_requires(
            """python_requires='>=2.7.12',""")
        environ = os.environ.copy().update(
            HOME=env.paths['home'],
        )
        code, data = environment.run_setup_py(
            cmd=['egg_info'],
            pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]),
            data_stream=1,
            env=environ,
        )
        egg_info_dir = os.path.join('.', 'foo.egg-info')
        with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file:
            pkg_info_lines = pkginfo_file.read().split('\n')
        assert 'Requires-Python: >=2.7.12' in pkg_info_lines
        assert 'Metadata-Version: 1.2' in pkg_info_lines

    def test_python_requires_install(self, tmpdir_cwd, env):
        self._setup_script_with_requires(
            """python_requires='>=1.2.3',""")
        self._run_install_command(tmpdir_cwd, env)
        egg_info_dir = self._find_egg_info_files(env.paths['lib']).base
        pkginfo = os.path.join(egg_info_dir, 'PKG-INFO')
        with open(pkginfo) as f:
            assert 'Requires-Python: >=1.2.3' in f.read().split('\n')

    def test_manifest_maker_warning_suppression(self):
        fixtures = [
            "standard file not found: should have one of foo.py, bar.py",
            "standard file 'setup.py' not found"
        ]

        for msg in fixtures:
            assert manifest_maker._should_suppress_warning(msg)

    def _run_install_command(self, tmpdir_cwd, env, cmd=None, output=None):
        environ = os.environ.copy().update(
            HOME=env.paths['home'],
        )
        if cmd is None:
            cmd = [
                'install',
                '--home', env.paths['home'],
                '--install-lib', env.paths['lib'],
                '--install-scripts', env.paths['scripts'],
                '--install-data', env.paths['data'],
            ]
        code, data = environment.run_setup_py(
            cmd=cmd,
            pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]),
            data_stream=1,
            env=environ,
        )
        if code:
            raise AssertionError(data)
        if output:
            assert output in data

    def _find_egg_info_files(self, root):
        class DirList(list):
            def __init__(self, files, base):
                super(DirList, self).__init__(files)
                self.base = base

        results = (
            DirList(filenames, dirpath)
            for dirpath, dirnames, filenames in os.walk(root)
            if os.path.basename(dirpath) == 'EGG-INFO'
        )
        # expect exactly one result
        result, = results
        return result
<commit_msg>Cleanup indentation<commit_after>import sys
import ast
import os
import glob
import re
import stat

from setuptools.command.egg_info import egg_info, manifest_maker
from setuptools.dist import Distribution
from setuptools.extern.six.moves import map

import pytest

from . import environment
from .files import build_files
from .textwrap import DALS
from . import contexts

class Environment(str):
    pass

class TestEggInfo(object):

    setup_script = DALS("""
        from setuptools import setup

        setup(
            name='foo',
            py_modules=['hello'],
            entry_points={'console_scripts': ['hi = hello.run']},
            zip_safe=False,
        )
        """)

    def _create_project(self):
        build_files({
            'setup.py': self.setup_script,
            'hello.py': DALS("""
                def run():
                    print('hello')
                """)
        })

    @pytest.yield_fixture
    def env(self):
        with contexts.tempdir(prefix='setuptools-test.') as env_dir:
            env = Environment(env_dir)
            os.chmod(env_dir, stat.S_IRWXU)
            subs = 'home', 'lib', 'scripts', 'data', 'egg-base'
            env.paths = dict(
                (dirname, os.path.join(env_dir, dirname))
                for dirname in subs
            )
            list(map(os.mkdir, env.paths.values()))
            build_files({
                env.paths['home']: {
                    '.pydistutils.cfg': DALS("""
                    [egg_info]
                    egg-base = %(egg-base)s
                    """ % env.paths)
                }
            })
            yield env

    def test_egg_info_save_version_info_setup_empty(self, tmpdir_cwd, env):
        """
        When the egg_info section is empty or not present, running
        save_version_info should add the settings to the setup.cfg
        in a deterministic order, consistent with the ordering found
        on Python 2.7 with PYTHONHASHSEED=0.
        """
        setup_cfg = os.path.join(env.paths['home'], 'setup.cfg')
        dist = Distribution()
        ei = egg_info(dist)
        ei.initialize_options()
        ei.save_version_info(setup_cfg)

        with open(setup_cfg, 'r') as f:
            content = f.read()

        assert '[egg_info]' in content
        assert 'tag_build =' in content
        assert 'tag_date = 0' in content

        expected_order = 'tag_build', 'tag_date',

        self._validate_content_order(content, expected_order)

    @staticmethod
    def _validate_content_order(content, expected):
        """
        Assert that the strings in expected appear in content
        in order.
        """
        pattern = '.*'.join(expected)
        flags = re.MULTILINE | re.DOTALL
        assert re.search(pattern, content, flags)

    def test_egg_info_save_version_info_setup_defaults(self, tmpdir_cwd, env):
        """
        When running save_version_info on an existing setup.cfg
        with the 'default' values present from a previous run,
        the file should remain unchanged.
        """
        setup_cfg = os.path.join(env.paths['home'], 'setup.cfg')
        build_files({
            setup_cfg: DALS("""
            [egg_info]
            tag_build =
            tag_date = 0
            """),
        })
        dist = Distribution()
        ei = egg_info(dist)
        ei.initialize_options()
        ei.save_version_info(setup_cfg)

        with open(setup_cfg, 'r') as f:
            content = f.read()

        assert '[egg_info]' in content
        assert 'tag_build =' in content
        assert 'tag_date = 0' in content

        expected_order = 'tag_build', 'tag_date',

        self._validate_content_order(content, expected_order)

    def test_egg_base_installed_egg_info(self, tmpdir_cwd, env):
        self._create_project()

        self._run_install_command(tmpdir_cwd, env)
        actual = self._find_egg_info_files(env.paths['lib'])

        expected = [
            'PKG-INFO',
            'SOURCES.txt',
            'dependency_links.txt',
            'entry_points.txt',
            'not-zip-safe',
            'top_level.txt',
        ]
        assert sorted(actual) == expected

    def test_manifest_template_is_read(self, tmpdir_cwd, env):
        self._create_project()
        build_files({
            'MANIFEST.in': DALS("""
                recursive-include docs *.rst
            """),
            'docs': {
                'usage.rst': "Run 'hi'",
            }
        })
        self._run_install_command(tmpdir_cwd, env)
        egg_info_dir = self._find_egg_info_files(env.paths['lib']).base
        sources_txt = os.path.join(egg_info_dir, 'SOURCES.txt')
        with open(sources_txt) as f:
            assert 'docs/usage.rst' in f.read().split('\n')

    def _setup_script_with_requires(self, requires, use_setup_cfg=False):
        setup_script = DALS(
            '''
            from setuptools import setup

            setup(name='foo', zip_safe=False, %s)
            '''
        ) % ('' if use_setup_cfg else requires)
        setup_config = requires if use_setup_cfg else ''
        build_files({'setup.py': setup_script,
                     'setup.cfg': setup_config})

    mismatch_marker = "python_version<'{this_ver}'".format(
        this_ver=sys.version_info[0],
    )
    # Alternate equivalent syntax.
    mismatch_marker_alternate = 'python_version < "{this_ver}"'.format(
        this_ver=sys.version_info[0],
    )
    invalid_marker = "<=>++"

    class RequiresTestHelper(object):

        @staticmethod
        def parametrize(*test_list, **format_dict):
            idlist = []
            argvalues = []
            for test in test_list:
                test_params = test.lstrip().split('\n\n', 3)
                name_kwargs = test_params.pop(0).split('\n')
                if len(name_kwargs) > 1:
                    val = name_kwargs[1].strip()
                    install_cmd_kwargs = ast.literal_eval(val)
                else:
                    install_cmd_kwargs = {}
                name = name_kwargs[0].strip()
                setup_py_requires, setup_cfg_requires, expected_requires = (
                    DALS(a).format(**format_dict) for a in test_params
                )
                for id_, requires, use_cfg in (
                    (name, setup_py_requires, False),
                    (name + '_in_setup_cfg', setup_cfg_requires, True),
                ):
                    idlist.append(id_)
                    marks = ()
                    if requires.startswith('@xfail\n'):
                        requires = requires[7:]
                        marks = pytest.mark.xfail
                    argvalues.append(pytest.param(requires, use_cfg,
                                                  expected_requires,
                                                  install_cmd_kwargs,
                                                  marks=marks))
            return pytest.mark.parametrize(
                'requires,use_setup_cfg,'
                'expected_requires,install_cmd_kwargs',
                argvalues, ids=idlist,
            )

    @RequiresTestHelper.parametrize(
        # Format of a test:
        #
        # id
        # install_cmd_kwargs [optional]
        #
        # requires block (when used in setup.py)
        #
        # requires block (when used in setup.cfg)
        #
        # expected contents of requires.txt

        '''
        install_requires_with_marker

        install_requires=["barbazquux;{mismatch_marker}"],

        [options]
        install_requires =
            barbazquux; {mismatch_marker}

        [:{mismatch_marker_alternate}]
        barbazquux
        ''',

        '''
        install_requires_with_extra
        {'cmd': ['egg_info']}

        install_requires=["barbazquux [test]"],

        [options]
        install_requires =
            barbazquux [test]

        barbazquux[test]
        ''',

        '''
        install_requires_with_extra_and_marker

        install_requires=["barbazquux [test]; {mismatch_marker}"],

        [options]
        install_requires =
            barbazquux [test]; {mismatch_marker}

        [:{mismatch_marker_alternate}]
        barbazquux[test]
        ''',

        '''
        setup_requires_with_markers

        setup_requires=["barbazquux;{mismatch_marker}"],

        [options]
        setup_requires =
            barbazquux; {mismatch_marker}

        ''',

        '''
        tests_require_with_markers
        {'cmd': ['test'], 'output': "Ran 0 tests in"}

        tests_require=["barbazquux;{mismatch_marker}"],

        [options]
        tests_require =
            barbazquux; {mismatch_marker}

        ''',

        '''
        extras_require_with_extra
        {'cmd': ['egg_info']}

        extras_require={{"extra": ["barbazquux [test]"]}},

        [options.extras_require]
        extra = barbazquux [test]

        [extra]
        barbazquux[test]
        ''',

        '''
        extras_require_with_extra_and_marker_in_req

        extras_require={{"extra": ["barbazquux [test]; {mismatch_marker}"]}},

        [options.extras_require]
        extra =
            barbazquux [test]; {mismatch_marker}

        [extra]

        [extra:{mismatch_marker_alternate}]
        barbazquux[test]
        ''',

        # FIXME: ConfigParser does not allow : in key names!
        '''
        extras_require_with_marker

        extras_require={{":{mismatch_marker}": ["barbazquux"]}},

        @xfail
        [options.extras_require]
        :{mismatch_marker} = barbazquux

        [:{mismatch_marker}]
        barbazquux
        ''',

        '''
        extras_require_with_marker_in_req

        extras_require={{"extra": ["barbazquux; {mismatch_marker}"]}},

        [options.extras_require]
        extra =
            barbazquux; {mismatch_marker}

        [extra]

        [extra:{mismatch_marker_alternate}]
        barbazquux
        ''',

        '''
        extras_require_with_empty_section

        extras_require={{"empty": []}},

        [options.extras_require]
        empty =

        [empty]
        ''',
        # Format arguments.
        invalid_marker=invalid_marker,
        mismatch_marker=mismatch_marker,
        mismatch_marker_alternate=mismatch_marker_alternate,
    )
    def test_requires(
            self, tmpdir_cwd, env, requires, use_setup_cfg,
            expected_requires, install_cmd_kwargs):
        self._setup_script_with_requires(requires, use_setup_cfg)
        self._run_install_command(tmpdir_cwd, env, **install_cmd_kwargs)
        egg_info_dir = os.path.join('.', 'foo.egg-info')
        requires_txt = os.path.join(egg_info_dir, 'requires.txt')
        if os.path.exists(requires_txt):
            with open(requires_txt) as fp:
                install_requires = fp.read()
        else:
            install_requires = ''
        assert install_requires.lstrip() == expected_requires
        assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == []

    def test_extras_require_with_invalid_marker(self, tmpdir_cwd, env):
        tmpl = 'extras_require={{":{marker}": ["barbazquux"]}},'
        req = tmpl.format(marker=self.invalid_marker)
        self._setup_script_with_requires(req)
        with pytest.raises(AssertionError):
            self._run_install_command(tmpdir_cwd, env)
        assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == []

    def test_extras_require_with_invalid_marker_in_req(self, tmpdir_cwd, env):
        tmpl = 'extras_require={{"extra": ["barbazquux; {marker}"]}},'
        req = tmpl.format(marker=self.invalid_marker)
        self._setup_script_with_requires(req)
        with pytest.raises(AssertionError):
            self._run_install_command(tmpdir_cwd, env)
        assert glob.glob(os.path.join(env.paths['lib'], 'barbazquux*')) == []

    def test_long_description_content_type(self, tmpdir_cwd, env):
        # Test that specifying a `long_description_content_type` keyword arg to
        # the `setup` function results in writing a `Description-Content-Type`
        # line to the `PKG-INFO` file in the `<distribution>.egg-info`
        # directory.
        # `Description-Content-Type` is described at
        # https://github.com/pypa/python-packaging-user-guide/pull/258

        self._setup_script_with_requires(
            """long_description_content_type='text/markdown',""")
        environ = os.environ.copy().update(
            HOME=env.paths['home'],
        )
        code, data = environment.run_setup_py(
            cmd=['egg_info'],
            pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]),
            data_stream=1,
            env=environ,
        )
        egg_info_dir = os.path.join('.', 'foo.egg-info')
        with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file:
            pkg_info_lines = pkginfo_file.read().split('\n')
        expected_line = 'Description-Content-Type: text/markdown'
        assert expected_line in pkg_info_lines

    def test_python_requires_egg_info(self, tmpdir_cwd, env):
        self._setup_script_with_requires(
            """python_requires='>=2.7.12',""")
        environ = os.environ.copy().update(
            HOME=env.paths['home'],
        )
        code, data = environment.run_setup_py(
            cmd=['egg_info'],
            pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]),
            data_stream=1,
            env=environ,
        )
        egg_info_dir = os.path.join('.', 'foo.egg-info')
        with open(os.path.join(egg_info_dir, 'PKG-INFO')) as pkginfo_file:
            pkg_info_lines = pkginfo_file.read().split('\n')
        assert 'Requires-Python: >=2.7.12' in pkg_info_lines
        assert 'Metadata-Version: 1.2' in pkg_info_lines

    def test_python_requires_install(self, tmpdir_cwd, env):
        self._setup_script_with_requires(
            """python_requires='>=1.2.3',""")
        self._run_install_command(tmpdir_cwd, env)
        egg_info_dir = self._find_egg_info_files(env.paths['lib']).base
        pkginfo = os.path.join(egg_info_dir, 'PKG-INFO')
        with open(pkginfo) as f:
            assert 'Requires-Python: >=1.2.3' in f.read().split('\n')

    def test_manifest_maker_warning_suppression(self):
        fixtures = [
            "standard file not found: should have one of foo.py, bar.py",
            "standard file 'setup.py' not found"
        ]

        for msg in fixtures:
            assert manifest_maker._should_suppress_warning(msg)

    def _run_install_command(self, tmpdir_cwd, env, cmd=None, output=None):
        environ = os.environ.copy().update(
            HOME=env.paths['home'],
        )
        if cmd is None:
            cmd = [
                'install',
                '--home', env.paths['home'],
                '--install-lib', env.paths['lib'],
                '--install-scripts', env.paths['scripts'],
                '--install-data', env.paths['data'],
            ]
        code, data = environment.run_setup_py(
            cmd=cmd,
            pypath=os.pathsep.join([env.paths['lib'], str(tmpdir_cwd)]),
            data_stream=1,
            env=environ,
        )
        if code:
            raise AssertionError(data)
        if output:
            assert output in data

    def _find_egg_info_files(self, root):
        class DirList(list):
            def __init__(self, files, base):
                super(DirList, self).__init__(files)
                self.base = base

        results = (
            DirList(filenames, dirpath)
            for dirpath, dirnames, filenames in os.walk(root)
            if os.path.basename(dirpath) == 'EGG-INFO'
        )
        # expect exactly one result
        result, = results
        return result
<|endoftext|>


<commit_before>"""
Models for the accounts application
"""

from django.contrib.auth.models import User
from django.contrib.localflavor.us.models import PhoneNumberField, USStateField
from django.core.mail import EmailMessage
from django.db import models
from django.template.loader import render_to_string

from muckrock.foia.models import FOIARequest
from muckrock.jurisdiction.models import Jurisdiction
from muckrock.organization.models import Organization
from muckrock.settings import MONTHLY_REQUESTS, STRIPE_SECRET_KEY
from muckrock.values import TextValue

from easy_thumbnails.fields import ThumbnailerImageField
from datetime import datetime
import dbsettings
from itertools import groupby
from random import choice
import string
import stripe
from urlauth.models import AuthKey

stripe.api_key = STRIPE_SECRET_KEY

class EmailOptions(dbsettings.Group):
    """DB settings for sending email"""
    email_footer = TextValue('email footer')

options = EmailOptions()

ACCT_TYPES = [
    ('admin', 'Admin'),
    ('beta', 'Beta'),
    ('community', 'Community'),
    ('pro', 'Professional'),
    ('proxy', 'Proxy'),
]

class Profile(models.Model):
    """User profile information for muckrock"""
    # pylint: disable=too-many-public-methods

    email_prefs = (
        ('instant', 'Instant'),
        ('daily', 'Daily'),
        ('weekly', 'Weekly'),
    )

    user = models.ForeignKey(User, unique=True)
    address1 = models.CharField(
        max_length=50,
        blank=True,
        verbose_name='address'
    )
    address2 = models.CharField(
        max_length=50,
        blank=True,
        verbose_name='address (line 2)'
    )
    city = models.CharField(max_length=60, blank=True)
    state = USStateField(
        blank=True,
        help_text=('Your state will be made public on this site.'
                   'If you do not want this information to be public,'
                   ' please leave blank.')
    )
    zip_code = models.CharField(max_length=10, blank=True)
    phone = PhoneNumberField(blank=True)
    follows_foia = models.ManyToManyField(
        FOIARequest,
        related_name='followed_by',
        blank=True
    )
    follows_question = models.ManyToManyField(
        'qanda.Question',
        related_name='followed_by',
        blank=True
    )
    notifications = models.ManyToManyField(
        FOIARequest,
        related_name='notify',
        blank=True
    )
    follow_questions = models.BooleanField(default=False)
    acct_type = models.CharField(max_length=10, choices=ACCT_TYPES)
    organization = models.ForeignKey(
        Organization,
        blank=True,
        null=True,
        related_name='members',
        on_delete=models.SET_NULL)

    # email confirmation
    email_confirmed = models.BooleanField(default=False)
    confirmation_key = models.CharField(max_length=24, blank=True)

    # extended information
    profile = models.TextField(blank=True)
    location = models.ForeignKey(Jurisdiction, blank=True, null=True)
    public_email = models.EmailField(max_length=255, blank=True)
    pgp_public_key = models.TextField(blank=True)
    website = models.URLField(
        max_length=255,
        blank=True,
        help_text='Begin with http://'
    )
    twitter = models.CharField(max_length=255, blank=True)
    linkedin = models.URLField(
        max_length=255,
        blank=True,
        help_text='Begin with http://'
    )
    avatar = ThumbnailerImageField(
        upload_to='account_images',
        blank=True, null=True,
        resize_source={'size': (200, 200), 'crop': 'smart'}
    )

    # prefrences
    email_pref = models.CharField(
        max_length=10,
        choices=email_prefs,
        default='daily',
        verbose_name='Email Preference',
        help_text=('Receive email updates to your requests instantly'
                   ' or in a daily or weekly digest')
    )
    use_autologin = models.BooleanField(
        default=True,
        help_text=('Links you receive in emails from us will contain'
                   ' a one time token to automatically log you in')
    )

    # paid for requests
    num_requests = models.IntegerField(default=0)
    # for limiting # of requests / month
    monthly_requests = models.IntegerField(default=0)
    date_update = models.DateField()
    # for stripe
    stripe_id = models.CharField(max_length=255, blank=True)

    def __unicode__(self):
        return u"%s's Profile" % unicode(self.user).capitalize()

    @models.permalink
    def get_absolute_url(self):
        """The url for this object"""
        # pylint: disable=E1101
        return ('acct-profile', [], {'user_name': self.user.username})

    def is_member_of(self, organization):
        """Answers whether the profile is a member of the passed organization"""
        return self.organization == organization

    def get_monthly_requests(self):
        """Get the number of requests left for this month"""
        not_this_month = self.date_update.month != datetime.now().month
        not_this_year = self.date_update.year != datetime.now().year
        # update requests if they have not yet been updated this month
        if not_this_month or not_this_year:
            self.date_update = datetime.now()
            self.monthly_requests = MONTHLY_REQUESTS.get(self.acct_type, 0)
            self.save()
        return self.monthly_requests

    def total_requests(self):
        """Get sum of paid for requests and monthly requests"""
        org_reqs = self.organization.get_requests() if self.organization else 0
        return self.num_requests + self.get_monthly_requests() + org_reqs

    def make_request(self):
        """Decrement the user's request amount by one"""
        organization = self.organization
        if organization and organization.get_requests() > 0:
            organization.num_requests -= 1
            organization.save()
            return True
        if self.get_monthly_requests() > 0:
            self.monthly_requests -= 1
        elif self.num_requests > 0:
            self.num_requests -= 1
        else:
            return False
        self.save()
        return True

    def multiple_requests(self, num):
        """How many requests of each type would be used for this user to make num requests"""
        request_dict = {'org_requests': 0, 'monthly_requests': 0,
                        'reg_requests': 0, 'extra_requests': 0}

        org_reqs = self.organization.get_requests()
        if org_reqs > num:
            request_dict['org_requests'] = num
            return request_dict
        else:
            request_dict['org_requests'] = org_reqs
            num -= org_reqs

        monthly = self.get_monthly_requests()
        if monthly > num:
            request_dict['monthly_requests'] = num
            return request_dict
        else:
            request_dict['monthly_requests'] = monthly
            num -= monthly

        if self.num_requests > num:
            request_dict['reg_requests'] = num
            return request_dict
        else:
            request_dict['reg_requests'] = self.num_requests
            request_dict['extra_requests'] = num - self.num_requests
            return request_dict

    def can_embargo(self):
        """Is this user allowed to embargo?"""
        return self.acct_type in ['admin', 'beta', 'pro', 'proxy'] or self.organization != None

    def can_view_emails(self):
        """Is this user allowed to view all emails and private contact information?"""
        return self.acct_type in ['admin', 'pro']

    def customer(self):
        """Get stripe customer or create one if it doesn't exist"""
        try:
            customer = stripe.Customer.retrieve(self.stripe_id)
        except stripe.InvalidRequestError:
            customer = stripe.Customer.create(
                description=self.user.username,
                email=self.user.email
            )
            self.stripe_id = customer.id
            self.save()
        return customer

    def start_pro_subscription(self):
        """Subscribe this profile to a pro plan"""
        customer = self.customer()
        customer.update_subscription(plan='pro')
        customer.save()
        self.acct_type = 'pro'
        self.date_update = datetime.now()
        self.monthly_requests = MONTHLY_REQUESTS.get('pro', 0)
        self.save()

    def cancel_pro_subscription(self):
        """Unsubscribe this profile form a pro plan"""
        customer = self.customer()
        customer.cancel_subscription()
        customer.save()
        self.acct_type = 'community'
        self.save()

    def pay(self, token, amount, desc):
        """Create a stripe charge for the user"""
        # pylint: disable=E1101
        stripe.Charge.create(
            amount=amount,
            currency='usd',
            card=token,
            description='%s: %s' % (self.user.username, desc)
        )

    def api_pay(self, amount, desc):
        """Create a stripe charge for the user through the API"""
        # pylint: disable=E1101
        customer = self.customer()
        desc = '%s: %s' % (self.user.username, desc)
        # always use card on file
        stripe.Charge.create(
            amount=amount,
            currency='usd',
            customer=customer.id,
            description=desc
        )

    def generate_confirmation_key(self):
        """Generate random key"""
        key = ''.join(choice(string.ascii_letters) for _ in range(24))
        self.confirmation_key = key
        self.save()
        return key

    def notify(self, foia):
        """Notify a user that foia has been updated or mark to be notified later
           according to preference"""
        # pylint: disable=E1101

        if self.email_pref == 'instant':
            link = self.wrap_url(foia.get_absolute_url())

            msg = render_to_string('text/foia/mail.txt', {
                'name': self.user.get_full_name(),
                'title': foia.title,
                'status': foia.get_status_display(),
                'link': link,
                'follow': self.user != foia.user,
                'footer': options.email_footer
            })
            email = EmailMessage(
                subject='[MuckRock] FOI request "%s" has been updated' % foia.title,
                body=msg,
                from_email='info@muckrock.com',
                to=[self.user.email],
                bcc=['diagnostics@muckrock.com']
            )
            email.send(fail_silently=False)

        else:
            self.notifications.add(foia)
            self.save()

    def send_notifications(self):
        """Send deferred notifications"""
        # pylint: disable=E1101

        subjects = {
            'done': "you've got new MuckRock docs!",
            'partial': "you've got new MuckRock docs!",
            'rejected': 'an agency rejected a MuckRock request - appeal?',
            'fix': 'we need help fixing a MuckRock request.',
            'payment': 'an agency wants payment for a MuckRock request.'
        }
        category = {
            'done': 'Completed Requests',
            'partial': 'Completed Requests',
            'rejected': 'Rejected Requests',
            'fix': 'Requests Needing Action',
            'payment': 'Requests Needing Action',
        }
        status_order = ['done', 'partial', 'rejected', 'fix', 'payment',
                        'no_docs', 'abandoned', 'appealing', 'started',
                        'submitted', 'ack', 'processed']

        def get_subject(status, total_foias):
            """Get subject for a given status"""
            if status in subjects:
                return subjects[status]
            elif total_foias > 1:
                return '%d MuckRock requests have updates' % total_foias
            else:
                return 'a MuckRock request has been updated'

        foias = sorted(
            self.notifications.all(),
            key=lambda f: status_order.index(f.status) if f.status in status_order else 100
        )
        grouped_foias = list((s, list(fs)) for s, fs in groupby(
            foias,
            lambda f: category.get(f.status, 'Recently Updated Requests')
        ))
        if not grouped_foias:
            return
        if len(grouped_foias) == 1:
            subject = '%s, %s' % (
                self.user.first_name,
                get_subject(grouped_foias[0][1][0].status, len(foias))
            )
        else:
            subject = '%s, %s  Plus, %s' % (
                self.user.first_name,
                get_subject(grouped_foias[0][1][0].status, len(foias)),
                get_subject(grouped_foias[1][1][0].status, len(foias))
            )

        msg = render_to_string('text/user/notify_mail.txt', {
            'name': self.user.get_full_name(),
            'foias': grouped_foias,
            'footer': options.email_footer
        })
        email = EmailMessage(
            subject=subject,
            body=msg,
            from_email='info@muckrock.com',
            to=[self.user.email],
            bcc=['diagnostics@muckrock.com']
        )
        email.send(fail_silently=False)

        self.notifications.clear()

    def wrap_url(self, link):
        """Wrap a URL for autologin"""
        if self.use_autologin:
            return AuthKey.objects.wrap_url(link, uid=self.user.pk)
        else:
            return link

class Statistics(models.Model):
    """Nightly statistics"""
    # pylint: disable=invalid-name
    date = models.DateField()
    total_requests = models.IntegerField()
    total_requests_success = models.IntegerField()
    total_requests_denied = models.IntegerField()
    total_requests_draft = models.IntegerField(null=True)
    total_requests_submitted = models.IntegerField(null=True)
    total_requests_awaiting_ack = models.IntegerField(null=True)
    total_requests_awaiting_response = models.IntegerField(null=True)
    total_requests_awaiting_appeal = models.IntegerField(null=True)
    total_requests_fix_required = models.IntegerField(null=True)
    total_requests_payment_required = models.IntegerField(null=True)
    total_requests_no_docs = models.IntegerField(null=True)
    total_requests_partial = models.IntegerField(null=True)
    total_requests_abandoned = models.IntegerField(null=True)

    orphaned_communications = models.IntegerField(null=True)

    total_agencies = models.IntegerField()
    stale_agencies = models.IntegerField(null=True)
    unapproved_agencies = models.IntegerField(null=True)

    total_pages = models.IntegerField()
    total_users = models.IntegerField()
    users_today = models.ManyToManyField(User)
    total_fees = models.IntegerField()
    pro_users = models.IntegerField(null=True)
    pro_user_names = models.TextField(blank=True)
    total_page_views = models.IntegerField(null=True)
    daily_requests_pro = models.IntegerField(null=True)
    daily_requests_community = models.IntegerField(null=True)
    daily_requests_beta = models.IntegerField(null=True)
    daily_articles = models.IntegerField(null=True)

    def __unicode__(self):
        return 'Stats for %s' % self.date

    class Meta:
        # pylint: disable=R0903
        ordering = ['-date']
        verbose_name_plural = 'statistics'

<commit_msg>fixes bug when calculating request counts for multirequests<commit_after>"""
Models for the accounts application
"""

from django.contrib.auth.models import User
from django.contrib.localflavor.us.models import PhoneNumberField, USStateField
from django.core.mail import EmailMessage
from django.db import models
from django.template.loader import render_to_string

from muckrock.foia.models import FOIARequest
from muckrock.jurisdiction.models import Jurisdiction
from muckrock.organization.models import Organization
from muckrock.settings import MONTHLY_REQUESTS, STRIPE_SECRET_KEY
from muckrock.values import TextValue

from easy_thumbnails.fields import ThumbnailerImageField
from datetime import datetime
import dbsettings
from itertools import groupby
from random import choice
import string
import stripe
from urlauth.models import AuthKey

stripe.api_key = STRIPE_SECRET_KEY

class EmailOptions(dbsettings.Group):
    """DB settings for sending email"""
    email_footer = TextValue('email footer')

options = EmailOptions()

ACCT_TYPES = [
    ('admin', 'Admin'),
    ('beta', 'Beta'),
    ('community', 'Community'),
    ('pro', 'Professional'),
    ('proxy', 'Proxy'),
]

class Profile(models.Model):
    """User profile information for muckrock"""
    # pylint: disable=too-many-public-methods

    email_prefs = (
        ('instant', 'Instant'),
        ('daily', 'Daily'),
        ('weekly', 'Weekly'),
    )

    user = models.ForeignKey(User, unique=True)
    address1 = models.CharField(
        max_length=50,
        blank=True,
        verbose_name='address'
    )
    address2 = models.CharField(
        max_length=50,
        blank=True,
        verbose_name='address (line 2)'
    )
    city = models.CharField(max_length=60, blank=True)
    state = USStateField(
        blank=True,
        help_text=('Your state will be made public on this site.'
                   'If you do not want this information to be public,'
                   ' please leave blank.')
    )
    zip_code = models.CharField(max_length=10, blank=True)
    phone = PhoneNumberField(blank=True)
    follows_foia = models.ManyToManyField(
        FOIARequest,
        related_name='followed_by',
        blank=True
    )
    follows_question = models.ManyToManyField(
        'qanda.Question',
        related_name='followed_by',
        blank=True
    )
    notifications = models.ManyToManyField(
        FOIARequest,
        related_name='notify',
        blank=True
    )
    follow_questions = models.BooleanField(default=False)
    acct_type = models.CharField(max_length=10, choices=ACCT_TYPES)
    organization = models.ForeignKey(
        Organization,
        blank=True,
        null=True,
        related_name='members',
        on_delete=models.SET_NULL)

    # email confirmation
    email_confirmed = models.BooleanField(default=False)
    confirmation_key = models.CharField(max_length=24, blank=True)

    # extended information
    profile = models.TextField(blank=True)
    location = models.ForeignKey(Jurisdiction, blank=True, null=True)
    public_email = models.EmailField(max_length=255, blank=True)
    pgp_public_key = models.TextField(blank=True)
    website = models.URLField(
        max_length=255,
        blank=True,
        help_text='Begin with http://'
    )
    twitter = models.CharField(max_length=255, blank=True)
    linkedin = models.URLField(
        max_length=255,
        blank=True,
        help_text='Begin with http://'
    )
    avatar = ThumbnailerImageField(
        upload_to='account_images',
        blank=True, null=True,
        resize_source={'size': (200, 200), 'crop': 'smart'}
    )

    # prefrences
    email_pref = models.CharField(
        max_length=10,
        choices=email_prefs,
        default='daily',
        verbose_name='Email Preference',
        help_text=('Receive email updates to your requests instantly'
                   ' or in a daily or weekly digest')
    )
    use_autologin = models.BooleanField(
        default=True,
        help_text=('Links you receive in emails from us will contain'
                   ' a one time token to automatically log you in')
    )

    # paid for requests
    num_requests = models.IntegerField(default=0)
    # for limiting # of requests / month
    monthly_requests = models.IntegerField(default=0)
    date_update = models.DateField()
    # for stripe
    stripe_id = models.CharField(max_length=255, blank=True)

    def __unicode__(self):
        return u"%s's Profile" % unicode(self.user).capitalize()

    @models.permalink
    def get_absolute_url(self):
        """The url for this object"""
        # pylint: disable=E1101
        return ('acct-profile', [], {'user_name': self.user.username})

    def is_member_of(self, organization):
        """Answers whether the profile is a member of the passed organization"""
        return self.organization == organization

    def get_monthly_requests(self):
        """Get the number of requests left for this month"""
        not_this_month = self.date_update.month != datetime.now().month
        not_this_year = self.date_update.year != datetime.now().year
        # update requests if they have not yet been updated this month
        if not_this_month or not_this_year:
            self.date_update = datetime.now()
            self.monthly_requests = MONTHLY_REQUESTS.get(self.acct_type, 0)
            self.save()
        return self.monthly_requests

    def total_requests(self):
        """Get sum of paid for requests and monthly requests"""
        org_reqs = self.organization.get_requests() if self.organization else 0
        return self.num_requests + self.get_monthly_requests() + org_reqs

    def make_request(self):
        """Decrement the user's request amount by one"""
        organization = self.organization
        if organization and organization.get_requests() > 0:
            organization.num_requests -= 1
            organization.save()
            return True
        if self.get_monthly_requests() > 0:
            self.monthly_requests -= 1
        elif self.num_requests > 0:
            self.num_requests -= 1
        else:
            return False
        self.save()
        return True

    def multiple_requests(self, num):
        """How many requests of each type would be used for this user to make num requests"""
        request_dict = {'org_requests': 0, 'monthly_requests': 0,
                        'reg_requests': 0, 'extra_requests': 0}

        if self.organization:
            org_reqs = self.organization.get_requests()
        else:
            org_reqs = 0

        if org_reqs > num:
            request_dict['org_requests'] = num
            return request_dict
        else:
            request_dict['org_requests'] = org_reqs
            num -= org_reqs

        monthly = self.get_monthly_requests()
        if monthly > num:
            request_dict['monthly_requests'] = num
            return request_dict
        else:
            request_dict['monthly_requests'] = monthly
            num -= monthly

        if self.num_requests > num:
            request_dict['reg_requests'] = num
            return request_dict
        else:
            request_dict['reg_requests'] = self.num_requests
            request_dict['extra_requests'] = num - self.num_requests
            return request_dict

    def can_embargo(self):
        """Is this user allowed to embargo?"""
        return self.acct_type in ['admin', 'beta', 'pro', 'proxy'] or self.organization != None

    def can_view_emails(self):
        """Is this user allowed to view all emails and private contact information?"""
        return self.acct_type in ['admin', 'pro']

    def customer(self):
        """Get stripe customer or create one if it doesn't exist"""
        try:
            customer = stripe.Customer.retrieve(self.stripe_id)
        except stripe.InvalidRequestError:
            customer = stripe.Customer.create(
                description=self.user.username,
                email=self.user.email
            )
            self.stripe_id = customer.id
            self.save()
        return customer

    def start_pro_subscription(self):
        """Subscribe this profile to a pro plan"""
        customer = self.customer()
        customer.update_subscription(plan='pro')
        customer.save()
        self.acct_type = 'pro'
        self.date_update = datetime.now()
        self.monthly_requests = MONTHLY_REQUESTS.get('pro', 0)
        self.save()

    def cancel_pro_subscription(self):
        """Unsubscribe this profile form a pro plan"""
        customer = self.customer()
        customer.cancel_subscription()
        customer.save()
        self.acct_type = 'community'
        self.save()

    def pay(self, token, amount, desc):
        """Create a stripe charge for the user"""
        # pylint: disable=E1101
        stripe.Charge.create(
            amount=amount,
            currency='usd',
            card=token,
            description='%s: %s' % (self.user.username, desc)
        )

    def api_pay(self, amount, desc):
        """Create a stripe charge for the user through the API"""
        # pylint: disable=E1101
        customer = self.customer()
        desc = '%s: %s' % (self.user.username, desc)
        # always use card on file
        stripe.Charge.create(
            amount=amount,
            currency='usd',
            customer=customer.id,
            description=desc
        )

    def generate_confirmation_key(self):
        """Generate random key"""
        key = ''.join(choice(string.ascii_letters) for _ in range(24))
        self.confirmation_key = key
        self.save()
        return key

    def notify(self, foia):
        """Notify a user that foia has been updated or mark to be notified later
           according to preference"""
        # pylint: disable=E1101

        if self.email_pref == 'instant':
            link = self.wrap_url(foia.get_absolute_url())

            msg = render_to_string('text/foia/mail.txt', {
                'name': self.user.get_full_name(),
                'title': foia.title,
                'status': foia.get_status_display(),
                'link': link,
                'follow': self.user != foia.user,
                'footer': options.email_footer
            })
            email = EmailMessage(
                subject='[MuckRock] FOI request "%s" has been updated' % foia.title,
                body=msg,
                from_email='info@muckrock.com',
                to=[self.user.email],
                bcc=['diagnostics@muckrock.com']
            )
            email.send(fail_silently=False)

        else:
            self.notifications.add(foia)
            self.save()

    def send_notifications(self):
        """Send deferred notifications"""
        # pylint: disable=E1101

        subjects = {
            'done': "you've got new MuckRock docs!",
            'partial': "you've got new MuckRock docs!",
            'rejected': 'an agency rejected a MuckRock request - appeal?',
            'fix': 'we need help fixing a MuckRock request.',
            'payment': 'an agency wants payment for a MuckRock request.'
        }
        category = {
            'done': 'Completed Requests',
            'partial': 'Completed Requests',
            'rejected': 'Rejected Requests',
            'fix': 'Requests Needing Action',
            'payment': 'Requests Needing Action',
        }
        status_order = ['done', 'partial', 'rejected', 'fix', 'payment',
                        'no_docs', 'abandoned', 'appealing', 'started',
                        'submitted', 'ack', 'processed']

        def get_subject(status, total_foias):
            """Get subject for a given status"""
            if status in subjects:
                return subjects[status]
            elif total_foias > 1:
                return '%d MuckRock requests have updates' % total_foias
            else:
                return 'a MuckRock request has been updated'

        foias = sorted(
            self.notifications.all(),
            key=lambda f: status_order.index(f.status) if f.status in status_order else 100
        )
        grouped_foias = list((s, list(fs)) for s, fs in groupby(
            foias,
            lambda f: category.get(f.status, 'Recently Updated Requests')
        ))
        if not grouped_foias:
            return
        if len(grouped_foias) == 1:
            subject = '%s, %s' % (
                self.user.first_name,
                get_subject(grouped_foias[0][1][0].status, len(foias))
            )
        else:
            subject = '%s, %s  Plus, %s' % (
                self.user.first_name,
                get_subject(grouped_foias[0][1][0].status, len(foias)),
                get_subject(grouped_foias[1][1][0].status, len(foias))
            )

        msg = render_to_string('text/user/notify_mail.txt', {
            'name': self.user.get_full_name(),
            'foias': grouped_foias,
            'footer': options.email_footer
        })
        email = EmailMessage(
            subject=subject,
            body=msg,
            from_email='info@muckrock.com',
            to=[self.user.email],
            bcc=['diagnostics@muckrock.com']
        )
        email.send(fail_silently=False)

        self.notifications.clear()

    def wrap_url(self, link):
        """Wrap a URL for autologin"""
        if self.use_autologin:
            return AuthKey.objects.wrap_url(link, uid=self.user.pk)
        else:
            return link

class Statistics(models.Model):
    """Nightly statistics"""
    # pylint: disable=invalid-name
    date = models.DateField()
    total_requests = models.IntegerField()
    total_requests_success = models.IntegerField()
    total_requests_denied = models.IntegerField()
    total_requests_draft = models.IntegerField(null=True)
    total_requests_submitted = models.IntegerField(null=True)
    total_requests_awaiting_ack = models.IntegerField(null=True)
    total_requests_awaiting_response = models.IntegerField(null=True)
    total_requests_awaiting_appeal = models.IntegerField(null=True)
    total_requests_fix_required = models.IntegerField(null=True)
    total_requests_payment_required = models.IntegerField(null=True)
    total_requests_no_docs = models.IntegerField(null=True)
    total_requests_partial = models.IntegerField(null=True)
    total_requests_abandoned = models.IntegerField(null=True)

    orphaned_communications = models.IntegerField(null=True)

    total_agencies = models.IntegerField()
    stale_agencies = models.IntegerField(null=True)
    unapproved_agencies = models.IntegerField(null=True)

    total_pages = models.IntegerField()
    total_users = models.IntegerField()
    users_today = models.ManyToManyField(User)
    total_fees = models.IntegerField()
    pro_users = models.IntegerField(null=True)
    pro_user_names = models.TextField(blank=True)
    total_page_views = models.IntegerField(null=True)
    daily_requests_pro = models.IntegerField(null=True)
    daily_requests_community = models.IntegerField(null=True)
    daily_requests_beta = models.IntegerField(null=True)
    daily_articles = models.IntegerField(null=True)

    def __unicode__(self):
        return 'Stats for %s' % self.date

    class Meta:
        # pylint: disable=R0903
        ordering = ['-date']
        verbose_name_plural = 'statistics'

<|endoftext|>