<commit_before>"""
Maximum entropy with marginal distribution constraints.

"""

from __future__ import print_function
from __future__ import division

import itertools
import bisect

import numpy as np
import dit

from dit.abstractdist import AbstractDenseDistribution

def as_full_rank(A, b):
    """
    From a linear system Ax = b, return Bx = c such that B has full rank.

    In CVXOPT, linear constraints are denoted as: Ax = b. A has shape (p, n)
    and must have full rank. x has shape (n, 1), and so b has shape (p, 1).
    Let's assume that we have:

        rank(A) = q <= n

    This is a typical situation if you are doing optimization, where you have
    an under-determined system and are using some criterion for selecting out
    a particular solution. Now, it may happen that q < p, which means that some
    of your constraint equations are not independent. Since CVXOPT requires
    that A have full rank, we must isolate an equivalent system Bx = c which
    does have full rank. We use SVD for this. So A = U \Sigma V^*, where
    U is (p, p), \Sigma is (p, n) and V^* is (n, n). Then:

        \Sigma V^* x = U^{-1} b

    We take B = \Sigma V^* and c = U^T b, where we use U^T instead of U^{-1}
    for computational efficiency (and since U is orthogonal). But note, we
    take only the cols of U and rows of \Sigma that have nonzero singular
    values.

    Parameters
    ----------
    A : array-like, shape (p, n)
        The LHS for the linear constraints.
    b : array-like, shape (p,) or (p, 1)
        The RHS for the linear constraints.

    Returns
    -------
    B : array-like, shape (q, n)
        The LHS for the linear constraints.
    c : array-like, shape (q,) or (q, 1)
        The RHS for the linear constraints.
    rank : int
        The rank of B.

    """
    try:
        from scipy import linalg
    except ImportError:
        from numpy import linalg

    A = np.atleast_2d(A)

    U, S, Vh = linalg.svd(A)

    tol = S.max() * max(A.shape) * np.finfo(S.dtype).eps
    rank = np.sum(S > tol)

    Ut = U[:, :rank].transpose()

    # See scipy.linalg.diagsvd for details.
    # Note that we only take the first 'rank' rows/cols of S.
    part = np.diag(S[:rank])
    typ = part.dtype.char
    D = np.r_['-1', part, np.zeros((rank, A.shape[1] - rank), typ)]

    B = np.dot(D, Vh)
    c = np.dot(Ut, b)

    return B, c, rank

class PhantomArray(object):
    """
    A simple wrapper around a sparse pmf specified by a lookup table.

    The wrapper provides NumPy

    """
    def __init__(self, lookup, n_variables, n_symbols):
        self.lookup = lookup
        self.n_variables = n_variables
        self.n_symbols = n_symbols
        self.n_elements = n_symbols ** n_variables

    def __getitem__(self, idx):
        try:
            idx[0]
        except:
            return self.lookup.get(idx, 0)
        else:
            # Iterable. Return a NumPy array of the elements.
            return np.array([self.lookup.get(i, 0) for i in idx])

    def __len__(self):
        return self.n_elements

def cartesian_product_view(dist):
    """
    Return a dense Cartesian product view of the `dist`.

    In a Cartesian product view, we union the sample space of each random
    variable and then use it in a Cartesian product that defines the sample
    space for the entire distribution.

    Parameters
    ----------
    dist : distribution
        The distribution to make dense.

    Returns
    -------
    pa : PhantomArray
        The dense, Cartesian product representation of the pmf of `dist`.
    n_variables : int
        The number of random variables in the distribution.
    n_symbols : int
        The number of symbols in the sample space for a single random variable.

    Examples
    --------
    >>> import dit
    >>> d = dit.Distribution(['00', '10'], [.5, .5])
    >>> pa, n, k = cartesian_product_view(d)
    >>> pa[[0,1,2,3]]
    array([ 0.5, 0., 0.5, 0. ])

    """
    symbols = list(sorted(set.union(*map(set, dist.alphabet))))
    n_variables = dist.outcome_length()
    n_symbols = len(symbols)

    lookup = {}
    for outcome, p in dist.zipped():
        index = 0
        for i, symbol in enumerate(outcome):
            idx = bisect.bisect_left(symbols, symbol)
            index += idx * n_symbols ** (n_variables - 1 - i)
        lookup[index] = p

    pmf = PhantomArray(lookup, n_variables, n_symbols)
    return pmf, n_variables, n_symbols

def linear_constraints(pmf, n_variables, n_symbols, m):
    """
    Returns `A` and `b` in `A x = b`, for a system of marginal constraints.

    The resulting matrix `A` is not guaranteed to have full rank.

    Parameters
    ----------
    pmf : array-like, shape ( n_symbols ** n_variables, )
        The probability mass function of the distribution. The pmf must have
        a Cartesian product sample space with the same sample space used for
        each random variable.
    n_variables : int
        The number of random variables.
    n_symbols : int
        The number of symbols that each random variable can be.
    m : int
        The size of the marginals to constrain. When `m=2`, pairwise marginals
        are constrained to equal the pairwise marginals in `pmf`. When `m=3`,
        three-way marginals are constrained to equal those in `pmf.

    Returns
    -------
    A : array-like, shape (p, q)
        The matrix defining the marginal equality constraints and also the
        normalization constraint. The number of rows is:
            p = C(n_variables, m) * n_symbols ** m + 1
        where C() is the choose formula.. The number of columns is:
            q = n_symbols ** n_variables

    b : array-like, (p,)
        The RHS of the linear equality constraints.

    """
    if m > n_variables:
        msg = "Cannot constrain {0}-way marginals"
        msg += " with only {1} random variables."
        msg = msg.format(m, n_variables)
        raise ValueError(msg)

    d = AbstractDenseDistribution(n_variables, n_symbols)

    # Begin with the normalization constraint.
    A = [ np.ones(d.n_elements) ]
    b = [ 1 ]

    # Now add all the marginal constraints.
    if m > 0:
        cache = {}
        for rvs in itertools.combinations(range(n_variables), m):
            for idx in d.parameter_array(rvs, cache=cache):
                bvec = np.zeros(d.n_elements)
                bvec[idx] = 1
                A.append(bvec)
                b.append(pmf[idx].sum())

    A = np.asarray(A)
    b = np.asarray(b)

    return A, b

def constraint_rank(dist, m):
    """
    Returns the rank of the linear constraint matrix.

    """
    pmf, n_variables, n_symbols = cartesian_product_view(dist)
    A, b = linear_constraints(pmf, n_variables, n_symbols, m)
    C, d, rank = as_full_rank(A, b)
    return rank
<commit_msg>Add maxent solver.<commit_after>"""
Maximum entropy with marginal distribution constraints.

Note: We are actually doing the maximum entropy optimization. So we have not
built in the fact that the solution is an exponential family.

Also, this doesn't seem to work that well in practice. The optimization
simply fails to converge. We might need to assume the exponential form
and then fit the params to match the marginals. Perhaps exact gradient
and Hessians might help, or maybe even some rescaling of the linear
constraints.

"""

from __future__ import print_function
from __future__ import division

import itertools
import bisect

import numpy as np
import numdifftools

import dit

from dit.abstractdist import AbstractDenseDistribution

def as_full_rank(A, b):
    """
    From a linear system Ax = b, return Bx = c such that B has full rank.

    In CVXOPT, linear constraints are denoted as: Ax = b. A has shape (p, n)
    and must have full rank. x has shape (n, 1), and so b has shape (p, 1).
    Let's assume that we have:

        rank(A) = q <= n

    This is a typical situation if you are doing optimization, where you have
    an under-determined system and are using some criterion for selecting out
    a particular solution. Now, it may happen that q < p, which means that some
    of your constraint equations are not independent. Since CVXOPT requires
    that A have full rank, we must isolate an equivalent system Bx = c which
    does have full rank. We use SVD for this. So A = U \Sigma V^*, where
    U is (p, p), \Sigma is (p, n) and V^* is (n, n). Then:

        \Sigma V^* x = U^{-1} b

    We take B = \Sigma V^* and c = U^T b, where we use U^T instead of U^{-1}
    for computational efficiency (and since U is orthogonal). But note, we
    take only the cols of U and rows of \Sigma that have nonzero singular
    values.

    Parameters
    ----------
    A : array-like, shape (p, n)
        The LHS for the linear constraints.
    b : array-like, shape (p,) or (p, 1)
        The RHS for the linear constraints.

    Returns
    -------
    B : array-like, shape (q, n)
        The LHS for the linear constraints.
    c : array-like, shape (q,) or (q, 1)
        The RHS for the linear constraints.
    rank : int
        The rank of B.

    """
    try:
        from scipy import linalg
    except ImportError:
        from numpy import linalg

    A = np.atleast_2d(A)

    U, S, Vh = linalg.svd(A)

    tol = S.max() * max(A.shape) * np.finfo(S.dtype).eps
    rank = np.sum(S > tol)

    Ut = U[:, :rank].transpose()

    # See scipy.linalg.diagsvd for details.
    # Note that we only take the first 'rank' rows/cols of S.
    part = np.diag(S[:rank])
    typ = part.dtype.char
    D = np.r_['-1', part, np.zeros((rank, A.shape[1] - rank), typ)]

    B = np.dot(D, Vh)
    c = np.dot(Ut, b)

    return B, c, rank

class PhantomArray(object):
    """
    A simple wrapper around a sparse pmf specified by a lookup table.

    The wrapper provides NumPy

    """
    def __init__(self, lookup, n_variables, n_symbols, symbols):
        self.lookup = lookup
        self.n_variables = n_variables
        self.n_symbols = n_symbols
        self.symbols = symbols
        self.n_elements = n_symbols ** n_variables

    def __getitem__(self, idx):
        try:
            idx[0]
        except:
            return self.lookup.get(idx, 0)
        else:
            # Iterable. Return a NumPy array of the elements.
            return np.array([self.lookup.get(i, 0) for i in idx])

    def __len__(self):
        return self.n_elements

def cartesian_product_view(dist):
    """
    Return a dense Cartesian product view of the `dist`.

    In a Cartesian product view, we union the sample space of each random
    variable and then use it in a Cartesian product that defines the sample
    space for the entire distribution.

    Parameters
    ----------
    dist : distribution
        The distribution to make dense.

    Returns
    -------
    pa : PhantomArray
        The dense, Cartesian product representation of the pmf of `dist`.
    n_variables : int
        The number of random variables in the distribution.
    n_symbols : int
        The number of symbols in the sample space for a single random variable.

    Examples
    --------
    >>> import dit
    >>> d = dit.Distribution(['00', '10'], [.5, .5])
    >>> pa, n, k = cartesian_product_view(d)
    >>> pa[[0,1,2,3]]
    array([ 0.5, 0., 0.5, 0. ])

    """
    symbols = list(sorted(set.union(*map(set, dist.alphabet))))
    n_variables = dist.outcome_length()
    n_symbols = len(symbols)

    pmf_original = dit.copypmf(dist, base='linear')

    lookup = {}
    for outcome, p in zip(dist.outcomes, pmf_original):
        index = 0
        for i, symbol in enumerate(outcome):
            idx = bisect.bisect_left(symbols, symbol)
            index += idx * n_symbols ** (n_variables - 1 - i)
        lookup[index] = p

    pmf = PhantomArray(lookup, n_variables, n_symbols, symbols)
    return pmf, n_variables, n_symbols

def linear_constraints(pmf, n_variables, n_symbols, m):
    """
    Returns `A` and `b` in `A x = b`, for a system of marginal constraints.

    The resulting matrix `A` is not guaranteed to have full rank.

    Parameters
    ----------
    pmf : array-like, shape ( n_symbols ** n_variables, )
        The probability mass function of the distribution. The pmf must have
        a Cartesian product sample space with the same sample space used for
        each random variable.
    n_variables : int
        The number of random variables.
    n_symbols : int
        The number of symbols that each random variable can be.
    m : int
        The size of the marginals to constrain. When `m=2`, pairwise marginals
        are constrained to equal the pairwise marginals in `pmf`. When `m=3`,
        three-way marginals are constrained to equal those in `pmf.

    Returns
    -------
    A : array-like, shape (p, q)
        The matrix defining the marginal equality constraints and also the
        normalization constraint. The number of rows is:
            p = C(n_variables, m) * n_symbols ** m + 1
        where C() is the choose formula.. The number of columns is:
            q = n_symbols ** n_variables

    b : array-like, (p,)
        The RHS of the linear equality constraints.

    """
    if m > n_variables:
        msg = "Cannot constrain {0}-way marginals"
        msg += " with only {1} random variables."
        msg = msg.format(m, n_variables)
        raise ValueError(msg)

    d = AbstractDenseDistribution(n_variables, n_symbols)

    # Begin with the normalization constraint.
    A = [ np.ones(d.n_elements) ]
    b = [ 1 ]

    # Now add all the marginal constraints.
    if m > 0:
        cache = {}
        for rvs in itertools.combinations(range(n_variables), m):
            for idx in d.parameter_array(rvs, cache=cache):
                bvec = np.zeros(d.n_elements)
                bvec[idx] = 1
                A.append(bvec)
                b.append(pmf[idx].sum())

    A = np.asarray(A, dtype=float)
    b = np.asarray(b, dtype=float)

    return A, b

def linear_constraint_rank(dist, m):
    """
    Returns the rank of the linear constraint matrix.

    """
    pmf, n_variables, n_symbols = cartesian_product_view(dist)
    A, b = linear_constraints(pmf, n_variables, n_symbols, m)
    C, d, rank = as_full_rank(A, b)
    return rank

def negentropy(p):
    """
    Entropy which operates on vectors of length N.

    """
    return np.nansum(p * np.log2(p))

class MaximumEntropy(object):
    """
    Find maximum entropy distribution subject to matching k-way marginals.

    """
    def __init__(self, dist, k, prng=None):
        """
        Initialize optimizer.

        Parameters
        ----------
        dist : distribution
            The distribution used to specify the marginal constraints.
        k : int
            The number of variables in the constrained marginals.

        """
        self.k = k
        if isinstance(dist, PhantomArray):
            self.dist = None
            self.pmf = dist
        else:
            self.dist = dist
            pmf, n_variables, n_symbols = cartesian_product_view(dist)
            self.pmf = pmf

        self.n_variables = self.pmf.n_variables
        self.n_symbols = self.pmf.n_symbols

        if prng is None:
            prng = np.random.RandomState()
        self.prng = prng

        self.tol = {}

        self.initial = None
        self.init()

    def init(self):

        # Dimension of optimization variable
        self.n = len(self.pmf)

        # Number of nonlinear constraints
        self.m = 0

        self.build_gradient_hessian()
        self.build_nonnegativity_constraints()
        self.build_linear_equality_constraints()
        self.build_F()

    def build_gradient_hessian(self):

        import numdifftools

        self.func = negentropy
        self.gradient = numdifftools.Gradient(negentropy)
        self.hessian = numdifftools.Hessian(negentropy)

    def build_nonnegativity_constraints(self):
        from cvxopt import matrix

        # Dimension of optimization variable
        n = self.n

        # Nonnegativity constraint
        #
        # We have M = N = 0 (no 2nd order cones or positive semidefinite cones)
        # So, K = l where l is the dimension of the nonnegative orthant. Thus,
        # we have l = n.
        G = matrix( -1 * np.eye(n) )   # G should have shape: (K,n) = (n,n)
        h = matrix( np.zeros((n,1)) )  # h should have shape: (K,1) = (n,1)

        self.G = G
        self.h = h

    def build_linear_equality_constraints(self):
        from cvxopt import matrix

        # Dimension of optimization variable
        n = self.n

        args = (self.pmf, self.n_variables, self.n_symbols, self.k)
        A, b = linear_constraints(*args)
        A, b, rank = as_full_rank(A, b)

        A = matrix(A)
        b = matrix(b)  # now a column vector

        self.A = A
        self.b = b

    def build_F(self):
        from cvxopt import matrix

        n = self.n
        m = self.m

        def F(x=None, z=None):
            # x has shape: (n,1)   and is the distribution
            # z has shape: (m+1,1) and is the Hessian of f_0

            if x is None and z is None:
                # Initial point is the original distribution.
                #d = self.pmf[np.arange(self.pmf.n_elements)]
                d = self.initial_dist()
                #d = np.ones(n) / n
                return (m, matrix(d))

            xarr = np.array(x)[:,0]

            # Verify that x is in domain.
            # Does G,h and A,b take care of this?
            #
            if np.any(xarr > 1) or np.any(xarr < 0):
                return None
            if not np.allclose(np.sum(xarr), 1, **self.tol):
                return None

            # Using automatic differentiators for now.
            f = self.func(xarr)
            Df = self.gradient(xarr)
            Df = matrix(Df.reshape((1, n)))

            if z is None:
                return (f, Df)
            H = self.hessian(xarr)
            H = matrix(H)

            return (f, Df, z[0] * H)

        self.F = F

    def initial_dist(self):
        return self.prng.dirichlet([1] * self.n)

    def optimize(self, show_progress=False):
        from cvxopt.solvers import cp, options

        old = options.get('show_progress', None)
        out = None

        try:
            options['show_progress'] = show_progress
            with np.errstate(divide='ignore', invalid='ignore'):
                result = cp(F=self.F,
                            G=self.G,
                            h=self.h,
                            dims={'l':self.n, 'q':[], 's':[]},
                            A=self.A,
                            b=self.b)
        except:
            raise
        else:
            self.result = result
            out = np.asarray(result['x'])
        finally:
            if old is None:
                del options['show_progress']
            else:
                options['show_progress'] = old

        return out

def maxent_dists(dist, k_max=None, show_progress=True):

    pmf, n_variables, n_symbols = cartesian_product_view(dist)

    if k_max is None:
        k_max = n_variables

    outcomes = list(dist._product(pmf.symbols, repeat=n_variables))

    dists = []
    for k in range(k_max + 1):
        print()
        print("Constraining maxent dist to match {0}-way marginals.".format(k))
        print()
        opt = MaximumEntropy(pmf, k)
        pmf_opt = opt.optimize(show_progress=show_progress)
        d = dit.Distribution(outcomes, pmf_opt)
        dists.append(d)

    return dists
<|endoftext|>


<commit_before>import os
import unittest
import multiprocessing
import time
from urllib.parse import urlparse

from werkzeug.security import generate_password_hash
from splinter import Browser

# Configure app to use the testing database
os.environ["CONFIG_PATH"] = "crossword.config.TestingConfig"

from crossword import app
from crossword.database import Base, engine, session, User, Entry

from datetime import datetime, timedelta

#...
class TestCase(unittest.TestCase):

    #...
    def test_followers(self):
        # make four users
        u1 = User(name='ptest1', email='ptest1@gmail.com')
        u2 = User(name='ptest2', email='ptest2@gmail.com')
        u3 = User(name='ptest3', email='ptest3@gmail.com')
        u4 = User(name='ptest4', email='ptest4@gmail.com')
        session.add(u1)
        session.add(u2)
        session.add(u3)
        session.add(u4)
        # make four entrys
        utcnow = datetime.utcnow()
        # setup the followers
        u1.follow(u1)
        u1.follow(u2)
        u1.follow(u4)
        u2.follow(u2)
        u2.follow(u3)
        u3.follow(u3)
        u3.follow(u4)
        u4.follow(u4)
        session.add(u1)
        session.add(u2)
        session.add(u3)
        session.add(u4)
        session.commit()
        # check the the amount of followers for each user
        #assert u1.follow(u2) is None
        assert u1.is_following(u2)
        assert u1.followers.count() == 1
        assert u1.followers.first().name == 'ptest1'
        assert u2.followers.count() == 2
        assert u2.followers.first().name == 'ptest1'
        u = u1.unfollow(u2)
        assert u is not None
        session.add(u)
        session.commit()
        assert not u1.is_following(u2)
        assert u1.followed.count() == 2
        assert u2.followed.count() == 2

    def tearDown(self):
        """ Test teardown """
        session.close()
        # Remove the tables and their data from the database
        Base.metadata.drop_all(engine)

if __name__ == "__main__":
    unittest.main()<commit_msg>cleaned up a bit<commit_after>import os
import unittest
import multiprocessing
import time
from urllib.parse import urlparse

from werkzeug.security import generate_password_hash
from splinter import Browser

# Configure app to use the testing database
os.environ["CONFIG_PATH"] = "crossword.config.TestingConfig"

from crossword import app
from crossword.database import Base, engine, session, User, Entry

from datetime import datetime, timedelta

#...
class TestCase(unittest.TestCase):

    #...
    def test_followers(self):
        # make four users
        u1 = User(name='ptest1', email='ptest1@gmail.com')
        u2 = User(name='ptest2', email='ptest2@gmail.com')
        u3 = User(name='ptest3', email='ptest3@gmail.com')
        u4 = User(name='ptest4', email='ptest4@gmail.com')
        session.add(u1)
        session.add(u2)
        session.add(u3)
        session.add(u4)
        utcnow = datetime.utcnow()
        # setup the followers
        u1.follow(u1)
        u1.follow(u2)
        u1.follow(u4)
        u2.follow(u2)
        u2.follow(u3)
        u3.follow(u3)
        u3.follow(u4)
        u4.follow(u4)
        session.add(u1)
        session.add(u2)
        session.add(u3)
        session.add(u4)
        session.commit()
        # check the the amount of followers for each user
        #assert u1.follow(u2) is None
        assert u1.is_following(u2)
        assert u1.followers.count() == 1
        assert u1.followers.first().name == 'ptest1'
        assert u2.followers.count() == 2
        assert u2.followers.first().name == 'ptest1'
        u = u1.unfollow(u2)
        assert u is not None
        session.add(u)
        session.commit()
        assert not u1.is_following(u2)
        assert u1.followed.count() == 2
        assert u2.followed.count() == 2

    def tearDown(self):
        """ Test teardown """
        session.close()
        # Remove the tables and their data from the database
        Base.metadata.drop_all(engine)

if __name__ == "__main__":
    unittest.main()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.db import models

# from documents.models import Languages

class WikiTexts(models.Model):
    """ Текст для вики """

    nazv = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    red_zag = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="редакционный заголовок"
    )
    author = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Автор"
    )
    translator = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Переводчик"
    )
    editor = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Редактор"
    )
    data_n = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    place_n = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    data_i = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    place_i = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    zhanr = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Жанр"
    )
    picture = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Изображение"
    )
    samizdat = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    categories = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Категории"
    )
    title = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    link = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Ссылка"
    )
    user = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Пользователь"
    )
    ruwiki = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="RU wiki"
    )
    enwiki = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="EN wiki"
    )
    timestamp = models.DateTimeField(
        auto_now_add=True, null=True,
        verbose_name="Дата")
    oborotka = models.TextField(
        blank=True, default='',
        verbose_name="Оборотка"
    )

    def __unicode__(self):
        return self.nazv

class TXTC(models.Model):
    """ TXTC """

    t_type = models.CharField(
        max_length=50,
        blank=True, default='',
        verbose_name="Тип"
    )

class XTC(models.Model):
    """ XTC """

    number = models.CharField(
        max_length=10,
        blank=True, default='',
        verbose_name="Номер ХТС"
    )
    pages = models.CharField(
        max_length=50,
        verbose_name="Номера страниц",
        help_text="Номера страниц хроники, на которых упомянут документ"
    )
    pages_from = models.IntegerField(
        blank=True, null=True,
        verbose_name="Начальная страница диапазона")
    pages_to = models.IntegerField(
        blank=True, null=True,
        verbose_name="Последняя страница диапазона")
    profile = models.CharField(
        max_length=20,
        blank=True,
        verbose_name="Профиль упоминания",
        default="упом."
    )
    notes = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Примечания"
    )
    catalog = models.ForeignKey("Catalog", verbose_name="Документ")
    operator = models.CharField(
        max_length=8,
        blank=True, default='',
        verbose_name="Оператор"
    )
    date = models.DateField(
        auto_now_add=True, null=True, verbose_name="Дата ввода")

    def __unicode__(self):
        return self.number

class Catalog(models.Model):
    """ Основной каталог ?? """

    ACNumber = models.CharField(
        max_length=28,
        blank=True, default='',
        verbose_name="Номер АС"
    )
    # language = models.CharField(
        # max_length=2,
        # blank=True, default='',
        # verbose_name="Язык"
    # )
    language = models.ForeignKey("documents.Languages", db_column="language")
    translated = models.CharField(
        max_length=2,
        blank=True, default='',
        verbose_name="Переведено"
    )
    author = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Автор"
    )
    auth_notes = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Примечания к автору"
    )
    auth_group = models.CharField(
        max_length=100,
        blank=True, default='',
        verbose_name="Группа авторов"
    )
    # auth_group_notes = models.CharField(
        # max_length=255,
        # blank=True, default='',
        # verbose_name="Примечания к группе авторов"
    # )
    group_members = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Состав группы"
    )
    # members_notes = models.CharField(
        # max_length=255,
        # blank=True, default='',
        # verbose_name="Примечания к составу группы"
    # )
    signers = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Подписанты"
    )
    signers_notes = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Примечания о подписантах"
    )
    complie_editors = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Редакторы_составители"
    )
    ce_notes = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Примечания о редакторах-составителях"
    )
    selfname = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Самоназвание"
    )
    name1 = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="name1"
    )
    #XXX Possible foreign key to TypeDoc
    typedoc = models.CharField(
        max_length=25,
        blank=True, default='',
        verbose_name="Тип документа"
    )
    name = models.TextField(blank=True, verbose_name="Название")
    name2 = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Name2"
    )
    place = models.CharField(
        max_length=100,
        blank=True, default='',
        verbose_name="Место"
    )
    m_ind = models.CharField(
        max_length=100,
        blank=True, default='',
        verbose_name="m-ind"
    )
    place_prim = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="PlacePrim"
    )
    date = models.CharField(
        max_length=125,
        blank=True, default='',
        verbose_name="Дата"
    )
    date_prim = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="DatePrim"
    )
    date1 = models.DateTimeField(blank=True, null=True, verbose_name="date1")
    date2 = models.DateTimeField(blank=True, null=True, verbose_name="date2")
    reproducing = models.CharField(
        max_length=15,
        blank=True, default='',
        verbose_name="Способ воспроизведения"
    )
    authencity = models.CharField(
        max_length=10,
        blank=True, default='',
        verbose_name="Подлинность"
    )
    num_copies = models.CharField(
        max_length=10,
        blank=True, null=True,
        verbose_name="Число экземпляров"
    )
    correction = models.CharField(
        max_length=255,
        blank=True, null=True,
        verbose_name="Правка"
    )
    medium = models.CharField(
        max_length=35,
        blank=True, null=True,
        verbose_name="Носитель"
    )
    pages = models.CharField(
        max_length=50,
        blank=True, null=True,
        verbose_name="Страниц"
    )
    archive_notes = models.CharField(
        max_length=50,
        blank=True, null=True,
        verbose_name="Архивные примечания"
    )
    notes = models.TextField(
        blank=True, null=True,
        verbose_name="Примечания"
    )
    published = models.TextField(
        blank=True, null=True,
        verbose_name="Опубликовано"
    )
    tome = models.CharField(
        max_length=15,
        blank=True, null=True,
        verbose_name="Том"
    )
    number_mc = models.CharField(
        max_length=10,
        blank=True, null=True,
        verbose_name="Номер МС"
    )
    year = models.CharField(
        max_length=20,
        blank=True, null=True,
        verbose_name="Год"
    )
    fund = models.CharField(
        max_length=70,
        blank=True, null=True,
        verbose_name="Фонд"
    )
    register = models.CharField(
        max_length=70,
        blank=True, null=True,
        verbose_name="Опись"
    )
    folder = models.CharField(
        max_length=10,
        blank=True, null=True,
        verbose_name="Дело"
    )
    sheets = models.CharField(
        max_length=50,
        blank=True, null=True,
        verbose_name="Листы"
    )
    annotation = models.TextField(
        blank=True, null=True,
        verbose_name="Аннотация"
    )
    web_address = models.CharField(
        max_length=255,
        blank=True, null=True,
        verbose_name="Адрес документа"
    )
    nas = models.IntegerField(
        blank=True, null=True,
        verbose_name="NAS"
    )
    nas_ind = models.CharField(
        max_length=28,
        blank=True, null=True,
        verbose_name="NAS-ind"
    )
    troubles = models.NullBooleanField(verbose_name="Troubles")
    hr = models.NullBooleanField(
        verbose_name="В хронике",
        help_text="Отметка о том, что документ упоминается в Хронике"
    )
    hr_search = models.NullBooleanField(
        verbose_name="hr_poisk",
        help_text="Отметка для фильтра по номеру ХТС"
    )
    operator = models.CharField(
        max_length=10,
        blank=True, null=True,
        verbose_name="Оператор",
        help_text="Имя оператора, вводящего запись"
    )
    registration_date = models.DateTimeField(
        blank=True, null=True,
        verbose_name="Дата ввода",
        help_text="Дата ввода оператором(проставляется автоматически)",
        auto_now_add=True
    )
    ready = models.NullBooleanField(
        verbose_name="Ready",
        help_text="Отметка для записей, обработанных на авторство по Именнику"
    )
    belles_lettres = models.NullBooleanField(
        verbose_name="Художка",
    )
    link = models.CharField(
        max_length=255,
        blank=True, null=True,
        verbose_name="Ссылка",
    )
    aka_name = models.CharField(
        max_length=255,
        blank=True, null=True,
        verbose_name="AKA_name",
    )

    def __unicode__(self):
        return self.name[0:60]
<commit_msg>Allow language to be NULL<commit_after># -*- coding: utf-8 -*-

from django.db import models

# from documents.models import Languages

class WikiTexts(models.Model):
    """ Текст для вики """

    nazv = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    red_zag = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="редакционный заголовок"
    )
    author = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Автор"
    )
    translator = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Переводчик"
    )
    editor = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Редактор"
    )
    data_n = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    place_n = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    data_i = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    place_i = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    zhanr = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Жанр"
    )
    picture = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Изображение"
    )
    samizdat = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    categories = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Категории"
    )
    title = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Название"
    )
    link = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Ссылка"
    )
    user = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Пользователь"
    )
    ruwiki = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="RU wiki"
    )
    enwiki = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="EN wiki"
    )
    timestamp = models.DateTimeField(
        auto_now_add=True, null=True,
        verbose_name="Дата")
    oborotka = models.TextField(
        blank=True, default='',
        verbose_name="Оборотка"
    )

    def __unicode__(self):
        return self.nazv

class TXTC(models.Model):
    """ TXTC """

    t_type = models.CharField(
        max_length=50,
        blank=True, default='',
        verbose_name="Тип"
    )

class XTC(models.Model):
    """ XTC """

    number = models.CharField(
        max_length=10,
        blank=True, default='',
        verbose_name="Номер ХТС"
    )
    pages = models.CharField(
        max_length=50,
        verbose_name="Номера страниц",
        help_text="Номера страниц хроники, на которых упомянут документ"
    )
    pages_from = models.IntegerField(
        blank=True, null=True,
        verbose_name="Начальная страница диапазона")
    pages_to = models.IntegerField(
        blank=True, null=True,
        verbose_name="Последняя страница диапазона")
    profile = models.CharField(
        max_length=20,
        blank=True,
        verbose_name="Профиль упоминания",
        default="упом."
    )
    notes = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Примечания"
    )
    catalog = models.ForeignKey("Catalog", verbose_name="Документ")
    operator = models.CharField(
        max_length=8,
        blank=True, default='',
        verbose_name="Оператор"
    )
    date = models.DateField(
        auto_now_add=True, null=True, verbose_name="Дата ввода")

    def __unicode__(self):
        return self.number

class Catalog(models.Model):
    """ Основной каталог ?? """

    ACNumber = models.CharField(
        max_length=28,
        blank=True, default='',
        verbose_name="Номер АС"
    )
    # language = models.CharField(
        # max_length=2,
        # blank=True, default='',
        # verbose_name="Язык"
    # )
    language = models.ForeignKey(
        "documents.Languages",
        db_column="language",
        null=True)
    translated = models.CharField(
        max_length=2,
        blank=True, default='',
        verbose_name="Переведено"
    )
    author = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Автор"
    )
    auth_notes = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Примечания к автору"
    )
    auth_group = models.CharField(
        max_length=100,
        blank=True, default='',
        verbose_name="Группа авторов"
    )
    # auth_group_notes = models.CharField(
        # max_length=255,
        # blank=True, default='',
        # verbose_name="Примечания к группе авторов"
    # )
    group_members = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Состав группы"
    )
    # members_notes = models.CharField(
        # max_length=255,
        # blank=True, default='',
        # verbose_name="Примечания к составу группы"
    # )
    signers = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Подписанты"
    )
    signers_notes = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Примечания о подписантах"
    )
    complie_editors = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Редакторы_составители"
    )
    ce_notes = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Примечания о редакторах-составителях"
    )
    selfname = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Самоназвание"
    )
    name1 = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="name1"
    )
    #XXX Possible foreign key to TypeDoc
    typedoc = models.CharField(
        max_length=25,
        blank=True, default='',
        verbose_name="Тип документа"
    )
    name = models.TextField(blank=True, verbose_name="Название")
    name2 = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="Name2"
    )
    place = models.CharField(
        max_length=100,
        blank=True, default='',
        verbose_name="Место"
    )
    m_ind = models.CharField(
        max_length=100,
        blank=True, default='',
        verbose_name="m-ind"
    )
    place_prim = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="PlacePrim"
    )
    date = models.CharField(
        max_length=125,
        blank=True, default='',
        verbose_name="Дата"
    )
    date_prim = models.CharField(
        max_length=255,
        blank=True, default='',
        verbose_name="DatePrim"
    )
    date1 = models.DateTimeField(blank=True, null=True, verbose_name="date1")
    date2 = models.DateTimeField(blank=True, null=True, verbose_name="date2")
    reproducing = models.CharField(
        max_length=15,
        blank=True, default='',
        verbose_name="Способ воспроизведения"
    )
    authencity = models.CharField(
        max_length=10,
        blank=True, default='',
        verbose_name="Подлинность"
    )
    num_copies = models.CharField(
        max_length=10,
        blank=True, null=True,
        verbose_name="Число экземпляров"
    )
    correction = models.CharField(
        max_length=255,
        blank=True, null=True,
        verbose_name="Правка"
    )
    medium = models.CharField(
        max_length=35,
        blank=True, null=True,
        verbose_name="Носитель"
    )
    pages = models.CharField(
        max_length=50,
        blank=True, null=True,
        verbose_name="Страниц"
    )
    archive_notes = models.CharField(
        max_length=50,
        blank=True, null=True,
        verbose_name="Архивные примечания"
    )
    notes = models.TextField(
        blank=True, null=True,
        verbose_name="Примечания"
    )
    published = models.TextField(
        blank=True, null=True,
        verbose_name="Опубликовано"
    )
    tome = models.CharField(
        max_length=15,
        blank=True, null=True,
        verbose_name="Том"
    )
    number_mc = models.CharField(
        max_length=10,
        blank=True, null=True,
        verbose_name="Номер МС"
    )
    year = models.CharField(
        max_length=20,
        blank=True, null=True,
        verbose_name="Год"
    )
    fund = models.CharField(
        max_length=70,
        blank=True, null=True,
        verbose_name="Фонд"
    )
    register = models.CharField(
        max_length=70,
        blank=True, null=True,
        verbose_name="Опись"
    )
    folder = models.CharField(
        max_length=10,
        blank=True, null=True,
        verbose_name="Дело"
    )
    sheets = models.CharField(
        max_length=50,
        blank=True, null=True,
        verbose_name="Листы"
    )
    annotation = models.TextField(
        blank=True, null=True,
        verbose_name="Аннотация"
    )
    web_address = models.CharField(
        max_length=255,
        blank=True, null=True,
        verbose_name="Адрес документа"
    )
    nas = models.IntegerField(
        blank=True, null=True,
        verbose_name="NAS"
    )
    nas_ind = models.CharField(
        max_length=28,
        blank=True, null=True,
        verbose_name="NAS-ind"
    )
    troubles = models.NullBooleanField(verbose_name="Troubles")
    hr = models.NullBooleanField(
        verbose_name="В хронике",
        help_text="Отметка о том, что документ упоминается в Хронике"
    )
    hr_search = models.NullBooleanField(
        verbose_name="hr_poisk",
        help_text="Отметка для фильтра по номеру ХТС"
    )
    operator = models.CharField(
        max_length=10,
        blank=True, null=True,
        verbose_name="Оператор",
        help_text="Имя оператора, вводящего запись"
    )
    registration_date = models.DateTimeField(
        blank=True, null=True,
        verbose_name="Дата ввода",
        help_text="Дата ввода оператором(проставляется автоматически)",
        auto_now_add=True
    )
    ready = models.NullBooleanField(
        verbose_name="Ready",
        help_text="Отметка для записей, обработанных на авторство по Именнику"
    )
    belles_lettres = models.NullBooleanField(
        verbose_name="Художка",
    )
    link = models.CharField(
        max_length=255,
        blank=True, null=True,
        verbose_name="Ссылка",
    )
    aka_name = models.CharField(
        max_length=255,
        blank=True, null=True,
        verbose_name="AKA_name",
    )

    def __unicode__(self):
        return self.name[0:60]
<|endoftext|>


<commit_before>import time
import socket
from multiprocessing import Process
from unittest.mock import patch

import zmq
import pytest

from zerolog.forwarder import start_forwarder

@pytest.fixture(scope='module')
def forwarder():
    """Return a process for forwarder"""
    p = Process(target=start_forwarder, args=(6002, 6001, 6500))
    p.start()
    time.sleep(1)
    return p

@pytest.fixture(scope='module')
def tcp_sender():
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    return sock

def test_forwarder(forwarder, tcp_sender, context):
    """Monitor should correctly send data"""
    sender = tcp_sender

    mon = context.socket(zmq.SUB)
    mon.setsockopt_string(zmq.SUBSCRIBE, "")
    mon.setsockopt(zmq.LINGER, 0)
    mon.connect("tcp://localhost:6500")

    recv = context.socket(zmq.SUB)
    recv.setsockopt_string(zmq.SUBSCRIBE, "")
    recv.setsockopt(zmq.LINGER, 0)
    recv.connect("tcp://localhost:6002")

    server_address = ('localhost', 6001)
    sender.connect(server_address)

    # waiting for warmup
    time.sleep(1)

    sender.sendall(b"test test")

    data = mon.recv()
    assert data is not None

    sender.sendall(b"test test")

    data = recv.recv()
    assert data is not None
    sender.close()
    forwarder.terminate()

@patch('zerolog.forwarder.zmq.proxy')
@patch('zerolog.forwarder.zmq.Context')
@patch('zerolog.forwarder.zmq.Context.socket')
@patch('zerolog.forwarder.zmq.Socket')
@patch('zerolog.forwarder.zmq.Socket.bind')
def test_forwarder_basic_run(*args):
    """forwarder should correctly run with different parameters"""
    start_forwarder(6010, 6011, 6012)
    start_forwarder(6020, 6021)
    start_forwarder(6030, 6031, "sub", "push")

@patch('zerolog.forwarder.zmq.proxy')
@patch('zerolog.forwarder.zmq.Context')
@patch('zerolog.forwarder.zmq.Context.socket')
@patch('zerolog.forwarder.zmq.Socket')
@patch('zerolog.forwarder.zmq.Socket.bind')
def test_forwarder_errors(*args):
    """Forwarder could correctly handle errors"""
    start_forwarder(6040, 6041, "bad_type", "push")
    start_forwarder(6050, 6051, "sub", "bad_type")
<commit_msg>improve test coverage<commit_after>import time
import socket
from multiprocessing import Process
from unittest.mock import patch

import zmq
import pytest

from zerolog.forwarder import start_forwarder

@pytest.fixture(scope='module')
def forwarder():
    """Return a process for forwarder"""
    p = Process(target=start_forwarder, args=(6002, 6001, 6500))
    p.start()
    time.sleep(1)
    return p

@pytest.fixture(scope='module')
def tcp_sender():
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    return sock

def test_forwarder(forwarder, tcp_sender, context):
    """Monitor should correctly send data"""
    sender = tcp_sender

    mon = context.socket(zmq.SUB)
    mon.setsockopt_string(zmq.SUBSCRIBE, "")
    mon.setsockopt(zmq.LINGER, 0)
    mon.connect("tcp://localhost:6500")

    recv = context.socket(zmq.SUB)
    recv.setsockopt_string(zmq.SUBSCRIBE, "")
    recv.setsockopt(zmq.LINGER, 0)
    recv.connect("tcp://localhost:6002")

    server_address = ('localhost', 6001)
    sender.connect(server_address)

    # waiting for warmup
    time.sleep(1)

    sender.sendall(b"test test")

    data = mon.recv()
    assert data is not None

    sender.sendall(b"test test")

    data = recv.recv()
    assert data is not None
    sender.close()
    forwarder.terminate()

@patch('zerolog.forwarder.zmq.proxy')
@patch('zerolog.forwarder.zmq.Context')
@patch('zerolog.forwarder.zmq.Context.socket')
@patch('zerolog.forwarder.zmq.Socket')
@patch('zerolog.forwarder.zmq.Socket.bind')
def test_forwarder_basic_run(*args):
    """forwarder should correctly run with different parameters"""
    start_forwarder(6010, 6011, 6012)
    start_forwarder(6020, 6021)
    start_forwarder(6030, 6031, "sub", "push")

@patch('zerolog.forwarder.zmq.proxy')
@patch('zerolog.forwarder.zmq.Context')
@patch('zerolog.forwarder.zmq.Context.socket')
@patch('zerolog.forwarder.zmq.Socket')
@patch('zerolog.forwarder.zmq.Socket.bind')
def test_forwarder_errors(*args):
    """Forwarder could correctly handle errors"""
    start_forwarder(6040, 6041, backend_socket="bad_type", frontend_socket="push")
    start_forwarder(6050, 6051, backend_socket="sub", frontend_socket="bad_type")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.test import TestCase
from django.http import HttpRequest, HttpResponse
from django.contrib.csrf.middleware import CsrfMiddleware, _make_token
from django.conf import settings

class CsrfMiddlewareTest(TestCase):

    _session_id = "1"

    def _get_no_session_request(self):
        return HttpRequest()

    def _get_session_request(self):
        req = self._get_no_session_request()
        req.COOKIES[settings.SESSION_COOKIE_NAME] = self._session_id
        return req

    def _get_post_form_response(self):
        resp = HttpResponse(content="""
<form method="POST"><input type="text" /></form>
""", mimetype="text/html")
        return resp

    def _get_new_session_response(self):
        resp = self._get_post_form_response()
        resp.cookies[settings.SESSION_COOKIE_NAME] = self._session_id
        return resp

    def _check_token_present(self, response):
        self.assertContains(response, "name='csrfmiddlewaretoken' value='%s'" % _make_token(self._session_id))

    def test_process_response_no_session(self):
        """
        Check the the post-processor does nothing if no session active
        """
        req = self._get_no_session_request()
        resp = self._get_post_form_response()
        resp_content = resp.content
        resp2 = CsrfMiddleware().process_response(req, resp)
        self.assertEquals(resp_content, resp2.content)

    def test_process_response_existing_session(self):
        """
        Check that the token is inserted if there is an existing session
        """
        req = self._get_session_request()
        resp = self._get_post_form_response()
        resp_content = resp.content
        resp2 = CsrfMiddleware().process_response(req, resp)
        self.assertNotEqual(resp_content, resp2.content)
        self._check_token_present(resp2)

    def test_process_response_new_session(self):
        """
        Check that the token is inserted if there is a new session being started
        """
        req = self._get_no_session_request() # no session in request
        resp = self._get_new_session_response() # but new session started
        resp_content = resp.content
        resp2 = CsrfMiddleware().process_response(req, resp)
        self.assertNotEqual(resp_content, resp2.content)
        self._check_token_present(resp2)
<commit_msg>More tests for the other half of CsrfMiddleware<commit_after># -*- coding: utf-8 -*-

from django.test import TestCase
from django.http import HttpRequest, HttpResponse, HttpResponseForbidden
from django.contrib.csrf.middleware import CsrfMiddleware, _make_token
from django.conf import settings

class CsrfMiddlewareTest(TestCase):

    _session_id = "1"

    def _get_GET_no_session_request(self):
        return HttpRequest()

    def _get_GET_session_request(self):
        req = self._get_GET_no_session_request()
        req.COOKIES[settings.SESSION_COOKIE_NAME] = self._session_id
        return req

    def _get_POST_session_request(self):
        req = self._get_GET_session_request()
        req.method = "POST"
        return req

    def _get_POST_no_session_request(self):
        req = self._get_GET_no_session_request()
        req.method = "POST"
        return req

    def _get_POST_session_request_with_token(self):
        req = self._get_POST_session_request()
        req.POST['csrfmiddlewaretoken'] = _make_token(self._session_id)
        return req

    def _get_post_form_response(self):
        resp = HttpResponse(content="""
<form method="POST"><input type="text" /></form>
""", mimetype="text/html")
        return resp

    def _get_new_session_response(self):
        resp = self._get_post_form_response()
        resp.cookies[settings.SESSION_COOKIE_NAME] = self._session_id
        return resp

    def _check_token_present(self, response):
        self.assertContains(response, "name='csrfmiddlewaretoken' value='%s'" % _make_token(self._session_id))

    # Check the post processing
    def test_process_response_no_session(self):
        """
        Check the the post-processor does nothing if no session active
        """
        req = self._get_GET_no_session_request()
        resp = self._get_post_form_response()
        resp_content = resp.content
        resp2 = CsrfMiddleware().process_response(req, resp)
        self.assertEquals(resp_content, resp2.content)

    def test_process_response_existing_session(self):
        """
        Check that the token is inserted if there is an existing session
        """
        req = self._get_GET_session_request()
        resp = self._get_post_form_response()
        resp_content = resp.content
        resp2 = CsrfMiddleware().process_response(req, resp)
        self.assertNotEqual(resp_content, resp2.content)
        self._check_token_present(resp2)

    def test_process_response_new_session(self):
        """
        Check that the token is inserted if there is a new session being started
        """
        req = self._get_GET_no_session_request() # no session in request
        resp = self._get_new_session_response() # but new session started
        resp_content = resp.content
        resp2 = CsrfMiddleware().process_response(req, resp)
        self.assertNotEqual(resp_content, resp2.content)
        self._check_token_present(resp2)

    # Check the request processing
    def test_process_request_no_session(self):
        """
        Check that if no session is present, the middleware does nothing.
        to the incoming request.
        """
        req = self._get_POST_no_session_request()
        req2 = CsrfMiddleware().process_request(req)
        self.assertEquals(None, req2)

    def test_process_request_session_no_token(self):
        """
        Check that if a session is present but no token, we get a 'forbidden'
        """
        req = self._get_POST_session_request()
        req2 = CsrfMiddleware().process_request(req)
        self.assertEquals(HttpResponseForbidden, req2.__class__)

    def test_process_request_session_and_token(self):
        """
        Check that if a session is present and a token, the middleware lets it through
        """
        req = self._get_POST_session_request_with_token()
        req2 = CsrfMiddleware().process_request(req)
        self.assertEquals(None, req2)
<|endoftext|>


<commit_before>import numpy as nm
import pytest

from sfepy import data_dir
from sfepy.base.base import assert_
import sfepy.base.testing as tst

filename_mesh = data_dir + '/meshes/2d/square_unit_tri.mesh'

def get_pars(ts, coors, mode=None, extra_arg=None,
             equations=None, term=None, problem=None, **kwargs):
    if mode == 'special':
        if extra_arg == 'hello!':
            ic = 0
        else:
            ic = 1
        coors = problem.get_mesh_coors()
        return {('x_%s' % ic) : coors[:,ic]}

    elif mode == 'qp':
        return {'a' : nm.tile(-2 + 1j, (coors.shape[0], 1, 1))}

def get_p_edge(ts, coors, bc=None, **kwargs):
    if bc.name == 'p_left':
        return nm.sin(nm.pi * coors[:,1])
    else:
        return nm.cos(nm.pi * coors[:,1])

def get_u_edge(ts, coors, bc=None, **kwargs):
    out = nm.zeros_like(coors)
    out[:, 1] = nm.arange(out.shape[0]) + 1.0
    return out

def get_circle(coors, domain=None):
    r = nm.sqrt(coors[:,0]**2.0 + coors[:,1]**2.0)
    return nm.where(r < 0.2)[0]

functions = {
    'get_pars' : (get_pars,),
    'get_pars1' : (lambda ts, coors, mode=None, **kwargs:
                   get_pars(ts, coors, mode, extra_arg='hello!', **kwargs),),
    'get_p_edge' : (get_p_edge,),
    'get_u_edge' : (get_u_edge,),
    'get_circle' : (get_circle,),
}

# Just another way of adding a function, besides 'functions' keyword.
function_1 = {
    'name' : 'get_pars2',
    'function' : lambda ts, coors, mode=None, **kwargs:
        get_pars(ts, coors, mode, extra_arg='hi!', **kwargs),
}

materials = {
    'mf1' : (None, 'get_pars1'),
    'mf2' : 'get_pars2',
    # Dot denotes a special value, that is not propagated to all QP.
    'mf3' : ({'a' : 10.0, 'b' : 2.0, '.c' : 'ahoj'},),
    # Complex values.
    'mf4' : 'get_pars',
    'mf5' : ({'a' : -2 - 1j},),
    'mf6' : ({'a' : {'Circle' : 1 + 1j, 'Rest' : 3j}},),
}

fields = {
    'pressure' : (nm.float64, 1, 'Omega', 2),
    'displacement' : (nm.float64, 2, 'Omega', 2),
    'complex' : (nm.complex128, 1, 'Omega', 2),
}

variables = {
    'p'   : ('unknown field', 'pressure', 0),
    'q'   : ('test field',    'pressure', 'p'),
    'u'   : ('unknown field', 'displacement', 1),
    'v'   : ('test field',    'displacement', 'u'),
}

wx = 0.499
regions = {
    'Omega' : 'all',
    'Left' : ('vertices in (x < -%.3f)' % wx, 'facet'),
    'Right' : ('vertices in (x > %.3f)' % wx, 'facet'),
    'Circle' : 'vertices by get_circle',
    'Rest' : 'r.Omega -c r.Circle',
}

ebcs = {
    'p_left' : ('Left', {'p.all' : 'get_p_edge'}),
    'p_right' : ('Right', {'p.all' : 'get_p_edge'}),
    'u_right' : ('Right', {'u.all' : 'get_u_edge'}),
}

equations = {
    'e1' : """dw_laplace.2.Omega( mf3.a, q, p ) = 0""",
    'e2' : """dw_div_grad.2.Omega( mf3.b, v, u ) = 0""",
}

variables2 = {
    'r'   : ('unknown field', 'complex', 2),
    's'   : ('test field',    'complex', 'r'),
}

equations2 = {
    'e3' : """dw_laplace.2.Omega(mf4.a, s, r) = 0""",
    'e4' : """dw_laplace.2.Omega(mf5.a, s, r) = 0""",
    'e5' : """dw_laplace.2.Circle(mf6.a, s, r)
            + dw_laplace.2.Rest(mf6.a, s, r) = 0""",
}

solver_0 = {
    'name' : 'ls',
    'kind' : 'ls.scipy_direct',
}

solver_1 = {
    'name' : 'newton',
    'kind' : 'nls.newton',
}

@pytest.fixture(scope='module')
def problem():
    import sys
    from sfepy.discrete import Problem
    from sfepy.base.conf import ProblemConf

    conf = ProblemConf.from_dict(globals(), sys.modules[__name__])

    problem = Problem.from_conf(conf)
    return problem

def test_material_functions(problem):
    from sfepy.discrete import Material
    from sfepy.base.conf import transform_variables

    conf = problem.conf

    ts = problem.get_default_ts(step=0)

    conf_mat1 = conf.get_item_by_name('materials', 'mf1')
    mat1 = Material.from_conf(conf_mat1, problem.functions)
    mat1.time_update(ts, None, mode='normal', problem=problem)

    coors = problem.domain.get_mesh_coors()
    assert_(nm.all(coors[:,0] == mat1.get_data(None, 'x_0')))

    conf_mat2 = conf.get_item_by_name('materials', 'mf2')
    mat2 = Material.from_conf(conf_mat2, problem.functions)
    mat2.time_update(ts, None, mode='normal', problem=problem)

    assert_(nm.all(coors[:,1] == mat2.get_data(None, 'x_1')))

    materials = problem.get_materials()
    materials.time_update(ts, problem.equations, mode='normal',
                          problem=problem)
    mat3 = materials['mf3']
    key = mat3.get_keys(region_name='Omega')[0]

    assert_(nm.all(mat3.get_data(key, 'a') == 10.0))
    assert_(nm.all(mat3.get_data(key, 'b') == 2.0))
    assert_(mat3.get_data(None, 'c') == 'ahoj')

    pb = problem.copy()
    pb.set_variables(transform_variables(conf.variables2))
    pb.set_equations(conf.equations2)
    materials = pb.get_materials()
    materials.time_update(ts, pb.equations, mode='normal', problem=pb)

    mat4 = materials['mf4']
    key = mat4.get_keys(region_name='Omega')[0]
    assert_(nm.all(mat4.get_data(key, 'a') == -2 + 1j))

    mat5 = materials['mf5']
    key = mat5.get_keys(region_name='Omega')[0]
    assert_(nm.all(mat5.get_data(key, 'a') == -2 - 1j))

    mat6 = materials['mf6']
    key = mat6.get_keys(region_name='Circle')[0]
    assert_(nm.all(mat6.get_data(key, 'a') == 1 + 1j))
    key = mat6.get_keys(region_name='Rest')[0]
    assert_(nm.all(mat6.get_data(key, 'a') == 3j))

def test_ebc_functions(problem, output_dir):
    import os.path as op

    problem.set_equations(problem.conf.equations)

    problem.time_update()
    state = problem.solve()
    name = op.join(output_dir,
                   op.splitext(op.basename(__file__))[0] + '_ebc.vtk')
    problem.save_state(name, state)

    ok = True
    domain = problem.domain

    vecs = state.get_state_parts()
    vec = vecs['p']

    iv = domain.regions['Left'].vertices
    coors = domain.get_mesh_coors()[iv]
    _ok = tst.compare_vectors(vec[iv], nm.sin(nm.pi * coors[:,1]),
                              label1='p_state_left',
                              label2='p_bc_left')
    ok = _ok and ok

    iv = domain.regions['Right'].vertices
    coors = domain.get_mesh_coors()[iv]
    _ok = tst.compare_vectors(vec[iv], nm.cos(nm.pi * coors[:,1]),
                              label1='p_state_right',
                              label2='p_bc_right')
    ok = _ok and ok

    vec = vecs['u']
    vec.shape = (-1, 2)
    ok = tst.compare_vectors(vec[iv, 0], nm.zeros(len(iv)),
                             label1='u_0_state_right',
                             label2='u_0_bc_right')
    ok = _ok and ok

    ok = tst.compare_vectors(vec[iv, 1], nm.arange(len(iv)) + 1.0,
                             label1='u_1_state_right',
                             label2='u_1_bc_right')
    ok = _ok and ok

    assert ok

def test_region_functions(problem, output_dir):
    import os.path as op

    name = op.join(output_dir,
                   op.splitext(op.basename(__file__))[0])
    problem.save_regions(name, ['Circle'])
<commit_msg>do not save state twice in test_ebc_functions(), update test_region_functions()<commit_after>import numpy as nm
import pytest

from sfepy import data_dir
from sfepy.base.base import assert_
import sfepy.base.testing as tst

filename_mesh = data_dir + '/meshes/2d/square_unit_tri.mesh'

def get_pars(ts, coors, mode=None, extra_arg=None,
             equations=None, term=None, problem=None, **kwargs):
    if mode == 'special':
        if extra_arg == 'hello!':
            ic = 0
        else:
            ic = 1
        coors = problem.get_mesh_coors()
        return {('x_%s' % ic) : coors[:,ic]}

    elif mode == 'qp':
        return {'a' : nm.tile(-2 + 1j, (coors.shape[0], 1, 1))}

def get_p_edge(ts, coors, bc=None, **kwargs):
    if bc.name == 'p_left':
        return nm.sin(nm.pi * coors[:,1])
    else:
        return nm.cos(nm.pi * coors[:,1])

def get_u_edge(ts, coors, bc=None, **kwargs):
    out = nm.zeros_like(coors)
    out[:, 1] = nm.arange(out.shape[0]) + 1.0
    return out

def get_circle(coors, domain=None):
    r = nm.sqrt(coors[:,0]**2.0 + coors[:,1]**2.0)
    return nm.where(r < 0.2)[0]

functions = {
    'get_pars' : (get_pars,),
    'get_pars1' : (lambda ts, coors, mode=None, **kwargs:
                   get_pars(ts, coors, mode, extra_arg='hello!', **kwargs),),
    'get_p_edge' : (get_p_edge,),
    'get_u_edge' : (get_u_edge,),
    'get_circle' : (get_circle,),
}

# Just another way of adding a function, besides 'functions' keyword.
function_1 = {
    'name' : 'get_pars2',
    'function' : lambda ts, coors, mode=None, **kwargs:
        get_pars(ts, coors, mode, extra_arg='hi!', **kwargs),
}

materials = {
    'mf1' : (None, 'get_pars1'),
    'mf2' : 'get_pars2',
    # Dot denotes a special value, that is not propagated to all QP.
    'mf3' : ({'a' : 10.0, 'b' : 2.0, '.c' : 'ahoj'},),
    # Complex values.
    'mf4' : 'get_pars',
    'mf5' : ({'a' : -2 - 1j},),
    'mf6' : ({'a' : {'Circle' : 1 + 1j, 'Rest' : 3j}},),
}

fields = {
    'pressure' : (nm.float64, 1, 'Omega', 2),
    'displacement' : (nm.float64, 2, 'Omega', 2),
    'complex' : (nm.complex128, 1, 'Omega', 2),
}

variables = {
    'p'   : ('unknown field', 'pressure', 0),
    'q'   : ('test field',    'pressure', 'p'),
    'u'   : ('unknown field', 'displacement', 1),
    'v'   : ('test field',    'displacement', 'u'),
}

wx = 0.499
regions = {
    'Omega' : 'all',
    'Left' : ('vertices in (x < -%.3f)' % wx, 'facet'),
    'Right' : ('vertices in (x > %.3f)' % wx, 'facet'),
    'Circle' : 'vertices by get_circle',
    'Rest' : 'r.Omega -c r.Circle',
}

ebcs = {
    'p_left' : ('Left', {'p.all' : 'get_p_edge'}),
    'p_right' : ('Right', {'p.all' : 'get_p_edge'}),
    'u_right' : ('Right', {'u.all' : 'get_u_edge'}),
}

equations = {
    'e1' : """dw_laplace.2.Omega( mf3.a, q, p ) = 0""",
    'e2' : """dw_div_grad.2.Omega( mf3.b, v, u ) = 0""",
}

variables2 = {
    'r'   : ('unknown field', 'complex', 2),
    's'   : ('test field',    'complex', 'r'),
}

equations2 = {
    'e3' : """dw_laplace.2.Omega(mf4.a, s, r) = 0""",
    'e4' : """dw_laplace.2.Omega(mf5.a, s, r) = 0""",
    'e5' : """dw_laplace.2.Circle(mf6.a, s, r)
            + dw_laplace.2.Rest(mf6.a, s, r) = 0""",
}

solver_0 = {
    'name' : 'ls',
    'kind' : 'ls.scipy_direct',
}

solver_1 = {
    'name' : 'newton',
    'kind' : 'nls.newton',
}

@pytest.fixture(scope='module')
def problem():
    import sys
    from sfepy.discrete import Problem
    from sfepy.base.conf import ProblemConf

    conf = ProblemConf.from_dict(globals(), sys.modules[__name__])

    problem = Problem.from_conf(conf)
    return problem

def test_material_functions(problem):
    from sfepy.discrete import Material
    from sfepy.base.conf import transform_variables

    conf = problem.conf

    ts = problem.get_default_ts(step=0)

    conf_mat1 = conf.get_item_by_name('materials', 'mf1')
    mat1 = Material.from_conf(conf_mat1, problem.functions)
    mat1.time_update(ts, None, mode='normal', problem=problem)

    coors = problem.domain.get_mesh_coors()
    assert_(nm.all(coors[:,0] == mat1.get_data(None, 'x_0')))

    conf_mat2 = conf.get_item_by_name('materials', 'mf2')
    mat2 = Material.from_conf(conf_mat2, problem.functions)
    mat2.time_update(ts, None, mode='normal', problem=problem)

    assert_(nm.all(coors[:,1] == mat2.get_data(None, 'x_1')))

    materials = problem.get_materials()
    materials.time_update(ts, problem.equations, mode='normal',
                          problem=problem)
    mat3 = materials['mf3']
    key = mat3.get_keys(region_name='Omega')[0]

    assert_(nm.all(mat3.get_data(key, 'a') == 10.0))
    assert_(nm.all(mat3.get_data(key, 'b') == 2.0))
    assert_(mat3.get_data(None, 'c') == 'ahoj')

    pb = problem.copy()
    pb.set_variables(transform_variables(conf.variables2))
    pb.set_equations(conf.equations2)
    materials = pb.get_materials()
    materials.time_update(ts, pb.equations, mode='normal', problem=pb)

    mat4 = materials['mf4']
    key = mat4.get_keys(region_name='Omega')[0]
    assert_(nm.all(mat4.get_data(key, 'a') == -2 + 1j))

    mat5 = materials['mf5']
    key = mat5.get_keys(region_name='Omega')[0]
    assert_(nm.all(mat5.get_data(key, 'a') == -2 - 1j))

    mat6 = materials['mf6']
    key = mat6.get_keys(region_name='Circle')[0]
    assert_(nm.all(mat6.get_data(key, 'a') == 1 + 1j))
    key = mat6.get_keys(region_name='Rest')[0]
    assert_(nm.all(mat6.get_data(key, 'a') == 3j))

def test_ebc_functions(problem, output_dir):
    import os.path as op

    problem.set_equations(problem.conf.equations)

    problem.time_update()
    state = problem.solve(save_results=False)
    name = op.join(output_dir, 'test_ebc_functions.vtk')
    problem.save_state(name, state)

    ok = True
    domain = problem.domain

    vecs = state.get_state_parts()
    vec = vecs['p']

    iv = domain.regions['Left'].vertices
    coors = domain.get_mesh_coors()[iv]
    _ok = tst.compare_vectors(vec[iv], nm.sin(nm.pi * coors[:,1]),
                              label1='p_state_left',
                              label2='p_bc_left')
    ok = _ok and ok

    iv = domain.regions['Right'].vertices
    coors = domain.get_mesh_coors()[iv]
    _ok = tst.compare_vectors(vec[iv], nm.cos(nm.pi * coors[:,1]),
                              label1='p_state_right',
                              label2='p_bc_right')
    ok = _ok and ok

    vec = vecs['u']
    vec.shape = (-1, 2)
    ok = tst.compare_vectors(vec[iv, 0], nm.zeros(len(iv)),
                             label1='u_0_state_right',
                             label2='u_0_bc_right')
    ok = _ok and ok

    ok = tst.compare_vectors(vec[iv, 1], nm.arange(len(iv)) + 1.0,
                             label1='u_1_state_right',
                             label2='u_1_bc_right')
    ok = _ok and ok

    assert ok

def test_region_functions(problem, output_dir):
    import os.path as op

    name = op.join(output_dir, 'test_region_functions.vtk')
    problem.save_regions_as_groups(name, ['Circle'])
<|endoftext|>


<commit_before>

import sys

# Test local version of goatools
sys.path.insert(0, '..')

from goatools.obo_parser import GODag

def prt_paths(paths, PRT=sys.stdout):
  for path in paths:
    PRT.write('\n')
    for GO in path:
      PRT.write('{}\n'.format(GO))

def chk_results(actual_paths, expected_paths):
  for actual_path in actual_paths:
    # GOTerm -> list of Strings
    actual = [GO.id for GO in actual_path]
    if actual not in expected_paths:
      raise Exception('ACTUAL {} NOT FOUND IN EXPECTED RESULTS\n'.format(actual))

def test_paths_to_top():
  #dag = GODag("./tests/data/mini_obo.obo")
  dag = GODag("./data/mini_obo.obo")
  expected_paths = [
    ['GO:0000001', 'GO:0000002', 'GO:0000005', 'GO:0000010'],
    ['GO:0000001', 'GO:0000003', 'GO:0000005', 'GO:0000010'],
    ['GO:0000001', 'GO:0000003', 'GO:0000006', 'GO:0000008', 'GO:0000010'] ]
  actual_paths = dag.paths_to_top("GO:0000010")
  chk_results(actual_paths, expected_paths)
  prt_paths(actual_paths)

if __name__ == '__main__':
  test_paths_to_top()
<commit_msg>Allowed the file to be call from nose. PEP8 changes.<commit_after>import sys
import os
from goatools.obo_parser import GODag

ROOT = os.path.dirname(os.path.abspath(__file__)) + "/data/"

def prt_paths(paths, PRT=sys.stdout):
    for path in paths:
        PRT.write('\n')
        for GO in path:
            PRT.write('{}\n'.format(GO))

def chk_results(actual_paths, expected_paths):
    for actual_path in actual_paths:
        # GOTerm -> list of Strings
        actual = [GO.id for GO in actual_path]
        if actual not in expected_paths:
            raise Exception('ACTUAL {} NOT FOUND IN EXPECTED RESULTS\n'.format(actual))

def test_paths_to_top():
    dag = GODag(ROOT + "mini_obo.obo")
    expected_paths = [
    ['GO:0000001', 'GO:0000002', 'GO:0000005', 'GO:0000010'],
    ['GO:0000001', 'GO:0000003', 'GO:0000005', 'GO:0000010'],
    ['GO:0000001', 'GO:0000003', 'GO:0000006', 'GO:0000008', 'GO:0000010'] ]
    actual_paths = dag.paths_to_top("GO:0000010")
    chk_results(actual_paths, expected_paths)
    prt_paths(actual_paths)

<|endoftext|>


<commit_before>
""" Unit tests for Config system. """

import unittest

from io import StringIO
import json

import sas

from sas.config_system.config import Config

from sas.config_system.config_meta import MalformedFile

from sas.config_system.schema_elements import \
    pairwise_schema_union, create_schema_element, \
    SchemaBool, SchemaInt, SchemaFloat, SchemaStr, \
    SchemaList, SchemaNonSpecified, \
    CoercionError

class TestConfig(unittest.TestCase):

    def a_test_dict(self) -> dict:
        """ A dictionary with field names from the config, but with different values"""

        config = Config()

        # config variables (exclude the ones used by the base class)
        all_variables = vars(config).copy()
        for skipped in config._meta_attributes:
            del all_variables[skipped]

        string_value_suffix = 42

        target_values = {}

        for key in all_variables:

            if isinstance(all_variables[key], bool):
                target_values[key] = not all_variables[key]

            elif isinstance(all_variables[key], int):
                target_values[key] = all_variables[key] + 42

            elif isinstance(all_variables[key], float):
                target_values[key] = all_variables[key] + 3.149999

            elif isinstance(all_variables[key], str):
                target_values[key] = all_variables[key] + f" with string making it end in the number {string_value_suffix}"
                string_value_suffix += 1

            elif all_variables[key] == []:
                target_values[key] = ["something else"]

            elif isinstance(all_variables[key], list):
                target_values[key] = all_variables[key] * 2

            elif all_variables[key] is None:
                target_values[key] = "something"

            else:
                raise TypeError(f"Config entry '{key}' is not bool, int, float, str ({all_variables[key]})")

        return target_values

    def test_valid_update(self):
        """ Test setting variables with known valid options"""

        config = Config()
        test_dict = self.a_test_dict()

        config.update(test_dict)

        for key in test_dict:
            var = getattr(config, key)
            self.assertEqual(var, test_dict[key])
            self.assertEqual(type(var), type(test_dict[key]))

    def test_invalid_update_bad_name(self):
        """ Check that an error is logged when there is a bad name in the config"""

        config = Config()

        # Create a variable that isn't in the config
        test_dict = self.a_test_dict()

        name = "x"
        while name in test_dict:
            name += "x"

        # try and set it
        with self.assertLogs('sas.config_system', level="ERROR") as cm:
            config.update({name: None})
            self.assertTrue(cm.output[0].startswith("ERROR:sas.config_system:"))

    def test_invalid_update_bad_type(self):

        """Check that bad types give an error, this tries a bunch of incompatable types with each of the
        existing entries in config

        For this test to be useful it requires config to have at least one default entry with a schematisable type
        """

        config = Config()
        test_dict = self.a_test_dict()
        for key in test_dict:

            # find types that should be incompatable
            for test_value in [False, 0, 1.0, "string", [1,2,3], [[["deep"]]]]:
                test_value_schema = create_schema_element("value not important", test_value)

                try:
                    config._schema[key].coerce(test_value)

                except CoercionError:

                    # Only test the ones that fail, i.e. cannot be coerced
                    if pairwise_schema_union(test_value_schema, config._schema[key]) is None:
                        with self.assertLogs('sas.config_system', level="ERROR") as cm:

                            # Try the bad value
                            config.update({key: test_value})

                            self.assertTrue(cm.output[0].startswith("ERROR:sas.config_system:"))

    def test_save_basics(self):
        file = StringIO()

        # Check saving with no changes, should be empty and
        config = Config()
        config.save_to_file_object(file)

        file.seek(0)

        observed = json.load(file)

        empty_file = {
            "sasview_version": sas.__version__,
             "config_data": {}
        }

        self.assertEqual(observed, empty_file)

    def test_save_changes(self):
        """ Check that save saves a change that has been made"""

        test_dict = self.a_test_dict()
        for key in test_dict:
            file = StringIO()
            config = Config()
            config.update({key: test_dict[key]})
            config.save_to_file_object(file)
            file.seek(0)

            observed = json.load(file)

            expected = {
                "sasview_version": sas.__version__,
                "config_data": {key: test_dict[key]}
            }

            self.assertEqual(observed, expected)

    def test_only_save_actual_changes(self):
        """ Check that if a field is set back to its default, it isn't saved in the config"""

        # (1) make a single change
        # (2) change back to the default value
        # (3) Check

        empty_file = {
            "sasview_version": sas.__version__,
             "config_data": {}
        }

        backup = Config()

        test_dict = self.a_test_dict()
        for key in test_dict:
            file = StringIO()
            config = Config()

            config.update({key: test_dict[key]})
            config.update({key: getattr(backup, key)})
            config.save_to_file_object(file)

            file.seek(0)
            observed = json.load(file)

            self.assertEqual(observed, empty_file)

    def test_bad_config_version(self):

        config = Config()
        file = StringIO()

        json.dump({
            "sasview_version": "1.2.3",
             "config_data": {}},
            file)

        file.seek(0)

        with self.assertLogs('sas.config_system', level="WARN") as cm:
            # Try the bad value
            config.load_from_file_object(file)

            self.assertTrue(cm.output[0].startswith("WARNING:sas.config_system:"))

    def test_bad_config_file_structure(self):

        config = Config()

        bad_structures = [
            {
                "sasview_version": sas.__version__,
                "quanfig_data": {}
            },
            {
                "sasview_version": "bad version",
                "config_data": {}
            },
            {
                "sassy_verberry": sas.__version__,
                "config_data": {}
            },
            {
                "sasview_version": sas.__version__,
                "config_data": []
            },
            {}
        ]

        for structure in bad_structures:
            file = StringIO()

            json.dump(structure,file)

            file.seek(0)

            with self.assertRaises(MalformedFile):
                config.load(file)

    def test_schema_union(self):
        """ Check the typing behaviour of the schema system"""

        anything = SchemaNonSpecified()

        boolean = SchemaBool()
        integer = SchemaInt()
        floating = SchemaFloat()
        string = SchemaStr()

        empty_list = SchemaList(anything)
        bool_list = SchemaList(boolean)
        int_list = SchemaList(integer)
        float_list = SchemaList(floating)
        str_list = SchemaList(string)

        empty_list2 = SchemaList(empty_list)
        bool_list2 = SchemaList(bool_list)
        int_list2 = SchemaList(int_list)
        float_list2 = SchemaList(float_list)
        str_list2 = SchemaList(str_list)

        all_base = [anything, boolean, integer, floating, string]
        all_level_1 = [empty_list, bool_list, int_list, float_list, str_list]
        all_level_2 = [empty_list2, bool_list2, int_list2, float_list2, str_list2]

        #
        # Unspecified
        #
        for x in all_base + all_level_1 + all_level_2:
            self.assertEqual(pairwise_schema_union(x, anything), x)
            self.assertEqual(pairwise_schema_union(anything, x), x)

        #
        # Variable classes
        #

        # Incompatible cases

        self.assertIsNone(pairwise_schema_union(integer, string))
        self.assertIsNone(pairwise_schema_union(integer, boolean))
        self.assertIsNone(pairwise_schema_union(string, integer))
        self.assertIsNone(pairwise_schema_union(boolean, integer))

        self.assertIsNone(pairwise_schema_union(floating, string))
        self.assertIsNone(pairwise_schema_union(floating, boolean))
        self.assertIsNone(pairwise_schema_union(string, floating))
        self.assertIsNone(pairwise_schema_union(boolean, floating))

        # Compatible cases
        self.assertEqual(pairwise_schema_union(floating, integer), floating)
        self.assertEqual(pairwise_schema_union(integer, floating), floating)

        #
        # Lists - a few important cases
        #

        # lists of non-compatible types should not be combined
        self.assertIsNone(pairwise_schema_union(str_list, int_list))
        self.assertIsNone(pairwise_schema_union(str_list, bool_list))
        self.assertIsNone(pairwise_schema_union(int_list, str_list))
        self.assertIsNone(pairwise_schema_union(bool_list, str_list))

        # Lists of the same kind but with different nesting should not combine
        self.assertIsNone(pairwise_schema_union(str_list, str_list2))
        self.assertIsNone(pairwise_schema_union(str_list2, str_list))

        # empty list should not care about level
        self.assertEqual(pairwise_schema_union(empty_list, str_list), str_list)
        self.assertEqual(pairwise_schema_union(empty_list, str_list2), str_list2)
        self.assertEqual(pairwise_schema_union(str_list, empty_list), str_list)
        self.assertEqual(pairwise_schema_union(str_list2, empty_list), str_list2)

        # lists of floats and ints should combine to float
        self.assertEqual(pairwise_schema_union(float_list, int_list), float_list)
        self.assertEqual(pairwise_schema_union(float_list2, int_list2), float_list2)
        self.assertEqual(pairwise_schema_union(int_list, float_list), float_list)
        self.assertEqual(pairwise_schema_union(int_list2, float_list2), float_list2)

        #... but only at the same level
        self.assertIsNone(pairwise_schema_union(float_list2, int_list))
        self.assertIsNone(pairwise_schema_union(float_list, int_list2))
        self.assertIsNone(pairwise_schema_union(int_list2, float_list))
        self.assertIsNone(pairwise_schema_union(int_list, float_list2))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fixed test<commit_after>
""" Unit tests for Config system. """

import unittest

from io import StringIO
import json

import sas

from sas.config_system.config import Config

from sas.config_system.config_meta import MalformedFile

from sas.config_system.schema_elements import \
    pairwise_schema_union, create_schema_element, \
    SchemaBool, SchemaInt, SchemaFloat, SchemaStr, \
    SchemaList, SchemaNonSpecified, \
    CoercionError

class TestConfig(unittest.TestCase):

    def a_test_dict(self) -> dict:
        """ A dictionary with field names from the config, but with different values"""

        config = Config()

        # config variables (exclude the ones used by the base class)
        all_variables = vars(config).copy()
        for skipped in config._meta_attributes:
            del all_variables[skipped]

        string_value_suffix = 42

        target_values = {}

        for key in all_variables:

            if isinstance(all_variables[key], bool):
                target_values[key] = not all_variables[key]

            elif isinstance(all_variables[key], int):
                target_values[key] = all_variables[key] + 42

            elif isinstance(all_variables[key], float):
                target_values[key] = all_variables[key] + 3.149999

            elif isinstance(all_variables[key], str):
                target_values[key] = all_variables[key] + f" with string making it end in the number {string_value_suffix}"
                string_value_suffix += 1

            elif all_variables[key] == []:
                target_values[key] = ["something else"]

            elif isinstance(all_variables[key], list):
                target_values[key] = all_variables[key] * 2

            elif all_variables[key] is None:
                target_values[key] = "something"

            else:
                raise TypeError(f"Config entry '{key}' is not bool, int, float, str ({all_variables[key]})")

        return target_values

    def test_valid_update(self):
        """ Test setting variables with known valid options"""

        config = Config()
        test_dict = self.a_test_dict()

        config.update(test_dict)

        for key in test_dict:
            var = getattr(config, key)
            self.assertEqual(var, test_dict[key])
            self.assertEqual(type(var), type(test_dict[key]))

    def test_invalid_update_bad_name(self):
        """ Check that an error is logged when there is a bad name in the config"""

        config = Config()

        # Create a variable that isn't in the config
        test_dict = self.a_test_dict()

        name = "x"
        while name in test_dict:
            name += "x"

        # try and set it
        with self.assertLogs('sas.config_system', level="ERROR") as cm:
            config.update({name: None})
            self.assertTrue(cm.output[0].startswith("ERROR:sas.config_system:"))

    def test_invalid_update_bad_type(self):

        """Check that bad types give an error, this tries a bunch of incompatable types with each of the
        existing entries in config

        For this test to be useful it requires config to have at least one default entry with a schematisable type
        """

        config = Config()
        test_dict = self.a_test_dict()
        for key in test_dict:

            # find types that should be incompatable
            for test_value in [False, 0, 1.0, "string", [1,2,3], [[["deep"]]]]:
                test_value_schema = create_schema_element("value not important", test_value)

                try:
                    config._schema[key].coerce(test_value)

                except CoercionError:

                    # Only test the ones that fail, i.e. cannot be coerced
                    if pairwise_schema_union(test_value_schema, config._schema[key]) is None:
                        with self.assertLogs('sas.config_system', level="ERROR") as cm:

                            # Try the bad value
                            config.update({key: test_value})

                            self.assertTrue(cm.output[0].startswith("ERROR:sas.config_system:"))

    def test_save_basics(self):
        file = StringIO()

        # Check saving with no changes, should be empty and
        config = Config()
        config.save_to_file_object(file)

        file.seek(0)

        observed = json.load(file)

        empty_file = {
            "sasview_version": sas.__version__,
             "config_data": {}
        }

        self.assertEqual(observed, empty_file)

    def test_save_changes(self):
        """ Check that save saves a change that has been made"""

        test_dict = self.a_test_dict()
        for key in test_dict:
            file = StringIO()
            config = Config()
            config.update({key: test_dict[key]})
            config.save_to_file_object(file)
            file.seek(0)

            observed = json.load(file)

            expected = {
                "sasview_version": sas.__version__,
                "config_data": {key: test_dict[key]}
            }

            self.assertEqual(observed, expected)

    def test_only_save_actual_changes(self):
        """ Check that if a field is set back to its default, it isn't saved in the config"""

        # (1) make a single change
        # (2) change back to the default value
        # (3) Check

        empty_file = {
            "sasview_version": sas.__version__,
             "config_data": {}
        }

        backup = Config()

        test_dict = self.a_test_dict()
        for key in test_dict:
            file = StringIO()
            config = Config()

            config.update({key: test_dict[key]})
            config.update({key: getattr(backup, key)})
            config.save_to_file_object(file)

            file.seek(0)
            observed = json.load(file)

            self.assertEqual(observed, empty_file)

    def test_bad_config_version(self):

        config = Config()
        file = StringIO()

        json.dump({
            "sasview_version": "1.2.3",
             "config_data": {}},
            file)

        file.seek(0)

        with self.assertLogs('sas.config_system', level="WARN") as cm:
            # Try the bad value
            config.load_from_file_object(file)

            self.assertTrue(cm.output[0].startswith("WARNING:sas.config_system:"))

    def test_bad_config_file_structure(self):

        config = Config()

        bad_structures = [
            {
                "sasview_version": sas.__version__,
                "quanfig_data": {}
            },
            {
                "sasview_version": "bad version",
                "config_data": {}
            },
            {
                "sassy_verberry": sas.__version__,
                "config_data": {}
            },
            {
                "sasview_version": sas.__version__,
                "config_data": []
            },
            {}
        ]

        for structure in bad_structures:
            file = StringIO()

            json.dump(structure,file)

            file.seek(0)

            with self.assertRaises(MalformedFile):
                config.load_from_file_object(file)

    def test_schema_union(self):
        """ Check the typing behaviour of the schema system"""

        anything = SchemaNonSpecified()

        boolean = SchemaBool()
        integer = SchemaInt()
        floating = SchemaFloat()
        string = SchemaStr()

        empty_list = SchemaList(anything)
        bool_list = SchemaList(boolean)
        int_list = SchemaList(integer)
        float_list = SchemaList(floating)
        str_list = SchemaList(string)

        empty_list2 = SchemaList(empty_list)
        bool_list2 = SchemaList(bool_list)
        int_list2 = SchemaList(int_list)
        float_list2 = SchemaList(float_list)
        str_list2 = SchemaList(str_list)

        all_base = [anything, boolean, integer, floating, string]
        all_level_1 = [empty_list, bool_list, int_list, float_list, str_list]
        all_level_2 = [empty_list2, bool_list2, int_list2, float_list2, str_list2]

        #
        # Unspecified
        #
        for x in all_base + all_level_1 + all_level_2:
            self.assertEqual(pairwise_schema_union(x, anything), x)
            self.assertEqual(pairwise_schema_union(anything, x), x)

        #
        # Variable classes
        #

        # Incompatible cases

        self.assertIsNone(pairwise_schema_union(integer, string))
        self.assertIsNone(pairwise_schema_union(integer, boolean))
        self.assertIsNone(pairwise_schema_union(string, integer))
        self.assertIsNone(pairwise_schema_union(boolean, integer))

        self.assertIsNone(pairwise_schema_union(floating, string))
        self.assertIsNone(pairwise_schema_union(floating, boolean))
        self.assertIsNone(pairwise_schema_union(string, floating))
        self.assertIsNone(pairwise_schema_union(boolean, floating))

        # Compatible cases
        self.assertEqual(pairwise_schema_union(floating, integer), floating)
        self.assertEqual(pairwise_schema_union(integer, floating), floating)

        #
        # Lists - a few important cases
        #

        # lists of non-compatible types should not be combined
        self.assertIsNone(pairwise_schema_union(str_list, int_list))
        self.assertIsNone(pairwise_schema_union(str_list, bool_list))
        self.assertIsNone(pairwise_schema_union(int_list, str_list))
        self.assertIsNone(pairwise_schema_union(bool_list, str_list))

        # Lists of the same kind but with different nesting should not combine
        self.assertIsNone(pairwise_schema_union(str_list, str_list2))
        self.assertIsNone(pairwise_schema_union(str_list2, str_list))

        # empty list should not care about level
        self.assertEqual(pairwise_schema_union(empty_list, str_list), str_list)
        self.assertEqual(pairwise_schema_union(empty_list, str_list2), str_list2)
        self.assertEqual(pairwise_schema_union(str_list, empty_list), str_list)
        self.assertEqual(pairwise_schema_union(str_list2, empty_list), str_list2)

        # lists of floats and ints should combine to float
        self.assertEqual(pairwise_schema_union(float_list, int_list), float_list)
        self.assertEqual(pairwise_schema_union(float_list2, int_list2), float_list2)
        self.assertEqual(pairwise_schema_union(int_list, float_list), float_list)
        self.assertEqual(pairwise_schema_union(int_list2, float_list2), float_list2)

        #... but only at the same level
        self.assertIsNone(pairwise_schema_union(float_list2, int_list))
        self.assertIsNone(pairwise_schema_union(float_list, int_list2))
        self.assertIsNone(pairwise_schema_union(int_list2, float_list))
        self.assertIsNone(pairwise_schema_union(int_list, float_list2))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>