<commit_before>from collections import OrderedDict
from flask import Blueprint, g, _request_ctx_stack, current_app
from werkzeug import LocalProxy
from operator import attrgetter
from types import FunctionType

flarf = LocalProxy(lambda: current_app.extensions['flarf'])

_fs = LocalProxy(lambda: current_app.extensions['flarf'].filters)

class FlarfError(Exception):
    pass

class FlarfFiltered(object):
    def __init__(self, filter_params, request):
        for arg in filter_params:
            if isinstance(arg, FunctionType):
                setattr(self, arg.__name__, arg(request))
            else:
                setattr(self, arg, getattr(request, arg, None))

class FlarfFilter(object):
    def __init__(self, filter_tag=None,
                       filter_precedence=100,
                       filtered_cls=FlarfFiltered,
                       filter_params=None,
                       filter_on=['all'],
                       filter_pass=['static'],
                       filter_skip=None):
        self.filter_tag = filter_tag
        self.filter_proxy_tag = "{}_context".format(filter_tag)
        self.filter_precedence = filter_precedence
        self.filtered_cls = filtered_cls
        self.filter_params = filter_params
        self.filter_on = filter_on
        self.filter_pass = filter_pass
        if filter_skip:
            self.filter_pass.extend(filter_skip)

    def filter_request(self, request):
        setattr(g, self.filter_tag, self.filtered_cls(self.filter_params, request))

def flarf_run_filters():
    r = _request_ctx_stack.top.request
    if r.url_rule:
        request_endpoint = str(r.url_rule.endpoint).rsplit('.')[-1]
        for f in _fs.itervalues():
            if request_endpoint not in f.filter_pass:
                if request_endpoint or 'all' in f.filter_on:
                    f.filter_request(r)

class Flarf(object):
    def __init__(self, app=None,
                       before_request_func=flarf_run_filters,
                       filter_cls=FlarfFilter,
                       filtered_cls=None,
                       filters=None):
        self.app = app
        self.before_request_func = before_request_func
        self.filter_cls = filter_cls
        self.filtered_cls = filtered_cls
        self.filters = self.set_filters(filters)

        if app is not None:
            self.app = app
            self.init_app(self.app)
        else:
            self.app = None

    def set_filters(self, filters):
        d = OrderedDict()
        fs = self.check_filters(filters)
        ofs = self.order_filters(fs)
        for f in ofs:
            d[f.filter_tag] = f
        return d

    def check_filters(self, filters):
        fs = []
        for f in filters:
            if isinstance(f, self.filter_cls):
                fs.append(f)
            elif isinstance(f, dict):
                if self.filtered_cls:
                    f.update({filtered_cls: self.filtered_cls})
                fc = self.filter_cls(**f)
                fs.append(fc)
            else:
                raise FlarfError("""
                                {}\n
                                filter must be a list of:\n
                                - instance of filter_cls given to Flarf extension\n
                                - instance of default filter_cls: FlarfFilter\n
                                - a dict of params for filter_cls\n
                                """.format(f))
        return fs

    def order_filters(self, filters):
        return sorted(filters, key=attrgetter('filter_precedence'))

    def init_app(self, app):
        app.before_request(self.before_request_func)
        app.extensions['flarf'] = self
<commit_msg>doc strings<commit_after>from collections import OrderedDict
from flask import Blueprint, g, _request_ctx_stack, current_app
from werkzeug import LocalProxy
from operator import attrgetter
from types import FunctionType

flarf = LocalProxy(lambda: current_app.extensions['flarf'])

_fs = LocalProxy(lambda: current_app.extensions['flarf'].filters)

class FlarfError(Exception):
    pass

class FlarfFiltered(object):
    """
    The class used to place information filtered from request.
    """
    def __init__(self, filter_params, request):
        for arg in filter_params:
            if isinstance(arg, FunctionType):
                setattr(self, arg.__name__, arg(request))
            else:
                setattr(self, arg, getattr(request, arg, None))

class FlarfFilter(object):
    """
    A single instance of a filter

    :param filter_tag:          The name of the filter
    :param filter_precedence:   If you need to order your filters in some way
                                defaults to 100. Filters will be ordered from
                                smallest number to largest
    :param filtered_cls:        The class used for recording the info from the
                                filter. Default is FlarfFiltered
    :param filter_params:       What to use to filter, strings must be attributes
                                of request, functions must take request as a
                                single argument.
    :param filter_on:           Which routes to use the filter on.
    :param filter_skip:         Which routes to NOT use the filter.
    """
    def __init__(self, filter_tag=None,
                       filter_precedence=100,
                       filtered_cls=FlarfFiltered,
                       filter_params=None,
                       filter_on=['all'],
                       filter_skip=None):
        self.filter_tag = filter_tag
        self.filter_proxy_tag = "{}_context".format(filter_tag)
        self.filter_precedence = filter_precedence
        self.filtered_cls = filtered_cls
        self.filter_params = filter_params
        self.filter_on = filter_on
        self.filter_pass = ['static']
        if filter_skip:
            self.filter_pass.extend(filter_skip)

    def filter_request(self, request):
        setattr(g, self.filter_tag, self.filtered_cls(self.filter_params, request))

def flarf_run_filters():
    """
    A before_request function registered on the application that runs each filter
    """
    r = _request_ctx_stack.top.request
    if r.url_rule:
        request_endpoint = str(r.endpoint).rsplit('.')[-1]
        for f in _fs.itervalues():
            if request_endpoint not in f.filter_pass:
                if request_endpoint or 'all' in f.filter_on:
                    f.filter_request(r)

class Flarf(object):
    """
    The Flarf extension object.

    :param app:                 The application to register the function on.
    :param before_request_func: The before request function to run the filters.
                                Defaults to flarf_run_filters
    :param filter_cls:          The default filter class to use in registering
                                filters. Defaults to FlarfFilter.
    :param filtered_cls:        The default class for filtered info, if none
                                FlarfFiltered will be used.
    :param filters:             The filters to be run per request
    """

    def __init__(self, app=None,
                       before_request_func=flarf_run_filters,
                       filter_cls=FlarfFilter,
                       filtered_cls=None,
                       filters=None):
        self.app = app
        self.before_request_func = before_request_func
        self.filter_cls = filter_cls
        self.filtered_cls = filtered_cls
        self.filters = self.set_filters(filters)

        if app is not None:
            self.app = app
            self.init_app(self.app)
        else:
            self.app = None

    def set_filters(self, filters):
        d = OrderedDict()
        fs = self.check_filters(filters)
        ofs = self.order_filters(fs)
        for f in ofs:
            d[f.filter_tag] = f
        return d

    def check_filters(self, filters):
        fs = []
        for f in filters:
            if isinstance(f, self.filter_cls):
                fs.append(f)
            elif isinstance(f, dict):
                if self.filtered_cls:
                    f.update({filtered_cls: self.filtered_cls})
                fc = self.filter_cls(**f)
                fs.append(fc)
            else:
                raise FlarfError("""
                                {}\n
                                filter must be a list of:\n
                                - instance of filter_cls given to Flarf extension\n
                                - instance of default filter_cls: FlarfFilter\n
                                - a dict of params for filter_cls\n
                                """.format(f))
        return fs

    def order_filters(self, filters):
        return sorted(filters, key=attrgetter('filter_precedence'))

    def init_app(self, app):
        app.before_request(self.before_request_func)
        app.extensions['flarf'] = self
<|endoftext|>


<commit_before>from itertools import product
import numpy as np

from ..distributions.discrete_family import discrete_family
from ..sampling.langevin import projected_langevin
from .bootstrap_target import bootstrapped_target_sampler

class multiple_views(object):

    def __init__(self, objectives):

        self.objectives = objectives

    def solve(self):
        for objective in self.objectives:
            # maybe just check if they have been solved
            # randomize first?
            objective.solve()

    def setup_sampler(self, form_covariances):

        nviews = self.nviews = len(self.objectives)

        self.num_opt_var = 0
        self.opt_slice = []
        self.score_info = []

        for objective in self.objectives:
            score_ = objective.setup_sampler() # shouldn't have to refit all the time as this function does
            self.score_info.append(score_)
            self.opt_slice.append(slice(self.num_opt_var, self.num_opt_var + objective.num_opt_var))
            self.num_opt_var += objective.num_opt_var

        self.observed_opt_state = np.zeros(self.num_opt_var)

        for i in range(nviews):
            self.observed_opt_state[self.opt_slice[i]] = self.objectives[i].observed_opt_state

        self.form_covariances = form_covariances

    def setup_target(self,
                    target_bootstrap,
                    observed_target_state,
                    target_set=None,
                    reference=None,
                    constructor=None):

        if constructor is None:
            constructor = targeted_sampler

            return constructor(self,
                   target_bootstrap,
                   observed_target_state,
                   target_set=target_set,
                   reference=reference)

    def setup_bootstrapped_target(self,
                     target_info,
                     observed_target_state,
                     boot_size,
                     target_alpha,
                     target_set=None,
                     reference=None,
                     constructor=None):

        if constructor is None:
            constructor = bootstrapped_target_sampler

        return constructor(self,
                           target_info,
                           observed_target_state,
                           self.form_covariances,
                           boot_size,
                           target_alpha,
                           target_set=target_set,
                           reference=reference)

class targeted_sampler(object):

    # make one of these for each hypothesis test

    def __init__(self,
                 multi_view,
                 target_info,
                 observed_target_state,
                 form_covariances,
                 boot_size,
                 target_alpha,
                 target_set=None,
                 reference=None):

        # sampler will draw samples for bootstrap
        # these are arguments to target_bootstrap and score_bootstrap
        # nonparamteric bootstrap is np.random.choice(n, size=(n,), replace=True)
        # residual bootstrap might be X_E.dot(\bar{\beta}_E) + np.random.choice(resid, size=(n,), replace=True)

        # if target_set is not None, we assume that these coordinates (specified by a list of coordinates) of target
        # is assumed to be independent of the rest
        # the corresponding block of `target_cov` is zeroed out

        # we need these attributes of multi_view

        self.nviews = len(multi_view.objectives)
        self.opt_slice = multi_view.opt_slice
        self.objectives = multi_view.objectives

        # for bootstrap
        self.boot_size = boot_size
        self.target_alpha = target_alpha
        #self.inv_mat = np.linalg.inv(np.dot(self.target_alpha, self.target_alpha.T))

        self.observed_target_state = observed_target_state

        covariances = form_covariances(target_info, cross_terms=multi_view.score_info)

        self.target_cov = np.atleast_2d(covariances[0])
        # zero out some coordinates of target_cov
        # to enforce independence of target and null statistics

        if target_set is not None:
            null_set = set(range(self.target_cov.shape[0])).difference(target_set)

            for t, n in product(target_set, null_set):
                self.target_cov[t, n] = 0.
                self.target_cov[n, t] = 0.

        self.score_cov = covariances[1:]

        self.target_inv_cov = np.linalg.inv(self.target_cov)

        # size of reference? should it only be target_set?
        if reference is None:
            reference = np.zeros(self.target_inv_cov.shape[0])

        self.target_transform = []
        self.boot_transform = [] # bootstrap

        for i in range(self.nviews):
            self.target_transform.append(self.objectives[i].linear_decomposition(self.score_cov[i],
                                                                                self.target_cov,
                                                                                self.observed_target_state))

            self.boot_transform.append(self.objectives[i].boot_decomposition(target_alpha,
                                                                             reference,
                                                                             self.score_cov[i],
                                                                             self.target_cov,
                                                                             self.observed_target_state))

        self.reference_inv = self.target_inv_cov.dot(reference)

        # need to vectorize the state for Langevin

        self.overall_opt_slice = slice(0, multi_view.num_opt_var)
        self.target_slice = slice(multi_view.num_opt_var, multi_view.num_opt_var + self.reference_inv.shape[0])
        self.keep_slice = self.target_slice

        # set the observed state

        self.observed_state = np.zeros(multi_view.num_opt_var + self.reference_inv.shape[0])
        self.observed_state[self.target_slice] = self.observed_target_state
        self.observed_state[self.overall_opt_slice] = multi_view.observed_opt_state

        # set the observed state for bootstrap

        self.boot_slice = slice(multi_view.num_opt_var, multi_view.num_opt_var + self.boot_size)
        self.boot_observed_state = np.zeros(multi_view.num_opt_var + self.boot_size)
        self.boot_observed_state[self.boot_slice] = np.ones(self.boot_size)
        #self.boot_observed_state[self.boot_slice] = np.random.normal(size=self.boot_size)
        self.boot_observed_state[self.overall_opt_slice] = multi_view.observed_opt_state

    def projection(self, state):
        target_state, opt_state = state[self.target_slice], state[self.overall_opt_slice]
        new_opt_state = np.zeros_like(opt_state)
        for i in range(self.nviews):
            new_opt_state[self.opt_slice[i]] = self.objectives[i].projection(opt_state[self.opt_slice[i]])
        state[self.overall_opt_slice] = new_opt_state
        return state

    def gradient(self, state):

        target_state, opt_state = state[self.target_slice], state[self.overall_opt_slice]
        target_grad, opt_grad = np.zeros_like(target_state), np.zeros_like(opt_state)
        full_grad = np.zeros_like(state)

        # randomization_gradient are gradients of a CONVEX function

        for i in range(self.nviews):
            target_grad_curr, opt_grad[self.opt_slice[i]] = \
                self.objectives[i].randomization_gradient(target_state, self.target_transform[i], opt_state[self.opt_slice[i]])
            target_grad += target_grad_curr.copy()

        target_grad = - target_grad
        target_grad += self.reference_inv - self.target_inv_cov.dot(target_state)
        full_grad[self.target_slice] = target_grad
        full_grad[self.overall_opt_slice] = -opt_grad

        return full_grad

    def boot_projection(self, state):
        boot_state, opt_state = state[self.boot_slice], state[self.overall_opt_slice]
        new_opt_state = np.zeros_like(opt_state)
        for i in range(self.nviews):
            new_opt_state[self.opt_slice[i]] = self.objectives[i].projection(opt_state[self.opt_slice[i]])
        state[self.overall_opt_slice] = new_opt_state
        return state

    def boot_gradient(self, state):

        boot_state, opt_state = state[self.boot_slice], state[self.overall_opt_slice]
        boot_grad, opt_grad = np.zeros_like(boot_state), np.zeros_like(opt_state)
        full_grad = np.zeros_like(state)

        # randomization_gradient are gradients of a CONVEX function

        for i in range(self.nviews):
            boot_grad_curr, opt_grad[self.opt_slice[i]] = \
                self.objectives[i].randomization_gradient(boot_state, self.boot_transform[i],
                                                          opt_state[self.opt_slice[i]])
            boot_grad += boot_grad_curr.copy()

        boot_grad = -boot_grad
        boot_grad -= boot_state
        #boot_grad -= np.dot(np.dot(self.target_alpha.T, self.inv_mat), self.target_alpha.dot(boot_state))
        #boot_grad -= np.dot(np.dot(self.target_alpha.T, self.target_inv_cov), self.target_alpha.dot(boot_state))

        full_grad[self.boot_slice] = boot_grad
        full_grad[self.overall_opt_slice] = -opt_grad

        return full_grad

    def sample(self, ndraw, burnin, bootstrap=False, stepsize = None):
        """
        assumes setup_sampler has been called
        """
        if stepsize is None:
            #stepsize = 1. /self.boot_size
            stepsize = 1. / self.observed_state.shape[0]
        target_langevin = projected_langevin(self.observed_state.copy(),
                                             self.gradient,
                                             self.projection,
                                             stepsize)

        bootstrap_langevin = projected_langevin(self.boot_observed_state.copy(),
                                                self.boot_gradient,
                                                self.boot_projection,
                                                stepsize)

        samples = []
        if bootstrap:
            for i in range(ndraw + burnin):
                bootstrap_langevin.next()
                if (i >= burnin):
                    samples.append(bootstrap_langevin.state[self.boot_slice].copy())
            return samples
        else:
            for i in range(ndraw + burnin):
                target_langevin.next()
                if (i >= burnin):
                    samples.append(target_langevin.state[self.keep_slice].copy())
            return samples

    def hypothesis_test(self,
                        test_stat,
                        observed_target,
                        ndraw=10000,
                        burnin=2000,
                        stepsize=None,
                        alternative='twosided'):

        if alternative not in ['greater', 'less', 'twosided']:
            raise ValueError("alternative should be one of ['greater', 'less', 'twosided']")

        samples = self.sample(ndraw, burnin, stepsize=stepsize)
        observed_stat = test_stat(observed_target)
        sample_test_stat = np.array([test_stat(x) for x in samples])

        family = discrete_family(sample_test_stat, np.ones_like(sample_test_stat))
        pval = family.cdf(0, observed_stat)

        if alternative == 'greater':
            return 1 - pval
        elif alternative == 'less':
            return pval
        else:
            return 2 * min(pval, 1 - pval)

    def boot_hypothesis_test(self,
                        test_stat,
                        observed_value,
                        ndraw = 10000,
                        burnin = 2000,
                        stepsize=None,
                        alternative='twosided'):

        if alternative not in ['greater', 'less', 'twosided']:
            raise ValueError("alternative should be one of ['greater', 'less', 'twosided']")

        samples = self.sample(ndraw, burnin, bootstrap=True, stepsize=stepsize)
        sample_test_stat = np.array([test_stat(x) for x in samples])

        family = discrete_family(sample_test_stat, np.ones_like(sample_test_stat))
        pval = family.cdf(0, observed_value)

        #print pval
        if alternative == 'greater':
            return 1 - pval
        elif alternative == 'less':
            return pval
        else:
            return 2 * min(pval, 1 - pval)

    def crude_lipschitz(self):
        result = np.linalg.svd(self.target_inv_cov)[1].max()
        for transform, objective in zip(self.target_transform, self.objectives):
            result += np.linalg.svd(transform[0])[1].max()**2 * objective.randomization.lipschitz
            result += np.linalg.svd(objective.score_transform[0])[1].max()**2 * objective.randomization.lipschitz
        return result

class conditional_targeted_sampler(targeted_sampler):
    # condition on the optimization variables -- don't move them...

    def __init__(self,
                 multi_view,
                 target_bootstrap,
                 observed_target_state,
                 target_set=None,
                 reference=None):
        targeted_sampler.__init__(self,
                                  multi_view,
                                  target_bootstrap,
                                  observed_target_state,
                                  target_set=target_set,
                                  reference=reference)

        # this is a hacky way to do things

        self._full_state = self.observed_state.copy()
        self._opt_state = self.observed_state[self.overall_opt_slice]
        self.observed_state = self.observed_state[self.target_slice]
        self.keep_slice = slice(None, None, None)

    def gradient(self, state):
        self._full_state[self.target_slice] = state
        full_grad = targeted_sampler.gradient(self, self._full_state)
        return full_grad[self.target_slice]

    def projection(self, state):
        return state

    def crude_lipschitz(self):
        result = np.linalg.svd(self.target_inv_cov)[1].max()
        for transform, objective in zip(self.target_transform, self.objectives):
            result += np.linalg.svd(transform[0])[1].max()**2 * objective.randomization.lipschitz
        return result
<commit_msg>put back targeted_sampler<commit_after>from itertools import product
import numpy as np

from ..distributions.discrete_family import discrete_family
from ..sampling.langevin import projected_langevin
from .bootstrap_target import bootstrapped_target_sampler

class multiple_views(object):

    def __init__(self, objectives):

        self.objectives = objectives

    def solve(self):
        for objective in self.objectives:
            # maybe just check if they have been solved
            # randomize first?
            objective.solve()

    def setup_sampler(self, form_covariances):

        nviews = self.nviews = len(self.objectives)

        self.num_opt_var = 0
        self.opt_slice = []
        self.score_info = []

        for objective in self.objectives:
            score_ = objective.setup_sampler() # shouldn't have to refit all the time as this function does
            self.score_info.append(score_)
            self.opt_slice.append(slice(self.num_opt_var, self.num_opt_var + objective.num_opt_var))
            self.num_opt_var += objective.num_opt_var

        self.observed_opt_state = np.zeros(self.num_opt_var)

        for i in range(nviews):
            self.observed_opt_state[self.opt_slice[i]] = self.objectives[i].observed_opt_state

        self.form_covariances = form_covariances

    def setup_target(self,
                    target_bootstrap,
                    observed_target_state,
                    target_set=None,
                    reference=None,
                    constructor=None):

        if constructor is None:
            constructor = targeted_sampler

            return constructor(self,
                             target_bootstrap,
                             observed_target_state,
                             target_set=target_set,
                             reference=reference)

    def setup_bootstrapped_target(self,
                     target_info,
                     observed_target_state,
                     boot_size,
                     target_alpha,
                     target_set=None,
                     reference=None,
                     constructor=None):

        if constructor is None:
            constructor = bootstrapped_target_sampler

        return constructor(self,
                           target_info,
                           observed_target_state,
                           self.form_covariances,
                           boot_size,
                           target_alpha,
                           target_set=target_set,
                           reference=reference)

class targeted_sampler(object):

    # make one of these for each hypothesis test

    def __init__(self,
                 multi_view,
                 target_bootstrap,
                 observed_target_state,
                 target_set=None,
                 reference=None):

        # sampler will draw samples for bootstrap
        # these are arguments to target_bootstrap and score_bootstrap
        # nonparamteric bootstrap is np.random.choice(n, size=(n,), replace=True)
        # residual bootstrap might be X_E.dot(\bar{\beta}_E) + np.random.choice(resid, size=(n,), replace=True)

        # if target_set is not None, we assume that these coordinates (specified by a list of coordinates) of target
        # is assumed to be independent of the rest
        # the corresponding block of `target_cov` is zeroed out

        # we need these attributes of multi_view

        self.nviews = len(multi_view.objectives)
        self.opt_slice = multi_view.opt_slice
        self.objectives = multi_view.objectives

        self.observed_target_state = observed_target_state

        covariances = bootstrap_cov(multi_view.sampler, target_bootstrap, cross_terms=multi_view.score_bootstrap)
        self.target_cov = np.atleast_2d(covariances[0])

        # zero out some coordinates of target_cov
        # to enforce independence of target and null statistics

        if target_set is not None:
            null_set = set(range(self.target_cov.shape[0])).difference(target_set)

            for t, n in product(target_set, null_set):
                self.target_cov[t, n] = 0.
                self.target_cov[n, t] = 0.

        self.score_cov = covariances[1:]

        self.target_transform = []
        for i in range(self.nviews):
            self.target_transform.append(self.objectives[i].condition(self.score_cov[i],
                                                                      self.target_cov,
                                                                      self.observed_target_state))
        self.target_inv_cov = np.linalg.inv(self.target_cov)
        # size of reference? should it only be target_set?
        if reference is None:
            reference = np.zeros(self.target_inv_cov.shape[0])
        self.reference_inv = self.target_inv_cov.dot(reference)

        # need to vectorize the state for Langevin

        self.overall_opt_slice = slice(0, multi_view.num_opt_var)
        self.target_slice = slice(multi_view.num_opt_var, multi_view.num_opt_var + self.reference_inv.shape[0])

        # set the observed state

        self.observed_state = np.zeros(multi_view.num_opt_var + self.reference_inv.shape[0])
        self.observed_state[self.target_slice] = self.observed_target_state
        self.observed_state[self.overall_opt_slice] = multi_view.observed_opt_state

    def projection(self, state):
        target_state, opt_state = state[self.target_slice], state[self.overall_opt_slice]
        new_opt_state = np.zeros_like(opt_state)
        for i in range(self.nviews):
            new_opt_state[self.opt_slice[i]] = self.objectives[i].projection(opt_state[self.opt_slice[i]])
        state[self.overall_opt_slice] = new_opt_state
        return state

    def gradient(self, state):

        target_state, opt_state = state[self.target_slice], state[self.overall_opt_slice]
        target_grad, opt_grad = np.zeros_like(target_state), np.zeros_like(opt_state)
        full_grad = np.zeros_like(state)

        # randomization_gradient are gradients of a CONVEX function

        for i in range(self.nviews):
            target_grad_curr, opt_grad[self.opt_slice[i]] = \
                self.objectives[i].randomization_gradient(target_state, self.target_transform[i], opt_state[self.opt_slice[i]])
            target_grad += target_grad_curr.copy()

        target_grad = - target_grad
        target_grad += self.reference_inv - self.target_inv_cov.dot(target_state)
        full_grad[self.target_slice] = target_grad
        full_grad[self.overall_opt_slice] = -opt_grad

        return full_grad

    def sample(self, ndraw, burnin, stepsize=None):
        """
        assumes setup_sampler has been called
        """
        if stepsize is None:
            stepsize = 2. / self.observed_state.shape[0]
        target_langevin = projected_langevin(self.observed_state.copy(),
                                             self.gradient,
                                             self.projection,
                                             stepsize)

        samples = []
        for i in range(ndraw + burnin):
            target_langevin.next()
            if (i >= burnin):
                samples.append(target_langevin.state[self.target_slice].copy())

        return samples

    def hypothesis_test(self,
                        test_stat,
                        observed_target,
                        ndraw=8000,
                        burnin=2000,
                        stepsize=None,
                        alternative='twosided'):

        if alternative not in ['greater', 'less', 'twosided']:
            raise ValueError("alternative should be one of ['greater', 'less', 'twosided']")

        samples = self.sample(ndraw, burnin, stepsize=stepsize)
        observed_stat = test_stat(observed_target)
        sample_test_stat = np.array([test_stat(x) for x in samples])

        family = discrete_family(sample_test_stat, np.ones_like(sample_test_stat))
        pval = family.cdf(0, observed_stat)

        if alternative == 'greater':
            return 1 - pval
        elif alternative == 'less':
            return pval
        else:
            return 2 * min(pval, 1 - pval)
<|endoftext|>


<commit_before># risk_bayes.py
# Ronald L. Rivest (with Karim Husayn Karimi)
# August 11, 2017
# python3

"""
Routines to compute Bayes risk for a contest, or for a set of contests.

Called by audit.py

This is designed to be compatible only with sampling by pbcid;
more elaborate sampling regimes, such as sampling by card number
or by reported vote, are yet to be implemented, and may require
a significant change to the code base.  (Some thoughts, albeit
primitive, are sketched in risk_bayes_2.py.)
"""

import copy
import numpy as np

import multi
import audit
import outcomes

##############################################################################
# Gamma distribution
# https://docs.scipy.org/doc/numpy-1.11.0/reference/generated/numpy.random.gamma.html
# from numpy.random import gamma
# To generate random gamma variate with mean k:
# gamma(k)  or rs.gamma(k) where rs is a numpy.random.RandomState object

def gamma(k, rs=None):
    """
    Return sample from gamma distribution with mean k.
    Differs from standard gamma distribution implementation
    in that that it allows k==0, and returns 0 in that case.
    Parameter rs, if present, is a numpy.random.RandomState object.
    """
    if rs == None:
        rs = audit.auditRandomState
    if k <= 0.0:
        return 0.0
    else:
        return rs.gamma(k)

# Dirichlet distribution

def dirichlet(tally):
    """
    Given tally dict mapping votes (tuples of selids) to nonnegative ints (counts),
    return dict mapping those votes to elements of Dirichlet distribution sample on
    those votes, where tally values are used as Dirichlet hyperparameters.
    The values produced sum to one.
    """

    # make sure order of applying gamma is deterministic, for reproducibility
    dir = {vote: gamma(tally[vote]) for vote in sorted(tally)}
    total = sum(dir.values())
    dir = {vote: dir[vote] / total for vote in dir}
    return dir

##############################################################################
# Risk measurement (Bayes risk)

def compute_risk(e, mid, sn_tcpra, trials=None):
    """
    Compute (estimate) Bayesian risk (chance that reported
    outcome is wrong for contest e.cid_m[mid]).
    We take sn_tcpra here as argument rather than just use e.sn_tcpra so
    we can call compute_contest_risk with modified sample counts.
    (This option not yet used, but might be later, when optimizing
    workload.)
    Here sn_tcpra is identical in structure to (and may in fact be
    identical to) e.sn_tcpra.
    Here trials is the number of trials to run to obtain the desired
    precision in the risk estimate.

    This method is the heart of the Bayesian post-election audit method.
    But it could be replaced by a frequentist approach instead, at
    least for those outcome rules and mixes of collection types for
    which a frequentist method is known.

    The comparison and ballot-polling audits are blended here; the
    reported election data just records a ("-noCVR",) vote for the
    reported vote in a noCVR paper ballot collection.
    """

    cid = e.cid_m[mid]
    wrong_outcome_count = 0
    if trials == None:
        trials = e.n_trials
    for trial in range(trials):
        test_tally = {vote: 0 for vote in e.votes_c[cid]}
        for pbcid in sorted(e.possible_pbcid_c[cid]):
            # Draw from posterior for each paper ballot collection, sum them.
            # Stratify by reported vote.
            for rv in sorted(sn_tcpra[e.stage_time][cid][pbcid]):
                tally = sn_tcpra[e.stage_time][cid][pbcid][rv].copy()
                for av in e.votes_c[cid]:
                    if av not in tally:
                        tally[av] = 0
                    tally[av] += (e.pseudocount_match if av==rv
                                  else e.pseudocount_base)
                dirichlet_dict = dirichlet(tally)
                stratum_size = e.rn_cpr[cid][pbcid][rv]
                # sample_size = sn_tcpr[e.stage_time][cid][pbcid][rv]
                sample_size = sum([sn_tcpra[e.stage_time][cid][pbcid][rv][av]
                                   for av in sn_tcpra[e.stage_time][cid][pbcid][rv]])
                nonsample_size = stratum_size - sample_size
                for av in sorted(tally):
                    test_tally[av] += tally[av]
                    test_tally[av] += dirichlet_dict[av] * nonsample_size
        if e.ro_c[cid] != outcomes.compute_outcome(e, cid, test_tally):
            wrong_outcome_count += 1
    risk = wrong_outcome_count / e.n_trials
    e.risk_tm[e.stage_time][mid] = risk
    return risk

def compute_risks(e, st):

    for mid in e.mids:
        compute_risk(e, mid, st)

def compute_slack_p(e):
    """
    Return dictionary showing amount by which sample in each
    pbcid can be increased.
    """

    mid = e.mids[0]         # sampling is same so far for all mids
    cid = e.cid_m[mid]
    slack_p = {}
    for pbcid in e.pbcids:
        slack_p[pbcid] = 0
        for rv in e.rn_cpr[cid][pbcid]:
            slack_p[pbcid] += e.rn_cpr[cid][pbcid][rv]
            slack_p[pbcid] -= e.sn_tcpr[e.stage_time][cid][pbcid][rv]
    return slack_p

def compute_risk_with_tweak(e, mid, slack_p, tweak):
    """
    Return computed risk if sample sizes were tweaked.

    Here tweak is a dict mapping pbcids to how much
    to increase sample size by in each pbcid.  We must have
        slack[pbcid] >= tweak[pbcid] >= 0
    for all pbcids.
    """

    cid = e.cid_m[mid]
    sn_tcpra = copy.deepcopy(e.sn_tcpra)
    sn_tcp = {}
    sn_tcp[e.stage_time] = {}
    sn_tcp[e.stage_time][cid] = {}
    for pbcid in e.pbcids:
        assert slack[pbcid] >= tweak[pbcid] >= 0
        sn_tcp[e.stage_time][cid][pbcid] = 0
        sn_tcpra = copy.deepcopy(e.sn_tcpra)
    return compute_risk(e, mid, sn_tcpra)

def tweak_all(e, mid):
    """
    Test routine to try all tweaks.
    """

    return

    # FIXME in below sn_tcp is not defined.
    risk = compute_risk(e, mid, e.sn_tcpra)
    print("Risk (no change):", risk)
    slack_p = compute_slack_p(e)
    cid = e.cid_m[mid]
    for pbcid in e.pbcids:
        for rv in e.sn_tcpra[e.stage_time][cid][pbcid]:
            for av in e.sn_tcpra[e.stage_time][cid][pbcid][rv]:
                if sn_tcp[e.stage_time][cid][pbcid] > 0:
                    sn_tcpra[e.stage_time][cid][pbcid][rv][av] += \
                        min(100,
                            slack_p[pbcid] *\
                            sn_tcpra[e.stage_time][cid][pbcid][rv][av] / \
                            sn_tcp[e.stage_time][cid][pbcid])
        risk = compute_risk_with_tweak(e, mid, slack_p, tweak)
        print("Risk (change {}):".format(pbcid),
              risk)

if __name__ == "__main__":

    pass

<commit_msg>Adjust risk_bayes.py to take trials as an argument.<commit_after># risk_bayes.py
# Ronald L. Rivest (with Karim Husayn Karimi)
# August 11, 2017
# python3

"""
Routines to compute Bayes risk for a contest, or for a set of contests.

Called by audit.py

This is designed to be compatible only with sampling by pbcid;
more elaborate sampling regimes, such as sampling by card number
or by reported vote, are yet to be implemented, and may require
a significant change to the code base.  (Some thoughts, albeit
primitive, are sketched in risk_bayes_2.py.)
"""

import copy
import numpy as np

import multi
import audit
import outcomes

##############################################################################
# Gamma distribution
# https://docs.scipy.org/doc/numpy-1.11.0/reference/generated/numpy.random.gamma.html
# from numpy.random import gamma
# To generate random gamma variate with mean k:
# gamma(k)  or rs.gamma(k) where rs is a numpy.random.RandomState object

def gamma(k, rs=None):
    """
    Return sample from gamma distribution with mean k.
    Differs from standard gamma distribution implementation
    in that that it allows k==0, and returns 0 in that case.
    Parameter rs, if present, is a numpy.random.RandomState object.
    """
    if rs == None:
        rs = audit.auditRandomState
    if k <= 0.0:
        return 0.0
    else:
        return rs.gamma(k)

# Dirichlet distribution

def dirichlet(tally):
    """
    Given tally dict mapping votes (tuples of selids) to nonnegative ints (counts),
    return dict mapping those votes to elements of Dirichlet distribution sample on
    those votes, where tally values are used as Dirichlet hyperparameters.
    The values produced sum to one.
    """

    # make sure order of applying gamma is deterministic, for reproducibility
    dir = {vote: gamma(tally[vote]) for vote in sorted(tally)}
    total = sum(dir.values())
    dir = {vote: dir[vote] / total for vote in dir}
    return dir

##############################################################################
# Risk measurement (Bayes risk)

def compute_risk(e, mid, sn_tcpra, trials=None):
    """
    Compute (estimate) Bayesian risk (chance that reported
    outcome is wrong for contest e.cid_m[mid]).
    We take sn_tcpra here as argument rather than just use e.sn_tcpra so
    we can call compute_contest_risk with modified sample counts.
    (This option not yet used, but might be later, when optimizing
    workload.)
    Here sn_tcpra is identical in structure to (and may in fact be
    identical to) e.sn_tcpra.
    Here trials is the number of trials to run to obtain the desired
    precision in the risk estimate.

    This method is the heart of the Bayesian post-election audit method.
    But it could be replaced by a frequentist approach instead, at
    least for those outcome rules and mixes of collection types for
    which a frequentist method is known.

    The comparison and ballot-polling audits are blended here; the
    reported election data just records a ("-noCVR",) vote for the
    reported vote in a noCVR paper ballot collection.
    """

    cid = e.cid_m[mid]
    wrong_outcome_count = 0
    if trials == None:
        trials = e.n_trials
    for trial in range(trials):
        test_tally = {vote: 0 for vote in e.votes_c[cid]}
        for pbcid in sorted(e.possible_pbcid_c[cid]):
            # Draw from posterior for each paper ballot collection, sum them.
            # Stratify by reported vote.
            for rv in sorted(sn_tcpra[e.stage_time][cid][pbcid]):
                tally = sn_tcpra[e.stage_time][cid][pbcid][rv].copy()
                for av in e.votes_c[cid]:
                    if av not in tally:
                        tally[av] = 0
                    tally[av] += (e.pseudocount_match if av==rv
                                  else e.pseudocount_base)
                dirichlet_dict = dirichlet(tally)
                stratum_size = e.rn_cpr[cid][pbcid][rv]
                # sample_size = sn_tcpr[e.stage_time][cid][pbcid][rv]
                sample_size = sum([sn_tcpra[e.stage_time][cid][pbcid][rv][av]
                                   for av in sn_tcpra[e.stage_time][cid][pbcid][rv]])
                nonsample_size = stratum_size - sample_size
                for av in sorted(tally):
                    test_tally[av] += tally[av]
                    test_tally[av] += dirichlet_dict[av] * nonsample_size
        if e.ro_c[cid] != outcomes.compute_outcome(e, cid, test_tally):
            wrong_outcome_count += 1
    risk = wrong_outcome_count / e.n_trials
    e.risk_tm[e.stage_time][mid] = risk
    return risk

def compute_risks(e, st, trials):
    """
    Compute risks for all measurements.
    """

    for mid in e.mids:
        compute_risk(e, mid, st, trials)

def compute_slack_p(e):
    """
    Return dictionary mapping pbcids to the amount by which
    sample in that pbcid can be increased.
    """

    # sampling so far doesn't depend on mid, so just use first one.
    mid = e.mids[0]
    cid = e.cid_m[mid]
    slack_p = {}
    for pbcid in e.pbcids:
        slack_p[pbcid] = 0
        for rv in e.rn_cpr[cid][pbcid]:
            slack_p[pbcid] += e.rn_cpr[cid][pbcid][rv]
            slack_p[pbcid] -= e.sn_tcpr[e.stage_time][cid][pbcid][rv]
    return slack_p

def compute_risk_with_tweak(e, mid, slack_p, tweak, trials):
    """
    Return computed risk for given mid
    if sample sizes were tweaked (increased).

    Here tweak is a dict mapping pbcids to how much
    to increase sample size by in each pbcid.  We must have
        0 <= tweak[pbcid] <= slack[pbcid]
    for all pbcids.
    """

    cid = e.cid_m[mid]
    sn_tcpra = copy.deepcopy(e.sn_tcpra)
    sn_tcp = {}
    sn_tcp[e.stage_time] = {}
    sn_tcp[e.stage_time][cid] = {}
    for pbcid in e.pbcids:
        assert 0 <= tweak[pbcid] <= slack[pbcid]
        sn_tcp[e.stage_time][cid][pbcid] = 0
        sn_tcpra = copy.deepcopy(e.sn_tcpra)
    return compute_risk(e, mid, sn_tcpra, trials)

def tweak_all(e, mid):
    """
    Test routine to try all possible tweaks.  That is,
    systematically vary each possible sample size.
    This was written as means of approximating gradient
    descent, which we might no longer do.
    Untested.
    """

    return

    risk = compute_risk(e, mid, e.sn_tcpra)
    print("Risk (no change):", risk)
    slack_p = compute_slack_p(e)
    cid = e.cid_m[mid]

    # Compute sn_tcp, as it is not otherwise defined.
    # (Perhaps this should be computed elsewhere/earlier?)
    sn_tcp = {}
    sn_tcp[e.stage_time] = {}
    sn_tcp[e.stage_time][cid] = {}
    for pbcid in e.pbcids:
        sn_tcp[e.stage_time][cid][pbcid] = 0
        for rv in e.sn_tcpra[e.stage_time][cid][pbcid]:
            for av in e.sn_tcpra[e.stage_time][cid][pbcid][rv]:
                sn_tcp[e.stage_time][cid][pbcid] += \
                    e.sn_tcpra[e.stage_time][cid][pbcid][rv][av]

    for pbcid in e.pbcids:
        for rv in e.sn_tcpra[e.stage_time][cid][pbcid]:
            for av in e.sn_tcpra[e.stage_time][cid][pbcid][rv]:
                if sn_tcp[e.stage_time][cid][pbcid] > 0:
                    sn_tcpra[e.stage_time][cid][pbcid][rv][av] += \
                        min(100,
                            slack_p[pbcid] *\
                            sn_tcpra[e.stage_time][cid][pbcid][rv][av] / \
                            sn_tcp[e.stage_time][cid][pbcid])
        risk = compute_risk_with_tweak(e, mid, slack_p, tweak)
        print("Risk (change {}):".format(pbcid),
              risk)

if __name__ == "__main__":

    pass

<|endoftext|>


<commit_before>import itertools
import sys
from struct import unpack

try:
    import cStringIO
    BytesIO = cStringIO.StringIO
except ImportError:
    from io import BytesIO  # noqa

PY2 = sys.version_info[0] == 2

# Kaitai Struct runtime streaming API version, defined as per PEP-0396
# standard. Used for two purposes:
#
# * .py files generated by ksc from .ksy check that they import proper
#   KS runtime library by this version number;
# * distribution utils (setup.py) use this when packaging for PyPI
#
__version__ = '0.7'

class KaitaiStruct(object):
    def __init__(self, stream):
        self._io = stream

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    @classmethod
    def from_file(cls, filename):
        f = open(filename, 'rb')
        try:
            return cls(KaitaiStream(f))
        except Exception:
            # close file descriptor, then reraise the exception
            f.close()
            raise

    def close(self):
        self._io.close()

class KaitaiStream(object):
    def __init__(self, io):
        self._io = io
        self.align_to_byte()

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def close(self):
        self._io.close()

    # ========================================================================
    # Stream positioning
    # ========================================================================

    def is_eof(self):
        io = self._io
        t = io.read(1)
        if t == b'':
            return True
        else:
            io.seek(io.tell() - 1)
            return False

    def seek(self, n):
        self._io.seek(n)

    def pos(self):
        return self._io.tell()

    def size(self):
        # Python has no internal File object API function to get
        # current file / StringIO size, thus we use the following
        # trick.
        io = self._io

        # Remember our current position
        cur_pos = io.tell()

        # Seek to the end of the File object
        io.seek(0, 2)

        # Remember position, which is equal to the full length
        full_size = io.tell()

        # Seek back to the current position
        io.seek(cur_pos)

        return full_size

    # ========================================================================
    # Integer numbers
    # ========================================================================

    # ------------------------------------------------------------------------
    # Signed
    # ------------------------------------------------------------------------

    def read_s1(self):
        return unpack('b', self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_s2be(self):
        return unpack('>h', self.read_bytes(2))[0]

    def read_s4be(self):
        return unpack('>i', self.read_bytes(4))[0]

    def read_s8be(self):
        return unpack('>q', self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_s2le(self):
        return unpack('<h', self.read_bytes(2))[0]

    def read_s4le(self):
        return unpack('<i', self.read_bytes(4))[0]

    def read_s8le(self):
        return unpack('<q', self.read_bytes(8))[0]

    # ------------------------------------------------------------------------
    # Unsigned
    # ------------------------------------------------------------------------

    def read_u1(self):
        return unpack('B', self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_u2be(self):
        return unpack('>H', self.read_bytes(2))[0]

    def read_u4be(self):
        return unpack('>I', self.read_bytes(4))[0]

    def read_u8be(self):
        return unpack('>Q', self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_u2le(self):
        return unpack('<H', self.read_bytes(2))[0]

    def read_u4le(self):
        return unpack('<I', self.read_bytes(4))[0]

    def read_u8le(self):
        return unpack('<Q', self.read_bytes(8))[0]

    # ========================================================================
    # Floating point numbers
    # ========================================================================

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_f4be(self):
        return unpack('>f', self.read_bytes(4))[0]

    def read_f8be(self):
        return unpack('>d', self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_f4le(self):
        return unpack('<f', self.read_bytes(4))[0]

    def read_f8le(self):
        return unpack('<d', self.read_bytes(8))[0]

    # ========================================================================
    # Unaligned bit values
    # ========================================================================

    def align_to_byte(self):
        self.bits = 0
        self.bits_left = 0

    def read_bits_int(self, n):
        bits_needed = n - self.bits_left
        if bits_needed > 0:
            # 1 bit  => 1 byte
            # 8 bits => 1 byte
            # 9 bits => 2 bytes
            bytes_needed = ((bits_needed - 1) // 8) + 1
            buf = self.read_bytes(bytes_needed)
            for byte in buf:
                # Python 2 will get "byte" as one-character str, thus
                # we need to convert it to integer manually; Python 3
                # is fine as is.
                if isinstance(byte, str):
                    byte = ord(byte)
                self.bits <<= 8
                self.bits |= byte
                self.bits_left += 8

        # raw mask with required number of 1s, starting from lowest bit
        mask = (1 << n) - 1
        # shift mask to align with highest bits available in self.bits
        shift_bits = self.bits_left - n
        mask <<= shift_bits
        # derive reading result
        res = (self.bits & mask) >> shift_bits
        # clear top bits that we've just read => AND with 1s
        self.bits_left -= n
        mask = (1 << self.bits_left) - 1
        self.bits &= mask

        return res

    # ========================================================================
    # Byte arrays
    # ========================================================================

    def read_bytes(self, n):
        r = self._io.read(n)
        if len(r) < n:
            raise EOFError(
                "requested %d bytes, but got only %d bytes" % (n, len(r))
            )
        return r

    def read_bytes_full(self):
        return self._io.read()

    def read_bytes_term(self, term, include_term, consume_term, eos_error):
        r = b''
        while True:
            c = self._io.read(1)
            if c == b'':
                if eos_error:
                    raise Exception(
                        "End of stream reached, but no terminator %d found" %
                        (term,)
                    )
                else:
                    return r
            elif ord(c) == term:
                if include_term:
                    r += c
                if not consume_term:
                    self._io.seek(self._io.tell() - 1)
                return r
            else:
                r += c

    def ensure_fixed_contents(self, expected):
        actual = self._io.read(len(expected))
        if actual != expected:
            raise Exception(
                "Unexpected fixed contents: got %s, was waiting for %s" %
                (str(actual), str(expected))
            )
        return actual

    @staticmethod
    def bytes_strip_right(src, pad_byte):
        new_len = len(src)
        if PY2:
            data = bytearray(src)
        else:
            data = src

        while new_len > 0 and data[new_len - 1] == pad_byte:
            new_len -= 1

        return data[:new_len]

    @staticmethod
    def bytes_terminate(src, term, include_term):
        new_len = 0
        max_len = len(src)
        if PY2:
            data = bytearray(src)
        else:
            data = src

        while new_len < max_len and data[new_len] != term:
            new_len += 1

        if include_term and new_len < max_len:
            new_len += 1

        return data[:new_len]

    # ========================================================================
    # Byte array processing
    # ========================================================================

    @staticmethod
    def process_xor_one(data, key):
        if PY2:
            r = bytearray(data)
            for i in range(len(r)):
                r[i] ^= key
            return bytes(r)
        else:
            return bytes(v ^ key for v in data)

    @staticmethod
    def process_xor_many(data, key):
        if PY2:
            r = bytearray(data)
            k = bytearray(key)
            ki = 0
            kl = len(k)
            for i in range(len(r)):
                r[i] ^= k[ki]
                ki += 1
                if ki >= kl:
                    ki = 0
            return bytes(r)
        else:
            return bytes(a ^ b for a, b in zip(data, itertools.cycle(key)))

    @staticmethod
    def process_rotate_left(data, amount, group_size):
        if group_size != 1:
            raise Exception(
                "unable to rotate group of %d bytes yet" %
                (group_size,)
            )

        mask = group_size * 8 - 1
        anti_amount = -amount & mask

        r = bytearray(data)
        for i in range(len(r)):
            r[i] = (r[i] << amount) & 0xff | (r[i] >> anti_amount)
        return bytes(r)
<commit_msg>More helper methods - `from_bytes` and `from_io`<commit_after>import itertools
import sys
from struct import unpack

try:
    import cStringIO
    BytesIO = cStringIO.StringIO
except ImportError:
    from io import BytesIO  # noqa

PY2 = sys.version_info[0] == 2

# Kaitai Struct runtime streaming API version, defined as per PEP-0396
# standard. Used for two purposes:
#
# * .py files generated by ksc from .ksy check that they import proper
#   KS runtime library by this version number;
# * distribution utils (setup.py) use this when packaging for PyPI
#
__version__ = '0.7'

class KaitaiStruct(object):
    def __init__(self, stream):
        self._io = stream

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    @classmethod
    def from_file(cls, filename):
        f = open(filename, 'rb')
        try:
            return cls(KaitaiStream(f))
        except Exception:
            # close file descriptor, then reraise the exception
            f.close()
            raise

    @classmethod
    def from_bytes(cls, buf):
        return cls(KaitaiStream(BytesIO(buf)))

    @classmethod
    def from_io(cls, io):
        return cls(KaitaiStream(io))

    def close(self):
        self._io.close()

class KaitaiStream(object):
    def __init__(self, io):
        self._io = io
        self.align_to_byte()

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def close(self):
        self._io.close()

    # ========================================================================
    # Stream positioning
    # ========================================================================

    def is_eof(self):
        io = self._io
        t = io.read(1)
        if t == b'':
            return True
        else:
            io.seek(io.tell() - 1)
            return False

    def seek(self, n):
        self._io.seek(n)

    def pos(self):
        return self._io.tell()

    def size(self):
        # Python has no internal File object API function to get
        # current file / StringIO size, thus we use the following
        # trick.
        io = self._io

        # Remember our current position
        cur_pos = io.tell()

        # Seek to the end of the File object
        io.seek(0, 2)

        # Remember position, which is equal to the full length
        full_size = io.tell()

        # Seek back to the current position
        io.seek(cur_pos)

        return full_size

    # ========================================================================
    # Integer numbers
    # ========================================================================

    # ------------------------------------------------------------------------
    # Signed
    # ------------------------------------------------------------------------

    def read_s1(self):
        return unpack('b', self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_s2be(self):
        return unpack('>h', self.read_bytes(2))[0]

    def read_s4be(self):
        return unpack('>i', self.read_bytes(4))[0]

    def read_s8be(self):
        return unpack('>q', self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_s2le(self):
        return unpack('<h', self.read_bytes(2))[0]

    def read_s4le(self):
        return unpack('<i', self.read_bytes(4))[0]

    def read_s8le(self):
        return unpack('<q', self.read_bytes(8))[0]

    # ------------------------------------------------------------------------
    # Unsigned
    # ------------------------------------------------------------------------

    def read_u1(self):
        return unpack('B', self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_u2be(self):
        return unpack('>H', self.read_bytes(2))[0]

    def read_u4be(self):
        return unpack('>I', self.read_bytes(4))[0]

    def read_u8be(self):
        return unpack('>Q', self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_u2le(self):
        return unpack('<H', self.read_bytes(2))[0]

    def read_u4le(self):
        return unpack('<I', self.read_bytes(4))[0]

    def read_u8le(self):
        return unpack('<Q', self.read_bytes(8))[0]

    # ========================================================================
    # Floating point numbers
    # ========================================================================

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_f4be(self):
        return unpack('>f', self.read_bytes(4))[0]

    def read_f8be(self):
        return unpack('>d', self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_f4le(self):
        return unpack('<f', self.read_bytes(4))[0]

    def read_f8le(self):
        return unpack('<d', self.read_bytes(8))[0]

    # ========================================================================
    # Unaligned bit values
    # ========================================================================

    def align_to_byte(self):
        self.bits = 0
        self.bits_left = 0

    def read_bits_int(self, n):
        bits_needed = n - self.bits_left
        if bits_needed > 0:
            # 1 bit  => 1 byte
            # 8 bits => 1 byte
            # 9 bits => 2 bytes
            bytes_needed = ((bits_needed - 1) // 8) + 1
            buf = self.read_bytes(bytes_needed)
            for byte in buf:
                # Python 2 will get "byte" as one-character str, thus
                # we need to convert it to integer manually; Python 3
                # is fine as is.
                if isinstance(byte, str):
                    byte = ord(byte)
                self.bits <<= 8
                self.bits |= byte
                self.bits_left += 8

        # raw mask with required number of 1s, starting from lowest bit
        mask = (1 << n) - 1
        # shift mask to align with highest bits available in self.bits
        shift_bits = self.bits_left - n
        mask <<= shift_bits
        # derive reading result
        res = (self.bits & mask) >> shift_bits
        # clear top bits that we've just read => AND with 1s
        self.bits_left -= n
        mask = (1 << self.bits_left) - 1
        self.bits &= mask

        return res

    # ========================================================================
    # Byte arrays
    # ========================================================================

    def read_bytes(self, n):
        r = self._io.read(n)
        if len(r) < n:
            raise EOFError(
                "requested %d bytes, but got only %d bytes" % (n, len(r))
            )
        return r

    def read_bytes_full(self):
        return self._io.read()

    def read_bytes_term(self, term, include_term, consume_term, eos_error):
        r = b''
        while True:
            c = self._io.read(1)
            if c == b'':
                if eos_error:
                    raise Exception(
                        "End of stream reached, but no terminator %d found" %
                        (term,)
                    )
                else:
                    return r
            elif ord(c) == term:
                if include_term:
                    r += c
                if not consume_term:
                    self._io.seek(self._io.tell() - 1)
                return r
            else:
                r += c

    def ensure_fixed_contents(self, expected):
        actual = self._io.read(len(expected))
        if actual != expected:
            raise Exception(
                "Unexpected fixed contents: got %s, was waiting for %s" %
                (str(actual), str(expected))
            )
        return actual

    @staticmethod
    def bytes_strip_right(src, pad_byte):
        new_len = len(src)
        if PY2:
            data = bytearray(src)
        else:
            data = src

        while new_len > 0 and data[new_len - 1] == pad_byte:
            new_len -= 1

        return data[:new_len]

    @staticmethod
    def bytes_terminate(src, term, include_term):
        new_len = 0
        max_len = len(src)
        if PY2:
            data = bytearray(src)
        else:
            data = src

        while new_len < max_len and data[new_len] != term:
            new_len += 1

        if include_term and new_len < max_len:
            new_len += 1

        return data[:new_len]

    # ========================================================================
    # Byte array processing
    # ========================================================================

    @staticmethod
    def process_xor_one(data, key):
        if PY2:
            r = bytearray(data)
            for i in range(len(r)):
                r[i] ^= key
            return bytes(r)
        else:
            return bytes(v ^ key for v in data)

    @staticmethod
    def process_xor_many(data, key):
        if PY2:
            r = bytearray(data)
            k = bytearray(key)
            ki = 0
            kl = len(k)
            for i in range(len(r)):
                r[i] ^= k[ki]
                ki += 1
                if ki >= kl:
                    ki = 0
            return bytes(r)
        else:
            return bytes(a ^ b for a, b in zip(data, itertools.cycle(key)))

    @staticmethod
    def process_rotate_left(data, amount, group_size):
        if group_size != 1:
            raise Exception(
                "unable to rotate group of %d bytes yet" %
                (group_size,)
            )

        mask = group_size * 8 - 1
        anti_amount = -amount & mask

        r = bytearray(data)
        for i in range(len(r)):
            r[i] = (r[i] << amount) & 0xff | (r[i] >> anti_amount)
        return bytes(r)
<|endoftext|>


<commit_before># Software License Agreement (BSD License)
#
# Copyright (c) 20092011, Eucalyptus Systems, Inc.
# All rights reserved.
#
# Redistribution and use of this software in source and binary forms, with or
# without modification, are permitted provided that the following conditions
# are met:
#
#   Redistributions of source code must retain the above
#   copyright notice, this list of conditions and the
#   following disclaimer.
#
#   Redistributions in binary form must reproduce the above
#   copyright notice, this list of conditions and the
#   following disclaimer in the documentation and/or other
#   materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# Author: Neil Soman neil@eucalyptus.com
#         Mitch Garnaat mgarnaat@eucalyptus.com

import euca2ools.commands.eucacommand
from boto.roboto.param import Param
import sys
import os
from xml.dom import minidom
from boto.exception import S3ResponseError, S3CreateError
from boto.s3.key import Key

class DownloadBundle(euca2ools.commands.eucacommand.EucaCommand):

    Description = 'Downloads a bundled image from a bucket.'
    Options = [Param(name='bucket', short_name='b', long_name='bucket',
                     optional=False, ptype='string',
                     doc='Name of the bucket to upload to.'),
               Param(name='manifest_path',
                     short_name='m', long_name='manifest',
                     optional=True, ptype='string',
                     doc='Path to the manifest file for bundled image.'),
               Param(name='prefix', short_name='p', long_name='prefix',
                     optional=True, ptype='string',
                     doc='Prefix used to identify the image in the bucket'),
               Param(name='directory',
                     short_name='d', long_name='directory',
                     optional=True, ptype='dir', default='/tmp',
                     doc='The directory to download the parts to.')]

    def ensure_bucket(self, bucket):
        bucket_instance = None
        s3conn = self.make_connection_cli('s3')
        try:
            bucket_instance = s3conn.get_bucket(bucket)
        except S3ResponseError, s3error:
            print 'Unable to get bucket %s' % bucket
            sys.exit()
        return bucket_instance

    def get_parts(self, manifest_filename):
        parts = []
        dom = minidom.parse(manifest_filename)
        manifest_elem = dom.getElementsByTagName('manifest')[0]
        parts_list = manifest_elem.getElementsByTagName('filename')
        for part_elem in parts_list:
            nodes = part_elem.childNodes
            for node in nodes:
                if node.nodeType == node.TEXT_NODE:
                    parts.append(node.data)
        return parts

    def get_manifests(self, bucket, image_prefix=None):
        manifests = []
        keys = bucket.get_all_keys()
        for k in keys:
            if k.name:
                if k.name.find('manifest') >= 0:
                    if image_prefix:
                        if k.name.startswith(image_prefix):
                            manifests.append(k.name)
                    else:
                        manifests.append(k.name)
        return manifests

    def download_manifests(self, bucket, manifests, directory):
        if len(manifests) > 0:
            if not os.path.exists(directory):
                os.makedirs(directory)
        for manifest in manifests:
            k = Key(bucket)
            k.key = manifest
            print 'Downloading', manifest
            manifest_file = open(os.path.join(directory, manifest), 'wb')
            k.get_contents_to_file(manifest_file)
            manifest_file.close()

    def download_parts(self, bucket, manifests, directory):
        for manifest in manifests:
            manifest_filename = os.path.join(directory, manifest)
            parts = get_parts(manifest_filename)
            for part in parts:
                k = Key(bucket)
                k.key = part
                print 'Downloading', part
                part_file = open(os.path.join(directory, part), 'wb')
                k.get_contents_to_file(part_file)
                part_file.close()

    def main(self):
        bucket_instance = self.ensure_bucket(self.bucket)
        manifests = self.get_manifests(bucket_instance, self.image_prefix)
        self.download_manifests(bucket_instance, manifests, self.directory)
        self.download_parts(bucket_instance, manifests, self.directory)

    def main_cli(self):
        self.main()
<commit_msg>Fixed incorrect reference to self.image_prefix rather than self.prefix<commit_after># Software License Agreement (BSD License)
#
# Copyright (c) 20092011, Eucalyptus Systems, Inc.
# All rights reserved.
#
# Redistribution and use of this software in source and binary forms, with or
# without modification, are permitted provided that the following conditions
# are met:
#
#   Redistributions of source code must retain the above
#   copyright notice, this list of conditions and the
#   following disclaimer.
#
#   Redistributions in binary form must reproduce the above
#   copyright notice, this list of conditions and the
#   following disclaimer in the documentation and/or other
#   materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# Author: Neil Soman neil@eucalyptus.com
#         Mitch Garnaat mgarnaat@eucalyptus.com

import euca2ools.commands.eucacommand
from boto.roboto.param import Param
import sys
import os
from xml.dom import minidom
from boto.exception import S3ResponseError, S3CreateError
from boto.s3.key import Key

class DownloadBundle(euca2ools.commands.eucacommand.EucaCommand):

    Description = 'Downloads a bundled image from a bucket.'
    Options = [Param(name='bucket', short_name='b', long_name='bucket',
                     optional=False, ptype='string',
                     doc='Name of the bucket to upload to.'),
               Param(name='manifest_path',
                     short_name='m', long_name='manifest',
                     optional=True, ptype='string',
                     doc='Path to the manifest file for bundled image.'),
               Param(name='prefix', short_name='p', long_name='prefix',
                     optional=True, ptype='string',
                     doc='Prefix used to identify the image in the bucket'),
               Param(name='directory',
                     short_name='d', long_name='directory',
                     optional=True, ptype='dir', default='/tmp',
                     doc='The directory to download the parts to.')]

    def ensure_bucket(self, bucket):
        bucket_instance = None
        s3conn = self.make_connection_cli('s3')
        try:
            bucket_instance = s3conn.get_bucket(bucket)
        except S3ResponseError, s3error:
            print 'Unable to get bucket %s' % bucket
            sys.exit()
        return bucket_instance

    def get_parts(self, manifest_filename):
        parts = []
        dom = minidom.parse(manifest_filename)
        manifest_elem = dom.getElementsByTagName('manifest')[0]
        parts_list = manifest_elem.getElementsByTagName('filename')
        for part_elem in parts_list:
            nodes = part_elem.childNodes
            for node in nodes:
                if node.nodeType == node.TEXT_NODE:
                    parts.append(node.data)
        return parts

    def get_manifests(self, bucket, image_prefix=None):
        manifests = []
        keys = bucket.get_all_keys()
        for k in keys:
            if k.name:
                if k.name.find('manifest') >= 0:
                    if image_prefix:
                        if k.name.startswith(image_prefix):
                            manifests.append(k.name)
                    else:
                        manifests.append(k.name)
        return manifests

    def download_manifests(self, bucket, manifests, directory):
        if len(manifests) > 0:
            if not os.path.exists(directory):
                os.makedirs(directory)
        for manifest in manifests:
            k = Key(bucket)
            k.key = manifest
            print 'Downloading', manifest
            manifest_file = open(os.path.join(directory, manifest), 'wb')
            k.get_contents_to_file(manifest_file)
            manifest_file.close()

    def download_parts(self, bucket, manifests, directory):
        for manifest in manifests:
            manifest_filename = os.path.join(directory, manifest)
            parts = get_parts(manifest_filename)
            for part in parts:
                k = Key(bucket)
                k.key = part
                print 'Downloading', part
                part_file = open(os.path.join(directory, part), 'wb')
                k.get_contents_to_file(part_file)
                part_file.close()

    def main(self):
        bucket_instance = self.ensure_bucket(self.bucket)
        manifests = self.get_manifests(bucket_instance, self.prefix)
        self.download_manifests(bucket_instance, manifests, self.directory)
        self.download_parts(bucket_instance, manifests, self.directory)

    def main_cli(self):
        self.main()
<|endoftext|>


<commit_before>from django.db import models
from django.utils.translation import gettext_lazy as _
from django.urls import reverse

from diventi.core.models import TimeStampedModel, PublishableModel, DiventiImageModel

class SectionQuerySet(models.QuerySet):

    # Select every dicetable related to the section
    def tables(self):
        section = self.select_related('table')
        section = section.prefetch_related('table__rows')
        return section

    # Select every list related to the section
    def lists(self):
        section = self.select_related('_list')
        section = section.prefetch_related('_list__items')
        return section

    # Select every item related to a character
    def characters(self):
        section = self.select_related('character')
        section = section.prefetch_related('character__items')
        return section

class Paper(TimeStampedModel):
    """
        A Paper is a document written in latex that contains fantastic adventures.
    """
    title = models.CharField(max_length=60, verbose_name=_('title'))
    description = models.TextField(max_length=250, verbose_name=_('description'))
    slug = models.SlugField(unique=True, verbose_name=_('slug'))

    class Meta:
        verbose_name = _('paper')

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return reverse('homebrew:detail', args=[str(self.slug)])

class DiceTable(models.Model):
    """
        A dice table has two columns: the first is an automatic value of the dice,
        the second is a free description.
    """
    DICES = [
        ('20', 'd20'),
        ('12', 'd12'),
        ('10', 'd10'),
        ('8', 'd8'),
        ('6', 'd6'),
        ('4', 'd4'),
    ]
    dice = models.CharField(max_length=3, choices=DICES, verbose_name=_('dice'))
    title = models.CharField(max_length=60, verbose_name=_('title'))

    def __str__(self):
        return self.title

class DiceTableRow(models.Model):
    description = models.TextField(verbose_name=_('description'))
    face = models.PositiveIntegerField(verbose_name=_('face'))
    dicetable = models.ForeignKey(DiceTable, on_delete=models.CASCADE, related_name='rows')

    def __str__(self):
        return self.description

class Itemize(models.Model):
    """
        An itemize object is a list of unordered items.
    """
    title = models.CharField(max_length=60, verbose_name=_('title'))

    def __str__(self):
        return self.title

class ItemizeItem(models.Model):
    description = models.TextField(verbose_name=_('description'))
    itemize = models.ForeignKey(Itemize, on_delete=models.CASCADE, related_name='items')

    def __str__(self):
        return self.description

class CharacterBlock(models.Model):
    """
        A special block that enunciates the characteristics of a character.
    """
    title = models.CharField(max_length=60, verbose_name=_('title'))
    description = models.TextField(verbose_name=_('description'))

    def __str__(self):
        return self.title

class CharacterItem(models.Model):
    title = models.CharField(max_length=60, verbose_name=_('title'))
    description = models.TextField(verbose_name=_('description'))
    characterblock = models.ForeignKey(CharacterBlock, on_delete=models.CASCADE, related_name='items')

    def __str__(self):
        return self.title

class Section(TimeStampedModel):
    """
        Each section type needs a function that renders the content of the paper
        in latex. If blank the Brew will display the title as plain string.
    """
    order_id = models.PositiveIntegerField(unique=True, verbose_name=_('order_id')) # This affects the display order of every piece of content
    title = models.CharField(max_length=60, verbose_name=_('title'))
    content = models.TextField(blank=True, verbose_name=_('content'))
    SECTION_TYPES = [
        ('section', _('section')),
        ('subsection', _('subsection')),
        ('subsubsection', _('subsubsection')),
        ('paragraph', _('paragraph')),
        ('commentbox', _('commentbox')),
        ('quotebox', _('quotebox')),
        ('paperbox', _('paperbox')),
        ('header', _('header')),
        ('phantom', _('phantom')),
    ]
    section_type = models.CharField(max_length=30, blank=True, choices=SECTION_TYPES, verbose_name=_('section type'))
    THEMES = [
        ('PhbLightGreen', _('light green')),
        ('PhbLightCyan', _('light chan')),
        ('PhbMauve', _('muave')),
        ('PhbTan', _('tan')),
        ('DmgLavender', _('lavender')),
        ('DmgCoral', _('coral')),
        ('DmgSlateGray', _('slate gray')),
        ('DmgLilac', _('lilac')),
    ]
    theme = models.CharField(max_length=30, blank=True, choices=THEMES, verbose_name=_('theme'))
    paper = models.ForeignKey(Paper, null=True, on_delete=models.SET_NULL, related_name=_('sections'))
    table = models.OneToOneField(DiceTable, null=True, blank=True, on_delete=models.SET_NULL, related_name=('section'), verbose_name=_('table'))
    _list = models.OneToOneField(Itemize, null=True, blank=True, on_delete=models.SET_NULL, related_name=('section'), verbose_name=_('list'))
    character = models.OneToOneField(CharacterBlock, null=True, blank=True, on_delete=models.SET_NULL, related_name=('section'), verbose_name=_('character'))
    new_page = models.BooleanField(default=False, verbose_name=_('new page'))
    title_page = models.BooleanField(default=False, verbose_name=_('title page'))

    objects = SectionQuerySet.as_manager()

    def __str__(self):
        return self.title

    def __repr__ (self):
        if self.section_type:
            return getattr(Section, self.section_type)(self)
        elif self.title_page:
            return ""
        else:
            return self.__str__()

    def phantom(self):
        return """
            \\phantom{Invisible text}"""

    def section(self):
        return """
            \\subsection{%s}
                %s""" % (self.title, self.content)

    def subsection(self):
        return """
            \\subsubsection{%s}
                %s""" % (self.title, self.content)

    def subsubsection(self):
        return """
            \\header{%s}
                %s""" % (self.title, self.content)

    def paragraph(self):
        return """
            \\textit{%s}
                %s""" % (self.title, self.content)

    def commentbox(self):
        return """
            \\begin{commentbox}{%s}[%s]
                %s
            \\end{commentbox}""" % (self.title, self.theme, self.content)

    def quotebox(self):
        return """
            \\begin{quotebox}[%s]
                %s
            \\end{quotebox}""" % (self.theme, self.content)

    def paperbox(self):
        return """
            \\begin{paperbox}{%s}[%s]
                %s
            \\end{paperbox}""" % (self.title, self.theme, self.content)

    def header(self):
        return """
            \\header{%s}""" % self.__str__() # The header of an object

class Watermark(TimeStampedModel):
    """
        Any page can be customized with cool backgrounds thanks to one or more
        watermarks.
    """
    title = models.CharField(max_length=60, verbose_name=_('title'))
    pages = models.CharField(max_length=10, verbose_name=_('pages'))
    scale = models.FloatField(default=1)
    xpos = models.IntegerField()
    ypos = models.IntegerField()
    figurename = models.CharField(max_length=60, verbose_name=_('figure name'))
    paper = models.ForeignKey(Paper, null=True, on_delete=models.SET_NULL, related_name='watermarks')

    def __str__(self):
        return self.title

<commit_msg>Adjustedy subsubsection format.<commit_after>from django.db import models
from django.utils.translation import gettext_lazy as _
from django.urls import reverse

from diventi.core.models import TimeStampedModel, PublishableModel, DiventiImageModel

class SectionQuerySet(models.QuerySet):

    # Select every dicetable related to the section
    def tables(self):
        section = self.select_related('table')
        section = section.prefetch_related('table__rows')
        return section

    # Select every list related to the section
    def lists(self):
        section = self.select_related('_list')
        section = section.prefetch_related('_list__items')
        return section

    # Select every item related to a character
    def characters(self):
        section = self.select_related('character')
        section = section.prefetch_related('character__items')
        return section

class Paper(TimeStampedModel):
    """
        A Paper is a document written in latex that contains fantastic adventures.
    """
    title = models.CharField(max_length=60, verbose_name=_('title'))
    description = models.TextField(max_length=250, verbose_name=_('description'))
    slug = models.SlugField(unique=True, verbose_name=_('slug'))

    class Meta:
        verbose_name = _('paper')

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return reverse('homebrew:detail', args=[str(self.slug)])

class DiceTable(models.Model):
    """
        A dice table has two columns: the first is an automatic value of the dice,
        the second is a free description.
    """
    DICES = [
        ('20', 'd20'),
        ('12', 'd12'),
        ('10', 'd10'),
        ('8', 'd8'),
        ('6', 'd6'),
        ('4', 'd4'),
    ]
    dice = models.CharField(max_length=3, choices=DICES, verbose_name=_('dice'))
    title = models.CharField(max_length=60, verbose_name=_('title'))

    def __str__(self):
        return self.title

class DiceTableRow(models.Model):
    description = models.TextField(verbose_name=_('description'))
    face = models.PositiveIntegerField(verbose_name=_('face'))
    dicetable = models.ForeignKey(DiceTable, on_delete=models.CASCADE, related_name='rows')

    def __str__(self):
        return self.description

class Itemize(models.Model):
    """
        An itemize object is a list of unordered items.
    """
    title = models.CharField(max_length=60, verbose_name=_('title'))

    def __str__(self):
        return self.title

class ItemizeItem(models.Model):
    description = models.TextField(verbose_name=_('description'))
    itemize = models.ForeignKey(Itemize, on_delete=models.CASCADE, related_name='items')

    def __str__(self):
        return self.description

class CharacterBlock(models.Model):
    """
        A special block that enunciates the characteristics of a character.
    """
    title = models.CharField(max_length=60, verbose_name=_('title'))
    description = models.TextField(verbose_name=_('description'))

    def __str__(self):
        return self.title

class CharacterItem(models.Model):
    title = models.CharField(max_length=60, verbose_name=_('title'))
    description = models.TextField(verbose_name=_('description'))
    characterblock = models.ForeignKey(CharacterBlock, on_delete=models.CASCADE, related_name='items')

    def __str__(self):
        return self.title

class Section(TimeStampedModel):
    """
        Each section type needs a function that renders the content of the paper
        in latex. If blank the Brew will display the title as plain string.
    """
    order_id = models.PositiveIntegerField(unique=True, verbose_name=_('order_id')) # This affects the display order of every piece of content
    title = models.CharField(max_length=60, verbose_name=_('title'))
    content = models.TextField(blank=True, verbose_name=_('content'))
    SECTION_TYPES = [
        ('section', _('section')),
        ('subsection', _('subsection')),
        ('subsubsection', _('subsubsection')),
        ('paragraph', _('paragraph')),
        ('commentbox', _('commentbox')),
        ('quotebox', _('quotebox')),
        ('paperbox', _('paperbox')),
        ('header', _('header')),
        ('phantom', _('phantom')),
    ]
    section_type = models.CharField(max_length=30, blank=True, choices=SECTION_TYPES, verbose_name=_('section type'))
    THEMES = [
        ('PhbLightGreen', _('light green')),
        ('PhbLightCyan', _('light chan')),
        ('PhbMauve', _('muave')),
        ('PhbTan', _('tan')),
        ('DmgLavender', _('lavender')),
        ('DmgCoral', _('coral')),
        ('DmgSlateGray', _('slate gray')),
        ('DmgLilac', _('lilac')),
    ]
    theme = models.CharField(max_length=30, blank=True, choices=THEMES, verbose_name=_('theme'))
    paper = models.ForeignKey(Paper, null=True, on_delete=models.SET_NULL, related_name=_('sections'))
    table = models.OneToOneField(DiceTable, null=True, blank=True, on_delete=models.SET_NULL, related_name=('section'), verbose_name=_('table'))
    _list = models.OneToOneField(Itemize, null=True, blank=True, on_delete=models.SET_NULL, related_name=('section'), verbose_name=_('list'))
    character = models.OneToOneField(CharacterBlock, null=True, blank=True, on_delete=models.SET_NULL, related_name=('section'), verbose_name=_('character'))
    new_page = models.BooleanField(default=False, verbose_name=_('new page'))
    title_page = models.BooleanField(default=False, verbose_name=_('title page'))

    objects = SectionQuerySet.as_manager()

    def __str__(self):
        return self.title

    def __repr__ (self):
        if self.section_type:
            return getattr(Section, self.section_type)(self)
        elif self.title_page:
            return ""
        else:
            return self.__str__()

    def phantom(self):
        return """
            \\phantom{Invisible text}"""

    def section(self):
        return """
            \\subsection{%s}
                %s""" % (self.title, self.content)

    def subsection(self):
        return """
            \\subsubsection{%s}
                %s""" % (self.title, self.content)

    def subsubsection(self):
        return """
            \\header{%s} \\newline
                %s""" % (self.title, self.content)

    def paragraph(self):
        return """
            \\textit{%s}
                %s""" % (self.title, self.content)

    def commentbox(self):
        return """
            \\begin{commentbox}{%s}[%s]
                %s
            \\end{commentbox}""" % (self.title, self.theme, self.content)

    def quotebox(self):
        return """
            \\begin{quotebox}[%s]
                %s
            \\end{quotebox}""" % (self.theme, self.content)

    def paperbox(self):
        return """
            \\begin{paperbox}{%s}[%s]
                %s
            \\end{paperbox}""" % (self.title, self.theme, self.content)

    def header(self):
        return """
            \\header{%s}""" % self.__str__() # The header of an object

class Watermark(TimeStampedModel):
    """
        Any page can be customized with cool backgrounds thanks to one or more
        watermarks.
    """
    title = models.CharField(max_length=60, verbose_name=_('title'))
    pages = models.CharField(max_length=10, verbose_name=_('pages'))
    scale = models.FloatField(default=1)
    xpos = models.IntegerField()
    ypos = models.IntegerField()
    figurename = models.CharField(max_length=60, verbose_name=_('figure name'))
    paper = models.ForeignKey(Paper, null=True, on_delete=models.SET_NULL, related_name='watermarks')

    def __str__(self):
        return self.title

<|endoftext|>