<commit_before>"""
A new rescoring technique based on sampling.
A complete publication describing the method and a comparative study is to be submitted in early September 2015 to TACL.

:Authors: - Wilker Aziz
"""

import logging
import numpy as np
from collections import defaultdict, Counter
from itertools import chain

from easyhg.recipes import timeit
from easyhg.report import EmptyReport

from easyhg.grammar.semiring import SumTimes
from easyhg.grammar.cfg import CFG, CFGProduction, Terminal, Nonterminal, TopSortTable
from easyhg.grammar.utils import make_nltk_tree, inlinetree

from easyhg.alg.exact.inference import total_weight
from easyhg.alg.exact import AncestralSampler, EarleyRescoring
from .slicevars import ExpSliceVariables, ConstantPrior, SymmetricGamma, AsymmetricGamma

from easyhg.recipes import progressbar

def make_span_conditions(d, semiring):
    return {r.lhs: semiring.as_real(r.weight) for r in d}

def gamma_priors(forest, semiring, percentile=None):
    """
    Return a table mapping a cell to a Gamma scale parameter.
    """
    priors = defaultdict(None)
    if percentile is None:
        for lhs, rules in forest.iteritems():
            thetas = np.array([semiring.as_real(r.weight) for r in rules])
            priors[lhs] = thetas.mean()
            # logging.debug('{0} min={1} max={2} mean={3} prior={4}'.format(lhs, np.min(thetas), np.max(thetas), np.mean(thetas), priors[lhs]))
    else:
        for lhs, rules in forest.iteritems():
            thetas = np.array([semiring.as_real(r.weight) for r in rules])
            priors[lhs] = np.percentile(thetas, percentile)
            # logging.debug('{0} min={1} max={2} mean={3} prior={4}'.format(lhs, np.min(thetas), np.max(thetas), np.mean(thetas), priors[lhs]))
    #logging.info('cell=%s incoming=%d sum=%s mean=%s median=%s prior=%s', lhs, len(rules), thetas.sum(), thetas.mean(), np.percentile(thetas, 50), priors[lhs])
    return priors

def prune_and_reweight(forest, tsort, slicevars, semiring, dead_terminal=Terminal('<null>')):
    """
    Prune a forest and compute a new weight function for edges.

    :param forest: weighted forest
    :param tsort: forest's top-sort table
    :param slicevars: an instance of SliceVariable
    :param semiring: the appropriate semiring
    :param dead_terminal: a Terminal symbol which represents a dead-end.
    :return: pruned CFG and weight table mapping edges to weights.
    """
    oforest = CFG()
    discovered = set([tsort.root()])
    weight_table = defaultdict(None)
    pruned = 0
    total_cells = 0
    n_uniform = []
    n_exponential = []
    for level in tsort.iterlevels(reverse=True):  # we go top-down level by level
        for lhs in chain(*level):  # flatten the buckets
            total_cells += 1
            if lhs not in discovered:  # symbols not yet discovered have been pruned
                continue
            cell = lhs
            n = 0
            incoming = forest.get(lhs)
            # we sort the incoming edges in order to find out which ones are not in the slice
            for rule in sorted(incoming, key=lambda r: r.weight, reverse=True):  # TODO: use semiring to sort
                u = semiring.from_real(slicevars[cell])
                if semiring.gt(rule.weight, u):  # inside slice
                    oforest.add(rule)
                    weight_table[rule] = slicevars.pdf_semiring(cell, rule.weight, semiring)
                    discovered.update(filter(lambda s: isinstance(s, Nonterminal), rule.rhs))
                    n += 1
                else:  # this edge and the remaining are pruned
                    pruned += 1
                    #logging.info('pruning: %s vs %s', p, slicevars[cell])
                    break
            if slicevars.has_conditions(cell):
                n_uniform.append(float(n)/len(incoming))
            else:
                n_exponential.append(float(n)/len(incoming))
            #logging.info('cell=%s slice=%d/%d conditioning=%s', cell, n, len(incoming), slicevars.has_conditions(cell))
            if n == 0:  # this is a dead-end, instead of pruning bottom-up, we add an edge with 0 weight for simplicity
                dead = CFGProduction(lhs, [dead_terminal], semiring.zero)
                oforest.add(dead)
                weight_table[dead] = semiring.zero
    #logging.info('Pruning: cells=%s/%s in=%s out=%s', pruned, total_cells, np.array(_in).mean(), np.array(_out).mean())
    return oforest, weight_table, np.mean(n_uniform), np.mean(n_exponential)

@timeit
def t_prune_and_reweight(forest, tsort, slicevars, semiring, dead_terminal=Terminal('<null>')):
    return prune_and_reweight(forest, tsort, slicevars, semiring, dead_terminal)

def get_ancestral_sampler(forest, tsort, generations, temperature):
    if temperature == 1.0:
        return AncestralSampler(forest, tsort, generations=generations)
    else:
        return AncestralSampler(forest, tsort, generations=generations, omega=lambda e: e.weight/temperature)

@timeit
def t_get_ancestral_sampler(forest, tsort, generations, temperature):
    return get_ancestral_sampler(forest, tsort, generations, temperature)

class SlicedRescoring(object):

    def __init__(self, forest,
                 tsort,
                 stateless,
                 stateful,
                 semiring=SumTimes,
                 generations=10,
                 do_nothing=set(),
                 temperature0=1.0,
                 report=EmptyReport()):
        self._forest = forest
        self._tsort = tsort
        self._stateless = stateless
        self._stateful = stateful
        self._do_nothing = do_nothing
        self._semiring = semiring
        self._generations = generations
        self._report = report

        dt, self._sampler0 = t_get_ancestral_sampler(self._forest, self._tsort, self._generations, temperature0)

        logging.info('l-forest: derivations=%d', self._sampler0.n_derivations())

        self._make_conditions = make_span_conditions

        self._report.init(terminals=self._forest.n_terminals(),
                          nonterminals=self._forest.n_nonterminals(),
                          edges=len(self._forest),
                          derivations=self._sampler0.n_derivations(),
                          ancestral_time=dt)

    def sample_d0(self):
        """Draw an initial derivation from the locally weighted forest."""
        return next(self._sampler0.sample(1))

    def _rescore_derivation(self, d):
        semiring = self._semiring
        w = semiring.one
        # stateless scorer goes edge by edge
        # TODO: stateless.total_score(derivation)
        for edge in filter(lambda e: e.lhs not in self._do_nothing, d):
            w = semiring.times(w, self._stateless.score(edge))
        w = semiring.times(w, self._stateful.score_derivation(d))
        return w

    def _make_slice_varialbes(self, conditions, prior_type, prior_parameter):
        if prior_type == 'const':
            prior = ConstantPrior(const=float(prior_parameter))
            logging.debug('Constant scale: %s', prior_parameter)
        elif prior_type == 'sym':
            prior = SymmetricGamma(scale=float(prior_parameter))
            logging.debug('Symmetric Gamma: %s', prior_parameter)
        elif prior_type == 'asym':
            if prior_parameter == 'mean':
                prior = AsymmetricGamma(scales=gamma_priors(self._forest, self._semiring))
            else:
                try:
                    percentile = float(prior_parameter)
                except ValueError:
                    raise ValueError("The parameter of the asymmetric "
                                     "Gamma must be the keyword 'mean' or a number between 0-100: %s" % percentile)
                if not (0 <= percentile <= 100):
                    raise ValueError("A percentile is a real number between 0 and 100: %s" % percentile)
                prior = AsymmetricGamma(scales=gamma_priors(self._forest, self._semiring, percentile=percentile))
            logging.debug('Asymmetric Gamma: %s', prior_parameter)

        return ExpSliceVariables(conditions=conditions, prior=prior)

    def _uniform(self, l_slice, g, batch_size):
        semiring = self._semiring
        # sample uniformly from the truncated support of l(d)
        # note the omega(e) function
        sampler = AncestralSampler(l_slice,
                                   TopSortTable(l_slice),
                                   omega=lambda e: semiring.one if g[e] != semiring.zero else semiring.zero,
                                   generations=self._generations)

        # sample a number of derivations and group them by identity
        counts = Counter(sampler.sample(batch_size))
        # compute the empirical distribution f(d|u) \propto \pi(d) * g(d|u)
        support = [None] * len(counts)
        numerators = np.zeros(len(counts))
        for i, (d, n) in enumerate(counts.items()):
            support[i] = d
            # score using the complete f(d|u)
            pi_d = self._rescore_derivation(d)  # this is \pi(d)
            g_d = semiring.times.reduce(list(g[e] for e in d))  # this is g(d)
            w = semiring.times(pi_d, g_d)
            numerators[i] = semiring.times(w, semiring.from_real(n))
        log_prob = numerators - np.logaddexp.reduce(numerators)
        prob = np.exp(log_prob)

        # sample a derivation
        i = np.random.choice(len(log_prob), p=prob)
        d_i = support[i]
        # make conditions based on the slice variables and l(d)
        c_i = self._make_conditions(d_i, semiring)
        return d_i, c_i, sampler.n_derivations()

    def _importance(self, l_slice, g, batch_size):
        semiring = self._semiring
        # sample from g(d) over the truncated support of l(d)
        sampler = AncestralSampler(l_slice,
                                   TopSortTable(l_slice),
                                   omega=lambda e: g[e],
                                   generations=self._generations)

        # sample a number of derivations and group them by identity
        counts = Counter(sampler.sample(batch_size))
        # compute the empirical (importance) distribution r(d) \propto f(d|u)/g(d|u)
        support = [None] * len(counts)
        numerators = np.zeros(len(counts))
        for i, (d, n) in enumerate(counts.items()):
            support[i] = d
            w = self._rescore_derivation(d)
            numerators[i] = semiring.times(w, semiring.from_real(n))
        log_prob = numerators - np.logaddexp.reduce(numerators)
        prob = np.exp(log_prob)

        # sample a derivation
        i = np.random.choice(len(log_prob), p=prob)
        d_i = support[i]
        # make conditions based on the slice variables and l(d)
        c_i = self._make_conditions(d_i, semiring)

        return d_i, c_i, sampler.n_derivations()

    def _ancestral(self, l_slice, g, batch_size):
        semiring = self._semiring
        # compute f(d|u) \propto \pi(d) g(d) exactly
        rescorer = EarleyRescoring(l_slice,
                                   semiring=semiring,
                                   omega=lambda e: g[e],
                                   stateful=self._stateful,
                                   stateless=self._stateless,
                                   do_nothing=self._do_nothing)
        f2l_edges = defaultdict(None)
        f_slice = rescorer.do(root=self._tsort.root(), edge_mapping=f2l_edges)

        # sample a derivation
        sampler = AncestralSampler(f_slice,
                                   TopSortTable(f_slice),
                                   generations=self._generations)

        # sample a derivation from f(d|u)
        d_f = next(sampler.sample(1))

        # get rid of state annotation yielding a derivation as from l_slice
        # this is necessary because the state annotation is not consistent across iterations
        # remember that states get mapped onto integers on demand
        d_i = tuple(f2l_edges[e] for e in d_f[1:])

        # make new conditions and reset slice variables
        # edges and weights must come from l
        c_i = {e.lhs: semiring.as_real(e.weight) for e in d_i}

        return d_i, c_i, sampler.n_derivations()

    def sample(self, args):

        semiring = self._semiring

        # configure the appropriate sampler
        if args.within == 'ancestral':
            @timeit
            def t_sampler(forest, wtable, batch_size):
                return self._ancestral(forest, wtable, batch_size)
        elif args.within == 'importance':
            @timeit
            def t_sampler(forest, wtable, batch_size):
                return self._importance(forest, wtable, batch_size)
        else:
            @timeit
            def t_sampler(forest, wtable, batch_size):
                return self._uniform(forest, wtable, batch_size)

        d0 = self.sample_d0()
        logging.info('Initial sample: prob=%s tree=%s', self._sampler0.prob(d0),
                     inlinetree(make_nltk_tree(d0)))
        # get slice variables
        slicevars = self._make_slice_varialbes(self._make_conditions(d0, semiring), args.prior[0], args.prior[1])

        markov_chain = []
        slices_sizes = [self._sampler0.n_derivations()]
        # mean ratio of sliced edges constrained by u either uniformly or exponentially distributed
        muni, mexp = 1.0, 1.0
        report_size = lambda: ' |D|={:5d} uni={:5f} exp={:5f}'.format(slices_sizes[-1], muni, mexp)

        if args.progress:
            bar = progressbar(range(args.burn + (args.samples * args.lag)), prefix='Sampling', dynsuffix=report_size)
        else:
            bar = range(args.burn + (args.samples * args.lag))

        for _ in bar:

            # get a truncated forest weighted by l(d)
            # and a weight table corresponding to g(d)
            dt_s, (l_slice, g, muni, mexp) = t_prune_and_reweight(self._forest,
                                                                self._tsort,
                                                                slicevars,
                                                                semiring)

            dt_r, (d_i, c_i, n_i) = t_sampler(l_slice, g, args.batch)

            # update the history
            markov_chain.append(d_i)
            # reset slice variables
            slicevars.reset(c_i)

            # reports
            slices_sizes.append(n_i)
            #logging.info('Sample: tree=%s', inlinetree(make_nltk_tree(d_i)))

            if n_i == 0:
                raise ValueError('I found an empty slice!')

            self._report.report(len(markov_chain),
                                slice_terminals=l_slice.n_terminals(),
                                slice_nonterminals=l_slice.n_nonterminals(),
                                slice_edges=len(l_slice),
                                slice_derivations=n_i,
                                slicing_time=dt_s,
                                rescoring_time=dt_r,
                                constrained_included=muni,
                                unconstrained_included=mexp)

        logging.info('Slice size: mean=%s std=%s', np.mean(slices_sizes[1:]), np.std(slices_sizes[1:]))
        return markov_chain<commit_msg>bugfix uniform sampler<commit_after>"""
A new rescoring technique based on sampling.
A complete publication describing the method and a comparative study is to be submitted in early September 2015 to TACL.

:Authors: - Wilker Aziz
"""

import logging
import numpy as np
from collections import defaultdict, Counter
from itertools import chain

from easyhg.recipes import timeit
from easyhg.report import EmptyReport

from easyhg.grammar.semiring import SumTimes
from easyhg.grammar.cfg import CFG, CFGProduction, Terminal, Nonterminal, TopSortTable
from easyhg.grammar.utils import make_nltk_tree, inlinetree

from easyhg.alg.exact.inference import total_weight
from easyhg.alg.exact import AncestralSampler, EarleyRescoring
from .slicevars import ExpSliceVariables, ConstantPrior, SymmetricGamma, AsymmetricGamma

from easyhg.recipes import progressbar

def make_span_conditions(d, semiring):
    return {r.lhs: semiring.as_real(r.weight) for r in d}

def gamma_priors(forest, semiring, percentile=None):
    """
    Return a table mapping a cell to a Gamma scale parameter.
    """
    priors = defaultdict(None)
    if percentile is None:
        for lhs, rules in forest.iteritems():
            thetas = np.array([semiring.as_real(r.weight) for r in rules])
            priors[lhs] = thetas.mean()
            # logging.debug('{0} min={1} max={2} mean={3} prior={4}'.format(lhs, np.min(thetas), np.max(thetas), np.mean(thetas), priors[lhs]))
    else:
        for lhs, rules in forest.iteritems():
            thetas = np.array([semiring.as_real(r.weight) for r in rules])
            priors[lhs] = np.percentile(thetas, percentile)
            # logging.debug('{0} min={1} max={2} mean={3} prior={4}'.format(lhs, np.min(thetas), np.max(thetas), np.mean(thetas), priors[lhs]))
    #logging.info('cell=%s incoming=%d sum=%s mean=%s median=%s prior=%s', lhs, len(rules), thetas.sum(), thetas.mean(), np.percentile(thetas, 50), priors[lhs])
    return priors

def prune_and_reweight(forest, tsort, slicevars, semiring, dead_terminal=Terminal('<null>')):
    """
    Prune a forest and compute a new weight function for edges.

    :param forest: weighted forest
    :param tsort: forest's top-sort table
    :param slicevars: an instance of SliceVariable
    :param semiring: the appropriate semiring
    :param dead_terminal: a Terminal symbol which represents a dead-end.
    :return: pruned CFG and weight table mapping edges to weights.
    """
    oforest = CFG()
    discovered = set([tsort.root()])
    weight_table = defaultdict(None)
    pruned = 0
    total_cells = 0
    n_uniform = []
    n_exponential = []
    for level in tsort.iterlevels(reverse=True):  # we go top-down level by level
        for lhs in chain(*level):  # flatten the buckets
            total_cells += 1
            if lhs not in discovered:  # symbols not yet discovered have been pruned
                continue
            cell = lhs
            n = 0
            incoming = forest.get(lhs)
            # we sort the incoming edges in order to find out which ones are not in the slice
            for rule in sorted(incoming, key=lambda r: r.weight, reverse=True):  # TODO: use semiring to sort
                u = semiring.from_real(slicevars[cell])
                if semiring.gt(rule.weight, u):  # inside slice
                    oforest.add(rule)
                    weight_table[rule] = slicevars.pdf_semiring(cell, rule.weight, semiring)
                    discovered.update(filter(lambda s: isinstance(s, Nonterminal), rule.rhs))
                    n += 1
                else:  # this edge and the remaining are pruned
                    pruned += 1
                    #logging.info('pruning: %s vs %s', p, slicevars[cell])
                    break
            if slicevars.has_conditions(cell):
                n_uniform.append(float(n)/len(incoming))
            else:
                n_exponential.append(float(n)/len(incoming))
            #logging.info('cell=%s slice=%d/%d conditioning=%s', cell, n, len(incoming), slicevars.has_conditions(cell))
            if n == 0:  # this is a dead-end, instead of pruning bottom-up, we add an edge with 0 weight for simplicity
                dead = CFGProduction(lhs, [dead_terminal], semiring.zero)
                oforest.add(dead)
                weight_table[dead] = semiring.zero
    #logging.info('Pruning: cells=%s/%s in=%s out=%s', pruned, total_cells, np.array(_in).mean(), np.array(_out).mean())
    return oforest, weight_table, np.mean(n_uniform), np.mean(n_exponential)

@timeit
def t_prune_and_reweight(forest, tsort, slicevars, semiring, dead_terminal=Terminal('<null>')):
    return prune_and_reweight(forest, tsort, slicevars, semiring, dead_terminal)

def get_ancestral_sampler(forest, tsort, generations, temperature):
    if temperature == 1.0:
        return AncestralSampler(forest, tsort, generations=generations)
    else:
        return AncestralSampler(forest, tsort, generations=generations, omega=lambda e: e.weight/temperature)

@timeit
def t_get_ancestral_sampler(forest, tsort, generations, temperature):
    return get_ancestral_sampler(forest, tsort, generations, temperature)

class SlicedRescoring(object):

    def __init__(self, forest,
                 tsort,
                 stateless,
                 stateful,
                 semiring=SumTimes,
                 generations=10,
                 do_nothing=set(),
                 temperature0=1.0,
                 report=EmptyReport()):
        self._forest = forest
        self._tsort = tsort
        self._stateless = stateless
        self._stateful = stateful
        self._do_nothing = do_nothing
        self._semiring = semiring
        self._generations = generations
        self._report = report

        dt, self._sampler0 = t_get_ancestral_sampler(self._forest, self._tsort, self._generations, temperature0)

        logging.info('l-forest: derivations=%d', self._sampler0.n_derivations())

        self._make_conditions = make_span_conditions

        self._report.init(terminals=self._forest.n_terminals(),
                          nonterminals=self._forest.n_nonterminals(),
                          edges=len(self._forest),
                          derivations=self._sampler0.n_derivations(),
                          ancestral_time=dt)

    def sample_d0(self):
        """Draw an initial derivation from the locally weighted forest."""
        return next(self._sampler0.sample(1))

    def _rescore_derivation(self, d):
        semiring = self._semiring
        w = semiring.one
        # stateless scorer goes edge by edge
        # TODO: stateless.total_score(derivation)
        for edge in filter(lambda e: e.lhs not in self._do_nothing, d):
            w = semiring.times(w, self._stateless.score(edge))
        w = semiring.times(w, self._stateful.score_derivation(d))
        return w

    def _make_slice_varialbes(self, conditions, prior_type, prior_parameter):
        if prior_type == 'const':
            prior = ConstantPrior(const=float(prior_parameter))
            logging.debug('Constant scale: %s', prior_parameter)
        elif prior_type == 'sym':
            prior = SymmetricGamma(scale=float(prior_parameter))
            logging.debug('Symmetric Gamma: %s', prior_parameter)
        elif prior_type == 'asym':
            if prior_parameter == 'mean':
                prior = AsymmetricGamma(scales=gamma_priors(self._forest, self._semiring))
            else:
                try:
                    percentile = float(prior_parameter)
                except ValueError:
                    raise ValueError("The parameter of the asymmetric "
                                     "Gamma must be the keyword 'mean' or a number between 0-100: %s" % percentile)
                if not (0 <= percentile <= 100):
                    raise ValueError("A percentile is a real number between 0 and 100: %s" % percentile)
                prior = AsymmetricGamma(scales=gamma_priors(self._forest, self._semiring, percentile=percentile))
            logging.debug('Asymmetric Gamma: %s', prior_parameter)

        return ExpSliceVariables(conditions=conditions, prior=prior)

    def _uniform(self, l_slice, g, batch_size):
        semiring = self._semiring
        # sample uniformly from the truncated support of l(d)
        # note the omega(e) function
        sampler = AncestralSampler(l_slice,
                                   TopSortTable(l_slice),
                                   omega=lambda e: semiring.one if g[e] != semiring.zero else semiring.zero,
                                   generations=self._generations)

        # sample a number of derivations and group them by identity
        counts = Counter(sampler.sample(batch_size))
        # compute the empirical distribution f(d|u) \propto \pi(d) * g(d|u)
        support = [None] * len(counts)
        numerators = np.zeros(len(counts))
        for i, (d, n) in enumerate(counts.items()):
            support[i] = d
            # score using the complete f(d|u)
            pi_d = self._rescore_derivation(d)  # this is \pi(d)
            g_d = semiring.times.reduce(list(g[e] for e in d))  # this is g(d)
            w = semiring.times(pi_d, g_d)
            #numerators[i] = semiring.times(w, semiring.from_real(n))
            numerators[i] = w
        log_prob = numerators - np.logaddexp.reduce(numerators)
        prob = np.exp(log_prob)

        # sample a derivation
        i = np.random.choice(len(log_prob), p=prob)
        d_i = support[i]
        # make conditions based on the slice variables and l(d)
        c_i = self._make_conditions(d_i, semiring)
        return d_i, c_i, sampler.n_derivations()

    def _importance(self, l_slice, g, batch_size):
        semiring = self._semiring
        # sample from g(d) over the truncated support of l(d)
        sampler = AncestralSampler(l_slice,
                                   TopSortTable(l_slice),
                                   omega=lambda e: g[e],
                                   generations=self._generations)

        # sample a number of derivations and group them by identity
        counts = Counter(sampler.sample(batch_size))
        # compute the empirical (importance) distribution r(d) \propto f(d|u)/g(d|u)
        support = [None] * len(counts)
        numerators = np.zeros(len(counts))
        for i, (d, n) in enumerate(counts.items()):
            support[i] = d
            w = self._rescore_derivation(d)
            numerators[i] = semiring.times(w, semiring.from_real(n))
        log_prob = numerators - np.logaddexp.reduce(numerators)
        prob = np.exp(log_prob)

        # sample a derivation
        i = np.random.choice(len(log_prob), p=prob)
        d_i = support[i]
        # make conditions based on the slice variables and l(d)
        c_i = self._make_conditions(d_i, semiring)

        return d_i, c_i, sampler.n_derivations()

    def _ancestral(self, l_slice, g, batch_size):
        semiring = self._semiring
        # compute f(d|u) \propto \pi(d) g(d) exactly
        rescorer = EarleyRescoring(l_slice,
                                   semiring=semiring,
                                   omega=lambda e: g[e],
                                   stateful=self._stateful,
                                   stateless=self._stateless,
                                   do_nothing=self._do_nothing)
        f2l_edges = defaultdict(None)
        f_slice = rescorer.do(root=self._tsort.root(), edge_mapping=f2l_edges)

        # sample a derivation
        sampler = AncestralSampler(f_slice,
                                   TopSortTable(f_slice),
                                   generations=self._generations)

        # sample a derivation from f(d|u)
        d_f = next(sampler.sample(1))

        # get rid of state annotation yielding a derivation as from l_slice
        # this is necessary because the state annotation is not consistent across iterations
        # remember that states get mapped onto integers on demand
        d_i = tuple(f2l_edges[e] for e in d_f[1:])

        # make new conditions and reset slice variables
        # edges and weights must come from l
        c_i = {e.lhs: semiring.as_real(e.weight) for e in d_i}

        return d_i, c_i, sampler.n_derivations()

    def sample(self, args):

        semiring = self._semiring

        # configure the appropriate sampler
        if args.within == 'ancestral':
            @timeit
            def t_sampler(forest, wtable, batch_size):
                return self._ancestral(forest, wtable, batch_size)
        elif args.within == 'importance':
            @timeit
            def t_sampler(forest, wtable, batch_size):
                return self._importance(forest, wtable, batch_size)
        else:
            @timeit
            def t_sampler(forest, wtable, batch_size):
                return self._uniform(forest, wtable, batch_size)

        d0 = self.sample_d0()
        logging.info('Initial sample: prob=%s tree=%s', self._sampler0.prob(d0),
                     inlinetree(make_nltk_tree(d0)))
        # get slice variables
        slicevars = self._make_slice_varialbes(self._make_conditions(d0, semiring), args.prior[0], args.prior[1])

        markov_chain = []
        slices_sizes = [self._sampler0.n_derivations()]
        # mean ratio of sliced edges constrained by u either uniformly or exponentially distributed
        muni, mexp = 1.0, 1.0
        report_size = lambda: ' |D|={:5d} uni={:5f} exp={:5f}'.format(slices_sizes[-1], muni, mexp)

        if args.progress:
            bar = progressbar(range(args.burn + (args.samples * args.lag)), prefix='Sampling', dynsuffix=report_size)
        else:
            bar = range(args.burn + (args.samples * args.lag))

        for _ in bar:

            # get a truncated forest weighted by l(d)
            # and a weight table corresponding to g(d)
            dt_s, (l_slice, g, muni, mexp) = t_prune_and_reweight(self._forest,
                                                                self._tsort,
                                                                slicevars,
                                                                semiring)

            dt_r, (d_i, c_i, n_i) = t_sampler(l_slice, g, args.batch)

            # update the history
            markov_chain.append(d_i)
            # reset slice variables
            slicevars.reset(c_i)

            # reports
            slices_sizes.append(n_i)
            #logging.info('Sample: tree=%s', inlinetree(make_nltk_tree(d_i)))

            if n_i == 0:
                raise ValueError('I found an empty slice!')

            self._report.report(len(markov_chain),
                                slice_terminals=l_slice.n_terminals(),
                                slice_nonterminals=l_slice.n_nonterminals(),
                                slice_edges=len(l_slice),
                                slice_derivations=n_i,
                                slicing_time=dt_s,
                                rescoring_time=dt_r,
                                constrained_included=muni,
                                unconstrained_included=mexp)

        logging.info('Slice size: mean=%s std=%s', np.mean(slices_sizes[1:]), np.std(slices_sizes[1:]))
        return markov_chain<|endoftext|>


<commit_before>import os
import binascii
import functools

from tornado.web import RequestHandler, HTTPError

from lib.session import DummySessionStore, Session

class BaseHandler(RequestHandler):
    def __init__(self, application, request, **kwargs):
        RequestHandler.__init__(self, application, request, **kwargs)
        self._session = None

    def prepare(self):
        if self.application.admin_settings is None and type(self) != 'InstallHandler':
            self.redirect('/install')

    def on_finish(self):
        if self._session:
            self._session.save()

    @property
    def session(self):
        if self._session:
            return self._session
        if self.application.admin_settings is None:
            return Session(DummySessionStore(), None)

        session_id = self.get_secure_cookie('sid')
        if session_id is None:
            session_id = binascii.hexlify(os.urandom(16)).decode('ascii')
            self.set_secure_cookie('sid', session_id)
        else:
            session_id = session_id.decode('ascii')

        self._session = Session(self.application.session_store, session_id)
        return self._session

    def render(self, template_name, **kwargs):
        self.write(
            self.application.renderer.render(
                template_name,
                logged_in=self.session['logged_in'] is not None,
                xsrf_token=self.xsrf_form_html(),
                **kwargs
            )
        )

    def redirect(self, url, permanent=False, status=None):
        RequestHandler.redirect(
            self,
            self.application.config.subdir + url,
            permanent,
            status
        )

def authenticated(method):
    @functools.wraps(method)
    def wrapper(self, *args, **kwargs):
        if not self.session['logged_in']:
            if self.request.method in ('GET', 'HEAD'):
                self.redirect(self.get_login_url())
                return
            raise HTTPError(403)
        return method(self, *args, **kwargs)
    return wrapper
<commit_msg>Actually fixed installation redirect<commit_after>import os
import binascii
import functools

from tornado.web import RequestHandler, HTTPError

from lib.session import DummySessionStore, Session

class BaseHandler(RequestHandler):
    def __init__(self, application, request, **kwargs):
        RequestHandler.__init__(self, application, request, **kwargs)
        self._session = None

    def prepare(self):
        if self.application.admin_settings is None and type(self).__name__ != 'InstallHandler':
            self.redirect('/install')

    def on_finish(self):
        if self._session:
            self._session.save()

    @property
    def session(self):
        if self._session:
            return self._session
        if self.application.admin_settings is None:
            return Session(DummySessionStore(), None)

        session_id = self.get_secure_cookie('sid')
        if session_id is None:
            session_id = binascii.hexlify(os.urandom(16)).decode('ascii')
            self.set_secure_cookie('sid', session_id)
        else:
            session_id = session_id.decode('ascii')

        self._session = Session(self.application.session_store, session_id)
        return self._session

    def render(self, template_name, **kwargs):
        self.write(
            self.application.renderer.render(
                template_name,
                logged_in=self.session['logged_in'] is not None,
                xsrf_token=self.xsrf_form_html(),
                **kwargs
            )
        )

    def redirect(self, url, permanent=False, status=None):
        RequestHandler.redirect(
            self,
            self.application.config.subdir + url,
            permanent,
            status
        )

def authenticated(method):
    @functools.wraps(method)
    def wrapper(self, *args, **kwargs):
        if not self.session['logged_in']:
            if self.request.method in ('GET', 'HEAD'):
                self.redirect(self.get_login_url())
                return
            raise HTTPError(403)
        return method(self, *args, **kwargs)
    return wrapper
<|endoftext|>


<commit_before>
import os
import datetime
import webapp2 as webapp

import ctrl.blog
import handlers

class BasePage(handlers.BaseHandler):
  pass

class HomePage(BasePage):
  def get(self):
    self.render('index.html', {})

class ShowcasePage(BasePage):
  def get(self):
    self.render('showcase.html', {})

class NotFoundPage(BasePage):
  def get(self):
    self.error(404)

app = webapp.WSGIApplication([('/?', HomePage),
                              ('/showcase', ShowcasePage),
                              ('.*', NotFoundPage)],
                             debug=os.environ['SERVER_SOFTWARE'].startswith('Development'))
<commit_msg>Added some hacks to the 'NotFound' page for common mistakes I see in the logs.<commit_after>
import os
import datetime
import webapp2 as webapp

from google.appengine.ext.webapp import blobstore_handlers
from google.appengine.ext import blobstore
from google.appengine.api import images
from google.appengine.api import users
from datetime import datetime, timedelta
import json

import ctrl.blog
import handlers

class BasePage(handlers.BaseHandler):
  pass

class HomePage(BasePage):
  def get(self):
    self.render('index.html', {})

class ShowcasePage(BasePage):
  def get(self):
    self.render('showcase.html', {})

class BlobUploadUrlPage(handlers.BaseHandler):
  def get(self):
    """Gets a new upload URL for uploading blobs."""
    if not users.get_current_user():
      self.response.set_status(403)
      return
    data = {'upload_url': blobstore.create_upload_url('/blob/upload-complete')}
    self.response.headers['Content-Type'] = 'application/json'
    self.response.write(json.dumps(data))

class BlobUploadCompletePage(blobstore_handlers.BlobstoreUploadHandler):
  def post(self):
    blob_info = self.get_uploads('file')[0]
    img = images.Image(blob_key=blob_info)
    img.im_feeling_lucky() # we have to do a transform in order to get dimensions
    img.execute_transforms()
    response = {'success': True,
                'blob_key': str(blob_info.key()),
                'size': blob_info.size,
                'filename': blob_info.filename,
                'width': img.width,
                'height': img.height,
                'url': images.get_serving_url(blob_info.key(), 100, 0)}
    self.response.headers['Content-Type'] = 'application/json'
    self.response.write(json.dumps(response))

class BlobPage(blobstore_handlers.BlobstoreDownloadHandler):
  def get(self, blob_key):
    if not blobstore.get(blob_key):
      self.error(404)
    else:
      self.response.headers['Cache-Control'] = 'public, max-age='+str(30*24*60*60) # 30 days
      self.response.headers['Expires'] = (datetime.now() + timedelta(days=30)).strftime('%a, %d %b %Y %H:%M:%S GMT')
      self.send_blob(blob_key)

class BlobInfoPage(handlers.BaseHandler):
  def get(self, blob_key):
    info = self._getBlobInfo(blob_key)
    if not info:
      self.error(404)
    self.response.headers['Content-Type'] = 'application/json'
    self.response.write(json.dumps(info))

  def post(self, blob_key):
    info = self._getBlobInfo(blob_key)
    if not info:
      self.error(404)
    self.response.headers['Content-Type'] = 'application/json'
    self.response.write(json.dumps(info))

  def _getBlobInfo(self, blob_key):
    blob_info = blobstore.get(blob_key)
    if not blob_info:
      return None

    size = None
    crop = None
    if self.request.POST.get('size'):
      size = int(self.request.POST.get('size'))
    if self.request.POST.get('crop'):
      crop = int(self.request.POST.get('crop'))

    return {'size': blob_info.size,
            'filename': blob_info.filename,
            'url': images.get_serving_url(blob_key, size, crop)}

class NotFoundPage(BasePage):
  """This is the 404 "not found" page. We'll try to do some guesses as to what you may have wanted though."""
  def get(self):
    if "index.php?tempskin=" in self.request.path_qs:
      self.redirect("/blog/rss")
    elif "index.php" in self.request.path_qs:
      self.redirect("/")
    elif self.request.path == "/war-worlds/":
      self.redirect("http://www.war-worlds.com/")
    else:
      self.error(404)

app = webapp.WSGIApplication([('/?', HomePage),
                              ('/showcase', ShowcasePage),
                              ('/blob/upload-url', BlobUploadUrlPage),
                              ('/blob/upload-complete', BlobUploadCompletePage),
                              ('/blob/([^/]+)', BlobPage),
                              ('/blob/([^/]+)/info', BlobInfoPage),
                              ('.*', NotFoundPage)],
                             debug=os.environ['SERVER_SOFTWARE'].startswith('Development'))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from yahdlc import *
from threading import Thread, Event, Lock
from queue import Queue, Full
from time import sleep, time

class HDLController:
	"""
	An HDLC controller based on python4yahdlc.
	"""

	MAX_SEQ_NO = 8
	MIN_SENDING_TIMEOUT = 0.5

	def __init__(self, read_func, write_func, sending_timeout=2, window=3, frames_queue_size=10):
		if not hasattr(read_func, '__call__'):
			raise TypeError('The read function parameter is not a callable object')
		if not hasattr(write_func, '__call__'):
			raise TypeError('The write function parameter is not a callable object')

		self.read = read_func
		self.write = write_func

		self.window = window
		self.senders = dict()
		self.send_lock = Lock()
		self.new_seq_no = 0

		self.send_callback = None
		self.receive_callback = None

		self.set_sending_timeout(sending_timeout)

		self.receiver = None
		self.frames_received = Queue(frames_queue_size)

	def start(self):
		"""
		Start HDLC controller's threads.
		"""

		self.receiver = self.Receiver(
			self.read,
			self.write,
			self.send_lock,
			self.senders,
			self.frames_received,
			callback=self.receive_callback,
		)

		self.receiver.start()

	def stop(self):
		"""
		Stop HDLC controller's threads.
		"""

		if self.receiver != None:
			self.receiver.join()

		for s in self.senders.values():
			s.join()

	def set_send_callback(self, callback):
		"""
		Set the send callback function.

		If the HDLC controller has already
		been started, the new callback
		function will be take into account
		for the next data frames to send.
		"""

		if not hasattr(callback, '__call__'):
			raise TypeError('The callback function parameter is not a callable object')

		self.send_callback = callback

	def set_receive_callback(self, callback):
		"""
		Set the receive callback function.

		This method has to be called before
		starting the HDLC controller.
		"""

		if not hasattr(callback, '__call__'):
			raise TypeError('The callback function parameter is not a callable object')

		self.receive_callback = callback

	def set_sending_timeout(self, sending_timeout):
		"""
		Set the sending timeout.
		"""

		if sending_timeout >= HDLController.MIN_SENDING_TIMEOUT:
			self.sending_timeout = sending_timeout

	def get_senders_number(self):
		"""
		Return the number of active
		senders.
		"""

		return len(self.senders)

	def send(self, data):
		"""
		Send a new data frame.

		This method will block until a new room is
		available for a new sender. This limit is
		determined by the size of the window.
		"""

		while len(self.senders) >= self.window:
			pass

		self.senders[self.new_seq_no] = self.Sender(
			self.write,
			self.send_lock,
			data,
			self.new_seq_no,
			timeout=self.sending_timeout,
			callback=self.send_callback,
		)

		self.senders[self.new_seq_no].start()
		self.new_seq_no = (self.new_seq_no + 1) % HDLController.MAX_SEQ_NO

	def get_data(self):
		"""
		Get the next frame received.

		This method will block until a new
		data frame is available.
		"""

		return self.frames_received.get()

	class Sender(Thread):
		"""
		Thread used to send HDLC frames.
		"""

		def __init__(self, write_func, send_lock, data, seq_no, timeout=2, callback=None):
			super().__init__()
			self.write = write_func
			self.send_lock = send_lock
			self.data = data
			self.seq_no = seq_no
			self.timeout = timeout
			self.callback = callback
			self.ack = Event()

		def run(self):
			with self.send_lock:
				self.__send_data()

			while not self.ack.isSet():
				if time() >= self.next_timeout:
					with self.send_lock:
						self.__send_data()

		def join(self, timeout=None):
			"""
			Stop the current thread.
			"""

			self.ack.set()
			super().join(timeout)

		def ack_received(self):
			"""
			Inform the sender that the ack frame
			has been received which has the
			consequence of stopping the current
			thread.
			"""

			self.join()

		def nack_received(self):
			"""
			Inform the sender that a nack frame
			has been received which has the
			consequence of resending the data
			frame.
			"""

			with self.send_lock:
				self.__send_data()

		def __send_data(self):
			"""
			Send a new data frame.
			"""

			if self.callback != None:
				self.callback(self.data)

			self.write(frame_data(self.data, FRAME_DATA, self.seq_no))
			self.next_timeout = time() + self.timeout

	class Receiver(Thread):
		"""
		Thread used to receive HDLC frames.
		"""

		def __init__(self, read_func, write_func, send_lock, senders_list, frames_received, callback=None):
			super().__init__()
			self.read = read_func
			self.write = write_func
			self.send_lock = send_lock
			self.senders = senders_list
			self.frames_received = frames_received
			self.callback = callback
			self.stop_receiver = Event()

		def run(self):
			while not self.stop_receiver.isSet():
				try:
					data, type, seq_no = get_data(self.read())

					if type == FRAME_DATA:
						with self.send_lock:
							if self.callback != None:
								self.callback(data)

							self.frames_received.put_nowait(data)
							self.__send_ack((seq_no + 1) % HDLController.MAX_SEQ_NO)
					elif type == FRAME_ACK:
						seq_no_sent = (seq_no - 1) % HDLController.MAX_SEQ_NO
						self.senders[seq_no_sent].ack_received()
						del self.senders[seq_no_sent]
					elif type == FRAME_NACK:
						self.senders[seq_no].nack_received()
					else:
						raise TypeError('Bad frame type received')
				except MessageError:
					# No HDLC frame detected
					pass
				except KeyError:
					# Drop bad (n)ack
					pass
				except Full:
					# Drop new data frame when
					# the frames received queue
					# is full
					pass
				except FCSError:
					with self.send_lock:
						self.__send_nack(seq_no)
				finally:
					# 200 µs
					sleep(200 / 1000000.0)

		def join(self, timeout=None):
			"""
			Stop the current thread.
			"""

			self.stop_receiver.set()
			super().join(timeout)

		def __send_ack(self, seq_no):
			"""
			Send a new ack frame.
			"""

			self.write(frame_data('', FRAME_ACK, seq_no))

		def __send_nack(self, seq_no):
			"""
			Send a new nack frame.
			"""

			self.write(frame_data('', FRAME_NACK, seq_no))

if __name__ == '__main__':
	import serial
	from sys import stdout, stderr
	from argparse import ArgumentParser

	ap = ArgumentParser(
		description='HDLC controller example',
		epilog="Example: ./hdlcontroller.py -d /dev/ttyUSB0 -b 115200 -m 'Hello world!'",
	)
	ap.add_argument('-d', '--device', default='/dev/ttyACM0', help='serial device to use (default: /dev/ttyACM0)')
	ap.add_argument('-b', '--baudrate', type=int, default='9600', help='serial baudrate value (default: 9600)')
	ap.add_argument('-t', '--timeout', type=int, default='0', help='serial read timeout value (default: 0)')
	ap.add_argument('-m', '--message', default='test', help='test message to send (default: test)')
	ap.add_argument('-i', '--interval', type=float, default='1.0', help='sending interval between two data frames (default: 1.0)')
	args = vars(ap.parse_args())

	# Serial port configuration
	ser = serial.Serial()
	ser.port = args['device']
	ser.baudrate = args['baudrate']
	ser.timeout = args['timeout']

	stdout.write('[*] Connection ...\n')

	try:
		ser.open()
	except serial.serialutil.SerialException as e:
		stderr.write('[x] Serial connection problem : {0}\n'.format(e))
		exit(1)

	def read_uart():
		return ser.read(ser.inWaiting())

	def send_callback(data):
		print('> {0}'.format(data))

	def receive_callback(data):
		print('< {0}'.format(data))

	try:
		hdlc_c = HDLController(read_uart, ser.write)
		hdlc_c.set_send_callback(send_callback)
		hdlc_c.set_receive_callback(receive_callback)
		hdlc_c.start()
		while True:
			hdlc_c.send(args['message'])
			sleep(args['interval'])
	except KeyboardInterrupt:
		stdout.write('[*] Bye !\n')
		hdlc_c.stop()
		ser.close()
<commit_msg>add argparse 'window' option<commit_after>#!/usr/bin/env python3

from yahdlc import *
from threading import Thread, Event, Lock
from queue import Queue, Full
from time import sleep, time

class HDLController:
	"""
	An HDLC controller based on python4yahdlc.
	"""

	MAX_SEQ_NO = 8
	MIN_SENDING_TIMEOUT = 0.5

	def __init__(self, read_func, write_func, sending_timeout=2, window=3, frames_queue_size=10):
		if not hasattr(read_func, '__call__'):
			raise TypeError('The read function parameter is not a callable object')
		if not hasattr(write_func, '__call__'):
			raise TypeError('The write function parameter is not a callable object')

		self.read = read_func
		self.write = write_func

		self.window = window
		self.senders = dict()
		self.send_lock = Lock()
		self.new_seq_no = 0

		self.send_callback = None
		self.receive_callback = None

		self.set_sending_timeout(sending_timeout)

		self.receiver = None
		self.frames_received = Queue(frames_queue_size)

	def start(self):
		"""
		Start HDLC controller's threads.
		"""

		self.receiver = self.Receiver(
			self.read,
			self.write,
			self.send_lock,
			self.senders,
			self.frames_received,
			callback=self.receive_callback,
		)

		self.receiver.start()

	def stop(self):
		"""
		Stop HDLC controller's threads.
		"""

		if self.receiver != None:
			self.receiver.join()

		for s in self.senders.values():
			s.join()

	def set_send_callback(self, callback):
		"""
		Set the send callback function.

		If the HDLC controller has already
		been started, the new callback
		function will be take into account
		for the next data frames to send.
		"""

		if not hasattr(callback, '__call__'):
			raise TypeError('The callback function parameter is not a callable object')

		self.send_callback = callback

	def set_receive_callback(self, callback):
		"""
		Set the receive callback function.

		This method has to be called before
		starting the HDLC controller.
		"""

		if not hasattr(callback, '__call__'):
			raise TypeError('The callback function parameter is not a callable object')

		self.receive_callback = callback

	def set_sending_timeout(self, sending_timeout):
		"""
		Set the sending timeout.
		"""

		if sending_timeout >= HDLController.MIN_SENDING_TIMEOUT:
			self.sending_timeout = sending_timeout

	def get_senders_number(self):
		"""
		Return the number of active
		senders.
		"""

		return len(self.senders)

	def send(self, data):
		"""
		Send a new data frame.

		This method will block until a new room is
		available for a new sender. This limit is
		determined by the size of the window.
		"""

		while len(self.senders) >= self.window:
			pass

		self.senders[self.new_seq_no] = self.Sender(
			self.write,
			self.send_lock,
			data,
			self.new_seq_no,
			timeout=self.sending_timeout,
			callback=self.send_callback,
		)

		self.senders[self.new_seq_no].start()
		self.new_seq_no = (self.new_seq_no + 1) % HDLController.MAX_SEQ_NO

	def get_data(self):
		"""
		Get the next frame received.

		This method will block until a new
		data frame is available.
		"""

		return self.frames_received.get()

	class Sender(Thread):
		"""
		Thread used to send HDLC frames.
		"""

		def __init__(self, write_func, send_lock, data, seq_no, timeout=2, callback=None):
			super().__init__()
			self.write = write_func
			self.send_lock = send_lock
			self.data = data
			self.seq_no = seq_no
			self.timeout = timeout
			self.callback = callback
			self.ack = Event()

		def run(self):
			with self.send_lock:
				self.__send_data()

			while not self.ack.isSet():
				if time() >= self.next_timeout:
					with self.send_lock:
						self.__send_data()

		def join(self, timeout=None):
			"""
			Stop the current thread.
			"""

			self.ack.set()
			super().join(timeout)

		def ack_received(self):
			"""
			Inform the sender that the ack frame
			has been received which has the
			consequence of stopping the current
			thread.
			"""

			self.join()

		def nack_received(self):
			"""
			Inform the sender that a nack frame
			has been received which has the
			consequence of resending the data
			frame.
			"""

			with self.send_lock:
				self.__send_data()

		def __send_data(self):
			"""
			Send a new data frame.
			"""

			if self.callback != None:
				self.callback(self.data)

			self.write(frame_data(self.data, FRAME_DATA, self.seq_no))
			self.next_timeout = time() + self.timeout

	class Receiver(Thread):
		"""
		Thread used to receive HDLC frames.
		"""

		def __init__(self, read_func, write_func, send_lock, senders_list, frames_received, callback=None):
			super().__init__()
			self.read = read_func
			self.write = write_func
			self.send_lock = send_lock
			self.senders = senders_list
			self.frames_received = frames_received
			self.callback = callback
			self.stop_receiver = Event()

		def run(self):
			while not self.stop_receiver.isSet():
				try:
					data, type, seq_no = get_data(self.read())

					if type == FRAME_DATA:
						with self.send_lock:
							if self.callback != None:
								self.callback(data)

							self.frames_received.put_nowait(data)
							self.__send_ack((seq_no + 1) % HDLController.MAX_SEQ_NO)
					elif type == FRAME_ACK:
						seq_no_sent = (seq_no - 1) % HDLController.MAX_SEQ_NO
						self.senders[seq_no_sent].ack_received()
						del self.senders[seq_no_sent]
					elif type == FRAME_NACK:
						self.senders[seq_no].nack_received()
					else:
						raise TypeError('Bad frame type received')
				except MessageError:
					# No HDLC frame detected
					pass
				except KeyError:
					# Drop bad (n)ack
					pass
				except Full:
					# Drop new data frame when
					# the frames received queue
					# is full
					pass
				except FCSError:
					with self.send_lock:
						self.__send_nack(seq_no)
				finally:
					# 200 µs
					sleep(200 / 1000000.0)

		def join(self, timeout=None):
			"""
			Stop the current thread.
			"""

			self.stop_receiver.set()
			super().join(timeout)

		def __send_ack(self, seq_no):
			"""
			Send a new ack frame.
			"""

			self.write(frame_data('', FRAME_ACK, seq_no))

		def __send_nack(self, seq_no):
			"""
			Send a new nack frame.
			"""

			self.write(frame_data('', FRAME_NACK, seq_no))

if __name__ == '__main__':
	import serial
	from sys import stdout, stderr
	from argparse import ArgumentParser

	ap = ArgumentParser(
		description='HDLC controller example',
		epilog="Example: ./hdlcontroller.py -d /dev/ttyUSB0 -b 115200 -m 'Hello world!'",
	)
	ap.add_argument('-d', '--device', default='/dev/ttyACM0', help='serial device to use (default: /dev/ttyACM0)')
	ap.add_argument('-b', '--baudrate', type=int, default='9600', help='serial baudrate value (default: 9600)')
	ap.add_argument('-t', '--timeout', type=int, default='0', help='serial read timeout value (default: 0)')
	ap.add_argument('-m', '--message', default='test', help='test message to send (default: test)')
	ap.add_argument('-i', '--interval', type=float, default='1.0', help='sending interval between two data frames (default: 1.0)')
	ap.add_argument('-w', '--window', type=int, default='3', help='sending window')
	args = vars(ap.parse_args())

	# Serial port configuration
	ser = serial.Serial()
	ser.port = args['device']
	ser.baudrate = args['baudrate']
	ser.timeout = args['timeout']

	stdout.write('[*] Connection ...\n')

	try:
		ser.open()
	except serial.serialutil.SerialException as e:
		stderr.write('[x] Serial connection problem : {0}\n'.format(e))
		exit(1)

	def read_uart():
		return ser.read(ser.inWaiting())

	def send_callback(data):
		print('> {0}'.format(data))

	def receive_callback(data):
		print('< {0}'.format(data))

	try:
		hdlc_c = HDLController(read_uart, ser.write, window=args['window'])
		hdlc_c.set_send_callback(send_callback)
		hdlc_c.set_receive_callback(receive_callback)
		hdlc_c.start()
		while True:
			hdlc_c.send(args['message'])
			sleep(args['interval'])
	except KeyboardInterrupt:
		stdout.write('[*] Bye !\n')
		hdlc_c.stop()
		ser.close()
<|endoftext|>


<commit_before>"""
Responsible for reading in configuration files, validating the proper
format and providing sane defaults for parts that don't have any.

"""
import json
import logging
from os import path
import platform
import sys
import yaml

from helper import NullHandler

(major, minor, rev) = platform.python_version_tuple()
if float('%s.%s' % (major, minor)) < 2.7:
    import logutils.dictconfig as logging_config
else:
    from logging import config as logging_config

LOGGER = logging.getLogger(__name__)

class Config(object):
    """The Config object holds the current state of the configuration for an
    application. If no configuration file is provided, it will used a set of
    defaults with very basic behavior for logging and daemonization.

    """
    APPLICATION = {'wake_interval': 60}
    DAEMON = {'user': None,
              'group': None,
              'pidfile': None,
              'prevent_core': True}
    LOGGING_FORMAT = ('%(levelname) -10s %(asctime)s %(process)-6d '
                      '%(processName) -20s %(threadName)-12s %(name) -30s '
                      '%(funcName) -25s L%(lineno)-6d: %(message)s')
    LOGGING = {'disable_existing_loggers': True,
               'filters': dict(),
               'formatters': {'verbose': {'datefmt': '%Y-%m-%d %H:%M:%S',
                                          'format': LOGGING_FORMAT}},
               'handlers': {'console': {'class': 'logging.StreamHandler',
                                        'debug_only': True,
                                        'formatter': 'verbose'}},
               'incremental': False,
               'loggers': {'helper': {'handlers': ['console'],
                                      'level': 'INFO',
                                      'propagate': True}},
               'root': {'handlers': [],
                        'level': logging.CRITICAL,
                        'propagate': True},
               'version': 1}

    def __init__(self, file_path=None):
        """Create a new instance of the configuration object, passing in the
        path to the configuration file.

        :param str file_path:

        """
        self.application = Data(self.APPLICATION)
        self.daemon = Data(self.DAEMON)
        self._file_path = None
        self._values = Data()
        if file_path:
            self._file_path = self._validate(file_path)
            self._values = self._load_config_file()

        app = self._values.get('Application', self._values.get('application'))
        self._assign_values(self.application, app)
        daemon = self._values.get('Daemon', self._values.get('daemon'))
        self._assign_values(self.daemon, daemon)

    @staticmethod
    def _assign_values(obj, values):
        """Assign values to the object passed in from the dictionary of values.

        :param Data obj: Data object to assign values to
        :param dict values: Values to assign

        """
        for key in values or dict():
            setattr(obj, key, values[key])

    def get(self, name, default=None):
        """Return the value for key if key is in the configuration, else default.

        :param str name: The key name to return
        :param mixed default: The default value for the key
        :return: mixed

        """
        return self._values.get(name, default)

    @property
    def logging(self):
        """Return the logging configuration in the form of a dictionary.

        :rtype: dict

        """
        config = self.LOGGING
        config_in = self._values.get('Logging',
                                     self._values.get('logging', {}))
        for section in ['formatters', 'handlers', 'loggers',
                        'filters', 'root']:
            if section in config_in:
                for key in config_in[section]:
                    config[section][key] = config_in[section][key]
        LOGGER.debug(config)
        return config

    def reload(self):
        """Reload the configuration from disk returning True if the
        configuration has changed from the previous values.

        """
        if self._file_path:

            # Try and reload the configuration file from disk
            try:
                values = Data(self._load_config_file())
            except ValueError as error:
                LOGGER.error('Could not reload configuration: %s', error)
                return False

            # Only update the configuration if the values differ
            if hash(values) != hash(self._values):
                self._values = values
                return True

        return False

    def _load_config_file(self):
        """Load the configuration file into memory, returning the content.

        """
        LOGGER.info('Loading configuration from %s', self._file_path)
        if self._file_path.endswith('json'):
            return self._load_json_config()
        # Keep the behavior as it was with regard to file extensions for YAML
        return self._load_yaml_config()

    def _load_json_config(self):
        """Load the configuration file in JSON format

        :rtype: dict

        """
        try:
            with open(self._file_path, 'r') as handle:
                return json.load(handle)
        except (OSError, ValueError) as error:
            raise ValueError('Could not read configuration file: %s' % error)

    def _load_yaml_config(self):
        """Loads the configuration file from a .yaml or .yml file

        :type: dict

        """
        try:
            config = open(self._file_path).read()
        except OSError as error:
            raise ValueError('Could not read configuration file: %s' % error)
        try:
            return yaml.safe_load(config)
        except yaml.YAMLError as error:
            message = '\n'.join(['    > %s' % line
                                 for line in str(error).split('\n')])
            sys.stderr.write("\n\n  Error in the configuration file:\n\n"
                             "%s\n\n" % message)
            sys.stderr.write("  Configuration should be a valid YAML file.\n")
            sys.stderr.write("  YAML format validation available at "
                             "http://yamllint.com\n")
            raise ValueError(error)

    @staticmethod
    def _validate(file_path):
        """Normalize the path provided and ensure the file path, raising a
        ValueError if the file does not exist.

        :param str file_path:
        :return: str
        :raises: ValueError

        """
        file_path = path.abspath(file_path)
        if not path.exists(file_path):
            raise ValueError('Configuration file not found: %s' % file_path)
        return file_path

class LoggingConfig(object):
    """The Logging class is used for abstracting away dictConfig logging
    semantics and can be used by sub-processes to ensure consistent logging
    rule application.

    """
    DEBUG_ONLY = 'debug_only'
    HANDLERS = 'handlers'
    LOGGERS = 'loggers'

    def __init__(self, configuration, debug=None):
        """Create a new instance of the Logging object passing in the
        DictConfig syntax logging configuration and a debug flag.

        :param dict configuration: The logging configuration
        :param bool debug: Toggles use of debug_only loggers

        """
        # Force a NullLogger for some libraries that require it
        root_logger = logging.getLogger()
        root_logger.addHandler(NullHandler())

        self.config = configuration
        self.debug = debug
        self.configure()

    def update(self, configuration, debug=None):
        """Update the internal configuration values, removing debug_only
        handlers if debug is False. Returns True if the configuration has
        changed from previous configuration values.

        :param dict configuration: The logging configuration
        :param bool debug: Toggles use of debug_only loggers
        :rtype: bool

        """

        if hash(self.config) != hash(configuration) and debug != self.debug:
            self.config = configuration
            self.debug = debug
            self.configure()
            return True
        return False

    def configure(self):
        """Configure the Python stdlib logger"""
        if self.debug is not None and not self.debug:
            self._remove_debug_handlers()
        self._remove_debug_only()
        logging_config.dictConfig(self.config)
        try:
            logging.captureWarnings(True)
        except AttributeError:
            pass

    def _remove_debug_only(self):
        """Iterate through each handler removing the invalid dictConfig key of
        debug_only.

        """
        LOGGER.debug('Removing debug only from handlers')
        for handler in self.config[self.HANDLERS]:
            if self.DEBUG_ONLY in self.config[self.HANDLERS][handler]:
                del self.config[self.HANDLERS][handler][self.DEBUG_ONLY]

    def _remove_debug_handlers(self):
        """Remove any handlers with an attribute of debug_only that is True and
        remove the references to said handlers from any loggers that are
        referencing them.

        """
        remove = list()
        for handler in self.config[self.HANDLERS]:
            if self.config[self.HANDLERS][handler].get('debug_only'):
                remove.append(handler)
        for handler in remove:
            del self.config[self.HANDLERS][handler]
            for logger in self.config[self.LOGGERS].keys():
                logger = self.config[self.LOGGERS][logger]
                if handler in logger[self.HANDLERS]:
                    logger[self.HANDLERS].remove(handler)
        self._remove_debug_only()

class Data(object):
    """Data object configuration is wrapped in, can be used as a object with
    attributes or as a dict.

    """
    def __init__(self, value=None):
        super(Data, self).__init__()
        if value and isinstance(value, dict):
            for name in value.keys():
                if isinstance(value[name], dict):
                    object.__setattr__(self, name, Data(value[name]))
                else:
                    object.__setattr__(self, name, value[name])

    def __contains__(self, name):
        return name in self.__dict__.keys()

    def __delattr__(self, name):
        object.__delattr__(self, name)

    def __delitem__(self, name):
        if name not in self.__dict__:
            raise KeyError(name)
        object.__delattr__(self, name)

    def __getattribute__(self, name):
        return object.__getattribute__(self, name)

    def __getitem__(self, name):
        return object.__getattribute__(self, name)

    def __setitem__(self, name, value):
        if isinstance(value, dict) and name != '__dict__':
            value = Data(value)
        object.__setattr__(self, name, value)

    def __setattr__(self, name, value):
        if isinstance(value, dict) and name != '__dict__':
            value = Data(value)
        object.__setattr__(self, name, value)

    def __repr__(self):
        return repr(self.__dict__)

    def __len__(self):
        return len(self.__dict__)

    def __iter__(self):
        for name in self.__dict__.keys():
            yield name

    def str(self):
        """Return a string representation of the data object.

        :rtype: str

        """
        return str(self.__dict__)

    def dict(self):
        """Return the data object as a dictionary, recursively casting children
        that are Data objects as well.

        :rtype: dict

        """
        output = dict()
        for key, value in self.items():
            if isinstance(value, Data):
                output[key] = value.dict()
            else:
                output[key] = value
        return output

    def get(self, name, default=None):
        """Return the value for key if key is in the dictionary, else default.
        If default is not given, it defaults to None, so that this method
        never raises a KeyError.

        :param str name: The key name to return
        :param mixed default: The default value for the key
        :return: mixed

        """
        return self.__dict__.get(name, default)

    def has_key(self, name):
        """Test for the presence of key in the data object. has_key() is
        deprecated in favor of key in d.

        :param name:
        :return: bool

        """
        return name in self.__dict__

    def items(self):
        """Return a copy of the dictionary's list of (key, value) pairs.

        :rtype: list

        """
        return self.__dict__.items()

    def itervalues(self):
        """Return an iterator over the data values. See the note for
        Data.items().

        Using itervalues() while adding or deleting entries in the data object
        may raise a RuntimeError or fail to iterate over all entries.

        :rtype: iterator
        :raises: RuntimeError

        """
        return self.__dict__.itervalues()

    def keys(self):
        """Return a copy of the dictionary's list of keys. See the note for
        Data.items()

        :rtype: list

        """
        return self.__dict__.keys()

    def pop(self, name, default=None):
        """If key is in the dictionary, remove it and return its value, else
        return default. If default is not given and key is not in the
        dictionary, a KeyError is raised.

        :param str name: The key name
        :param mixed default: The default value
        :raises: KeyError

        """
        return self.__dict__.pop(name, default)

    def setdefault(self, name, default=None):
        """If key is in the dictionary, return its value. If not, insert key
        with a value of default and return default. default defaults to None.

        :param str name: The key
        :param mixed default: The value
        :return: mixed

        """
        return self.__dict__.setdefault(name, default)

    def update(self, other=None, **kwargs):
        """Update the dictionary with the key/value pairs from other,
        overwriting existing keys. update() accepts either another dictionary
        object or an iterable of key/value pairs (as tuples or other iterables
        of length two). If keyword arguments are specified, the dictionary is
        then updated with those key/value pairs: d.update(red=1, blue=2).

        :param dict other: Dict or other iterable
        :param dict kwargs: Key/value pairs to update
        :rtype: None

        """
        self.__dict__.update(other, **kwargs)

    def values(self):
        """Return the configuration values

        :rtype: list

        """
        return self.__dict__.values()
<commit_msg>fix config reloading<commit_after>"""
Responsible for reading in configuration files, validating the proper
format and providing sane defaults for parts that don't have any.

"""
import json
import logging
from os import path
import platform
import sys
import yaml

from helper import NullHandler

(major, minor, rev) = platform.python_version_tuple()
if float('%s.%s' % (major, minor)) < 2.7:
    import logutils.dictconfig as logging_config
else:
    from logging import config as logging_config

LOGGER = logging.getLogger(__name__)

class Config(object):
    """The Config object holds the current state of the configuration for an
    application. If no configuration file is provided, it will used a set of
    defaults with very basic behavior for logging and daemonization.

    """
    APPLICATION = {'wake_interval': 60}
    DAEMON = {'user': None,
              'group': None,
              'pidfile': None,
              'prevent_core': True}
    LOGGING_FORMAT = ('%(levelname) -10s %(asctime)s %(process)-6d '
                      '%(processName) -20s %(threadName)-12s %(name) -30s '
                      '%(funcName) -25s L%(lineno)-6d: %(message)s')
    LOGGING = {'disable_existing_loggers': True,
               'filters': dict(),
               'formatters': {'verbose': {'datefmt': '%Y-%m-%d %H:%M:%S',
                                          'format': LOGGING_FORMAT}},
               'handlers': {'console': {'class': 'logging.StreamHandler',
                                        'debug_only': True,
                                        'formatter': 'verbose'}},
               'incremental': False,
               'loggers': {'helper': {'handlers': ['console'],
                                      'level': 'INFO',
                                      'propagate': True}},
               'root': {'handlers': [],
                        'level': logging.CRITICAL,
                        'propagate': True},
               'version': 1}

    def __init__(self, file_path=None):
        """Create a new instance of the configuration object, passing in the
        path to the configuration file.

        :param str file_path:

        """
        self.application = Data(self.APPLICATION)
        self.daemon = Data(self.DAEMON)
        self._file_path = None
        self._values = Data()
        if file_path:
            self._file_path = self._validate(file_path)
            self._values = Data(self._load_config_file())
            self.apply_config('application')
            self.apply_config('daemon')

    def apply_config(self, name):
        """Apply raw loaded config to running config.  This starts with the base
        config, applies any defined config changes to it, then swaps it in.
        """
        base = Data(getattr(self, name.upper()))
        updates = self._values.get(name.capitalize(), self._values.get(name))
        self._assign_values(base, updates)
        setattr(self, name, base)

    @staticmethod
    def _assign_values(obj, values):
        """Assign values to the object passed in from the dictionary of values.

        :param Data obj: Data object to assign values to
        :param dict values: Values to assign

        """
        for key in values or dict():
            setattr(obj, key, values[key])

    def get(self, name, default=None):
        """Return the value for key if key is in the configuration, else default.

        :param str name: The key name to return
        :param mixed default: The default value for the key
        :return: mixed

        """
        return self._values.get(name, default)

    @property
    def logging(self):
        """Return the logging configuration in the form of a dictionary.

        :rtype: dict

        """
        config = self.LOGGING
        config_in = self._values.get('Logging',
                                     self._values.get('logging', {}))
        for section in ['formatters', 'handlers', 'loggers',
                        'filters', 'root']:
            if section in config_in:
                for key in config_in[section]:
                    config[section][key] = config_in[section][key]
        LOGGER.debug(config)
        return config

    def reload(self):
        """Reload the configuration from disk returning True if the
        configuration has changed from the previous values.

        """
        if self._file_path:

            # Try and reload the configuration file from disk
            try:
                values = Data(self._load_config_file())
            except ValueError as error:
                LOGGER.error('Could not reload configuration: %s', error)
                return False

            # Only update the configuration if the values differ
            if values != self._values:
                self._values = values
                return True

        return False

    def _load_config_file(self):
        """Load the configuration file into memory, returning the content.

        """
        LOGGER.info('Loading configuration from %s', self._file_path)
        if self._file_path.endswith('json'):
            return self._load_json_config()
        # Keep the behavior as it was with regard to file extensions for YAML
        return self._load_yaml_config()

    def _load_json_config(self):
        """Load the configuration file in JSON format

        :rtype: dict

        """
        try:
            with open(self._file_path, 'r') as handle:
                return json.load(handle)
        except (OSError, ValueError) as error:
            raise ValueError('Could not read configuration file: %s' % error)

    def _load_yaml_config(self):
        """Loads the configuration file from a .yaml or .yml file

        :type: dict

        """
        try:
            config = open(self._file_path).read()
        except OSError as error:
            raise ValueError('Could not read configuration file: %s' % error)
        try:
            return yaml.safe_load(config)
        except yaml.YAMLError as error:
            message = '\n'.join(['    > %s' % line
                                 for line in str(error).split('\n')])
            sys.stderr.write("\n\n  Error in the configuration file:\n\n"
                             "%s\n\n" % message)
            sys.stderr.write("  Configuration should be a valid YAML file.\n")
            sys.stderr.write("  YAML format validation available at "
                             "http://yamllint.com\n")
            raise ValueError(error)

    @staticmethod
    def _validate(file_path):
        """Normalize the path provided and ensure the file path, raising a
        ValueError if the file does not exist.

        :param str file_path:
        :return: str
        :raises: ValueError

        """
        file_path = path.abspath(file_path)
        if not path.exists(file_path):
            raise ValueError('Configuration file not found: %s' % file_path)
        return file_path

class LoggingConfig(object):
    """The Logging class is used for abstracting away dictConfig logging
    semantics and can be used by sub-processes to ensure consistent logging
    rule application.

    """
    DEBUG_ONLY = 'debug_only'
    HANDLERS = 'handlers'
    LOGGERS = 'loggers'

    def __init__(self, configuration, debug=None):
        """Create a new instance of the Logging object passing in the
        DictConfig syntax logging configuration and a debug flag.

        :param dict configuration: The logging configuration
        :param bool debug: Toggles use of debug_only loggers

        """
        # Force a NullLogger for some libraries that require it
        root_logger = logging.getLogger()
        root_logger.addHandler(NullHandler())

        self.config = configuration
        self.debug = debug
        self.configure()

    def update(self, configuration, debug=None):
        """Update the internal configuration values, removing debug_only
        handlers if debug is False. Returns True if the configuration has
        changed from previous configuration values.

        :param dict configuration: The logging configuration
        :param bool debug: Toggles use of debug_only loggers
        :rtype: bool

        """

        if self.config != configuration and debug != self.debug:
            self.config = configuration
            self.debug = debug
            self.configure()
            return True
        return False

    def configure(self):
        """Configure the Python stdlib logger"""
        if self.debug is not None and not self.debug:
            self._remove_debug_handlers()
        self._remove_debug_only()
        logging_config.dictConfig(self.config)
        try:
            logging.captureWarnings(True)
        except AttributeError:
            pass

    def _remove_debug_only(self):
        """Iterate through each handler removing the invalid dictConfig key of
        debug_only.

        """
        LOGGER.debug('Removing debug only from handlers')
        for handler in self.config[self.HANDLERS]:
            if self.DEBUG_ONLY in self.config[self.HANDLERS][handler]:
                del self.config[self.HANDLERS][handler][self.DEBUG_ONLY]

    def _remove_debug_handlers(self):
        """Remove any handlers with an attribute of debug_only that is True and
        remove the references to said handlers from any loggers that are
        referencing them.

        """
        remove = list()
        for handler in self.config[self.HANDLERS]:
            if self.config[self.HANDLERS][handler].get('debug_only'):
                remove.append(handler)
        for handler in remove:
            del self.config[self.HANDLERS][handler]
            for logger in self.config[self.LOGGERS].keys():
                logger = self.config[self.LOGGERS][logger]
                if handler in logger[self.HANDLERS]:
                    logger[self.HANDLERS].remove(handler)
        self._remove_debug_only()

class Data(object):
    """Data object configuration is wrapped in, can be used as a object with
    attributes or as a dict.

    """
    __hash__ = False

    def __init__(self, value=None):
        super(Data, self).__init__()
        if value and isinstance(value, dict):
            for name in value.keys():
                if isinstance(value[name], dict):
                    object.__setattr__(self, name, Data(value[name]))
                else:
                    object.__setattr__(self, name, value[name])

    def __contains__(self, name):
        return name in self.__dict__.keys()

    def __delattr__(self, name):
        object.__delattr__(self, name)

    def __delitem__(self, name):
        if name not in self.__dict__:
            raise KeyError(name)
        object.__delattr__(self, name)

    def __getattribute__(self, name):
        return object.__getattribute__(self, name)

    def __getitem__(self, name):
        return object.__getattribute__(self, name)

    def __setitem__(self, name, value):
        if isinstance(value, dict) and name != '__dict__':
            value = Data(value)
        object.__setattr__(self, name, value)

    def __setattr__(self, name, value):
        if isinstance(value, dict) and name != '__dict__':
            value = Data(value)
        object.__setattr__(self, name, value)

    def __repr__(self):
        return repr(self.__dict__)

    def __len__(self):
        return len(self.__dict__)

    def __iter__(self):
        for name in self.__dict__.keys():
            yield name

    def __eq__(self, other):
        if isinstance(other, Data):
            return self.dict() == other.dict()
        return self.dict() == other

    def __ne__(self, other):
        return not self.__eq__(other)

    def str(self):
        """Return a string representation of the data object.

        :rtype: str

        """
        return str(self.__dict__)

    def dict(self):
        """Return the data object as a dictionary, recursively casting children
        that are Data objects as well.

        :rtype: dict

        """
        output = dict()
        for key, value in self.items():
            if isinstance(value, Data):
                output[key] = value.dict()
            else:
                output[key] = value
        return output

    def get(self, name, default=None):
        """Return the value for key if key is in the dictionary, else default.
        If default is not given, it defaults to None, so that this method
        never raises a KeyError.

        :param str name: The key name to return
        :param mixed default: The default value for the key
        :return: mixed

        """
        return self.__dict__.get(name, default)

    def has_key(self, name):
        """Test for the presence of key in the data object. has_key() is
        deprecated in favor of key in d.

        :param name:
        :return: bool

        """
        return name in self.__dict__

    def items(self):
        """Return a copy of the dictionary's list of (key, value) pairs.

        :rtype: list

        """
        return self.__dict__.items()

    def itervalues(self):
        """Return an iterator over the data values. See the note for
        Data.items().

        Using itervalues() while adding or deleting entries in the data object
        may raise a RuntimeError or fail to iterate over all entries.

        :rtype: iterator
        :raises: RuntimeError

        """
        return self.__dict__.itervalues()

    def keys(self):
        """Return a copy of the dictionary's list of keys. See the note for
        Data.items()

        :rtype: list

        """
        return self.__dict__.keys()

    def pop(self, name, default=None):
        """If key is in the dictionary, remove it and return its value, else
        return default. If default is not given and key is not in the
        dictionary, a KeyError is raised.

        :param str name: The key name
        :param mixed default: The default value
        :raises: KeyError

        """
        return self.__dict__.pop(name, default)

    def setdefault(self, name, default=None):
        """If key is in the dictionary, return its value. If not, insert key
        with a value of default and return default. default defaults to None.

        :param str name: The key
        :param mixed default: The value
        :return: mixed

        """
        return self.__dict__.setdefault(name, default)

    def update(self, other=None, **kwargs):
        """Update the dictionary with the key/value pairs from other,
        overwriting existing keys. update() accepts either another dictionary
        object or an iterable of key/value pairs (as tuples or other iterables
        of length two). If keyword arguments are specified, the dictionary is
        then updated with those key/value pairs: d.update(red=1, blue=2).

        :param dict other: Dict or other iterable
        :param dict kwargs: Key/value pairs to update
        :rtype: None

        """
        self.__dict__.update(other, **kwargs)

    def values(self):
        """Return the configuration values

        :rtype: list

        """
        return self.__dict__.values()
<|endoftext|>


<commit_before>from django import forms

class PlaceholderForm(forms.Form):
    """
    A base form for automatically adding placeholder text.

    Forms that extend this form will by default have all text, password, and
    date input widgets display placeholder text equal to their label. This can
    be overridden per form field by simply specifying a different placeholder
    attribute.
    """
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        for field_name in self.fields:
            field = self.fields.get(field_name)
            if field:
                if type(field.widget) in (forms.TextInput, forms.PasswordInput, forms.EmailInput, forms.DateInput):
                    field.widget.attrs.update(
                            { 'placeholder': field.widget.attrs.get('placeholder', field.label or field_name) }
                            )

<commit_msg>New mixin for placeholder forms<commit_after>from django import forms

class PlaceholderForm(forms.Form):
    """
    A base form for automatically adding placeholder text.

    Forms that extend this form will by default have all text, password, and
    date input widgets display placeholder text equal to their label. This can
    be overridden per form field by simply specifying a different placeholder
    attribute.
    """
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        for field_name in self.fields:
            field = self.fields.get(field_name)
            if field:
                if type(field.widget) in (forms.TextInput, forms.PasswordInput, forms.EmailInput, forms.DateInput):
                    field.widget.attrs.update(
                            { 'placeholder': field.widget.attrs.get('placeholder', field.label or field_name) }
                            )

class PlaceholderFormMixin(object):
    """
    A form mixin for automatically adding placeholder text.

    Forms that use this mixin will by default have all text, password, and
    date input widgets display placeholder text equal to their label. This can
    be overridden per form field by simply specifying a different placeholder
    attribute.
    """
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        for field_name in self.fields:
            field = self.fields.get(field_name)
            if field:
                if type(field.widget) in (forms.TextInput, forms.PasswordInput, forms.EmailInput, forms.DateInput):
                    field.widget.attrs.update(
                            { 'placeholder': field.widget.attrs.get('placeholder', field.label or field_name) }
                            )

<|endoftext|>