<commit_before>
def handle(slice):
	from core.models import Controller, ControllerSiteDeployments, ControllerSlices,Controller,Network,NetworkSlice,NetworkTemplate
	from collections import defaultdict
	ctrl_site_deployments = ControllerSiteDeployments.objects.all()
	site_deploy_lookup = defaultdict(list)
	for ctrl_site_deployment in ctrl_site_deployments:
		site_deploy_lookup[ctrl_site_deployment.site_deployment].append(ctrl_site_deployment)

	slice_controllers = ControllerSlices.objects.all()
	slice_deploy_lookup = defaultdict(list)
	for slice_controller in slice_controllers:
		slice_deploy_lookup[slice_controller.slice].append(slice_controller)

	#expected_controllers = site_deploy_lookup[slice.site]
        all_controllers = Controller.objects.all()
	for expected_controller in all_controllers:
		if slice not in slice_deploy_lookup or \
		   expected_controller not in slice_deploy_lookup[slice]:
			sd = ControllerSlices(slice=slice, controller=expected_controller)
			sd.save()

	# make sure slice has at least 1 public and 1 private networkd
	public_nets = []
	private_net = None
	networks = Network.objects.filter(owner=slice)
	for network in networks:
		if network.template.name == 'Public dedicated IPv4':
			public_nets.append(network)
		elif network.template.name == 'Public shared IPv4':
			public_nets.append(network)
		elif network.template.name == 'Private':
			private_net = network
	if not public_nets:
                # ensure there is at least one public network, and default it to dedicated
		nat_net = Network(
          	    name = slice.name+'-nat',
             	    template = NetworkTemplate.objects.get(name='Public shared IPv4'),
          	    owner = slice
      		    )
		nat_net.save()
                public_nets.append(nat_net)

	if not private_net:
        	private_net = Network(
          	name = slice.name+'-private',
          	template = NetworkTemplate.objects.get(name='Private'),
          	owner = slice
      		)
      		private_net.save()
	# create slice networks
	public_net_slice = None
	private_net_slice = None
	net_slices = NetworkSlice.objects.filter(slice=slice, network__in=[private_net]+public_nets)
	for net_slice in net_slices:
		if net_slice.network in public_nets:
			public_net_slice = net_slice
		elif net_slice.network == private_net:
			private_net_slice = net_slice
	if not public_net_slice:
		public_net_slice = NetworkSlice(slice=slice, network=public_nets[0])
		public_net_slice.save()
	if not private_net_slice:
		private_net_slice = NetworkSlice(slice=slice, network=private_net)
		private_net_slice.save()

<commit_msg>fix bugs<commit_after>
def handle(slice):
	from core.models import Controller,ControllerSlices,Controller,Network,NetworkSlice,NetworkTemplate
	from collections import defaultdict
	ctrl_site_deployments = SiteDeployments.objects.all()

	slice_controllers = ControllerSlices.objects.all()
	slice_deploy_lookup = defaultdict(list)
	for slice_controller in slice_controllers:
		slice_deploy_lookup[slice_controller.slice].append(slice_controller)

        all_controllers = Controller.objects.all()
	for expected_controller in all_controllers:
		if slice not in slice_deploy_lookup or \
		   expected_controller not in slice_deploy_lookup[slice]:
			sd = ControllerSlices(slice=slice, controller=expected_controller)
			sd.save()

	# make sure slice has at least 1 public and 1 private networkd
	public_nets = []
	private_net = None
	networks = Network.objects.filter(owner=slice)
	for network in networks:
		if network.template.name == 'Public dedicated IPv4':
			public_nets.append(network)
		elif network.template.name == 'Public shared IPv4':
			public_nets.append(network)
		elif network.template.name == 'Private':
			private_net = network
	if not public_nets:
                # ensure there is at least one public network, and default it to dedicated
		nat_net = Network(
          	    name = slice.name+'-nat',
             	    template = NetworkTemplate.objects.get(name='Public shared IPv4'),
          	    owner = slice
      		    )
		nat_net.save()
                public_nets.append(nat_net)

	if not private_net:
        	private_net = Network(
          	name = slice.name+'-private',
          	template = NetworkTemplate.objects.get(name='Private'),
          	owner = slice
      		)
      		private_net.save()
	# create slice networks
	public_net_slice = None
	private_net_slice = None
	net_slices = NetworkSlice.objects.filter(slice=slice, network__in=[private_net]+public_nets)
	for net_slice in net_slices:
		if net_slice.network in public_nets:
			public_net_slice = net_slice
		elif net_slice.network == private_net:
			private_net_slice = net_slice
	if not public_net_slice:
		public_net_slice = NetworkSlice(slice=slice, network=public_nets[0])
		public_net_slice.save()
	if not private_net_slice:
		private_net_slice = NetworkSlice(slice=slice, network=private_net)
		private_net_slice.save()

<|endoftext|>


<commit_before>'''given a 1D sample of observation, find a matching distribution

* estimate maximum likelihood paramater for each distribution
* rank estimated distribution by Kolmogorov-Smirnov and Anderson-Darling
  test statistics

Author: Josef Pktd
License: Simplified BSD
original December 2008

TODO:

* refactor to result class
* split estimation by support, add option and choose automatically
*

'''

from scipy import stats
import numpy as np
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt

def plothist(x,distfn, args, loc, scale, right=1):

    plt.figure()
    # the histogram of the data
    n, bins, patches = plt.hist(x, 25, normed=1, facecolor='green', alpha=0.75)
    maxheight = max([p.get_height() for p in patches])
    print maxheight
    axlim = list(plt.axis())
    #print axlim
    axlim[-1] = maxheight*1.05
    #plt.axis(tuple(axlim))
##    print bins
##    print 'args in plothist', args
    # add a 'best fit' line
    #yt = stats.norm.pdf( bins, loc=loc, scale=scale)
    yt = distfn.pdf( bins, loc=loc, scale=scale, *args)
    yt[yt>maxheight]=maxheight
    lt = plt.plot(bins, yt, 'r--', linewidth=1)
    ys = stats.t.pdf( bins, 10,scale=10,)*right
    ls = plt.plot(bins, ys, 'b-', linewidth=1)

    plt.xlabel('Smarts')
    plt.ylabel('Probability')
    plt.title(r'$\mathrm{Testing: %s :}\ \mu=%f,\ \sigma=%f$'%(distfn.name,loc,scale))

    #plt.axis([bins[0], bins[-1], 0, 0.134+0.05])

    plt.grid(True)
    plt.draw()
    #plt.show()
    #plt.close()

#targetdist = ['norm','t','truncnorm','johnsonsu','johnsonsb',
targetdist = ['norm','alpha', 'anglit', 'arcsine',
           'beta', 'betaprime', 'bradford', 'burr', 'fisk', 'cauchy',
           'chi', 'chi2', 'cosine', 'dgamma', 'dweibull', 'erlang',
           'expon', 'exponweib', 'exponpow', 'fatiguelife', 'foldcauchy',
           'f', 'foldnorm', 'frechet_r', 'weibull_min', 'frechet_l',
           'weibull_max', 'genlogistic', 'genpareto', 'genexpon', 'genextreme',
           'gamma', 'gengamma', 'genhalflogistic', 'gompertz', 'gumbel_r',
           'gumbel_l', 'halfcauchy', 'halflogistic', 'halfnorm', 'hypsecant',
           'gausshyper', 'invgamma', 'invnorm', 'invweibull', 'johnsonsb',
           'johnsonsu', 'laplace', 'levy', 'levy_l',
           'logistic', 'loggamma', 'loglaplace', 'lognorm', 'gilbrat',
           'maxwell', 'mielke', 'nakagami', 'ncx2', 'ncf', 't',
           'nct', 'pareto', 'lomax', 'powerlaw', 'powerlognorm', 'powernorm',
           'rdist', 'rayleigh', 'reciprocal', 'rice', 'recipinvgauss',
           'semicircular', 'triang', 'truncexpon', 'truncnorm',
           'tukeylambda', 'uniform', 'vonmises', 'wald', 'wrapcauchy',

           'binom', 'bernoulli', 'nbinom', 'geom', 'hypergeom', 'logser',
           'poisson', 'planck', 'boltzmann', 'randint', 'zipf', 'dlaplace']

left = []
right = []
finite = []
unbound = []
other = []
contdist = []
discrete = []

categ = {('open','open'):'unbound', ('0','open'):'right',('open','0',):'left',
             ('finite','finite'):'finite',('oth','oth'):'other'}
categ = {('open','open'):unbound, ('0','open'):right,('open','0',):left,
             ('finite','finite'):finite,('oth','oth'):other}

categ2 = {
    ('open', '0') : ['frechet_l', 'weibull_max', 'levy_l'],
    ('finite', 'finite') : ['anglit', 'cosine', 'rdist', 'semicircular'],
    ('0', 'open') : ['alpha', 'burr', 'fisk', 'chi', 'chi2', 'erlang',
                'expon', 'exponweib', 'exponpow', 'fatiguelife', 'foldcauchy', 'f',
                'foldnorm', 'frechet_r', 'weibull_min', 'genpareto', 'genexpon',
                'gamma', 'gengamma', 'genhalflogistic', 'gompertz', 'halfcauchy',
                'halflogistic', 'halfnorm', 'invgamma', 'invnorm', 'invweibull',
                'levy', 'loglaplace', 'lognorm', 'gilbrat', 'maxwell', 'mielke',
                'nakagami', 'ncx2', 'ncf', 'lomax', 'powerlognorm', 'rayleigh',
                'rice', 'recipinvgauss', 'truncexpon', 'wald'],
    ('open', 'open') : ['cauchy', 'dgamma', 'dweibull', 'genlogistic', 'genextreme',
                'gumbel_r', 'gumbel_l', 'hypsecant', 'johnsonsu', 'laplace',
                'logistic', 'loggamma', 't', 'nct', 'powernorm', 'reciprocal',
                'truncnorm', 'tukeylambda', 'vonmises'],
    ('0', 'finite') : ['arcsine', 'beta', 'betaprime', 'bradford', 'gausshyper',
                'johnsonsb', 'powerlaw', 'triang', 'uniform', 'wrapcauchy'],
    ('finite', 'open') : ['pareto']
    }

#Note: weibull_max == frechet_l

right_incorrect = ['genextreme']

right_all = categ2[('0', 'open')] + categ2[('0', 'finite')] + categ2[('finite', 'open')]\
            + right_incorrect

for distname in targetdist:
    distfn = getattr(stats,distname)
    if hasattr(distfn,'_pdf'):
        if np.isinf(distfn.a):
            low = 'open'
        elif distfn.a == 0:
            low = '0'
        else:
            low = 'finite'
        if np.isinf(distfn.b):
            high = 'open'
        elif distfn.b == 0:
            high = '0'
        else:
            high = 'finite'
        contdist.append(distname)
        categ.setdefault((low,high),[]).append(distname)

not_good = ['genextreme', 'reciprocal', 'vonmises']
# 'genextreme' is right (or left?), 'reciprocal' requires 0<a<b, 'vonmises' no a,b
targetdist = [f for f in categ[('open', 'open')] if not f in not_good]
not_good = ['wrapcauchy']
not_good = ['vonmises']
not_good = ['genexpon','vonmises']
#'wrapcauchy' requires additional parameter (scale) in argcheck
targetdist = [f for f in contdist if not f in not_good]
#targetdist = contdist
#targetdist = not_good
#targetdist = ['t', 'f']
#targetdist = ['norm','burr']

if __name__ == '__main__':

    #TODO: calculate correct tail probability for mixture
    prefix = 'run_conv500_1_'
    convol = 0.75
    n = 500
    dgp_arg = 10
    dgp_scale = 10
    results = []
    for i in range(1):
        rvs_orig = stats.t.rvs(dgp_arg,scale=dgp_scale,size=n*convol)
        rvs_orig = np.hstack((rvs_orig,stats.halflogistic.rvs(loc=0.4, scale=5.0,size =n*(1-convol))))
        rvs_abs = np.absolute(rvs_orig)
        rvs_pos = rvs_orig[rvs_orig>0]
        rightfactor = 1
        rvs_right = rvs_pos
        print '='*50
        print 'samplesize = ', n
        for distname in targetdist:
            distfn = getattr(stats,distname)
            if distname in right_all:
                rvs = rvs_right
                rind = rightfactor

            else:
                rvs = rvs_orig
                rind = 1
            print '-'*30
            print 'target = %s' % distname
            sm = rvs.mean()
            sstd = np.sqrt(rvs.var())
            ssupp = (rvs.min(), rvs.max())
            if distname in ['truncnorm','betaprime','reciprocal']:

                par0 = (sm-2*sstd,sm+2*sstd)
                par_est = tuple(distfn.fit(rvs,loc=sm,scale=sstd,*par0))
            elif distname == 'norm':
                par_est = tuple(distfn.fit(rvs,loc=sm,scale=sstd))
            elif distname == 'genextreme':
                par_est = tuple(distfn.fit(rvs,-5,loc=sm,scale=sstd))
            elif distname == 'wrapcauchy':
                par_est = tuple(distfn.fit(rvs,0.5,loc=0,scale=sstd))
            elif distname == 'f':\
                par_est = tuple(distfn.fit(rvs,10,15,loc=0,scale=1))

            elif distname in right:
                sm = rvs.mean()
                sstd = np.sqrt(rvs.var())
                par_est = tuple(distfn.fit(rvs,loc=0,scale=1))
            else:
                sm = rvs.mean()
                sstd = np.sqrt(rvs.var())
                par_est = tuple(distfn.fit(rvs,loc=sm,scale=sstd))

            print 'fit', par_est
            arg_est = par_est[:-2]
            loc_est = par_est[-2]
            scale_est = par_est[-1]
            rvs_normed = (rvs-loc_est)/scale_est
            ks_stat, ks_pval = stats.kstest(rvs_normed,distname, arg_est)
            print 'kstest', ks_stat, ks_pval
            quant = 0.1
            crit = distfn.ppf(1-quant*float(rind), loc=loc_est, scale=scale_est,*par_est)
            tail_prob = stats.t.sf(crit,dgp_arg,scale=dgp_scale)
            print 'crit, prob', quant, crit, tail_prob
            #if distname == 'norm':
                #plothist(rvs,loc_est,scale_est)
                #args = tuple()
            results.append([distname,ks_stat, ks_pval,arg_est,loc_est,scale_est,crit,tail_prob ])
            #plothist(rvs,distfn,arg_est,loc_est,scale_est)

    #plothist(rvs,distfn,arg_est,loc_est,scale_est)
    #plt.show()
    #plt.close()
    #TODO: collect results and compare tail quantiles

    from operator import itemgetter

    res_sort = sorted(results, key = itemgetter(2))

    res_sort.reverse()  #kstest statistic: smaller is better, pval larger is better

    print 'number of distributions', len(res_sort)
    imagedir = 'matchresults'
    import os
    if not os.path.exists(imagedir):
        os.makedirs(imagedir)

    for ii,di in enumerate(res_sort):
        distname,ks_stat, ks_pval,arg_est,loc_est,scale_est,crit,tail_prob = di[:]
        distfn = getattr(stats,distname)
        if distname in right_all:
            rvs = rvs_right
            rind = rightfactor
            ri = 'r'
        else:
            rvs = rvs_orig
            ri = ''
            rind = 1
        print '%s ks-stat = %f, ks-pval = %f tail_prob = %f)' % \
              (distname, ks_stat, ks_pval, tail_prob)
    ##    print 'arg_est = %s, loc_est = %f scale_est = %f)' % \
    ##          (repr(arg_est),loc_est,scale_est)
        plothist(rvs,distfn,arg_est,loc_est,scale_est,right = rind)
        plt.savefig(os.path.join(imagedir,'%s%s%02d_%s.png'% (prefix, ri,ii, distname)))
    ##plt.show()
    ##plt.close()

<commit_msg>matchdist.py, temporarily monkeypatch beta_gen so script runs<commit_after>'''given a 1D sample of observation, find a matching distribution

* estimate maximum likelihood paramater for each distribution
* rank estimated distribution by Kolmogorov-Smirnov and Anderson-Darling
  test statistics

Author: Josef Pktd
License: Simplified BSD
original December 2008

TODO:

* refactor to result class
* split estimation by support, add option and choose automatically
*

'''

from scipy import stats
import numpy as np
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt

stats.distributions.beta_gen._fitstart = lambda self, data : (5,5,0,1)

def plothist(x,distfn, args, loc, scale, right=1):

    plt.figure()
    # the histogram of the data
    n, bins, patches = plt.hist(x, 25, normed=1, facecolor='green', alpha=0.75)
    maxheight = max([p.get_height() for p in patches])
    print maxheight
    axlim = list(plt.axis())
    #print axlim
    axlim[-1] = maxheight*1.05
    #plt.axis(tuple(axlim))
##    print bins
##    print 'args in plothist', args
    # add a 'best fit' line
    #yt = stats.norm.pdf( bins, loc=loc, scale=scale)
    yt = distfn.pdf( bins, loc=loc, scale=scale, *args)
    yt[yt>maxheight]=maxheight
    lt = plt.plot(bins, yt, 'r--', linewidth=1)
    ys = stats.t.pdf( bins, 10,scale=10,)*right
    ls = plt.plot(bins, ys, 'b-', linewidth=1)

    plt.xlabel('Smarts')
    plt.ylabel('Probability')
    plt.title(r'$\mathrm{Testing: %s :}\ \mu=%f,\ \sigma=%f$'%(distfn.name,loc,scale))

    #plt.axis([bins[0], bins[-1], 0, 0.134+0.05])

    plt.grid(True)
    plt.draw()
    #plt.show()
    #plt.close()

#targetdist = ['norm','t','truncnorm','johnsonsu','johnsonsb',
targetdist = ['norm','alpha', 'anglit', 'arcsine',
           'beta', 'betaprime', 'bradford', 'burr', 'fisk', 'cauchy',
           'chi', 'chi2', 'cosine', 'dgamma', 'dweibull', 'erlang',
           'expon', 'exponweib', 'exponpow', 'fatiguelife', 'foldcauchy',
           'f', 'foldnorm', 'frechet_r', 'weibull_min', 'frechet_l',
           'weibull_max', 'genlogistic', 'genpareto', 'genexpon', 'genextreme',
           'gamma', 'gengamma', 'genhalflogistic', 'gompertz', 'gumbel_r',
           'gumbel_l', 'halfcauchy', 'halflogistic', 'halfnorm', 'hypsecant',
           'gausshyper', 'invgamma', 'invnorm', 'invweibull', 'johnsonsb',
           'johnsonsu', 'laplace', 'levy', 'levy_l',
           'logistic', 'loggamma', 'loglaplace', 'lognorm', 'gilbrat',
           'maxwell', 'mielke', 'nakagami', 'ncx2', 'ncf', 't',
           'nct', 'pareto', 'lomax', 'powerlaw', 'powerlognorm', 'powernorm',
           'rdist', 'rayleigh', 'reciprocal', 'rice', 'recipinvgauss',
           'semicircular', 'triang', 'truncexpon', 'truncnorm',
           'tukeylambda', 'uniform', 'vonmises', 'wald', 'wrapcauchy',

           'binom', 'bernoulli', 'nbinom', 'geom', 'hypergeom', 'logser',
           'poisson', 'planck', 'boltzmann', 'randint', 'zipf', 'dlaplace']

left = []
right = []
finite = []
unbound = []
other = []
contdist = []
discrete = []

categ = {('open','open'):'unbound', ('0','open'):'right',('open','0',):'left',
             ('finite','finite'):'finite',('oth','oth'):'other'}
categ = {('open','open'):unbound, ('0','open'):right,('open','0',):left,
             ('finite','finite'):finite,('oth','oth'):other}

categ2 = {
    ('open', '0') : ['frechet_l', 'weibull_max', 'levy_l'],
    ('finite', 'finite') : ['anglit', 'cosine', 'rdist', 'semicircular'],
    ('0', 'open') : ['alpha', 'burr', 'fisk', 'chi', 'chi2', 'erlang',
                'expon', 'exponweib', 'exponpow', 'fatiguelife', 'foldcauchy', 'f',
                'foldnorm', 'frechet_r', 'weibull_min', 'genpareto', 'genexpon',
                'gamma', 'gengamma', 'genhalflogistic', 'gompertz', 'halfcauchy',
                'halflogistic', 'halfnorm', 'invgamma', 'invnorm', 'invweibull',
                'levy', 'loglaplace', 'lognorm', 'gilbrat', 'maxwell', 'mielke',
                'nakagami', 'ncx2', 'ncf', 'lomax', 'powerlognorm', 'rayleigh',
                'rice', 'recipinvgauss', 'truncexpon', 'wald'],
    ('open', 'open') : ['cauchy', 'dgamma', 'dweibull', 'genlogistic', 'genextreme',
                'gumbel_r', 'gumbel_l', 'hypsecant', 'johnsonsu', 'laplace',
                'logistic', 'loggamma', 't', 'nct', 'powernorm', 'reciprocal',
                'truncnorm', 'tukeylambda', 'vonmises'],
    ('0', 'finite') : ['arcsine', 'beta', 'betaprime', 'bradford', 'gausshyper',
                'johnsonsb', 'powerlaw', 'triang', 'uniform', 'wrapcauchy'],
    ('finite', 'open') : ['pareto']
    }

#Note: weibull_max == frechet_l

right_incorrect = ['genextreme']

right_all = categ2[('0', 'open')] + categ2[('0', 'finite')] + categ2[('finite', 'open')]\
            + right_incorrect

for distname in targetdist:
    distfn = getattr(stats,distname)
    if hasattr(distfn,'_pdf'):
        if np.isinf(distfn.a):
            low = 'open'
        elif distfn.a == 0:
            low = '0'
        else:
            low = 'finite'
        if np.isinf(distfn.b):
            high = 'open'
        elif distfn.b == 0:
            high = '0'
        else:
            high = 'finite'
        contdist.append(distname)
        categ.setdefault((low,high),[]).append(distname)

not_good = ['genextreme', 'reciprocal', 'vonmises']
# 'genextreme' is right (or left?), 'reciprocal' requires 0<a<b, 'vonmises' no a,b
targetdist = [f for f in categ[('open', 'open')] if not f in not_good]
not_good = ['wrapcauchy']
not_good = ['vonmises']
not_good = ['genexpon','vonmises']
#'wrapcauchy' requires additional parameter (scale) in argcheck
targetdist = [f for f in contdist if not f in not_good]
#targetdist = contdist
#targetdist = not_good
#targetdist = ['t', 'f']
#targetdist = ['norm','burr']

if __name__ == '__main__':

    #TODO: calculate correct tail probability for mixture
    prefix = 'run_conv500_1_'
    convol = 0.75
    n = 500
    dgp_arg = 10
    dgp_scale = 10
    results = []
    for i in range(1):
        rvs_orig = stats.t.rvs(dgp_arg,scale=dgp_scale,size=n*convol)
        rvs_orig = np.hstack((rvs_orig,stats.halflogistic.rvs(loc=0.4, scale=5.0,size =n*(1-convol))))
        rvs_abs = np.absolute(rvs_orig)
        rvs_pos = rvs_orig[rvs_orig>0]
        rightfactor = 1
        rvs_right = rvs_pos
        print '='*50
        print 'samplesize = ', n
        for distname in targetdist:
            distfn = getattr(stats,distname)
            if distname in right_all:
                rvs = rvs_right
                rind = rightfactor

            else:
                rvs = rvs_orig
                rind = 1
            print '-'*30
            print 'target = %s' % distname
            sm = rvs.mean()
            sstd = np.sqrt(rvs.var())
            ssupp = (rvs.min(), rvs.max())
            if distname in ['truncnorm','betaprime','reciprocal']:

                par0 = (sm-2*sstd,sm+2*sstd)
                par_est = tuple(distfn.fit(rvs,loc=sm,scale=sstd,*par0))
            elif distname == 'norm':
                par_est = tuple(distfn.fit(rvs,loc=sm,scale=sstd))
            elif distname == 'genextreme':
                par_est = tuple(distfn.fit(rvs,-5,loc=sm,scale=sstd))
            elif distname == 'wrapcauchy':
                par_est = tuple(distfn.fit(rvs,0.5,loc=0,scale=sstd))
            elif distname == 'f':\
                par_est = tuple(distfn.fit(rvs,10,15,loc=0,scale=1))

            elif distname in right:
                sm = rvs.mean()
                sstd = np.sqrt(rvs.var())
                par_est = tuple(distfn.fit(rvs,loc=0,scale=1))
            else:
                sm = rvs.mean()
                sstd = np.sqrt(rvs.var())
                par_est = tuple(distfn.fit(rvs,loc=sm,scale=sstd))

            print 'fit', par_est
            arg_est = par_est[:-2]
            loc_est = par_est[-2]
            scale_est = par_est[-1]
            rvs_normed = (rvs-loc_est)/scale_est
            ks_stat, ks_pval = stats.kstest(rvs_normed,distname, arg_est)
            print 'kstest', ks_stat, ks_pval
            quant = 0.1
            crit = distfn.ppf(1-quant*float(rind), loc=loc_est, scale=scale_est,*par_est)
            tail_prob = stats.t.sf(crit,dgp_arg,scale=dgp_scale)
            print 'crit, prob', quant, crit, tail_prob
            #if distname == 'norm':
                #plothist(rvs,loc_est,scale_est)
                #args = tuple()
            results.append([distname,ks_stat, ks_pval,arg_est,loc_est,scale_est,crit,tail_prob ])
            #plothist(rvs,distfn,arg_est,loc_est,scale_est)

    #plothist(rvs,distfn,arg_est,loc_est,scale_est)
    #plt.show()
    #plt.close()
    #TODO: collect results and compare tail quantiles

    from operator import itemgetter

    res_sort = sorted(results, key = itemgetter(2))

    res_sort.reverse()  #kstest statistic: smaller is better, pval larger is better

    print 'number of distributions', len(res_sort)
    imagedir = 'matchresults'
    import os
    if not os.path.exists(imagedir):
        os.makedirs(imagedir)

    for ii,di in enumerate(res_sort):
        distname,ks_stat, ks_pval,arg_est,loc_est,scale_est,crit,tail_prob = di[:]
        distfn = getattr(stats,distname)
        if distname in right_all:
            rvs = rvs_right
            rind = rightfactor
            ri = 'r'
        else:
            rvs = rvs_orig
            ri = ''
            rind = 1
        print '%s ks-stat = %f, ks-pval = %f tail_prob = %f)' % \
              (distname, ks_stat, ks_pval, tail_prob)
    ##    print 'arg_est = %s, loc_est = %f scale_est = %f)' % \
    ##          (repr(arg_est),loc_est,scale_est)
        plothist(rvs,distfn,arg_est,loc_est,scale_est,right = rind)
        plt.savefig(os.path.join(imagedir,'%s%s%02d_%s.png'% (prefix, ri,ii, distname)))
    ##plt.show()
    ##plt.close()

<|endoftext|>


<commit_before># Import the SDK
import boto3
import uuid
import sys
import getopt

elbclient = boto3.client('elb')

def usage ():
   print "elb_helper.py -l <LB name> -i <instance> <-d[eregister]|-r[egister]|-s[status]>"

if(len(sys.argv) < 6):
   usage()
   sys.exit(2)

#Let's print the status of our instances
def printinstances( loadblancer, instance ):
  list_elb_resp = elbclient.describe_load_balancers(LoadBalancerNames=[loadblancer])
  for list_instance in (list_elb_resp['LoadBalancerDescriptions'][0]['Instances' ]):
            if (instance[0]['InstanceId']==list_instance['InstanceId']):
              print ('Instance {1} registered with load balancer {0}'.format(loadblancer,list_instance['InstanceId']))
              return;
  print ('Instance {1} IS NOT registered with load balancer {0}'.format(loadblancer,list_instance['InstanceId']))
  return;

try:
    opts, args = getopt.getopt(sys.argv[1:], 'l:i:rds', ['loadbalancer=', 'instance=', 'help', 'r|d|s'])
except getopt.GetoptError:
    usage()
    sys.exit(2)
for opt, arg in opts:
    if opt in ('-h', '--help'):
        usage()
        sys.exit(2)
    elif opt in ('-l', '--loadbalancer'):
        LB = arg
    elif opt in ('-i', '--instance'):
        InstanceID=[ {'InstanceId': arg } ]
    elif opt in ('-r', '--redister'):
	response = elbclient.register_instances_with_load_balancer(
        LoadBalancerName=LB,
    	Instances= InstanceID
	)
    elif opt in ('-d', '--deregister'):
	 response = elbclient.deregister_instances_from_load_balancer(
         LoadBalancerName=LB,
         Instances= InstanceID
         )
    elif opt in ('-s', '--status'):
         printinstances(LB,InstanceID)
    else:
        usage()
        sys.exit(2)

<commit_msg>added env<commit_after>#!/usr/bin/env python
# Import the SDK
import boto3
import uuid
import sys
import getopt

elbclient = boto3.client('elb')

def usage ():
   print "elb_helper.py -l <LB name> -i <instance> <-d[eregister]|-r[egister]|-s[status]>"

if(len(sys.argv) < 6):
   usage()
   sys.exit(2)

#Let's print the status of our instances
def printinstances( loadblancer, instance ):
  list_elb_resp = elbclient.describe_load_balancers(LoadBalancerNames=[loadblancer])
  for list_instance in (list_elb_resp['LoadBalancerDescriptions'][0]['Instances' ]):
            if (instance[0]['InstanceId']==list_instance['InstanceId']):
              print ('Instance {1} registered with load balancer {0}'.format(loadblancer,list_instance['InstanceId']))
              return;
  print ('Instance {1} IS NOT registered with load balancer {0}'.format(loadblancer,list_instance['InstanceId']))
  return;

try:
    opts, args = getopt.getopt(sys.argv[1:], 'l:i:rds', ['loadbalancer=', 'instance=', 'help', 'r|d|s'])
except getopt.GetoptError:
    usage()
    sys.exit(2)
for opt, arg in opts:
    if opt in ('-h', '--help'):
        usage()
        sys.exit(2)
    elif opt in ('-l', '--loadbalancer'):
        LB = arg
    elif opt in ('-i', '--instance'):
        InstanceID=[ {'InstanceId': arg } ]
    elif opt in ('-r', '--redister'):
	response = elbclient.register_instances_with_load_balancer(
        LoadBalancerName=LB,
    	Instances= InstanceID
	)
    elif opt in ('-d', '--deregister'):
	 response = elbclient.deregister_instances_from_load_balancer(
         LoadBalancerName=LB,
         Instances= InstanceID
         )
    elif opt in ('-s', '--status'):
         printinstances(LB,InstanceID)
    else:
        usage()
        sys.exit(2)

<|endoftext|>


<commit_before>def pb_settings_filename():
    return 'Package Bundler.sublime-settings'

def st_settings_filename():
    return 'Preferences.sublime-settings'<commit_msg>Update settings.py<commit_after>def pb_settings_filename():
    return 'Package Bundler.sublime-settings'

def st_settings_filename():
    if int(sublime.version()) >= 2174:
        return 'Preferences.sublime-settings'
    return 'Global.sublime-settings
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Keras neural network explanations"""

import numpy as np
import keras
import keras.backend as K
import tensorflow as tf
from keras.preprocessing import image
from keras.models import Model
from keras.layers.core import Lambda

from eli5.base import Explanation
from eli5.explain import explain_prediction

# note that Sequential subclasses Model, so we can just register the Model type
# Model subclasses Network, but is using Network with this function valid?
@explain_prediction.register(Model)
def explain_prediction_keras(estimator, doc, # model, image
                             top=None, # NOT SUPPORTED
                             top_targets=None, # NOT SUPPORTED
                             target_names=None, # rename / provide prediction labels
                             targets=None, # prediction(s) to focus on, if None take top prediction
                             feature_names=None, # NOT SUPPORTED
                             feature_re=None, # NOT SUPPORTED
                             feature_filter=None, # NOT SUPPORTED
                             # new parameters:
                             layer=None, # which layer to focus on,
                             prediction_decoder=None, # target prediction decoding function
                            ):
    """Explain prediction of a Keras model
    doc : image,
        one of: must be an input acceptable by estimator,
        preprocessing can be done with helper functions.
    targets: predictions
        a list of predictions
        integer for ImageNet classification
    layer: valid target layer in the model to Grad-CAM on,
        one of: a valid keras layer name (str) or index (int),
        a callable function that returns True when the desired layer is matched for the model
        if None, automatically use a helper callable function to get the last suitable Conv layer
    """
    explanation = Explanation(
        repr(estimator), # might want to replace this with something else, eg: estimator.summary()
        description='',
        error='',
        method='gradcam',
        is_regression=False, # classification vs regression model
        highlight_spaces=None, # might be relevant later when explaining text models
    )
    # TODO: grad-cam on multiple layers by passing a list of layers
    if layer is None:
        # Automatically get the layer if not provided
        # this might not be a good idea from transparency / user point of view
        layer = get_last_activation_maps
    target_layer = get_target_layer(estimator, layer)

    # get prediction to focus on
    if targets is None:
        predicted = get_target_prediction(estimator, doc, decoder=prediction_decoder)
    else:
        predicted = targets[0]
        # TODO: take in a single target as well, not just a list
        # does it make sense to take a list of targets. You can only Grad-CAM a single target?
        # TODO: consider changing signature / types for explain_prediction generic function
        # TODO: need to find a way to show the label for the passed prediction as well as its probability
    heatmap = grad_cam(estimator, doc, predicted, target_layer)
    heatmap = image.array_to_img(heatmap)
    original = image.array_to_img(doc[0])
    explanation.heatmap = heatmap
    explanation.image = original
    return explanation

def get_target_prediction(model, x, decoder=None):
    predictions = model.predict(x)
    if decoder is not None:
        # TODO: check if decoder is callable?
        # FIXME: it is not certain that we need such indexing into the decoder's output
        top_1 = decoder(predictions)[0][0]
        ncode, label, proba = top_1
        # TODO: do I print, log, or append to 'description' this output?
        print('Predicted class:')
        print('%s (%s) with probability %.2f' % (label, ncode, proba))
    # FIXME: non-classification tasks
    predicted_class = np.argmax(predictions)
    return predicted_class

def get_target_layer(estimator, desired_layer):
    """
    Return instance of the desired layer in the model.
    estimator: model whose layer is to be gotten
    desired_layer: one of: integer index, string name of the layer,
    callable that returns True if a layer instance matches.
    """
    if isinstance(desired_layer, int):
        # conv_output =  [l for l in model.layers if l.name is layer_name]
        # conv_output = conv_output[0]
        # bottom-up horizontal graph traversal
        target_layer = estimator.get_layer(index=desired_layer)
        # These can raise ValueError if the layer index / name specified is not found
    elif isinstance(desired_layer, str):
        target_layer = estimator.get_layer(name=desired_layer)
    elif callable(desired_layer):
        # is 'callable()' the right check to use here?
        l = estimator.get_layer(index=-4)
        # FIXME: don't hardcode four
        # actually iterate through the list of layers backwards (using negative indexing with get_layer()) until find the desired layer
        target_layer = l if desired_layer(l) else None
        if target_layer is None:
            # If can't find, raise error
            raise ValueError('Target layer could not be found using callable %s' % desired_layer)
    else:
        raise ValueError('Invalid desired_layer (must be str, int, or callable): "%s"' % desired_layer)

    # TODO: check target_layer dimensions (is it possible to do Grad-CAM on it?)
    return target_layer

def get_last_activation_maps(estimator):
    # TODO: automatically get last Conv layer if layer_name and layer_index are None
    # Some ideas:
    # 1. look at layer name, exclude things like "softmax", "global average pooling",
    # etc, include things like "conv" (but watch for false positives)
    # 2. look at layer input/output dimensions, to ensure they match
    return True

# path to a single image, directory containing images,
# PIL image object, or an array can also be multiple images.

def preprocess_image(img, estimator=None, preprocessing=None):
    """
    Returns a single image as an array for an estimator's input
    img: one of: path to a single image file, PIL Image object, numpy array
    estimator: model instance, for resizing the image to the required input dimensions
    preprocessing: one of: callable (called with the image array as an argument),
    "auto" (attempts to use a preprocessing function from keras.applications, requires estimator argument)
    """
    xDims = None
    if estimator is not None:
        xDims = estimator.input_shape[1:3]
    im = image.load_img(img, target_size=xDims)
    x = image.img_to_array(im)

    # we need to insert an axis at the 0th position to indicate the batch size
    # this is required by the keras predict() function
    x = np.expand_dims(x, axis=0)

    if callable(preprocessing):
        x = preprocessing(x)
    elif preprocessing == "auto":
        # Consider removing this "auto" option (the user can pass the function themselves)
        try:
            f = getattr(keras.applications, estimator.name.lower()).preprocess_input
        except AttributeError:
            print('Could not get preprocessing automatically')
        else:
            x = f(x)
    return x

# Credits to Jacob Gildenblat for https://github.com/jacobgil/keras-grad-cam
# (and https://github.com/PowerOfCreation/keras-grad-cam)

def target_category_loss(x, category_index, nb_classes):
    return tf.multiply(x, K.one_hot([category_index], nb_classes))
    # return x * K.one_hot([category_index], nb_classes) # this works as well

def target_category_loss_output_shape(input_shape):
    return input_shape

def normalize(x):
    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)

def _compute_gradients(tensor, var_list):
    grads = tf.gradients(tensor, var_list)
    return [grad if grad is not None else tf.zeros_like(var) for var, grad in zip(var_list, grads)]
    # grads = K.gradients(tensor, var_list)
    # return [grad if grad is not None else K.zeros_like(var) for var, grad in zip(var_list, grads)]

def grad_cam(input_model, image, category_index, layer):
    # FIXME: this assumes that we are doing classification
    # nb_classes = 1000 # FIXME: number of classes can be variable
    nb_classes = input_model.output_shape[1] # TODO: test this

    # FIXME: rename these "layer" variables
    target_layer = lambda x: target_category_loss(x, category_index, nb_classes)
    x = Lambda(target_layer, output_shape = target_category_loss_output_shape)(input_model.output)
    model = Model(inputs=input_model.input, outputs=x)
    # model.summary() # remove this later
    loss = K.sum(model.output)

    # we need to get the output attribute, else we get a TypeError: Failed to convert object to tensor
    conv_output = layer.output

    grads = normalize(_compute_gradients(loss, [conv_output])[0])
    gradient_function = K.function([model.input], [conv_output, grads])

    output, grads_val = gradient_function([image]) # work happens here
    output, grads_val = output[0, :], grads_val[0, :, :, :] # FIXME: this probably assumes that the layer is a width*height filter

    weights = np.mean(grads_val, axis = (0, 1))
    # cam = np.ones(output.shape[0 : 2], dtype = np.float32)
    heatmap = np.ones(output.shape[0 : 2], dtype = np.float32)

    for i, w in enumerate(weights):
        # cam += w * output[:, :, i]
        heatmap += w * output[:, :, i]

    heatmap = np.maximum(heatmap, 0) # ReLU
    heatmap = heatmap / np.max(heatmap) # probability
    heatmap = 255*heatmap # 0...255 float
    # we need to insert a channels axis to have an image (channels last by default)
    heatmap = np.expand_dims(heatmap, axis=-1)
    return heatmap<commit_msg>Replace tensorflow operations with keras.backend operations<commit_after># -*- coding: utf-8 -*-
"""Keras neural network explanations"""

import numpy as np
import keras
import keras.backend as K
from keras.preprocessing import image
from keras.models import Model
from keras.layers.core import Lambda

from eli5.base import Explanation
from eli5.explain import explain_prediction

# note that Sequential subclasses Model, so we can just register the Model type
# Model subclasses Network, but is using Network with this function valid?
@explain_prediction.register(Model)
def explain_prediction_keras(estimator, doc, # model, image
                             top=None, # NOT SUPPORTED
                             top_targets=None, # NOT SUPPORTED
                             target_names=None, # rename / provide prediction labels
                             targets=None, # prediction(s) to focus on, if None take top prediction
                             feature_names=None, # NOT SUPPORTED
                             feature_re=None, # NOT SUPPORTED
                             feature_filter=None, # NOT SUPPORTED
                             # new parameters:
                             layer=None, # which layer to focus on,
                             prediction_decoder=None, # target prediction decoding function
                            ):
    """Explain prediction of a Keras model
    doc : image,
        one of: must be an input acceptable by estimator,
        preprocessing can be done with helper functions.
    targets: predictions
        a list of predictions
        integer for ImageNet classification
    layer: valid target layer in the model to Grad-CAM on,
        one of: a valid keras layer name (str) or index (int),
        a callable function that returns True when the desired layer is matched for the model
        if None, automatically use a helper callable function to get the last suitable Conv layer
    """
    explanation = Explanation(
        repr(estimator), # might want to replace this with something else, eg: estimator.summary()
        description='',
        error='',
        method='gradcam',
        is_regression=False, # classification vs regression model
        highlight_spaces=None, # might be relevant later when explaining text models
    )
    # TODO: grad-cam on multiple layers by passing a list of layers
    if layer is None:
        # Automatically get the layer if not provided
        # this might not be a good idea from transparency / user point of view
        layer = get_last_activation_maps
    target_layer = get_target_layer(estimator, layer)

    # get prediction to focus on
    if targets is None:
        predicted = get_target_prediction(estimator, doc, decoder=prediction_decoder)
    else:
        predicted = targets[0]
        # TODO: take in a single target as well, not just a list
        # does it make sense to take a list of targets. You can only Grad-CAM a single target?
        # TODO: consider changing signature / types for explain_prediction generic function
        # TODO: need to find a way to show the label for the passed prediction as well as its probability
    heatmap = grad_cam(estimator, doc, predicted, target_layer)
    heatmap = image.array_to_img(heatmap)
    original = image.array_to_img(doc[0])
    explanation.heatmap = heatmap
    explanation.image = original
    return explanation

def get_target_prediction(model, x, decoder=None):
    predictions = model.predict(x)
    if decoder is not None:
        # TODO: check if decoder is callable?
        # FIXME: it is not certain that we need such indexing into the decoder's output
        top_1 = decoder(predictions)[0][0]
        ncode, label, proba = top_1
        # TODO: do I print, log, or append to 'description' this output?
        print('Predicted class:')
        print('%s (%s) with probability %.2f' % (label, ncode, proba))
    # FIXME: non-classification tasks
    predicted_class = np.argmax(predictions)
    return predicted_class

def get_target_layer(estimator, desired_layer):
    """
    Return instance of the desired layer in the model.
    estimator: model whose layer is to be gotten
    desired_layer: one of: integer index, string name of the layer,
    callable that returns True if a layer instance matches.
    """
    if isinstance(desired_layer, int):
        # conv_output =  [l for l in model.layers if l.name is layer_name]
        # conv_output = conv_output[0]
        # bottom-up horizontal graph traversal
        target_layer = estimator.get_layer(index=desired_layer)
        # These can raise ValueError if the layer index / name specified is not found
    elif isinstance(desired_layer, str):
        target_layer = estimator.get_layer(name=desired_layer)
    elif callable(desired_layer):
        # is 'callable()' the right check to use here?
        l = estimator.get_layer(index=-4)
        # FIXME: don't hardcode four
        # actually iterate through the list of layers backwards (using negative indexing with get_layer()) until find the desired layer
        target_layer = l if desired_layer(l) else None
        if target_layer is None:
            # If can't find, raise error
            raise ValueError('Target layer could not be found using callable %s' % desired_layer)
    else:
        raise ValueError('Invalid desired_layer (must be str, int, or callable): "%s"' % desired_layer)

    # TODO: check target_layer dimensions (is it possible to do Grad-CAM on it?)
    return target_layer

def get_last_activation_maps(estimator):
    # TODO: automatically get last Conv layer if layer_name and layer_index are None
    # Some ideas:
    # 1. look at layer name, exclude things like "softmax", "global average pooling",
    # etc, include things like "conv" (but watch for false positives)
    # 2. look at layer input/output dimensions, to ensure they match
    return True

# path to a single image, directory containing images,
# PIL image object, or an array can also be multiple images.

def preprocess_image(img, estimator=None, preprocessing=None):
    """
    Returns a single image as an array for an estimator's input
    img: one of: path to a single image file, PIL Image object, numpy array
    estimator: model instance, for resizing the image to the required input dimensions
    preprocessing: one of: callable (called with the image array as an argument),
    "auto" (attempts to use a preprocessing function from keras.applications, requires estimator argument)
    """
    xDims = None
    if estimator is not None:
        xDims = estimator.input_shape[1:3]
    im = image.load_img(img, target_size=xDims)
    x = image.img_to_array(im)

    # we need to insert an axis at the 0th position to indicate the batch size
    # this is required by the keras predict() function
    x = np.expand_dims(x, axis=0)

    if callable(preprocessing):
        x = preprocessing(x)
    elif preprocessing == "auto":
        # Consider removing this "auto" option (the user can pass the function themselves)
        try:
            f = getattr(keras.applications, estimator.name.lower()).preprocess_input
        except AttributeError:
            print('Could not get preprocessing automatically')
        else:
            x = f(x)
    return x

# Credits to Jacob Gildenblat for https://github.com/jacobgil/keras-grad-cam
# (and https://github.com/PowerOfCreation/keras-grad-cam)

def target_category_loss(x, category_index, nb_classes):
    # return tf.multiply(x, K.one_hot([category_index], nb_classes))
    return x * K.one_hot([category_index], nb_classes) # this works as well

def target_category_loss_output_shape(input_shape):
    return input_shape

def normalize(x):
    return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)

def _compute_gradients(tensor, var_list):
    # grads = tf.gradients(tensor, var_list)
    # return [grad if grad is not None else tf.zeros_like(var) for var, grad in zip(var_list, grads)]
    grads = K.gradients(tensor, var_list)
    return [grad if grad is not None else K.zeros_like(var) for var, grad in zip(var_list, grads)]

def grad_cam(input_model, image, category_index, layer):
    # FIXME: this assumes that we are doing classification
    # nb_classes = 1000 # FIXME: number of classes can be variable
    nb_classes = input_model.output_shape[1] # TODO: test this

    # FIXME: rename these "layer" variables
    target_layer = lambda x: target_category_loss(x, category_index, nb_classes)
    x = Lambda(target_layer, output_shape = target_category_loss_output_shape)(input_model.output)
    model = Model(inputs=input_model.input, outputs=x)
    # model.summary() # remove this later
    loss = K.sum(model.output)

    # we need to get the output attribute, else we get a TypeError: Failed to convert object to tensor
    conv_output = layer.output

    grads = normalize(_compute_gradients(loss, [conv_output])[0])
    gradient_function = K.function([model.input], [conv_output, grads])

    output, grads_val = gradient_function([image]) # work happens here
    output, grads_val = output[0, :], grads_val[0, :, :, :] # FIXME: this probably assumes that the layer is a width*height filter

    weights = np.mean(grads_val, axis = (0, 1))
    # cam = np.ones(output.shape[0 : 2], dtype = np.float32)
    heatmap = np.ones(output.shape[0 : 2], dtype = np.float32)

    for i, w in enumerate(weights):
        # cam += w * output[:, :, i]
        heatmap += w * output[:, :, i]

    heatmap = np.maximum(heatmap, 0) # ReLU
    heatmap = heatmap / np.max(heatmap) # probability
    heatmap = 255*heatmap # 0...255 float
    # we need to insert a channels axis to have an image (channels last by default)
    heatmap = np.expand_dims(heatmap, axis=-1)
    return heatmap<|endoftext|>


<commit_before>import argparse
import json
import sys
from datetime import datetime

DEFAULT_IGNORED_KEYS = set([
    u'action_status', u'action_type', u'task_level', u'task_uuid'])

def _truncate_value(value, limit=100):
    """
    Truncate long values.
    """
    values = value.split('\n')
    value = values[0]
    if len(value) > limit or len(values) > 1:
        return '{}...'.format(value[:limit])
    return value

def _task_name(task):
    """
    Compute the task name for an Eliot task.
    """
    level = u','.join(map(unicode, task[u'task_level']))
    message_type = task.get('message_type', None)
    if message_type is not None:
        status = None
    elif message_type is None:
        message_type = task['action_type']
        status = task['action_status']
    return u'{message_type}@{level}/{status}'.format(
        message_type=message_type,
        level=level,
        status=status)

class TaskNode(object):
    def __init__(self, task, name=None, sorter=None):
        self.task = task
        self._children = {}
        if name is None:
            name = _task_name(task)
        self.name = name
        self.sorter = sorter

    def __repr__(self):
        if self.task is None:
            task_uuid = 'root'
        else:
            # XXX: This is probably wrong in a bunch of places.
            task_uuid = self.task[u'task_uuid'].encode('utf-8')
        return '<{type} {task_uuid} {name} children={children}>'.format(
            type=type(self).__name__,
            task_uuid=task_uuid,
            # XXX: This is probably wrong in a bunch of places.
            name=self.name.encode('utf-8'),
            children=len(self._children))

    def add_child(self, node, levels=None):
        """
        Add a child ``TaskNode``.
        """
        if levels is None:
            levels = node.task['task_level']
        levels = list(levels)
        level = levels.pop(0)
        children = self._children
        if level in children:
            return children[level].add_child(node, levels)
        assert level not in children
        children[level] = node

    def children(self):
        """
        Get an ordered ``list`` of child ``TaskNode``s.
        """
        return sorted(
            self._children.values(), key=lambda n: n.task[u'task_level'])

def _indented_write(write):
    """
    Indent a write.
    """
    def _write(data):
        write('    ' + data)
    return _write

def _format_value(value):
    """
    Format a value for a task tree.
    """
    if isinstance(value, datetime):
        return value.isoformat(' ')
    elif isinstance(value, unicode):
        # XXX: This is probably wrong in a bunch of places.
        return value.encode('utf-8')
    elif isinstance(value, dict):
        return value
    # XXX: This is probably wrong in a bunch of places.
    return str(value)

def _render_task(write, task, ignored_task_keys):
    """
    Render a single ``TaskNode`` as an ``ASCII`` tree.

    :param write: Callable taking a single ``bytes`` argument to write the
        output.
    :param task: Eliot task ``dict`` to render.
    :param ignored_task_keys: ``set`` of task key names to ignore.
    """
    _write = _indented_write(write)
    num_items = len(task)
    for i, (key, value) in enumerate(sorted(task.items()), 1):
        if key not in ignored_task_keys:
            tree_char = '`' if i == num_items else '|'
            _value = _format_value(value)
            if isinstance(value, dict):
                write(
                    '{tree_char}-- {key}:\n'.format(
                        tree_char=tree_char,
                        key=key.encode('utf-8')))
                _render_task(_write, _value, {})
            else:
                write(
                    '{tree_char}-- {key}: {value}\n'.format(
                        tree_char=tree_char,
                        key=key.encode('utf-8'),
                        value=_truncate_value(_value)))

def _render_task_node(write, node, ignored_task_keys):
    """
    Render a single ``TaskNode`` as an ``ASCII`` tree.

    :param write: Callable taking a single ``bytes`` argument to write the
        output.
    :param node: ``TaskNode`` to render.
    :param ignored_task_keys: ``set`` of task key names to ignore.
    """
    _child_write = _indented_write(write)
    if node.task is not None:
        write(
            '+-- {name}\n'.format(
                # XXX: This is probably wrong in a bunch of places.
                name=node.name.encode('utf-8')))
        _render_task(_child_write, node.task, ignored_task_keys)

    for child in node.children():
        _render_task_node(_child_write, child, ignored_task_keys)

def render_task_tree(write, tasktree, ignored_task_keys=DEFAULT_IGNORED_KEYS):
    """
    Render a task tree as an ``ASCII`` tree.

    :param write: Callable taking a single ``bytes`` argument to write the
        output.
    :param tasktree: ``list`` of ``(task_uuid, task_node)``.
    :param ignored_task_keys: ``set`` of task key names to ignore.
    """
    for task_uuid, node in tasktree:
        write('{name}\n'.format(
            # XXX: This is probably wrong in a bunch of places.
            name=node.name.encode('utf-8')))
        _render_task_node(write, node, ignored_task_keys)
        write('\n')

def merge_tasktree(tasktree, fd, process_task=None):
    """
    Merge Eliot tasks specified in ``fd`` with ``tasktree`.

    :type tasktree: ``dict``
    :type fd: ``file``-like
    :param process_task: Callable taking a single ``dict`` argument, the task
        read from ``fd``, that returns a transformed ``dict``.
    :return: Newly merged task tree.
    """
    if process_task is None:
        process_task = lambda task: task
    for line in fd:
        task = process_task(json.loads(line))
        key = task[u'task_uuid']
        node = tasktree.get(key)
        if node is None:
            node = tasktree[key] = TaskNode(None, key, task[u'timestamp'])
        node.add_child(TaskNode(task))
    return tasktree

def _convert_timestamp(task):
    """
    Convert a ``timestamp`` key to a ``datetime``.
    """
    task['timestamp'] = datetime.fromtimestamp(task['timestamp'])
    return task

def display_task_tree(args):
    """
    Read the input files, apply any command-line-specified behaviour and
    display the task tree.
    """
    if not args.files:
        args.files.append(sys.stdin)
    process_task = None
    if args.human_readable:
        process_task = _convert_timestamp

    tasktree = {}
    for fd in args.files:
        tasktree = merge_tasktree(tasktree, fd, process_task)

    if args.task_uuid is not None:
        node = tasktree.get(args.task_uuid)
        if node is None:
            tasktree = {}
        else:
            tasktree = {args.task_uuid: node}

    tasktree = sorted(tasktree.items(), key=lambda (_, n): n.sorter)
    render_task_tree(
        write=sys.stdout.write,
        tasktree=tasktree,
        ignored_task_keys=set(args.ignored_task_keys) or DEFAULT_IGNORED_KEYS)

def main():
    parser = argparse.ArgumentParser(
        description='Display an Eliot log as a tree of tasks.')
    parser.add_argument('files',
                        metavar='FILE',
                        nargs='*',
                        type=argparse.FileType('r'),
                        help='''Files to process''')
    parser.add_argument('-u', '--task-uuid',
                        dest='task_uuid',
                        metavar='UUID',
                        help='''Select a specific task by UUID''')
    parser.add_argument('-i', '--ignore-task-key',
                        action='append',
                        default=[],
                        dest='ignored_task_keys',
                        metavar='KEY',
                        help='''Ignore a task key, use multiple times to ignore
                        multiple keys. Defaults to ignoring most Eliot standard
                        keys.''')
    parser.add_argument('--raw',
                        action='store_false',
                        dest='human_readable',
                        help='''Do not format some task values (such as
                        timestamps) as human-readable''')
    args = parser.parse_args()
    display_task_tree(args)
<commit_msg>Add an option to disable truncating long values.<commit_after>import argparse
import json
import sys
from datetime import datetime

DEFAULT_IGNORED_KEYS = set([
    u'action_status', u'action_type', u'task_level', u'task_uuid'])

def _truncate_value(value, limit=100):
    """
    Truncate long values.
    """
    values = value.split('\n')
    value = values[0]
    if len(value) > limit or len(values) > 1:
        return '{} [...]'.format(value[:limit])
    return value

def _task_name(task):
    """
    Compute the task name for an Eliot task.
    """
    level = u','.join(map(unicode, task[u'task_level']))
    message_type = task.get('message_type', None)
    if message_type is not None:
        status = None
    elif message_type is None:
        message_type = task['action_type']
        status = task['action_status']
    return u'{message_type}@{level}/{status}'.format(
        message_type=message_type,
        level=level,
        status=status)

class TaskNode(object):
    def __init__(self, task, name=None, sorter=None):
        self.task = task
        self._children = {}
        if name is None:
            name = _task_name(task)
        self.name = name
        self.sorter = sorter

    def __repr__(self):
        if self.task is None:
            task_uuid = 'root'
        else:
            # XXX: This is probably wrong in a bunch of places.
            task_uuid = self.task[u'task_uuid'].encode('utf-8')
        return '<{type} {task_uuid} {name} children={children}>'.format(
            type=type(self).__name__,
            task_uuid=task_uuid,
            # XXX: This is probably wrong in a bunch of places.
            name=self.name.encode('utf-8'),
            children=len(self._children))

    def add_child(self, node, levels=None):
        """
        Add a child ``TaskNode``.
        """
        if levels is None:
            levels = node.task['task_level']
        levels = list(levels)
        level = levels.pop(0)
        children = self._children
        if level in children:
            return children[level].add_child(node, levels)
        assert level not in children
        children[level] = node

    def children(self):
        """
        Get an ordered ``list`` of child ``TaskNode``s.
        """
        return sorted(
            self._children.values(), key=lambda n: n.task[u'task_level'])

def _indented_write(write):
    """
    Indent a write.
    """
    def _write(data):
        write('    ' + data)
    return _write

def _format_value(value):
    """
    Format a value for a task tree.
    """
    if isinstance(value, datetime):
        return value.isoformat(' ')
    elif isinstance(value, unicode):
        # XXX: This is probably wrong in a bunch of places.
        return value.encode('utf-8')
    elif isinstance(value, dict):
        return value
    # XXX: This is probably wrong in a bunch of places.
    return str(value)

def _render_task(write, task, ignored_task_keys, truncate):
    """
    Render a single ``TaskNode`` as an ``ASCII`` tree.

    :param write: Callable taking a single ``bytes`` argument to write the
        output.
    :param task: Eliot task ``dict`` to render.
    :param ignored_task_keys: ``set`` of task key names to ignore.
    """
    _write = _indented_write(write)
    num_items = len(task)
    for i, (key, value) in enumerate(sorted(task.items()), 1):
        if key not in ignored_task_keys:
            tree_char = '`' if i == num_items else '|'
            _value = _format_value(value)
            if isinstance(value, dict):
                write(
                    '{tree_char}-- {key}:\n'.format(
                        tree_char=tree_char,
                        key=key.encode('utf-8')))
                _render_task(_write, _value, {})
            else:
                if truncate:
                    first_line = _truncate_value(_value)
                else:
                    lines = _value.splitlines() or [u'']
                    first_line = lines.pop(0)
                write(
                    '{tree_char}-- {key}: {value}\n'.format(
                        tree_char=tree_char,
                        key=key.encode('utf-8'),
                        value=first_line))
                if not truncate:
                    for line in lines:
                        _write(line + '\n')

def _render_task_node(write, node, ignored_task_keys, truncate):
    """
    Render a single ``TaskNode`` as an ``ASCII`` tree.

    :param write: Callable taking a single ``bytes`` argument to write the
        output.
    :param node: ``TaskNode`` to render.
    :param ignored_task_keys: ``set`` of task key names to ignore.
    """
    _child_write = _indented_write(write)
    if node.task is not None:
        write(
            '+-- {name}\n'.format(
                # XXX: This is probably wrong in a bunch of places.
                name=node.name.encode('utf-8')))
        _render_task(_child_write, node.task, ignored_task_keys, truncate)

    for child in node.children():
        _render_task_node(_child_write, child, ignored_task_keys, truncate)

def render_task_tree(write, tasktree, ignored_task_keys=DEFAULT_IGNORED_KEYS,
                     truncate=False):
    """
    Render a task tree as an ``ASCII`` tree.

    :param write: Callable taking a single ``bytes`` argument to write the
        output.
    :param tasktree: ``list`` of ``(task_uuid, task_node)``.
    :param ignored_task_keys: ``set`` of task key names to ignore.
    """
    for task_uuid, node in tasktree:
        write('{name}\n'.format(
            # XXX: This is probably wrong in a bunch of places.
            name=node.name.encode('utf-8')))
        _render_task_node(write, node, ignored_task_keys, truncate)
        write('\n')

def merge_tasktree(tasktree, fd, process_task=None):
    """
    Merge Eliot tasks specified in ``fd`` with ``tasktree`.

    :type tasktree: ``dict``
    :type fd: ``file``-like
    :param process_task: Callable taking a single ``dict`` argument, the task
        read from ``fd``, that returns a transformed ``dict``.
    :return: Newly merged task tree.
    """
    if process_task is None:
        process_task = lambda task: task
    for line in fd:
        task = process_task(json.loads(line))
        key = task[u'task_uuid']
        node = tasktree.get(key)
        if node is None:
            node = tasktree[key] = TaskNode(None, key, task[u'timestamp'])
        node.add_child(TaskNode(task))
    return tasktree

def _convert_timestamp(task):
    """
    Convert a ``timestamp`` key to a ``datetime``.
    """
    task['timestamp'] = datetime.fromtimestamp(task['timestamp'])
    return task

def display_task_tree(args):
    """
    Read the input files, apply any command-line-specified behaviour and
    display the task tree.
    """
    if not args.files:
        args.files.append(sys.stdin)
    process_task = None
    if args.human_readable:
        process_task = _convert_timestamp

    tasktree = {}
    for fd in args.files:
        tasktree = merge_tasktree(tasktree, fd, process_task)

    if args.task_uuid is not None:
        node = tasktree.get(args.task_uuid)
        if node is None:
            tasktree = {}
        else:
            tasktree = {args.task_uuid: node}

    tasktree = sorted(tasktree.items(), key=lambda (_, n): n.sorter)
    render_task_tree(
        write=sys.stdout.write,
        tasktree=tasktree,
        ignored_task_keys=set(args.ignored_task_keys) or DEFAULT_IGNORED_KEYS,
        truncate=args.truncate)

def main():
    parser = argparse.ArgumentParser(
        description='Display an Eliot log as a tree of tasks.')
    parser.add_argument('files',
                        metavar='FILE',
                        nargs='*',
                        type=argparse.FileType('r'),
                        help='''Files to process''')
    parser.add_argument('-u', '--task-uuid',
                        dest='task_uuid',
                        metavar='UUID',
                        help='''Select a specific task by UUID''')
    parser.add_argument('-i', '--ignore-task-key',
                        action='append',
                        default=[],
                        dest='ignored_task_keys',
                        metavar='KEY',
                        help='''Ignore a task key, use multiple times to ignore
                        multiple keys. Defaults to ignoring most Eliot standard
                        keys.''')
    parser.add_argument('--raw',
                        action='store_false',
                        dest='human_readable',
                        help='''Do not format some task values (such as
                        timestamps) as human-readable''')
    parser.add_argument('--no-truncate',
                        action='store_false',
                        dest='truncate',
                        help='''Do not truncate long values''')
    args = parser.parse_args()
    display_task_tree(args)
<|endoftext|>


<commit_before>from django import forms
from django.forms.models import BaseModelFormSet, modelformset_factory
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.utils.html import conditional_escape
from django.utils.safestring import mark_safe

from localtv import models

class TagWidgetMixin:
    def render(self, name, value, attrs=None):
        if isinstance(value, basestring):
            return self.__class__.__bases__[-1].render(self, name, value,
                                                       attrs)
        if value is None:
            value = []
        return self.__class__.__bases__[-1].render(
            self, name,
            ', '.join(models.Tag.objects.filter(pk__in=value).values_list(
                    'name', flat=True)),
            attrs)

class TagWidget(TagWidgetMixin, forms.TextInput):
    pass

class TagAreaWidget(TagWidgetMixin, forms.Textarea):
    pass

class TagField(forms.CharField):
    widget = TagWidget

    def clean(self, value):
        if not value:
            return []
        names = [name.strip() for name in value.split(',')]
        tags = []
        for name in names:
            tag, created = models.Tag.objects.get_or_create(name=name)
            tags.append(tag)
        return tags

class EditVideoForm(forms.ModelForm):
    """
    """
    thumbnail = forms.ImageField(required=False)
    class Meta:
        model = models.Video
        fields = ('thumbnail', 'thumbnail_url', )

    def save(self, *args, **kwargs):
        if 'thumbnail' in self.cleaned_data:
            thumbnail = self.cleaned_data.pop('thumbnail')
            if thumbnail:
                self.instance.thumbnail_url = \
                    self.cleaned_data['thumbnail_url'] = ''
                # since we're no longer using
                # that URL for a thumbnail
                self.instance.save_thumbnail_from_file(thumbnail)
        if 'thumbnail_url' in self.cleaned_data:
            thumbnail_url = self.cleaned_data.pop('thumbnail_url')
            if thumbnail_url and thumbnail_url != self.instance.thumbnail_url:
                self.instance.thumbnail_url = thumbnail_url
                self.instance.save_thumbnail()
        return forms.ModelForm.save(self, *args, **kwargs)

class BulkChecklistField(forms.ModelMultipleChoiceField):
    widget = forms.CheckboxSelectMultiple

    def label_from_instance(self, instance):
        if isinstance(instance, User):
            if instance.first_name:
                name = '%s %s' % (instance.first_name,
                                  instance.last_name)
            else:
                name = instance.username
        else:
            name = instance.name
        return mark_safe(u'%s' % (
                conditional_escape(name)))

class BulkEditVideoForm(EditVideoForm):
    bulk = forms.BooleanField(required=False)
    name = forms.CharField(widget=forms.TextInput(
            attrs={'class': 'large_field'}),
                           required=False)
    file_url = forms.CharField(widget=forms.TextInput(
            attrs={'class': 'large_field'}),
                               required=False)
    thumbnail_url = forms.CharField(widget=forms.TextInput(
            attrs={'class': 'large_field'}),
                                    required=False)
    tags = TagField(required=False,
                    widget=TagAreaWidget)
    categories = BulkChecklistField(models.Category.objects, required=False)
    authors = BulkChecklistField(User.objects, required=False)
    when_published = forms.DateTimeField(required=False,
                                         widget=forms.TextInput(
            attrs={'class': 'large_field'}))

    class Meta:
        model = models.Video
        fields = ('name', 'description', 'thumbnail', 'thumbnail_url', 'tags',
                  'categories', 'authors', 'when_published', 'file_url')

    def __init__(self, *args, **kwargs):
        EditVideoForm.__init__(self, *args, **kwargs)
        self.fields['categories'].queryset = \
            self.fields['categories'].queryset.filter(
            site=self.instance.site)

VideoFormSet = modelformset_factory(models.Video,
                                    form=BulkEditVideoForm,
                                    can_delete=True,
                                    extra=1)

class EditTitleForm(forms.Form):
    """
    """
    title = forms.CharField(label="Site Title")
    tagline = forms.CharField(label="Site Tagline")
    about = forms.CharField(label="About Us Page (use html)",
                            widget=forms.Textarea, required=False)

    @classmethod
    def create_from_sitelocation(cls, sitelocation):
        self = cls()
        self.initial['title'] = sitelocation.site.name
        self.initial['tagline'] = sitelocation.tagline
        self.initial['about'] = sitelocation.about_html
        return self

    def save_to_sitelocation(self, sitelocation):
        if not self.is_valid():
            raise RuntimeError("cannot save invalid form")
        sitelocation.site.name = self.cleaned_data['title']
        sitelocation.tagline = self.cleaned_data.get('tagline', '')
        sitelocation.about_html = self.cleaned_data.get('about', '')
        sitelocation.site.save()
        sitelocation.save()

class EditSidebarForm(forms.Form):
    sidebar = forms.CharField(label="Sidebar Blurb (use html)",
                            widget=forms.Textarea, required=False)
    footer = forms.CharField(label="Footer Blurb (use html)",
                             widget=forms.Textarea, required=False,
                             help_text="In addition to any footer text you "
                             "would like to add, we suggest using this space "
                             "to paste in a Google Analytics tracking code, "
                             "which will provide excellent statistics on "
                             "usage of your site.")

    @classmethod
    def create_from_sitelocation(cls, sitelocation):
        self = cls()
        self.initial['sidebar'] = sitelocation.sidebar_html
        self.initial['footer'] = sitelocation.footer_html
        return self

    def save_to_sitelocation(self, sitelocation):
        if not self.is_valid():
            raise RuntimeError("cannot save invalid form")
        sitelocation.sidebar_html = self.cleaned_data.get('sidebar', '')
        sitelocation.footer_html = self.cleaned_data.get('footer', '')
        sitelocation.save()

class EditMiscDesignForm(forms.Form):
    logo = forms.ImageField(label="Logo Image", required=False)
    background = forms.ImageField(label="Background Image", required=False)
    #theme = forms.ChoiceField(label="Color Theme", choices=(
    #        ("day", "Day"),
    #        ("night", "Night")))
    layout = forms.ChoiceField(label="Front Page Layout", choices=(
            ("scrolling", "Scrolling big features"),
            ("list", "List style"),
            ("categorized", "Categorized layout")),
                               help_text=" (note: with the scrolling and categorized layouts, you will need to provide hi-quality images for each featured video)")
    display_submit_button = forms.BooleanField(
        label="Display the 'submit a video' nav item",
        required=False)
    submission_requires_login = forms.BooleanField(
        label="Require users to login to submit a video",
        required=False)
    css = forms.CharField(label="Custom CSS",
                          help_text="Here you can append your own CSS to customize your site.",
                          widget=forms.Textarea, required=False)

    @classmethod
    def create_from_sitelocation(cls, sitelocation):
        self = cls()
        self.initial['css'] = sitelocation.css
        self.initial['layout'] = sitelocation.frontpage_style
        self.initial['display_submit_button'] = sitelocation.display_submit_button
        self.initial['submission_requires_login'] = sitelocation.submission_requires_login
        return self

    def save_to_sitelocation(self, sitelocation):
        if not self.is_valid():
            raise RuntimeError("cannot save invalid form")
        logo = self.cleaned_data.get('logo')
        if logo is not None:
            sitelocation.logo.save(logo.name, logo, save=False)
        background = self.cleaned_data.get('background')
        if background is not None:
            sitelocation.background.save(background.name, background, save=False)
        sitelocation.css = self.cleaned_data.get('css', '')
        sitelocation.frontpage_style = self.cleaned_data['layout']
        sitelocation.display_submit_button = self.cleaned_data['display_submit_button']
        sitelocation.submission_requires_login = self.cleaned_data['submission_requires_login']
        sitelocation.save()

class CategoryForm(forms.ModelForm):
    class Meta:
        model = models.Category
        exclude = ['site']

    def __init__(self, *args, **kwargs):
        forms.ModelForm.__init__(self, *args, **kwargs)

        site = Site.objects.get_current()
        self.fields['parent'].queryset = models.Category.objects.filter(
            site=site)

class AuthorForm(forms.ModelForm):
    logo = forms.ImageField(required=False)

    class Meta:
        model = User
        fields = ['first_name', 'last_name', 'logo']

    def save(self, **kwargs):
        author = forms.ModelForm.save(self, **kwargs)
        if self.cleaned_data.get('logo'):
            logo = self.cleaned_data['logo']
            print 'going to save', logo
            try:
                profile = author.get_profile()
            except models.Profile.DoesNotExist:
                profile = models.Profile.objects.create(
                    user=author)

            profile.logo = logo
            profile.save()
        return author

class AddUserForm(forms.Form):
    user = forms.CharField(
        help_text=("You can enter a user name, an OpenID, or an e-mail "
                   "address in this field"))

    def clean_user(self):
        value = self.cleaned_data['user']

        if value.startswith('http://') or value.startswith('https://'):
            # possibly an OpenID
            openid_users = models.OpenIdUser.objects.filter(url=value)
            if openid_users:
                return [oid.user for oid in openid_users]

        if '@' in value:
            # possibly an e-mail address
            users = User.objects.filter(email=value)
            if users:
                return users

        users = User.objects.filter(username=value)
        if users:
            return users
        else:
            raise forms.ValidationError('Could not find a matching user')

class VideoServiceForm(forms.Form):
    URLS = (
        ('YouTube', 'http://www.youtube.com/rss/user/%s/videos.rss'),
        ('blip.tv', 'http://%s.blip.tv/rss'),
        ('Vimeo', 'http://www.vimeo.com/%s/videos/rss'),
         )
    service = forms.ChoiceField(
        choices = enumerate([service for service, url in URLS]))
    username = forms.CharField(
        initial="e.g. openvideoalliance")

    def __init__(self, *args, **kwargs):
        forms.Form.__init__(self, *args, **kwargs)
        self.fields['username'].widget.attrs.update(
            {'onfocus': 'clearText(this)',
             'class': 'large_field'})

    def feed_url(self):
        index = int(self.cleaned_data['service'])
        return self.URLS[index][1] % self.cleaned_data['username']
<commit_msg>blank instance won't have a Site, so look it up instead<commit_after>from django import forms
from django.forms.models import modelformset_factory
from django.contrib.auth.models import User
from django.contrib.sites.models import Site
from django.utils.html import conditional_escape
from django.utils.safestring import mark_safe

from localtv import models

class TagWidgetMixin:
    def render(self, name, value, attrs=None):
        if isinstance(value, basestring):
            return self.__class__.__bases__[-1].render(self, name, value,
                                                       attrs)
        if value is None:
            value = []
        return self.__class__.__bases__[-1].render(
            self, name,
            ', '.join(models.Tag.objects.filter(pk__in=value).values_list(
                    'name', flat=True)),
            attrs)

class TagWidget(TagWidgetMixin, forms.TextInput):
    pass

class TagAreaWidget(TagWidgetMixin, forms.Textarea):
    pass

class TagField(forms.CharField):
    widget = TagWidget

    def clean(self, value):
        if not value:
            return []
        names = [name.strip() for name in value.split(',')]
        tags = []
        for name in names:
            tag, created = models.Tag.objects.get_or_create(name=name)
            tags.append(tag)
        return tags

class EditVideoForm(forms.ModelForm):
    """
    """
    thumbnail = forms.ImageField(required=False)
    class Meta:
        model = models.Video
        fields = ('thumbnail', 'thumbnail_url', )

    def save(self, *args, **kwargs):
        if 'thumbnail' in self.cleaned_data:
            thumbnail = self.cleaned_data.pop('thumbnail')
            if thumbnail:
                self.instance.thumbnail_url = \
                    self.cleaned_data['thumbnail_url'] = ''
                # since we're no longer using
                # that URL for a thumbnail
                self.instance.save_thumbnail_from_file(thumbnail)
        if 'thumbnail_url' in self.cleaned_data:
            thumbnail_url = self.cleaned_data.pop('thumbnail_url')
            if thumbnail_url and thumbnail_url != self.instance.thumbnail_url:
                self.instance.thumbnail_url = thumbnail_url
                self.instance.save_thumbnail()
        return forms.ModelForm.save(self, *args, **kwargs)

class BulkChecklistField(forms.ModelMultipleChoiceField):
    widget = forms.CheckboxSelectMultiple

    def label_from_instance(self, instance):
        if isinstance(instance, User):
            if instance.first_name:
                name = '%s %s' % (instance.first_name,
                                  instance.last_name)
            else:
                name = instance.username
        else:
            name = instance.name
        return mark_safe(u'%s' % (
                conditional_escape(name)))

class BulkEditVideoForm(EditVideoForm):
    bulk = forms.BooleanField(required=False)
    name = forms.CharField(widget=forms.TextInput(
            attrs={'class': 'large_field'}),
                           required=False)
    file_url = forms.CharField(widget=forms.TextInput(
            attrs={'class': 'large_field'}),
                               required=False)
    thumbnail_url = forms.CharField(widget=forms.TextInput(
            attrs={'class': 'large_field'}),
                                    required=False)
    tags = TagField(required=False,
                    widget=TagAreaWidget)
    categories = BulkChecklistField(models.Category.objects, required=False)
    authors = BulkChecklistField(User.objects, required=False)
    when_published = forms.DateTimeField(required=False,
                                         widget=forms.TextInput(
            attrs={'class': 'large_field'}))

    class Meta:
        model = models.Video
        fields = ('name', 'description', 'thumbnail', 'thumbnail_url', 'tags',
                  'categories', 'authors', 'when_published', 'file_url')

    def __init__(self, *args, **kwargs):
        EditVideoForm.__init__(self, *args, **kwargs)
        site = Site.objects.get_current()
        self.fields['categories'].queryset = \
            self.fields['categories'].queryset.filter(
            site=site)

VideoFormSet = modelformset_factory(models.Video,
                                    form=BulkEditVideoForm,
                                    can_delete=True,
                                    extra=1)

class EditTitleForm(forms.Form):
    """
    """
    title = forms.CharField(label="Site Title")
    tagline = forms.CharField(label="Site Tagline")
    about = forms.CharField(label="About Us Page (use html)",
                            widget=forms.Textarea, required=False)

    @classmethod
    def create_from_sitelocation(cls, sitelocation):
        self = cls()
        self.initial['title'] = sitelocation.site.name
        self.initial['tagline'] = sitelocation.tagline
        self.initial['about'] = sitelocation.about_html
        return self

    def save_to_sitelocation(self, sitelocation):
        if not self.is_valid():
            raise RuntimeError("cannot save invalid form")
        sitelocation.site.name = self.cleaned_data['title']
        sitelocation.tagline = self.cleaned_data.get('tagline', '')
        sitelocation.about_html = self.cleaned_data.get('about', '')
        sitelocation.site.save()
        sitelocation.save()

class EditSidebarForm(forms.Form):
    sidebar = forms.CharField(label="Sidebar Blurb (use html)",
                            widget=forms.Textarea, required=False)
    footer = forms.CharField(label="Footer Blurb (use html)",
                             widget=forms.Textarea, required=False,
                             help_text="In addition to any footer text you "
                             "would like to add, we suggest using this space "
                             "to paste in a Google Analytics tracking code, "
                             "which will provide excellent statistics on "
                             "usage of your site.")

    @classmethod
    def create_from_sitelocation(cls, sitelocation):
        self = cls()
        self.initial['sidebar'] = sitelocation.sidebar_html
        self.initial['footer'] = sitelocation.footer_html
        return self

    def save_to_sitelocation(self, sitelocation):
        if not self.is_valid():
            raise RuntimeError("cannot save invalid form")
        sitelocation.sidebar_html = self.cleaned_data.get('sidebar', '')
        sitelocation.footer_html = self.cleaned_data.get('footer', '')
        sitelocation.save()

class EditMiscDesignForm(forms.Form):
    logo = forms.ImageField(label="Logo Image", required=False)
    background = forms.ImageField(label="Background Image", required=False)
    #theme = forms.ChoiceField(label="Color Theme", choices=(
    #        ("day", "Day"),
    #        ("night", "Night")))
    layout = forms.ChoiceField(label="Front Page Layout", choices=(
            ("scrolling", "Scrolling big features"),
            ("list", "List style"),
            ("categorized", "Categorized layout")),
                               help_text=" (note: with the scrolling and categorized layouts, you will need to provide hi-quality images for each featured video)")
    display_submit_button = forms.BooleanField(
        label="Display the 'submit a video' nav item",
        required=False)
    submission_requires_login = forms.BooleanField(
        label="Require users to login to submit a video",
        required=False)
    css = forms.CharField(label="Custom CSS",
                          help_text="Here you can append your own CSS to customize your site.",
                          widget=forms.Textarea, required=False)

    @classmethod
    def create_from_sitelocation(cls, sitelocation):
        self = cls()
        self.initial['css'] = sitelocation.css
        self.initial['layout'] = sitelocation.frontpage_style
        self.initial['display_submit_button'] = sitelocation.display_submit_button
        self.initial['submission_requires_login'] = sitelocation.submission_requires_login
        return self

    def save_to_sitelocation(self, sitelocation):
        if not self.is_valid():
            raise RuntimeError("cannot save invalid form")
        logo = self.cleaned_data.get('logo')
        if logo is not None:
            sitelocation.logo.save(logo.name, logo, save=False)
        background = self.cleaned_data.get('background')
        if background is not None:
            sitelocation.background.save(background.name, background, save=False)
        sitelocation.css = self.cleaned_data.get('css', '')
        sitelocation.frontpage_style = self.cleaned_data['layout']
        sitelocation.display_submit_button = self.cleaned_data['display_submit_button']
        sitelocation.submission_requires_login = self.cleaned_data['submission_requires_login']
        sitelocation.save()

class CategoryForm(forms.ModelForm):
    class Meta:
        model = models.Category
        exclude = ['site']

    def __init__(self, *args, **kwargs):
        forms.ModelForm.__init__(self, *args, **kwargs)

        site = Site.objects.get_current()
        self.fields['parent'].queryset = models.Category.objects.filter(
            site=site)

class AuthorForm(forms.ModelForm):
    logo = forms.ImageField(required=False)

    class Meta:
        model = User
        fields = ['first_name', 'last_name', 'logo']

    def save(self, **kwargs):
        author = forms.ModelForm.save(self, **kwargs)
        if self.cleaned_data.get('logo'):
            logo = self.cleaned_data['logo']
            try:
                profile = author.get_profile()
            except models.Profile.DoesNotExist:
                profile = models.Profile.objects.create(
                    user=author)

            profile.logo = logo
            profile.save()
        return author

class AddUserForm(forms.Form):
    user = forms.CharField(
        help_text=("You can enter a user name, an OpenID, or an e-mail "
                   "address in this field"))

    def clean_user(self):
        value = self.cleaned_data['user']

        if value.startswith('http://') or value.startswith('https://'):
            # possibly an OpenID
            openid_users = models.OpenIdUser.objects.filter(url=value)
            if openid_users:
                return [oid.user for oid in openid_users]

        if '@' in value:
            # possibly an e-mail address
            users = User.objects.filter(email=value)
            if users:
                return users

        users = User.objects.filter(username=value)
        if users:
            return users
        else:
            raise forms.ValidationError('Could not find a matching user')

class VideoServiceForm(forms.Form):
    URLS = (
        ('YouTube', 'http://www.youtube.com/rss/user/%s/videos.rss'),
        ('blip.tv', 'http://%s.blip.tv/rss'),
        ('Vimeo', 'http://www.vimeo.com/%s/videos/rss'),
         )
    service = forms.ChoiceField(
        choices = enumerate([service for service, url in URLS]))
    username = forms.CharField(
        initial="e.g. openvideoalliance")

    def __init__(self, *args, **kwargs):
        forms.Form.__init__(self, *args, **kwargs)
        self.fields['username'].widget.attrs.update(
            {'onfocus': 'clearText(this)',
             'class': 'large_field'})

    def feed_url(self):
        index = int(self.cleaned_data['service'])
        return self.URLS[index][1] % self.cleaned_data['username']
<|endoftext|>