<commit_before>import click
import cv2
import numpy as np

@click.group()
@click.option('--out', '-o', default=None)
@click.pass_context
def cli(ctx, out):
    ctx.obj['out'] = out

@click.pass_context
def save(ctx, img):
    if ctx.obj.get('out') is not None:
        click.echo('output to {}'.format(ctx.obj.get('out')))
        cv2.imwrite(ctx.obj.get('out'), img)
    else:
        cv2.imshow('image', img)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

@cli.command()
@click.argument('image', type=click.Path(exists=True), required=True)
def grayscale(image):
    im = cv2.imread(image)
    im = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
    save(im)

@cli.group()
def filter():
    pass

@filter.command()
@click.argument('image', type=click.Path(exists=True), required=True)
@click.option('--ksize', '-k', nargs=2, type=click.Tuple([int, int]), default=(3, 3))
def blur(image, ksize):
    im = cv2.imread(image)
    im = cv2.blur(im, ksize=ksize)
    save(im)

@filter.command()
@click.argument('image', type=click.Path(exists=True), required=True)
@click.option('--pillow_kernel', '-p', is_flag=True, default=False)
def sharpen(image, pillow_kernel):
    if pillow_kernel is False:
        # https://en.wikipedia.org/wiki/Kernel_(image_processing)
        kernel = np.array([
            [0, -1, 0],
            [-1, 5, -1],
            [0, -1, 0],
        ])
    else:
        # pillow's sharpen kernel
        kernel = np.array([
            [-2, -2, -2],
            [-2, 32, -2],
            [-2, -2, -2],
        ]) / 16
    im = cv2.imread(image)
    im = cv2.filter2D(
        im,
        -1,
        kernel
    )
    save(im)

@filter.command()
@click.argument('image', type=click.Path(exists=True), required=True)
@click.option('--neiborhood', '-n', type=click.Choice([4, 8]), default=4)
def erosion(image, neiborhood):
    im = cv2.imread(image, 0)
    if neiborhood == 4:
        kernel = np.array([
            [0, 1, 0],
            [1, 1, 1],
            [0, 1, 0],
        ], np.uint8)
    else:
        kernel = np.ones((5, 5), np.uint8)
    im = cv2.erode(im, kernel, iterations=1)
    save(im)

def main():
    cli(obj={})

if __name__ == '__main__':
    main()
<commit_msg>phase correlate from two image<commit_after>import click
import cv2
import math
import numpy as np

@click.group()
@click.option('--out', '-o', default=None)
@click.pass_context
def cli(ctx, out):
    ctx.obj['out'] = out

@click.pass_context
def save(ctx, img):
    if ctx.obj.get('out') is not None:
        click.echo('output to {}'.format(ctx.obj.get('out')))
        cv2.imwrite(ctx.obj.get('out'), img)
    else:
        cv2.imshow('image', img)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

@cli.command()
@click.argument('image', type=click.Path(exists=True), required=True)
def grayscale(image):
    im = cv2.imread(image)
    im = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
    save(im)

@cli.command()
@click.argument('base_image', type=click.Path(exists=True), required=True)
@click.argument('correlate_image', type=click.Path(exists=True), required=True)
def correlate(base_image, correlate_image):
    base_im_gray = cv2.imread(base_image, 0)
    save(base_im_gray)
    correlate_im_gray = cv2.imread(correlate_image, 0)
    (dx, dy), _ = cv2.phaseCorrelate(np.float32(correlate_im_gray), np.float32(base_im_gray))
    M = np.float32([[1, 0, dx], [0, 1, dy]])
    rows, cols = base_im_gray.shape
    im_moved = cv2.warpAffine(cv2.imread(correlate_image, 1), M, (cols, rows))

    # fill outline with dummy color
    mask = np.full(base_im_gray.shape, 255, dtype=np.uint8)
    mask[math.ceil(abs(dy)):rows, math.ceil(abs(dx)):cols] = 0
    if dx < 0:
        mask = mask.T[::-1].T
    if dy < 0:
        mask = mask[::-1]
    im_moved = cv2.inpaint(im_moved, mask, 3, cv2.INPAINT_TELEA)
    save(im_moved)

@cli.group()
def filter():
    pass

@filter.command()
@click.argument('image', type=click.Path(exists=True), required=True)
@click.option('--ksize', '-k', nargs=2, type=click.Tuple([int, int]), default=(3, 3))
def blur(image, ksize):
    im = cv2.imread(image)
    im = cv2.blur(im, ksize=ksize)
    save(im)

@filter.command()
@click.argument('image', type=click.Path(exists=True), required=True)
@click.option('--pillow_kernel', '-p', is_flag=True, default=False)
def sharpen(image, pillow_kernel):
    if pillow_kernel is False:
        # https://en.wikipedia.org/wiki/Kernel_(image_processing)
        kernel = np.array([
            [0, -1, 0],
            [-1, 5, -1],
            [0, -1, 0],
        ])
    else:
        # pillow's sharpen kernel
        kernel = np.array([
            [-2, -2, -2],
            [-2, 32, -2],
            [-2, -2, -2],
        ]) / 16
    im = cv2.imread(image)
    im = cv2.filter2D(
        im,
        -1,
        kernel
    )
    save(im)

@filter.command()
@click.argument('image', type=click.Path(exists=True), required=True)
@click.option('--neiborhood', '-n', type=click.Choice([4, 8]), default=4)
def erosion(image, neiborhood):
    im = cv2.imread(image, 0)
    if neiborhood == 4:
        kernel = np.array([
            [0, 1, 0],
            [1, 1, 1],
            [0, 1, 0],
        ], np.uint8)
    else:
        kernel = np.ones((5, 5), np.uint8)
    im = cv2.erode(im, kernel, iterations=1)
    save(im)

def main():
    cli(obj={})

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># regress <master> <inputFile_Y> <inputFile_X> <outputFile> <analMode> <outputMode>
#
# time series regression on a data matrix
# each row is (x,y,z,timeseries)
# inputs are signals to regress against
#
# can return results either by:
#		- doing dimensionality reduction on coeffieints
# 	- fitting a parametric model to the coefficients
#   - computing norms of coefficients (individuals or groups)
#

import sys
import os
from numpy import *
from scipy.linalg import *
from scipy.io import *
from scipy.optimize import curve_fit
from scipy.stats import vonmises
from pyspark import SparkContext
import logging

argsIn = sys.argv[1:]

if len(argsIn) < 6:
  print >> sys.stderr, \
  "(regress) usage: regress <master> <inputFile_Y> <inputFile_X> <outputFile> <regressMode> <outputMode> <opts>"
  exit(-1)

def parseVector(line,mode="raw",xyz=0,inds=None):
	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:]) # get tseries
	if inds is not None :
		ts = ts[inds[0]:inds[1]]
	if mode == "dff" :
		meanVal = mean(ts)
		ts = (ts - meanVal) / (meanVal + 0.1)
	if xyz == 1 :
		return ((int(vec[0]),int(vec[1]),int(vec[2])),ts)
	else :
		return ts

def gaussian(x, *p) :
    gain, mu, sigma = p
    return gain*exp(-(x-mu)**2/(2.*sigma**2))

def inRange(val,rng1,rng2) :
	if (val > rng1) & (val < rng2):
		return True
	else:
		return False

def getRegression(y,model) :

	if model.regressMode == 'mean' :
		b = dot(model.X,y)
		return b

	if model.regressMode == 'linear' :
		b = dot(model.Xhat,y)[1:]
		if model.outputMode == 'pca' :
			for ig in range(0,model.nG) :
				ginds = model.g==ig
				b[ginds] = b[ginds] - mean(b[ginds])
		return b

	if model.regressMode == 'linear-shuffle' :
		b = dot(model.Xhat,y)
		predic = dot(b,model.X)
		sse = sum((predic-y) ** 2)
		sst = sum((y-mean(y)) ** 2)
		r2 = 1 - sse/sst
		r2shuffle = zeros((5,1))
		# for iShuf in range(0,5) :
		# 	X = model.X
		# 	for ix in range(0,shape(X)[0]) :
		# 		shift = int(round(random.rand(1)*shape(X)[1]))
		# 		X[ix,:] = roll(X[ix,:],shift)
		# 	Xhat = dot(inv(dot(X,transpose(X))),X)
		# 	b = dot(Xhat,y)
		# 	predic = dot(b,model.X)
		# 	sse = sum((predic-y) ** 2)
		# 	r2shuffle[iShuf] = 1 - sse/sst
		# p = sum(r2shuffle > r2) / 5
		p = 0
		return (b[1:],r2,p)

	if model.regressMode == 'bilinear' :
		b1 = dot(model.X1hat,y)
		b1 = b1 - min(b1)
		b1hat = dot(transpose(model.X1),b1)
		X3 = X2 * b1hat
		X3 = concatenate((ones((1,shape(X3)[1])),X3))
		X3hat = dot(inv(dot(X3,transpose(X3))),X3)
		b2 = dot(X3hat,y)
		b = b2[1:]
		if model.outputMode == 'tuning' :
			return b
		if model.outputMode == 'pca' :
			return (b1 - mean(b1))

def getTuning(y,model) :

	if model.tuningMode == 'circular' :
		y = y - min(y)
		y = y/sum(y)
		r = inner(y,exp(1j*model.s))
		mu = angle(r)
		v = absolute(r)/sum(y)
		if v < 0.53 :
			k = 2*v + (v**3) + 5*(v**5)/6
		elif (v>=0.53) & (v<0.85) :
			k = -.4 + 1.39*v + 0.43/(1-v)
		else :
			k = 1/(v**3 - 4*(v**2) + 3*v)
		return (mu,k)

	if model.tuningMode == 'gaussian' :
		y[y<0] = 0
		coeff,varMat = curve_fit(gaussian, model.s, y, p0=[1., mean(model.s), 1.], maxfev=100000,gtol=0.01)
		return (coeff[1],coeff[2]) # return mu and sigma

def getNorm(y,model) :
	b = getRegression(y,model)
	n = zeros((model.nG,))
	for ig in range(0,model.nG) :
		n[ig] = norm(b[model.g==ig])
	return n

# parse inputs
sc = SparkContext(argsIn[0], "regress")
inputFile_Y = str(argsIn[1])
inputFile_X = str(argsIn[2])
outputFile = str(argsIn[3]) + "-regress"
regressMode = str(argsIn[4])
outputMode = str(argsIn[5])
opts = str(argsIn[6])

if not os.path.exists(outputFile) :
	os.makedirs(outputFile)
logging.basicConfig(filename=outputFile+'/'+'stdout.log',level=logging.INFO,format='%(asctime)s %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p')

# parse data
logging.info("(regress) loading data")
lines_Y = sc.textFile(inputFile_Y)
Y = lines_Y.map(lambda y : parseVector(y,"dff")).cache()

# parse model
class model : pass
model.regressMode = regressMode
model.outputMode = outputMode
if regressMode == 'mean' :
	X = loadmat(inputFile_X + "_X.mat")['X']
	X = X.astype(float)
	model.X = X
if (regressMode == 'linear') | (regressMode == 'linear-shuffle') :
	X = loadmat(inputFile_X + "_X.mat")['X']
	X = X.astype(float)
	g = loadmat(inputFile_X + "_g.mat")['g']
	g = g.astype(float)[0]
	Xhat = dot(inv(dot(X,transpose(X))),X)
	model.X = X
	model.Xhat = Xhat
	model.g = g
	model.nG = len(unique(model.g))
if regressMode == 'bilinear' :
	X1 = loadmat(inputFile_X + "_X1.mat")['X1']
	X2 = loadmat(inputFile_X + "_X2.mat")['X2']
	X1hat = dot(inv(dot(X1,transpose(X1))),X1)
	model.X1 = X1
	model.X2 = X2
	model.X1hat = X1hat
if outputMode == 'tuning' :
	s = loadmat(inputFile_X + "_s.mat")['s']
	model.s = transpose(s)[0]
	model.tuningMode = opts
if outputMode == 'pca' :
	model.k = int(opts)

# get simple measure of response strength
r = Y.map(lambda y : float16(norm(y-mean(y)))).collect()
savemat(outputFile+"/"+"r.mat",mdict={'r':r},oned_as='column',do_compression='true')

# reduce dimensionality of regression coefficients using pca
if outputMode == 'pca' :
	B = Y.map(lambda y : getRegression(y,model))
	k = model.k
	n = B.count()
	cov = B.map(lambda b : outer(b,b)).reduce(lambda x,y : (x + y)) / n
	w, v = eig(cov)
	w = real(w)
	v = real(v)
	inds = argsort(w)[::-1]
	comps = transpose(v[:,inds[0:k]])
	savemat(outputFile+"/"+"comps.mat",mdict={'comps':comps},oned_as='column',do_compression='true')
	latent = w
	savemat(outputFile+"/"+"latent.mat",mdict={'latent':latent},oned_as='column',do_compression='true')

	for ik in range(0,k):
		scores = Y.map(lambda y : float16(inner(getRegression(y,model),comps[ik,:]))).collect()
		savemat(outputFile+"/"+"scores-"+str(ik)+".mat",mdict={'scores':scores},oned_as='column',do_compression='true')

	traj = Y.map(lambda y : outer(y,inner(getRegression(y,model),comps))).reduce(lambda x,y : x + y) / n
	savemat(outputFile+"/"+"traj.mat",mdict={'traj':traj},oned_as='column',do_compression='true')

# process output with a parametric tuning curves
if outputMode == 'tuning' :
	B = Y.map(lambda y : getRegression(y,model))
	stats = B.map(lambda b : float16(b[1:])).collect()
	savemat(outputFile+"/"+"stats.mat",mdict={'stats':stats},oned_as='column',do_compression='true')
	p = B.map(lambda b : float16(getTuning(b[0],model))).collect()
	savemat(outputFile+"/"+"p.mat",mdict={'p':p},oned_as='column',do_compression='true')
	# get average tuning for groups of pixels
	vals = linspace(min(model.s),max(model.s),len(model.s))
	tuningCurves = zeros((len(model.s)-1),len(B.first()[0]))
	for iv in range(0,len(model.s)-1) :
		subset = B.filter(lambda b : inRange(getTuning(b[0],model),vals[iv],vals[iv+1]))
		tuningCurves[iv,:] = subset.map(lambda b : b[0]).reduce(lambda x,y : x + y) / subset.count()
		savemat(outputFile+"/"+"tuningCurves.mat",mdict={'tuningCurves':tuningCurves},oned_as='column',do_compression='true')

# get norms of coefficients to make a contrast map
if outputMode == 'norm' :
	B = Y.map(lambda y : (y,getNorm(y,model)))
	n = B.count()
	m = len(Y.first())
	traj = zeros((2,m))
	for ic in range(0,2) :
		traj[ic,:] = B.filter(lambda (y,b) : (b[ic] - b[1-ic])>0.01).map(lambda (y,b) : y * b[ic]).reduce(lambda x,y : x + y) / n
	norms = B.map(lambda (y,b) : float16(b)).collect()
	savemat(outputFile+"/"+"traj.mat",mdict={'traj':traj},oned_as='column',do_compression='true')
	savemat(outputFile+"/"+"norms.mat",mdict={'norms':norms},oned_as='column',do_compression='true')

<commit_msg>Testing<commit_after># regress <master> <inputFile_Y> <inputFile_X> <outputFile> <analMode> <outputMode>
#
# time series regression on a data matrix
# each row is (x,y,z,timeseries)
# inputs are signals to regress against
#
# can return results either by:
#		- doing dimensionality reduction on coeffieints
# 	- fitting a parametric model to the coefficients
#   - computing norms of coefficients (individuals or groups)
#

import sys
import os
from numpy import *
from scipy.linalg import *
from scipy.io import *
from scipy.optimize import curve_fit
from scipy.stats import vonmises
from pyspark import SparkContext
import logging

argsIn = sys.argv[1:]

if len(argsIn) < 6:
  print >> sys.stderr, \
  "(regress) usage: regress <master> <inputFile_Y> <inputFile_X> <outputFile> <regressMode> <outputMode> <opts>"
  exit(-1)

def parseVector(line,mode="raw",xyz=0,inds=None):
	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:]) # get tseries
	if inds is not None :
		ts = ts[inds[0]:inds[1]]
	if mode == "dff" :
		meanVal = mean(ts)
		ts = (ts - meanVal) / (meanVal + 0.1)
	if xyz == 1 :
		return ((int(vec[0]),int(vec[1]),int(vec[2])),ts)
	else :
		return ts

def gaussian(x, *p) :
    gain, mu, sigma = p
    return gain*exp(-(x-mu)**2/(2.*sigma**2))

def inRange(val,rng1,rng2) :
	if (val > rng1) & (val < rng2):
		return True
	else:
		return False

def getRegression(y,model) :

	if model.regressMode == 'mean' :
		b = dot(model.X,y)
		return b

	if model.regressMode == 'linear' :
		b = dot(model.Xhat,y)[1:]
		if model.outputMode == 'pca' :
			for ig in range(0,model.nG) :
				ginds = model.g==ig
				b[ginds] = b[ginds] - mean(b[ginds])
		return b

	if model.regressMode == 'linear-shuffle' :
		b = dot(model.Xhat,y)
		predic = dot(b,model.X)
		sse = sum((predic-y) ** 2)
		sst = sum((y-mean(y)) ** 2)
		r2 = 1 - sse/sst
		r2shuffle = zeros((5,1))
		# for iShuf in range(0,5) :
		# 	X = model.X
		# 	for ix in range(0,shape(X)[0]) :
		# 		shift = int(round(random.rand(1)*shape(X)[1]))
		# 		X[ix,:] = roll(X[ix,:],shift)
		# 	Xhat = dot(inv(dot(X,transpose(X))),X)
		# 	b = dot(Xhat,y)
		# 	predic = dot(b,model.X)
		# 	sse = sum((predic-y) ** 2)
		# 	r2shuffle[iShuf] = 1 - sse/sst
		# p = sum(r2shuffle > r2) / 5
		p = 0
		return (b[1:],r2,p)

	if model.regressMode == 'bilinear' :
		b1 = dot(model.X1hat,y)
		b1 = b1 - min(b1)
		b1hat = dot(transpose(model.X1),b1)
		X3 = X2 * b1hat
		X3 = concatenate((ones((1,shape(X3)[1])),X3))
		X3hat = dot(inv(dot(X3,transpose(X3))),X3)
		b2 = dot(X3hat,y)
		b = b2[1:]
		if model.outputMode == 'tuning' :
			return b
		if model.outputMode == 'pca' :
			return (b1 - mean(b1))

def getTuning(y,model) :

	if model.tuningMode == 'circular' :
		y = y - min(y)
		y = y/sum(y)
		r = inner(y,exp(1j*model.s))
		mu = angle(r)
		v = absolute(r)/sum(y)
		if v < 0.53 :
			k = 2*v + (v**3) + 5*(v**5)/6
		elif (v>=0.53) & (v<0.85) :
			k = -.4 + 1.39*v + 0.43/(1-v)
		else :
			k = 1/(v**3 - 4*(v**2) + 3*v)
		return (mu,k)

	if model.tuningMode == 'gaussian' :
		y[y<0] = 0
		coeff,varMat = curve_fit(gaussian, model.s, y, p0=[1., mean(model.s), 1.], maxfev=100000)
		return (coeff[1],coeff[2]) # return mu and sigma

def getNorm(y,model) :
	b = getRegression(y,model)
	n = zeros((model.nG,))
	for ig in range(0,model.nG) :
		n[ig] = norm(b[model.g==ig])
	return n

# parse inputs
sc = SparkContext(argsIn[0], "regress")
inputFile_Y = str(argsIn[1])
inputFile_X = str(argsIn[2])
outputFile = str(argsIn[3]) + "-regress"
regressMode = str(argsIn[4])
outputMode = str(argsIn[5])
opts = str(argsIn[6])

if not os.path.exists(outputFile) :
	os.makedirs(outputFile)
logging.basicConfig(filename=outputFile+'/'+'stdout.log',level=logging.INFO,format='%(asctime)s %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p')

# parse data
logging.info("(regress) loading data")
lines_Y = sc.textFile(inputFile_Y)
Y = lines_Y.map(lambda y : parseVector(y,"dff")).cache()

# parse model
class model : pass
model.regressMode = regressMode
model.outputMode = outputMode
if regressMode == 'mean' :
	X = loadmat(inputFile_X + "_X.mat")['X']
	X = X.astype(float)
	model.X = X
if (regressMode == 'linear') | (regressMode == 'linear-shuffle') :
	X = loadmat(inputFile_X + "_X.mat")['X']
	X = X.astype(float)
	g = loadmat(inputFile_X + "_g.mat")['g']
	g = g.astype(float)[0]
	Xhat = dot(inv(dot(X,transpose(X))),X)
	model.X = X
	model.Xhat = Xhat
	model.g = g
	model.nG = len(unique(model.g))
if regressMode == 'bilinear' :
	X1 = loadmat(inputFile_X + "_X1.mat")['X1']
	X2 = loadmat(inputFile_X + "_X2.mat")['X2']
	X1hat = dot(inv(dot(X1,transpose(X1))),X1)
	model.X1 = X1
	model.X2 = X2
	model.X1hat = X1hat
if outputMode == 'tuning' :
	s = loadmat(inputFile_X + "_s.mat")['s']
	model.s = transpose(s)[0]
	model.tuningMode = opts
if outputMode == 'pca' :
	model.k = int(opts)

# get simple measure of response strength
r = Y.map(lambda y : float16(norm(y-mean(y)))).collect()
savemat(outputFile+"/"+"r.mat",mdict={'r':r},oned_as='column',do_compression='true')

# reduce dimensionality of regression coefficients using pca
if outputMode == 'pca' :
	B = Y.map(lambda y : getRegression(y,model))
	k = model.k
	n = B.count()
	cov = B.map(lambda b : outer(b,b)).reduce(lambda x,y : (x + y)) / n
	w, v = eig(cov)
	w = real(w)
	v = real(v)
	inds = argsort(w)[::-1]
	comps = transpose(v[:,inds[0:k]])
	savemat(outputFile+"/"+"comps.mat",mdict={'comps':comps},oned_as='column',do_compression='true')
	latent = w
	savemat(outputFile+"/"+"latent.mat",mdict={'latent':latent},oned_as='column',do_compression='true')

	for ik in range(0,k):
		scores = Y.map(lambda y : float16(inner(getRegression(y,model),comps[ik,:]))).collect()
		savemat(outputFile+"/"+"scores-"+str(ik)+".mat",mdict={'scores':scores},oned_as='column',do_compression='true')

	traj = Y.map(lambda y : outer(y,inner(getRegression(y,model),comps))).reduce(lambda x,y : x + y) / n
	savemat(outputFile+"/"+"traj.mat",mdict={'traj':traj},oned_as='column',do_compression='true')

# process output with a parametric tuning curves
if outputMode == 'tuning' :
	B = Y.map(lambda y : getRegression(y,model))
	stats = B.map(lambda b : float16(b[1:])).collect()
	savemat(outputFile+"/"+"stats.mat",mdict={'stats':stats},oned_as='column',do_compression='true')
	p = B.map(lambda b : float16(getTuning(b[0],model))).collect()
	savemat(outputFile+"/"+"p.mat",mdict={'p':p},oned_as='column',do_compression='true')
	# get average tuning for groups of pixels
	vals = linspace(min(model.s),max(model.s),len(model.s))
	tuningCurves = zeros((len(model.s)-1),len(B.first()[0]))
	for iv in range(0,len(model.s)-1) :
		subset = B.filter(lambda b : inRange(getTuning(b[0],model),vals[iv],vals[iv+1]))
		tuningCurves[iv,:] = subset.map(lambda b : b[0]).reduce(lambda x,y : x + y) / subset.count()
		savemat(outputFile+"/"+"tuningCurves.mat",mdict={'tuningCurves':tuningCurves},oned_as='column',do_compression='true')

# get norms of coefficients to make a contrast map
if outputMode == 'norm' :
	B = Y.map(lambda y : (y,getNorm(y,model)))
	n = B.count()
	m = len(Y.first())
	traj = zeros((2,m))
	for ic in range(0,2) :
		traj[ic,:] = B.filter(lambda (y,b) : (b[ic] - b[1-ic])>0.01).map(lambda (y,b) : y * b[ic]).reduce(lambda x,y : x + y) / n
	norms = B.map(lambda (y,b) : float16(b)).collect()
	savemat(outputFile+"/"+"traj.mat",mdict={'traj':traj},oned_as='column',do_compression='true')
	savemat(outputFile+"/"+"norms.mat",mdict={'norms':norms},oned_as='column',do_compression='true')

<|endoftext|>


<commit_before># regress <master> <inputFile_Y> <inputFile_X> <outputFile> <analMode> <outputMode>
#
# time series regression on a data matrix
# each row is (x,y,z,timeseries)
# inputs are signals to regress against
#
# can return results either by:
#		- doing dimensionality reduction on coeffieints
# 	- fitting a parametric model to the coefficients
#   - computing norms of coefficients (individuals or groups)
#

import sys
import os
from numpy import *
from scipy.linalg import *
from scipy.io import *
from scipy.optimize import curve_fit
from scipy.stats import vonmises
from pyspark import SparkContext
import logging

argsIn = sys.argv[1:]

if len(argsIn) < 6:
  print >> sys.stderr, \
  "(regress) usage: regress <master> <inputFile_Y> <inputFile_X> <outputFile> <regressMode> <outputMode> <opts>"
  exit(-1)

def parseVector(line,mode="raw",xyz=0,inds=None):
	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:]) # get tseries
	if inds is not None :
		ts = ts[inds[0]:inds[1]]
	if mode == "dff" :
		meanVal = mean(ts)
		ts = (ts - meanVal) / (meanVal + 0.1)
	if xyz == 1 :
		return ((int(vec[0]),int(vec[1]),int(vec[2])),ts)
	else :
		return ts

def gaussian(x, *p) :
    gain, mu, sigma = p
    return gain*exp(-(x-mu)**2/(2.*sigma**2))

def inRange(val,rng1,rng2) :
	if (val > rng1) & (val < rng2):
		return True
	else:
		return False

def getRegression(y,model) :

	if model.regressMode == 'mean' :
		b = dot(model.X,y)
		return b

	if model.regressMode == 'linear' :
		b = dot(model.Xhat,y)[1:]
		if model.outputMode == 'pca' :
			for ig in range(0,model.nG) :
				ginds = model.g==ig
				b[ginds] = b[ginds] - mean(b[ginds])
		return b

	if model.regressMode == 'linear-shuffle' :
		#b = dot(model.Xhat,y)
		b = lstsq(transpose(model.X),transpose(y))
		predic = dot(b,model.X)
		sse = sum((predic-y) ** 2)
		sst = sum((y-mean(y)) ** 2)
		r2 = 1 - sse/sst
		# r2shuffle = zeros((1,1))
		# for iShuf in range(0,1) :
		# 	X = copy(model.X)
		# 	for ix in range(0,shape(X)[0]) :
		# 		shift = int(round(random.rand(1)*shape(X)[1]))
		# 		X[ix,:] = roll(X[ix,:],shift)
		# 	b = lstsq(X,y)
		# 	predic = dot(b,X)
		# 	sse = sum((predic-y) ** 2)
		# 	r2shuffle[iShuf] = 1 - sse/sst
		#p = sum(r2shuffle > r2) / 1
		p = 0
		return (b[1:],r2,p)

	if model.regressMode == 'bilinear' :
		b1 = dot(model.X1hat,y)
		b1 = b1 - min(b1)
		b1hat = dot(transpose(model.X1),b1)
		X3 = X2 * b1hat
		X3 = concatenate((ones((1,shape(X3)[1])),X3))
		X3hat = dot(inv(dot(X3,transpose(X3))),X3)
		b2 = dot(X3hat,y)
		b = b2[1:]
		if model.outputMode == 'tuning' :
			return b
		if model.outputMode == 'pca' :
			return (b1 - mean(b1))

def getTuning(y,model) :

	if model.tuningMode == 'circular' :
		y = y - min(y)
		y = y/sum(y)
		r = inner(y,exp(1j*model.s))
		mu = angle(r)
		v = absolute(r)/sum(y)
		if v < 0.53 :
			k = 2*v + (v**3) + 5*(v**5)/6
		elif (v>=0.53) & (v<0.85) :
			k = -.4 + 1.39*v + 0.43/(1-v)
		else :
			k = 1/(v**3 - 4*(v**2) + 3*v)
		return (mu,k)

	if model.tuningMode == 'gaussian' :
		#y[y<0] = 0
		#coeff,varMat = curve_fit(gaussian, model.s, y, p0=[1., mean(model.s), 1.], maxfev=100000,gtol=0.01)
		#return (coeff[1],coeff[2]) # return mu and sigma
		y[y<0] = 0
		y = y/sum(y)
		mu = dot(model.s,y)
		sigma = dot(y,(model.s-mu)**2)
		return (mu,sigma)

def getNorm(y,model) :
	b = getRegression(y,model)
	n = zeros((model.nG,))
	for ig in range(0,model.nG) :
		n[ig] = norm(b[model.g==ig])
	return n

# parse inputs
sc = SparkContext(argsIn[0], "regress")
inputFile_Y = str(argsIn[1])
inputFile_X = str(argsIn[2])
outputFile = str(argsIn[3]) + "-regress"
regressMode = str(argsIn[4])
outputMode = str(argsIn[5])
opts = str(argsIn[6])

if not os.path.exists(outputFile) :
	os.makedirs(outputFile)
logging.basicConfig(filename=outputFile+'/'+'stdout.log',level=logging.INFO,format='%(asctime)s %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p')

# parse data
logging.info("(regress) loading data")
lines_Y = sc.textFile(inputFile_Y)
Y = lines_Y.map(lambda y : parseVector(y,"dff")).cache()

# parse model
class model : pass
model.regressMode = regressMode
model.outputMode = outputMode
if regressMode == 'mean' :
	X = loadmat(inputFile_X + "_X.mat")['X']
	X = X.astype(float)
	model.X = X
if (regressMode == 'linear') | (regressMode == 'linear-shuffle') :
	X = loadmat(inputFile_X + "_X.mat")['X']
	X = X.astype(float)
	g = loadmat(inputFile_X + "_g.mat")['g']
	g = g.astype(float)[0]
	Xhat = dot(inv(dot(X,transpose(X))),X)
	model.X = X
	model.Xhat = Xhat
	model.g = g
	model.nG = len(unique(model.g))
if regressMode == 'bilinear' :
	X1 = loadmat(inputFile_X + "_X1.mat")['X1']
	X2 = loadmat(inputFile_X + "_X2.mat")['X2']
	X1hat = dot(inv(dot(X1,transpose(X1))),X1)
	model.X1 = X1
	model.X2 = X2
	model.X1hat = X1hat
if outputMode == 'tuning' :
	s = loadmat(inputFile_X + "_s.mat")['s']
	model.s = transpose(s)[0]
	model.tuningMode = opts
if outputMode == 'pca' :
	model.k = int(opts)

# get simple measure of response strength
r = Y.map(lambda y : float16(norm(y-mean(y)))).collect()
savemat(outputFile+"/"+"r.mat",mdict={'r':r},oned_as='column',do_compression='true')

# reduce dimensionality of regression coefficients using pca
if outputMode == 'pca' :
	B = Y.map(lambda y : getRegression(y,model))
	k = model.k
	n = B.count()
	cov = B.map(lambda b : outer(b,b)).reduce(lambda x,y : (x + y)) / n
	w, v = eig(cov)
	w = real(w)
	v = real(v)
	inds = argsort(w)[::-1]
	comps = transpose(v[:,inds[0:k]])
	savemat(outputFile+"/"+"comps.mat",mdict={'comps':comps},oned_as='column',do_compression='true')
	latent = w
	savemat(outputFile+"/"+"latent.mat",mdict={'latent':latent},oned_as='column',do_compression='true')

	for ik in range(0,k):
		scores = Y.map(lambda y : float16(inner(getRegression(y,model),comps[ik,:]))).collect()
		savemat(outputFile+"/"+"scores-"+str(ik)+".mat",mdict={'scores':scores},oned_as='column',do_compression='true')

	traj = Y.map(lambda y : outer(y,inner(getRegression(y,model),comps))).reduce(lambda x,y : x + y) / n
	savemat(outputFile+"/"+"traj.mat",mdict={'traj':traj},oned_as='column',do_compression='true')

# process output with a parametric tuning curves
if outputMode == 'tuning' :
	B = Y.map(lambda y : getRegression(y,model)).cache()
	stats = B.map(lambda b : float16(b[1:])).collect()
	savemat(outputFile+"/"+"stats.mat",mdict={'stats':stats},oned_as='column',do_compression='true')
	p = B.map(lambda b : float16(getTuning(b[0],model))).collect()
	savemat(outputFile+"/"+"p.mat",mdict={'p':p},oned_as='column',do_compression='true')
	# get average tuning for groups of pixels
	vals = linspace(min(model.s),max(model.s),len(model.s))
	tuningCurves = zeros((len(model.s)-1,len(model.s)))
	for iv in range(0,len(model.s)-1) :
		subset = B.filter(lambda b : inRange(getTuning(b[0],model)[0],vals[iv],vals[iv+1]))
		tuningCurves[iv,:] = subset.map(lambda b : b[0]).reduce(lambda x,y : x + y) / subset.count()
		savemat(outputFile+"/"+"tuningCurves.mat",mdict={'tuningCurves':tuningCurves},oned_as='column',do_compression='true')

# get norms of coefficients to make a contrast map
if outputMode == 'norm' :
	B = Y.map(lambda y : (y,getNorm(y,model)))
	n = B.count()
	m = len(Y.first())
	traj = zeros((2,m))
	for ic in range(0,2) :
		traj[ic,:] = B.filter(lambda (y,b) : (b[ic] - b[1-ic])>0.01).map(lambda (y,b) : y * b[ic]).reduce(lambda x,y : x + y) / n
	norms = B.map(lambda (y,b) : float16(b)).collect()
	savemat(outputFile+"/"+"traj.mat",mdict={'traj':traj},oned_as='column',do_compression='true')
	savemat(outputFile+"/"+"norms.mat",mdict={'norms':norms},oned_as='column',do_compression='true')

<commit_msg>Testing<commit_after># regress <master> <inputFile_Y> <inputFile_X> <outputFile> <analMode> <outputMode>
#
# time series regression on a data matrix
# each row is (x,y,z,timeseries)
# inputs are signals to regress against
#
# can return results either by:
#		- doing dimensionality reduction on coeffieints
# 	- fitting a parametric model to the coefficients
#   - computing norms of coefficients (individuals or groups)
#

import sys
import os
from numpy import *
from scipy.linalg import *
from scipy.io import *
from scipy.optimize import curve_fit
from scipy.stats import vonmises
from pyspark import SparkContext
import logging

argsIn = sys.argv[1:]

if len(argsIn) < 6:
  print >> sys.stderr, \
  "(regress) usage: regress <master> <inputFile_Y> <inputFile_X> <outputFile> <regressMode> <outputMode> <opts>"
  exit(-1)

def parseVector(line,mode="raw",xyz=0,inds=None):
	vec = [float(x) for x in line.split(' ')]
	ts = array(vec[3:]) # get tseries
	if inds is not None :
		ts = ts[inds[0]:inds[1]]
	if mode == "dff" :
		meanVal = mean(ts)
		ts = (ts - meanVal) / (meanVal + 0.1)
	if xyz == 1 :
		return ((int(vec[0]),int(vec[1]),int(vec[2])),ts)
	else :
		return ts

def gaussian(x, *p) :
    gain, mu, sigma = p
    return gain*exp(-(x-mu)**2/(2.*sigma**2))

def inRange(val,rng1,rng2) :
	if (val > rng1) & (val < rng2):
		return True
	else:
		return False

def getRegression(y,model) :

	if model.regressMode == 'mean' :
		b = dot(model.X,y)
		return b

	if model.regressMode == 'linear' :
		b = dot(model.Xhat,y)[1:]
		if model.outputMode == 'pca' :
			for ig in range(0,model.nG) :
				ginds = model.g==ig
				b[ginds] = b[ginds] - mean(b[ginds])
		return b

	if model.regressMode == 'linear-shuffle' :
		#b = dot(model.Xhat,y)
		b = array(lstsq(transpose(model.X),transpose(y)))
		predic = dot(b,model.X)
		sse = sum((predic-y) ** 2)
		sst = sum((y-mean(y)) ** 2)
		r2 = 1 - sse/sst
		# r2shuffle = zeros((1,1))
		# for iShuf in range(0,1) :
		# 	X = copy(model.X)
		# 	for ix in range(0,shape(X)[0]) :
		# 		shift = int(round(random.rand(1)*shape(X)[1]))
		# 		X[ix,:] = roll(X[ix,:],shift)
		# 	b = lstsq(X,y)
		# 	predic = dot(b,X)
		# 	sse = sum((predic-y) ** 2)
		# 	r2shuffle[iShuf] = 1 - sse/sst
		#p = sum(r2shuffle > r2) / 1
		p = 0
		return (b[1:],r2,p)

	if model.regressMode == 'bilinear' :
		b1 = dot(model.X1hat,y)
		b1 = b1 - min(b1)
		b1hat = dot(transpose(model.X1),b1)
		X3 = X2 * b1hat
		X3 = concatenate((ones((1,shape(X3)[1])),X3))
		X3hat = dot(inv(dot(X3,transpose(X3))),X3)
		b2 = dot(X3hat,y)
		b = b2[1:]
		if model.outputMode == 'tuning' :
			return b
		if model.outputMode == 'pca' :
			return (b1 - mean(b1))

def getTuning(y,model) :

	if model.tuningMode == 'circular' :
		y = y - min(y)
		y = y/sum(y)
		r = inner(y,exp(1j*model.s))
		mu = angle(r)
		v = absolute(r)/sum(y)
		if v < 0.53 :
			k = 2*v + (v**3) + 5*(v**5)/6
		elif (v>=0.53) & (v<0.85) :
			k = -.4 + 1.39*v + 0.43/(1-v)
		else :
			k = 1/(v**3 - 4*(v**2) + 3*v)
		return (mu,k)

	if model.tuningMode == 'gaussian' :
		#y[y<0] = 0
		#coeff,varMat = curve_fit(gaussian, model.s, y, p0=[1., mean(model.s), 1.], maxfev=100000,gtol=0.01)
		#return (coeff[1],coeff[2]) # return mu and sigma
		y[y<0] = 0
		y = y/sum(y)
		mu = dot(model.s,y)
		sigma = dot(y,(model.s-mu)**2)
		return (mu,sigma)

def getNorm(y,model) :
	b = getRegression(y,model)
	n = zeros((model.nG,))
	for ig in range(0,model.nG) :
		n[ig] = norm(b[model.g==ig])
	return n

# parse inputs
sc = SparkContext(argsIn[0], "regress")
inputFile_Y = str(argsIn[1])
inputFile_X = str(argsIn[2])
outputFile = str(argsIn[3]) + "-regress"
regressMode = str(argsIn[4])
outputMode = str(argsIn[5])
opts = str(argsIn[6])

if not os.path.exists(outputFile) :
	os.makedirs(outputFile)
logging.basicConfig(filename=outputFile+'/'+'stdout.log',level=logging.INFO,format='%(asctime)s %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p')

# parse data
logging.info("(regress) loading data")
lines_Y = sc.textFile(inputFile_Y)
Y = lines_Y.map(lambda y : parseVector(y,"dff")).cache()

# parse model
class model : pass
model.regressMode = regressMode
model.outputMode = outputMode
if regressMode == 'mean' :
	X = loadmat(inputFile_X + "_X.mat")['X']
	X = X.astype(float)
	model.X = X
if (regressMode == 'linear') | (regressMode == 'linear-shuffle') :
	X = loadmat(inputFile_X + "_X.mat")['X']
	X = X.astype(float)
	g = loadmat(inputFile_X + "_g.mat")['g']
	g = g.astype(float)[0]
	Xhat = dot(inv(dot(X,transpose(X))),X)
	model.X = X
	model.Xhat = Xhat
	model.g = g
	model.nG = len(unique(model.g))
if regressMode == 'bilinear' :
	X1 = loadmat(inputFile_X + "_X1.mat")['X1']
	X2 = loadmat(inputFile_X + "_X2.mat")['X2']
	X1hat = dot(inv(dot(X1,transpose(X1))),X1)
	model.X1 = X1
	model.X2 = X2
	model.X1hat = X1hat
if outputMode == 'tuning' :
	s = loadmat(inputFile_X + "_s.mat")['s']
	model.s = transpose(s)[0]
	model.tuningMode = opts
if outputMode == 'pca' :
	model.k = int(opts)

# get simple measure of response strength
r = Y.map(lambda y : float16(norm(y-mean(y)))).collect()
savemat(outputFile+"/"+"r.mat",mdict={'r':r},oned_as='column',do_compression='true')

# reduce dimensionality of regression coefficients using pca
if outputMode == 'pca' :
	B = Y.map(lambda y : getRegression(y,model))
	k = model.k
	n = B.count()
	cov = B.map(lambda b : outer(b,b)).reduce(lambda x,y : (x + y)) / n
	w, v = eig(cov)
	w = real(w)
	v = real(v)
	inds = argsort(w)[::-1]
	comps = transpose(v[:,inds[0:k]])
	savemat(outputFile+"/"+"comps.mat",mdict={'comps':comps},oned_as='column',do_compression='true')
	latent = w
	savemat(outputFile+"/"+"latent.mat",mdict={'latent':latent},oned_as='column',do_compression='true')

	for ik in range(0,k):
		scores = Y.map(lambda y : float16(inner(getRegression(y,model),comps[ik,:]))).collect()
		savemat(outputFile+"/"+"scores-"+str(ik)+".mat",mdict={'scores':scores},oned_as='column',do_compression='true')

	traj = Y.map(lambda y : outer(y,inner(getRegression(y,model),comps))).reduce(lambda x,y : x + y) / n
	savemat(outputFile+"/"+"traj.mat",mdict={'traj':traj},oned_as='column',do_compression='true')

# process output with a parametric tuning curves
if outputMode == 'tuning' :
	B = Y.map(lambda y : getRegression(y,model)).cache()
	stats = B.map(lambda b : float16(b[1:])).collect()
	savemat(outputFile+"/"+"stats.mat",mdict={'stats':stats},oned_as='column',do_compression='true')
	p = B.map(lambda b : float16(getTuning(b[0],model))).collect()
	savemat(outputFile+"/"+"p.mat",mdict={'p':p},oned_as='column',do_compression='true')
	# get average tuning for groups of pixels
	vals = linspace(min(model.s),max(model.s),len(model.s))
	tuningCurves = zeros((len(model.s)-1,len(model.s)))
	for iv in range(0,len(model.s)-1) :
		subset = B.filter(lambda b : inRange(getTuning(b[0],model)[0],vals[iv],vals[iv+1]))
		tuningCurves[iv,:] = subset.map(lambda b : b[0]).reduce(lambda x,y : x + y) / subset.count()
		savemat(outputFile+"/"+"tuningCurves.mat",mdict={'tuningCurves':tuningCurves},oned_as='column',do_compression='true')

# get norms of coefficients to make a contrast map
if outputMode == 'norm' :
	B = Y.map(lambda y : (y,getNorm(y,model)))
	n = B.count()
	m = len(Y.first())
	traj = zeros((2,m))
	for ic in range(0,2) :
		traj[ic,:] = B.filter(lambda (y,b) : (b[ic] - b[1-ic])>0.01).map(lambda (y,b) : y * b[ic]).reduce(lambda x,y : x + y) / n
	norms = B.map(lambda (y,b) : float16(b)).collect()
	savemat(outputFile+"/"+"traj.mat",mdict={'traj':traj},oned_as='column',do_compression='true')
	savemat(outputFile+"/"+"norms.mat",mdict={'norms':norms},oned_as='column',do_compression='true')

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
This module contains a container for stream manifest data.

A container object for the media stream (video only / audio only / video+audio
combined). This was referred to as ``Video`` in the legacy pytube version, but
has been renamed to accommodate DASH (which serves the audio and video
separately).
"""
from __future__ import absolute_import

import io
import logging
import os
import pprint

from pytube import extract
from pytube import request
from pytube.helpers import safe_filename
from pytube.itags import get_format_profile

logger = logging.getLogger(__name__)

class Stream(object):
    """Container for stream manifest data."""

    def __init__(self, stream, player_config_args, monostate):
        """Construct a :class:`Stream <Stream>`.

        :param dict stream:
            The unscrambled data extracted from YouTube.
        :param dict player_config_args:
            The data object containing video media data like title and
            keywords.
        :param dict monostate:
            Dictionary of data shared across all instances of
            :class:`Stream <Stream>`.
        """
        # A dictionary shared between all instances of :class:`Stream <Stream>`
        # (Borg pattern).
        self._monostate = monostate

        self.abr = None   # average bitrate (audio streams only)
        self.fps = None   # frames per second (video streams only)
        self.itag = None  # stream format id (youtube nomenclature)
        self.res = None   # resolution (e.g.: 480p, 720p, 1080p)
        self.url = None   # signed download url

        self.mime_type = None  # content identifier (e.g.: video/mp4)
        self.type = None       # the part of the mime before the slash
        self.subtype = None    # the part of the mime after the slash

        self.codecs = []         # audio/video encoders (e.g.: vp8, mp4a)
        self.audio_codec = None  # audio codec of the stream (e.g.: vorbis)
        self.video_codec = None  # video codec of the stream (e.g.: vp8)

        # Iterates over the key/values of stream and sets them as class
        # attributes. This is an anti-pattern and should be removed.
        self.set_attributes_from_dict(stream)

        # Additional information about the stream format, such as resolution,
        # frame rate, and whether the stream is live (HLS) or 3D.
        self.fmt_profile = get_format_profile(self.itag)

        # Same as above, except for the format profile attributes.
        self.set_attributes_from_dict(self.fmt_profile)

        # The player configuration which contains information like the video
        # title.
        # TODO(nficano): this should be moved to the monostate.
        self.player_config_args = player_config_args

        # 'video/webm; codecs="vp8, vorbis"' -> 'video/webm', ['vp8', 'vorbis']
        self.mime_type, self.codecs = extract.mime_type_codec(self.type)

        # 'video/webm' -> 'video', 'webm'
        self.type, self.subtype = self.mime_type.split('/')

        # ['vp8', 'vorbis'] -> video_codec: vp8, audio_codec: vorbis. DASH
        # streams return NoneType for audio/video depending.
        self.video_codec, self.audio_codec = self.parse_codecs()

    def set_attributes_from_dict(self, dct):
        """Set class attributes from dictionary items.

        :rtype: None
        """
        for key, val in dct.items():
            setattr(self, key, val)

    @property
    def is_adaptive(self):
        """Whether the stream is DASH.

        :rtype: bool
        """
        # if codecs has two elements (e.g.: ['vp8', 'vorbis']): 2 % 2 = 0
        # if codecs has one element (e.g.: ['vp8']) 1 % 2 = 1
        return len(self.codecs) % 2

    @property
    def is_progressive(self):
        """Whether the stream is progressive.

        :rtype: bool
        """
        return not self.is_adaptive

    @property
    def includes_audio_track(self):
        """Whether the stream only contains audio.

        :rtype: bool
        """
        if self.is_progressive:
            return True
        return self.type == 'audio'

    @property
    def includes_video_track(self):
        """Whether the stream only contains video.

        :rtype: bool
        """
        if self.is_progressive:
            return True
        return self.type == 'video'

    def parse_codecs(self):
        """Get the video/audio codecs from list of codecs.

        Parse a variable length sized list of codecs and returns a
        constant two element tuple, with the video codec as the first element
        and audio as the second. Returns None if one is not available
        (adaptive only).

        :rtype: tuple
        :returns:
            A two element tuple with audio and video codecs.

        """
        video = None
        audio = None
        if not self.is_adaptive:
            video, audio = self.codecs
        elif self.includes_video_track:
            video = self.codecs[0]
        elif self.includes_audio_track:
            audio = self.codecs[0]
        return video, audio

    @property
    def filesize(self):
        """File size of the media stream in bytes.

        :rtype: int
        :returns:
            Filesize (in bytes) of the stream.
        """
        headers = request.get(self.url, headers=True)
        return int(headers['content-length'])

    @property
    def default_filename(self):
        """Generate filename based on the video title.

        :rtype: str
        :returns:
            An os file system compatible filename.
        """
        title = self.player_config_args['title']
        filename = safe_filename(title)
        return '{filename}.{s.subtype}'.format(filename=filename, s=self)

    def download(self, output_path=None, filename=None):
        """Write the media stream to disk.

        :param output_path:
            (optional) Output path for writing media file. If one is not
            specified, defaults to the current working directory.
        :type output_path: str or None
        :param filename:
            (optional) Output filename (stem only) for writing media file.
            If one is not specified, the default filename is used.
        :type filename: str or None

        :rtype: None

        """
        output_path = output_path or os.getcwd()
        if filename:
            safe = safe_filename(filename)
            filename = '{filename}.{s.subtype}'.format(filename=safe, s=self)
        filename = filename or self.default_filename

        # file path
        fp = os.path.join(output_path, filename)
        bytes_remaining = self.filesize
        logger.debug(
            'downloading (%s total bytes) file to %s',
            self.filesize, fp,
        )

        with open(fp, 'wb') as fh:
            for chunk in request.get(self.url, streaming=True):
                # reduce the (bytes) remainder by the length of the chunk.
                bytes_remaining -= len(chunk)
                # send to the on_progress callback.
                self.on_progress(chunk, fh, bytes_remaining)
            self.on_complete(fh)

    def stream_to_buffer(self):
        """Write the media stream to buffer

        :rtype: io.BytesIO buffer
        """
        buffer = io.BytesIO()
        bytes_remaining = self.filesize
        logger.debug(
            'downloading (%s total bytes) file to BytesIO buffer',
            self.filesize,
        )

        for chunk in request.get(self.url, streaming=True):
            # reduce the (bytes) remainder by the length of the chunk.
            bytes_remaining -= len(chunk)
            # send to the on_progress callback.
            self.on_progress(chunk, buffer, bytes_remaining)
        self.on_complete(buffer)
        return buffer

    def on_progress(self, chunk, file_handler, bytes_remaining):
        """On progress callback function.

        This function writes the binary data to the file, then checks if an
        additional callback is defined in the monostate. This is exposed to
        allow things like displaying a progress bar.

        :param str chunk:
            Segment of media file binary data, not yet written to disk.
        :param file_handler:
            The file handle where the media is being written to.
        :type file_handler:
            :py:class:`io.BufferedWriter`
        :param int bytes_remaining:
            The delta between the total file size in bytes and amount already
            downloaded.

        :rtype: None

        """
        file_handler.write(chunk)
        logger.debug(
            'download progress\n%s',
            pprint.pformat(
                {
                    'chunk_size': len(chunk),
                    'bytes_remaining': bytes_remaining,
                }, indent=2,
            ),
        )
        on_progress = self._monostate['on_progress']
        if on_progress:
            logger.debug('calling on_progress callback %s', on_progress)
            on_progress(self, chunk, file_handler, bytes_remaining)

    def on_complete(self, file_handle):
        """On download complete handler function.

        :param file_handle:
            The file handle where the media is being written to.
        :type file_handle:
            :py:class:`io.BufferedWriter`

        :rtype: None

        """
        logger.debug('download finished')
        on_complete = self._monostate['on_complete']
        if on_complete:
            logger.debug('calling on_complete callback %s', on_complete)
            on_complete(self, file_handle)

    def __repr__(self):
        """Printable object representation.

        :rtype: str
        :returns:
            A string representation of a :class:`Stream <Stream>` object.
        """
        # TODO(nficano): this can probably be written better.
        parts = ['itag="{s.itag}"', 'mime_type="{s.mime_type}"']
        if self.includes_video_track:
            parts.extend(['res="{s.resolution}"', 'fps="{s.fps}fps"'])
            if not self.is_adaptive:
                parts.extend([
                    'vcodec="{s.video_codec}"',
                    'acodec="{s.audio_codec}"',
                ])
            else:
                parts.extend(['vcodec="{s.video_codec}"'])
        else:
            parts.extend(['abr="{s.abr}"', 'acodec="{s.audio_codec}"'])
        parts = ' '.join(parts).format(s=self)
        return '<Stream: {parts}>'.format(parts=parts)
<commit_msg>Return downloaded file path at the end of download<commit_after># -*- coding: utf-8 -*-
"""
This module contains a container for stream manifest data.

A container object for the media stream (video only / audio only / video+audio
combined). This was referred to as ``Video`` in the legacy pytube version, but
has been renamed to accommodate DASH (which serves the audio and video
separately).
"""
from __future__ import absolute_import

import io
import logging
import os
import pprint

from pytube import extract
from pytube import request
from pytube.helpers import safe_filename
from pytube.itags import get_format_profile

logger = logging.getLogger(__name__)

class Stream(object):
    """Container for stream manifest data."""

    def __init__(self, stream, player_config_args, monostate):
        """Construct a :class:`Stream <Stream>`.

        :param dict stream:
            The unscrambled data extracted from YouTube.
        :param dict player_config_args:
            The data object containing video media data like title and
            keywords.
        :param dict monostate:
            Dictionary of data shared across all instances of
            :class:`Stream <Stream>`.
        """
        # A dictionary shared between all instances of :class:`Stream <Stream>`
        # (Borg pattern).
        self._monostate = monostate

        self.abr = None   # average bitrate (audio streams only)
        self.fps = None   # frames per second (video streams only)
        self.itag = None  # stream format id (youtube nomenclature)
        self.res = None   # resolution (e.g.: 480p, 720p, 1080p)
        self.url = None   # signed download url

        self.mime_type = None  # content identifier (e.g.: video/mp4)
        self.type = None       # the part of the mime before the slash
        self.subtype = None    # the part of the mime after the slash

        self.codecs = []         # audio/video encoders (e.g.: vp8, mp4a)
        self.audio_codec = None  # audio codec of the stream (e.g.: vorbis)
        self.video_codec = None  # video codec of the stream (e.g.: vp8)

        # Iterates over the key/values of stream and sets them as class
        # attributes. This is an anti-pattern and should be removed.
        self.set_attributes_from_dict(stream)

        # Additional information about the stream format, such as resolution,
        # frame rate, and whether the stream is live (HLS) or 3D.
        self.fmt_profile = get_format_profile(self.itag)

        # Same as above, except for the format profile attributes.
        self.set_attributes_from_dict(self.fmt_profile)

        # The player configuration which contains information like the video
        # title.
        # TODO(nficano): this should be moved to the monostate.
        self.player_config_args = player_config_args

        # 'video/webm; codecs="vp8, vorbis"' -> 'video/webm', ['vp8', 'vorbis']
        self.mime_type, self.codecs = extract.mime_type_codec(self.type)

        # 'video/webm' -> 'video', 'webm'
        self.type, self.subtype = self.mime_type.split('/')

        # ['vp8', 'vorbis'] -> video_codec: vp8, audio_codec: vorbis. DASH
        # streams return NoneType for audio/video depending.
        self.video_codec, self.audio_codec = self.parse_codecs()

    def set_attributes_from_dict(self, dct):
        """Set class attributes from dictionary items.

        :rtype: None
        """
        for key, val in dct.items():
            setattr(self, key, val)

    @property
    def is_adaptive(self):
        """Whether the stream is DASH.

        :rtype: bool
        """
        # if codecs has two elements (e.g.: ['vp8', 'vorbis']): 2 % 2 = 0
        # if codecs has one element (e.g.: ['vp8']) 1 % 2 = 1
        return len(self.codecs) % 2

    @property
    def is_progressive(self):
        """Whether the stream is progressive.

        :rtype: bool
        """
        return not self.is_adaptive

    @property
    def includes_audio_track(self):
        """Whether the stream only contains audio.

        :rtype: bool
        """
        if self.is_progressive:
            return True
        return self.type == 'audio'

    @property
    def includes_video_track(self):
        """Whether the stream only contains video.

        :rtype: bool
        """
        if self.is_progressive:
            return True
        return self.type == 'video'

    def parse_codecs(self):
        """Get the video/audio codecs from list of codecs.

        Parse a variable length sized list of codecs and returns a
        constant two element tuple, with the video codec as the first element
        and audio as the second. Returns None if one is not available
        (adaptive only).

        :rtype: tuple
        :returns:
            A two element tuple with audio and video codecs.

        """
        video = None
        audio = None
        if not self.is_adaptive:
            video, audio = self.codecs
        elif self.includes_video_track:
            video = self.codecs[0]
        elif self.includes_audio_track:
            audio = self.codecs[0]
        return video, audio

    @property
    def filesize(self):
        """File size of the media stream in bytes.

        :rtype: int
        :returns:
            Filesize (in bytes) of the stream.
        """
        headers = request.get(self.url, headers=True)
        return int(headers['content-length'])

    @property
    def default_filename(self):
        """Generate filename based on the video title.

        :rtype: str
        :returns:
            An os file system compatible filename.
        """
        title = self.player_config_args['title']
        filename = safe_filename(title)
        return '{filename}.{s.subtype}'.format(filename=filename, s=self)

    def download(self, output_path=None, filename=None):
        """Write the media stream to disk.

        :param output_path:
            (optional) Output path for writing media file. If one is not
            specified, defaults to the current working directory.
        :type output_path: str or None
        :param filename:
            (optional) Output filename (stem only) for writing media file.
            If one is not specified, the default filename is used.
        :type filename: str or None

        :rtype: str

        """
        output_path = output_path or os.getcwd()
        if filename:
            safe = safe_filename(filename)
            filename = '{filename}.{s.subtype}'.format(filename=safe, s=self)
        filename = filename or self.default_filename

        # file path
        fp = os.path.join(output_path, filename)
        bytes_remaining = self.filesize
        logger.debug(
            'downloading (%s total bytes) file to %s',
            self.filesize, fp,
        )

        with open(fp, 'wb') as fh:
            for chunk in request.get(self.url, streaming=True):
                # reduce the (bytes) remainder by the length of the chunk.
                bytes_remaining -= len(chunk)
                # send to the on_progress callback.
                self.on_progress(chunk, fh, bytes_remaining)
            self.on_complete(fh)
        return fp

    def stream_to_buffer(self):
        """Write the media stream to buffer

        :rtype: io.BytesIO buffer
        """
        buffer = io.BytesIO()
        bytes_remaining = self.filesize
        logger.debug(
            'downloading (%s total bytes) file to BytesIO buffer',
            self.filesize,
        )

        for chunk in request.get(self.url, streaming=True):
            # reduce the (bytes) remainder by the length of the chunk.
            bytes_remaining -= len(chunk)
            # send to the on_progress callback.
            self.on_progress(chunk, buffer, bytes_remaining)
        self.on_complete(buffer)
        return buffer

    def on_progress(self, chunk, file_handler, bytes_remaining):
        """On progress callback function.

        This function writes the binary data to the file, then checks if an
        additional callback is defined in the monostate. This is exposed to
        allow things like displaying a progress bar.

        :param str chunk:
            Segment of media file binary data, not yet written to disk.
        :param file_handler:
            The file handle where the media is being written to.
        :type file_handler:
            :py:class:`io.BufferedWriter`
        :param int bytes_remaining:
            The delta between the total file size in bytes and amount already
            downloaded.

        :rtype: None

        """
        file_handler.write(chunk)
        logger.debug(
            'download progress\n%s',
            pprint.pformat(
                {
                    'chunk_size': len(chunk),
                    'bytes_remaining': bytes_remaining,
                }, indent=2,
            ),
        )
        on_progress = self._monostate['on_progress']
        if on_progress:
            logger.debug('calling on_progress callback %s', on_progress)
            on_progress(self, chunk, file_handler, bytes_remaining)

    def on_complete(self, file_handle):
        """On download complete handler function.

        :param file_handle:
            The file handle where the media is being written to.
        :type file_handle:
            :py:class:`io.BufferedWriter`

        :rtype: None

        """
        logger.debug('download finished')
        on_complete = self._monostate['on_complete']
        if on_complete:
            logger.debug('calling on_complete callback %s', on_complete)
            on_complete(self, file_handle)

    def __repr__(self):
        """Printable object representation.

        :rtype: str
        :returns:
            A string representation of a :class:`Stream <Stream>` object.
        """
        # TODO(nficano): this can probably be written better.
        parts = ['itag="{s.itag}"', 'mime_type="{s.mime_type}"']
        if self.includes_video_track:
            parts.extend(['res="{s.resolution}"', 'fps="{s.fps}fps"'])
            if not self.is_adaptive:
                parts.extend([
                    'vcodec="{s.video_codec}"',
                    'acodec="{s.audio_codec}"',
                ])
            else:
                parts.extend(['vcodec="{s.video_codec}"'])
        else:
            parts.extend(['abr="{s.abr}"', 'acodec="{s.audio_codec}"'])
        parts = ' '.join(parts).format(s=self)
        return '<Stream: {parts}>'.format(parts=parts)
<|endoftext|>


<commit_before>from broadcaster import Broadcaster
from light_strand import LightStrand
from server import Server
import Queue
import time

  try:
command_queue = Queue.Queue()
strands = LightStrand.connect_all_strands()

#start the server
server = Server(command_queue)
server.start()

#start the broadcaster
broadcaster = Broadcaster(strands, command_queue)
broadcaster.start()

while True:
    try:
        time.sleep(1)
    except (KeyboardInterrupt, SystemExit):
        print "Exiting..."
        server.stop()
        broadcaster.stop()
<commit_msg>remove extra try<commit_after>from broadcaster import Broadcaster
from light_strand import LightStrand
from server import Server
import Queue
import time

command_queue = Queue.Queue()
strands = LightStrand.connect_all_strands()

#start the server
server = Server(command_queue)
server.start()

#start the broadcaster
broadcaster = Broadcaster(strands, command_queue)
broadcaster.start()

while True:
    try:
        time.sleep(1)
    except (KeyboardInterrupt, SystemExit):
        print "Exiting..."
        server.stop()
        broadcaster.stop()
<|endoftext|>


<commit_before>import os.path
import re

from .trie import Trie
from .utils import hexstrings2int

COLL_ELEMENT_PATTERN = r"""
    \[
    (\*|\.)
    ([0-9A-Fa-f]{4})
    \.
    ([0-9A-Fa-f]{4})
    \.
    ([0-9A-Fa-f]{4})
    (?:\.([0-9A-Fa-f]{4}))?
\]
"""

class Collator:

    def __init__(self, filename=None):

        if filename is None:
            filename = os.path.join(os.path.dirname(__file__), "allkeys.txt")
        self.table = Trie()
        self.load(filename)

    def load(self, filename):
        with open(filename) as keys_file:
            for line in keys_file:
                line = line.split("#")[0].split("%")[0].strip()

                if not line:
                    continue

                if line.startswith("@version"):
                    pass
                else:
                    a, b = line.split(";")
                    char_list = hexstrings2int(a.split())
                    coll_elements = []
                    for x in re.finditer(COLL_ELEMENT_PATTERN, b.strip(), re.X):
                        alt, weight1, weight2, weight3, weight4 = x.groups()
                        weights = [weight1, weight2, weight3]
                        if weight4:
                            weights.append(weight4)
                        coll_elements.append(hexstrings2int(weights))
                    self.table.add(char_list, coll_elements)

    def collation_elements(self, string):
        collation_elements = []

        lookup_key = [ord(ch) for ch in string]
        while lookup_key:
            value, lookup_key = self.table.find_prefix(lookup_key)
            if not value:
                # Calculate implicit weighting for CJK Ideographs
                # http://www.unicode.org/reports/tr10/#Implicit_Weights
                key = lookup_key[0]
                value = [
                    (0xFB40 + (key >> 15), 0x0020, 0x0002, 0x0001),
                    ((key & 0x7FFF) | 0x8000, 0x0000, 0x0000, 0x0000)
                ]
                lookup_key = lookup_key[1:]
            collation_elements.extend(value)

        return collation_elements

    def sort_key_from_collation_elements(self, collation_elements):
        sort_key = []

        for level in range(4):
            if level:
                sort_key.append(0)  # level separator
            for element in collation_elements:
                if len(element) > level:
                    ce_l = element[level]
                    if ce_l:
                        sort_key.append(ce_l)

        return tuple(sort_key)

    def sort_key(self, string):

        collation_elements = self.collation_elements(string)
        return self.sort_key_from_collation_elements(collation_elements)
<commit_msg>compile regex<commit_after>import os.path
import re

from .trie import Trie
from .utils import hexstrings2int

COLL_ELEMENT_PATTERN = re.compile(r"""
    \[
    (\*|\.)
    ([0-9A-Fa-f]{4})
    \.
    ([0-9A-Fa-f]{4})
    \.
    ([0-9A-Fa-f]{4})
    (?:\.([0-9A-Fa-f]{4}))?
\]
""", re.X)

class Collator:

    def __init__(self, filename=None):

        if filename is None:
            filename = os.path.join(os.path.dirname(__file__), "allkeys.txt")
        self.table = Trie()
        self.load(filename)

    def load(self, filename):
        with open(filename) as keys_file:
            for line in keys_file:
                line = line.split("#")[0].split("%")[0].strip()

                if not line:
                    continue

                if line.startswith("@version"):
                    pass
                else:
                    a, b = line.split(";")
                    char_list = hexstrings2int(a.split())
                    coll_elements = []
                    for x in COLL_ELEMENT_PATTERN.finditer(b.strip()):
                        alt, weight1, weight2, weight3, weight4 = x.groups()
                        weights = [weight1, weight2, weight3]
                        if weight4:
                            weights.append(weight4)
                        coll_elements.append(hexstrings2int(weights))
                    self.table.add(char_list, coll_elements)

    def collation_elements(self, string):
        collation_elements = []

        lookup_key = [ord(ch) for ch in string]
        while lookup_key:
            value, lookup_key = self.table.find_prefix(lookup_key)
            if not value:
                # Calculate implicit weighting for CJK Ideographs
                # http://www.unicode.org/reports/tr10/#Implicit_Weights
                key = lookup_key[0]
                value = [
                    (0xFB40 + (key >> 15), 0x0020, 0x0002, 0x0001),
                    ((key & 0x7FFF) | 0x8000, 0x0000, 0x0000, 0x0000)
                ]
                lookup_key = lookup_key[1:]
            collation_elements.extend(value)

        return collation_elements

    def sort_key_from_collation_elements(self, collation_elements):
        sort_key = []

        for level in range(4):
            if level:
                sort_key.append(0)  # level separator
            for element in collation_elements:
                if len(element) > level:
                    ce_l = element[level]
                    if ce_l:
                        sort_key.append(ce_l)

        return tuple(sort_key)

    def sort_key(self, string):

        collation_elements = self.collation_elements(string)
        return self.sort_key_from_collation_elements(collation_elements)
<|endoftext|>


<commit_before>"""
                                      '||
... ...  .... ... ... ... ...  ....    ||
 ||'  ||  '|.  |   ||  ||  |  '' .||   ||
 ||    |   '|.|     ||| |||   .|' ||   ||
 ||...'     '|       |   |    '|..'|' .||.
 ||      .. |
''''      ''
Created by Dylan Araps.
"""

import pathlib

__version__ = "0.5.4"

HOME = pathlib.Path.home()
CACHE_DIR = HOME / ".cache/wal/"
MODULE_DIR = pathlib.Path(__file__).parent
COLOR_COUNT = 16
<commit_msg>version: bump<commit_after>"""
                                      '||
... ...  .... ... ... ... ...  ....    ||
 ||'  ||  '|.  |   ||  ||  |  '' .||   ||
 ||    |   '|.|     ||| |||   .|' ||   ||
 ||...'     '|       |   |    '|..'|' .||.
 ||      .. |
''''      ''
Created by Dylan Araps.
"""

import pathlib

__version__ = "0.5.5"

HOME = pathlib.Path.home()
CACHE_DIR = HOME / ".cache/wal/"
MODULE_DIR = pathlib.Path(__file__).parent
COLOR_COUNT = 16
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Poio Tools for Linguists
#
# Copyright (C) 2009-2013 Poio Project
# Author: António Lopes <alopes@cidles.eu>
# URL: <http://media.cidles.eu/poio/>
# For license information, see LICENSE.TXT

import os

import poioapi.io.pickle
import poioapi.io.graf
import poioapi.data

class TestPickle:

    def setup(self):
        self.data_structure = ['utterance',
            ['clause_unit',
                ['word', 'wfw', 'graid1'],
             'graid2'], 'translation', 'comment']

        filename = os.path.join(os.path.dirname(__file__), "sample_files",
            "balochi_graf", "balochi.pickle")

        self.pickle_parser = poioapi.io.pickle.Parser(filename, poioapi.data.GRAID)

    def test_find_childs_in_hierarchy(self):
        utterance_childs = ['clause_unit', 'translation', 'comment']

        clause_unit = ['clause_unit',
            ['word', 'wfw', 'graid1'],
                       'graid2']
        clause_unit_childs = ['word', 'graid2']

        word = ['word', 'wfw', 'graid1']
        word_childs = ['wfw', 'graid1']

        utterance_strct = self.pickle_parser._find_childs_from_structure(self.data_structure)
        assert utterance_strct == utterance_childs

        clause_unit_strct = self.pickle_parser._find_childs_from_structure(clause_unit)
        assert clause_unit_strct == clause_unit_childs

        word_strct = self.pickle_parser._find_childs_from_structure(word)
        assert word_strct == word_childs

    def test_get_child_tiers_for_tier(self):

        tier = poioapi.io.graf.Tier('utterance')
        expected_child_tiers = ['clause_unit', 'translation', 'comment']

        assert self.pickle_parser._find_tier_in_structure(tier, self.data_structure) == \
               expected_child_tiers

        tier = poioapi.io.graf.Tier('clause_unit')
        expected_child_tiers = ['word', 'graid2']

        assert self.pickle_parser._find_tier_in_structure(tier, self.data_structure) == \
               expected_child_tiers<commit_msg>Test_pickle updated to latest parser.<commit_after># -*- coding: utf-8 -*-
#
# Poio Tools for Linguists
#
# Copyright (C) 2009-2013 Poio Project
# Author: António Lopes <alopes@cidles.eu>
# URL: <http://media.cidles.eu/poio/>
# For license information, see LICENSE.TXT

import os

import poioapi.io.pickle
import poioapi.io.graf
import poioapi.data

class TestPickle:

    def setup(self):
        self.data_structure = ['utterance',
            ['clause_unit',
                ['word', 'wfw', 'graid1'],
             'graid2'], 'translation', 'comment']

        filename = os.path.join(os.path.dirname(__file__), "sample_files",
            "balochi_graf", "balochi.pickle")

        self.pickle_parser = poioapi.io.pickle.Parser(filename)

    def test_find_childs_in_hierarchy(self):
        utterance_childs = ['clause_unit', 'translation', 'comment']

        clause_unit = ['clause_unit',
            ['word', 'wfw', 'graid1'],
                       'graid2']
        clause_unit_childs = ['word', 'graid2']

        word = ['word', 'wfw', 'graid1']
        word_childs = ['wfw', 'graid1']

        utterance_strct = self.pickle_parser._find_childs_from_structure(self.data_structure)
        assert utterance_strct == utterance_childs

        clause_unit_strct = self.pickle_parser._find_childs_from_structure(clause_unit)
        assert clause_unit_strct == clause_unit_childs

        word_strct = self.pickle_parser._find_childs_from_structure(word)
        assert word_strct == word_childs

    def test_get_child_tiers_for_tier(self):

        tier = poioapi.io.graf.Tier('utterance')
        expected_child_tiers = ['clause_unit', 'translation', 'comment']

        assert self.pickle_parser._find_tier_in_structure(tier, self.data_structure) == \
               expected_child_tiers

        tier = poioapi.io.graf.Tier('clause_unit')
        expected_child_tiers = ['word', 'graid2']

        assert self.pickle_parser._find_tier_in_structure(tier, self.data_structure) == \
               expected_child_tiers<|endoftext|>


<commit_before>from typing import List, Type, Union

import numpy as np
from rx import subjects

from data import meta, observable_meta
from puzzle.heuristics import analyze
from puzzle.problems import problem
from puzzle.puzzlepedia import meta_problem, solution_stream

PuzzleSources = Union['Puzzle', str, problem.ProblemData]

class Puzzle(subjects.Subject):
  _meta_problems: List[meta_problem.MetaProblem]
  _child_streams: List[solution_stream.SolutionStream]

  def __init__(
      self, name: str,
      source: PuzzleSources,
      hint: str = None,
      **kwargs) -> None:
    super(Puzzle, self).__init__()
    self._meta_problems = []
    self._child_streams = []
    if isinstance(source, str):
      data = [line for line in source.split('\n') if line]
    elif isinstance(source, list):
      data = source
    elif isinstance(source, Puzzle):
      data = source.solutions()
    elif isinstance(source, np.ndarray):
      data = source
    else:
      raise NotImplementedError(
          'Puzzle source type %s unsupported' % type(source))
    for i, (mp, consumed) in enumerate(
        analyze.identify_problems(data, hint=hint)):
      p = _reify(mp, '#%s' % (i + 1), consumed, **kwargs)
      self._meta_problems.append(p)
      self._child_streams.append(solution_stream.SolutionStream(str(i), p))
    self._observable = solution_stream.SolutionStream(
        name, observable_meta.ObservableMeta(), self._child_streams)
    self._observable.subscribe(self)

  def problem(self, index: int) -> meta_problem.MetaProblem:
    return self._meta_problems[index]

  def problems(self) -> List['problem.Problem']:
    return [p.active for p in self._meta_problems]

  def solutions(self) -> List[str]:
    return [p.solution for p in self._meta_problems]

  def get_next_stage(self) -> 'Puzzle':
    return Puzzle('meta', self)

def _reify(
    mp: meta.Meta[Type[problem.Problem]],
    name: str,
    lines: problem.ProblemData,
    **kwargs,
) -> meta_problem.MetaProblem:
  result = meta_problem.MetaProblem()
  for value, weight in mp.items():
    result[value(name, lines, **kwargs)] = weight
  return result
<commit_msg>Expose inner "MetaProblem" objects.<commit_after>from typing import Iterable, List, Type, Union

import numpy as np
from rx import subjects

from data import meta, observable_meta
from puzzle.heuristics import analyze
from puzzle.problems import problem
from puzzle.puzzlepedia import meta_problem, solution_stream

PuzzleSources = Union['Puzzle', str, problem.ProblemData]

class Puzzle(subjects.Subject):
  _meta_problems: List[meta_problem.MetaProblem]
  _child_streams: List[solution_stream.SolutionStream]

  def __init__(
      self, name: str,
      source: PuzzleSources,
      hint: str = None,
      **kwargs) -> None:
    super(Puzzle, self).__init__()
    self._meta_problems = []
    self._child_streams = []
    if isinstance(source, str):
      data = [line for line in source.split('\n') if line]
    elif isinstance(source, list):
      data = source
    elif isinstance(source, Puzzle):
      data = source.solutions()
    elif isinstance(source, np.ndarray):
      data = source
    else:
      raise NotImplementedError(
          'Puzzle source type %s unsupported' % type(source))
    for i, (mp, consumed) in enumerate(
        analyze.identify_problems(data, hint=hint)):
      p = _reify(mp, '#%s' % (i + 1), consumed, **kwargs)
      self._meta_problems.append(p)
      self._child_streams.append(solution_stream.SolutionStream(str(i), p))
    self._observable = solution_stream.SolutionStream(
        name, observable_meta.ObservableMeta(), self._child_streams)
    self._observable.subscribe(self)

  def meta_problems(self) -> Iterable[meta_problem.MetaProblem]:
    yield from self._meta_problems

  def problem(self, index: int) -> meta_problem.MetaProblem:
    return self._meta_problems[index]

  def problems(self) -> List['problem.Problem']:
    return [p.active for p in self._meta_problems]

  def solutions(self) -> List[str]:
    return [p.solution for p in self._meta_problems]

  def get_next_stage(self) -> 'Puzzle':
    return Puzzle('meta', self)

def _reify(
    mp: meta.Meta[Type[problem.Problem]],
    name: str,
    lines: problem.ProblemData,
    **kwargs,
) -> meta_problem.MetaProblem:
  result = meta_problem.MetaProblem()
  for value, weight in mp.items():
    result[value(name, lines, **kwargs)] = weight
  return result
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import aiohttp
import asyncio

import json

api_baseurl = "https://hacker-news.firebaseio.com/v0"
categories = ["ask", "best", "job", "new", "show", "top"]
worker_count = 30

# NOTE: asyncio.gather orders the result values the same way as the input
#       awaitables

class ProgressState:
    def __init__(self, category, total):
        self.category = category
        self.total = total
        self.done = 0
        self.output = []

    def story_done(self, item):
        self.done = self.done + 1
        self.output.append(item)

        print(
            f'Fetching "{self.category}" stories...[{self.done}/{self.total}]',
            end="\r",
        )

        if self.done == self.total:
            print()

async def fetch(session, url):
    async with session.get(url) as response:
        return await response.json()

async def fetch_stories_ids(session, category):
    return await fetch(session, f"{api_baseurl}/{category}stories.json")

async def fetch_item(session, id):
    return await fetch(session, f"{api_baseurl}/item/{id}.json")

async def worker(session, queue, shared_state):
    while True:
        story = await fetch_item(session, await queue.get())
        stack = [story]

        while stack:
            parent = stack.pop()

            if not parent or "kids" not in parent:
                continue

            kids = [fetch_item(session, id) for id in parent["kids"]]
            parent["_kids"] = await asyncio.gather(*kids)

            stack.extend(parent["_kids"])

        shared_state.story_done(story)

        queue.task_done()

async def fetch_stories(category):
    async with aiohttp.ClientSession() as session:
        ids = await fetch_stories_ids(session, category)

        work_queue = asyncio.Queue()

        for id in ids:
            work_queue.put_nowait(id)

        tasks = []
        shared_state = ProgressState(category, len(ids))

        for _ in range(worker_count):
            task = asyncio.create_task(
                worker(session, work_queue, shared_state)
            )
            tasks.append(task)

        await work_queue.join()

        for task in tasks:
            task.cancel()

        await asyncio.gather(*tasks, return_exceptions=True)

        return shared_state.output

async def main():
    for category in categories:
        with open(f"{category}_stories.json", "w", encoding="utf8") as f:
            json.dump(await fetch_stories(category), f, indent=2)

if __name__ == "__main__":
    asyncio.run(main())
<commit_msg>Update stories dump script<commit_after>#!/usr/bin/env python3

import aiohttp
import asyncio

import json

api_baseurl = "https://hacker-news.firebaseio.com/v0"
categories = ["ask", "best", "job", "new", "show", "top"]
worker_count = 10
stories_count = 10

# NOTE: asyncio.gather orders the result values the same way as the input
#       awaitables

class ProgressState:
    def __init__(self, category, total):
        self.category = category
        self.total = total
        self.done = 0
        self.output = []

    def story_done(self, item):
        self.done = self.done + 1
        self.output.append(item)

        print(
            f'Fetching "{self.category}" stories... [{self.done}/{self.total}]',
            end="\r",
        )

        if self.done == self.total:
            print()

async def fetch(session, url):
    async with session.get(url) as response:
        return await response.json()

async def fetch_stories_ids(session, category):
    return await fetch(session, f"{api_baseurl}/{category}stories.json")

async def fetch_item(session, id):
    return await fetch(session, f"{api_baseurl}/item/{id}.json")

async def worker(session, queue, shared_state):
    while True:
        story = await fetch_item(session, await queue.get())
        stack = [story]

        while stack:
            parent = stack.pop()

            if not parent or "kids" not in parent:
                continue

            kids_tasks = [fetch_item(session, id) for id in parent["kids"]]
            kids = await asyncio.gather(*kids_tasks)
            parent["_kids"] = [kid for kid in kids if kid]

            stack.extend(parent["_kids"])

        shared_state.story_done(story)

        queue.task_done()

async def fetch_stories(category):
    async with aiohttp.ClientSession() as session:
        all_ids = await fetch_stories_ids(session, category)
        ids = all_ids[:stories_count]

        work_queue = asyncio.Queue()

        for id in ids:
            work_queue.put_nowait(id)

        tasks = []
        shared_state = ProgressState(category, len(ids))

        for _ in range(worker_count):
            task = asyncio.create_task(worker(session, work_queue, shared_state))
            tasks.append(task)

        await work_queue.join()

        for task in tasks:
            task.cancel()

        await asyncio.gather(*tasks, return_exceptions=True)

        return shared_state.output

async def main():
    for category in categories:
        with open(f"{category}_stories.json", "w", encoding="utf8") as f:
            json.dump(await fetch_stories(category), f, indent=2)

if __name__ == "__main__":
    asyncio.run(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from pyramid.events import BeforeRender, ContextFound, NewRequest, NewResponse
from pyramid.settings import asbool

from .campaign import get_campaign_url
from .csrf import validate_against_csrf
from .flash import get_joined_flash
from .flat import add_flatten_functions
from .hsts import hsts_redirect_to_https
from .hsts import set_hsts_header
from .hsts import secure_application_url
from .hsts import secure_resource_url
from .hsts import secure_route_url
from .i18n import add_underscore_translation
from .markdown import markdown_to_html
from .nav import add_is_active_function
from .seen import set_seen_cookie
from .seen import get_has_been_seen
from .serve import get_serve_spec
from .session import get_session_id
from .snip import add_snip_functions
from .track import get_track_event
from .utils import *

def includeme(config):
    """Allow developers to use ``config.include('pyramid_weblayer')`` to register
      the ``add_underscore_translation`` subscriber::

      Setup::

          >>> from mock import Mock
          >>> mock_config = Mock()
          >>> includeme(mock_config)

      CSRF validation::

          >>> mock_config.add_subscriber.assert_any_call(validate_against_csrf,
          ...         ContextFound)

      Provide `_` template namespace::

          >>> mock_config.add_subscriber.assert_any_call(add_underscore_translation,
          ...         BeforeRender)
          >>> mock_config.add_subscriber.assert_any_call(add_is_active_function,
          ...         BeforeRender)

      Add flatten and snip functions to template namespace::

          >>> mock_config.add_subscriber.assert_any_call(add_flatten_functions,
          ...         BeforeRender)
          >>> mock_config.add_subscriber.assert_any_call(add_snip_functions,
          ...         BeforeRender)

      Optionally force https::

          >>> mock_config.registry.settings = {'hsts.force_https': True}
          >>> includeme(mock_config)
          >>> mock_config.add_subscriber.assert_any_call(hsts_redirect_to_https,
          ...         NewRequest)
          >>> mock_config.add_subscriber.assert_any_call(set_hsts_header,
          ...         NewResponse)
          >>> mock_config.set_request_property.assert_any_call(secure_route_url,
          ...         'route_url', reify=True)

      Has been seen flag.::

          >>> mock_config.add_subscriber.assert_any_call(set_seen_cookie,
          ...         NewResponse)
          >>> mock_config.set_request_property.assert_any_call(get_has_been_seen,
          ...         'has_been_seen', reify=True)

      Session ID::

          >>> mock_config.set_request_property.assert_any_call(get_session_id,
          ...         'session_id', reify=True)

      Joined session flash::

          >>> mock_config.set_request_property.assert_any_call(get_joined_flash,
          ...         'joined_flash', reify=True)

      Markdown rendering::

          >>> mock_config.set_request_property.assert_any_call(markdown_to_html,
          ...         'markdown_to_html', reify=True)

      Track event::

          >>> mock_config.set_request_property.assert_any_call(get_track_event,
          ...         'track_event', reify=True)

      Prereq routes::

          >>> mock_config.add_route.assert_any_call('favicon_ico', 'favicon.ico')
          >>> mock_config.add_route.assert_any_call('robots_txt', 'robots.txt')

      Scan::

          >>> mock_config.scan.assert_called_with('pyramid_weblayer')

    """

    # CSRF validation.
    config.add_subscriber(validate_against_csrf, ContextFound)

    # Provide `_` template namespace.
    config.add_subscriber(add_underscore_translation, BeforeRender)
    config.add_subscriber(add_is_active_function, BeforeRender)

    # Add snip and flatten functions to the template namespace.
    config.add_subscriber(add_flatten_functions, BeforeRender)
    config.add_subscriber(add_snip_functions, BeforeRender)

    # Optionally force https://
    settings = config.registry.settings
    should_force_https = asbool(settings.get('hsts.force_https', False))
    if should_force_https:
        config.add_subscriber(hsts_redirect_to_https, NewRequest)
        config.add_subscriber(set_hsts_header, NewResponse)
        config.set_request_property(secure_application_url, 'application_url',
                reify=True)
        config.set_request_property(secure_resource_url, 'resource_url',
                reify=True)
        config.set_request_property(secure_route_url, 'route_url', reify=True)
        config.add_tween('pyramid_weblayer.hsts.secure_redirect_tween')

    # Has been seen flag.
    config.add_subscriber(set_seen_cookie, NewResponse)
    config.set_request_property(get_has_been_seen, 'has_been_seen', reify=True)

    # Session id.
    config.set_request_property(get_session_id, 'session_id', reify=True)

    # Provide ``request.campaign_url``.
    config.set_request_property(get_campaign_url, 'campaign_url', reify=True)

    # Provide ``request.joined_flash``.
    config.set_request_property(get_joined_flash, 'joined_flash', reify=True)

    # Provide ``request.markdown_to_html``.
    config.set_request_property(markdown_to_html, 'markdown_to_html', reify=True)

    # Provide ``request.serve_spec``.
    config.set_request_property(get_serve_spec, 'serve_spec', reify=True)

    # Provide ``request.track_event``.
    config.set_request_property(get_track_event, 'track_event', reify=True)
    config.set_request_property(get_track_page, 'track_page', reify=True)

    # Favicon and robots.txt.
    config.add_route('favicon_ico', 'favicon.ico')
    config.add_route('robots_txt', 'robots.txt')
    config.scan('pyramid_weblayer')

<commit_msg>init: fix import bug registering track_page method.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from pyramid.events import BeforeRender, ContextFound, NewRequest, NewResponse
from pyramid.settings import asbool

from .campaign import get_campaign_url
from .csrf import validate_against_csrf
from .flash import get_joined_flash
from .flat import add_flatten_functions
from .hsts import hsts_redirect_to_https
from .hsts import set_hsts_header
from .hsts import secure_application_url
from .hsts import secure_resource_url
from .hsts import secure_route_url
from .i18n import add_underscore_translation
from .markdown import markdown_to_html
from .nav import add_is_active_function
from .seen import set_seen_cookie
from .seen import get_has_been_seen
from .serve import get_serve_spec
from .session import get_session_id
from .snip import add_snip_functions
from .track import get_track_event
from .track import get_track_page
from .utils import *

def includeme(config):
    """Allow developers to use ``config.include('pyramid_weblayer')`` to register
      the ``add_underscore_translation`` subscriber::

      Setup::

          >>> from mock import Mock
          >>> mock_config = Mock()
          >>> includeme(mock_config)

      CSRF validation::

          >>> mock_config.add_subscriber.assert_any_call(validate_against_csrf,
          ...         ContextFound)

      Provide `_` template namespace::

          >>> mock_config.add_subscriber.assert_any_call(add_underscore_translation,
          ...         BeforeRender)
          >>> mock_config.add_subscriber.assert_any_call(add_is_active_function,
          ...         BeforeRender)

      Add flatten and snip functions to template namespace::

          >>> mock_config.add_subscriber.assert_any_call(add_flatten_functions,
          ...         BeforeRender)
          >>> mock_config.add_subscriber.assert_any_call(add_snip_functions,
          ...         BeforeRender)

      Optionally force https::

          >>> mock_config.registry.settings = {'hsts.force_https': True}
          >>> includeme(mock_config)
          >>> mock_config.add_subscriber.assert_any_call(hsts_redirect_to_https,
          ...         NewRequest)
          >>> mock_config.add_subscriber.assert_any_call(set_hsts_header,
          ...         NewResponse)
          >>> mock_config.set_request_property.assert_any_call(secure_route_url,
          ...         'route_url', reify=True)

      Has been seen flag.::

          >>> mock_config.add_subscriber.assert_any_call(set_seen_cookie,
          ...         NewResponse)
          >>> mock_config.set_request_property.assert_any_call(get_has_been_seen,
          ...         'has_been_seen', reify=True)

      Session ID::

          >>> mock_config.set_request_property.assert_any_call(get_session_id,
          ...         'session_id', reify=True)

      Joined session flash::

          >>> mock_config.set_request_property.assert_any_call(get_joined_flash,
          ...         'joined_flash', reify=True)

      Markdown rendering::

          >>> mock_config.set_request_property.assert_any_call(markdown_to_html,
          ...         'markdown_to_html', reify=True)

      Track event::

          >>> mock_config.set_request_property.assert_any_call(get_track_event,
          ...         'track_event', reify=True)

      Prereq routes::

          >>> mock_config.add_route.assert_any_call('favicon_ico', 'favicon.ico')
          >>> mock_config.add_route.assert_any_call('robots_txt', 'robots.txt')

      Scan::

          >>> mock_config.scan.assert_called_with('pyramid_weblayer')

    """

    # CSRF validation.
    config.add_subscriber(validate_against_csrf, ContextFound)

    # Provide `_` template namespace.
    config.add_subscriber(add_underscore_translation, BeforeRender)
    config.add_subscriber(add_is_active_function, BeforeRender)

    # Add snip and flatten functions to the template namespace.
    config.add_subscriber(add_flatten_functions, BeforeRender)
    config.add_subscriber(add_snip_functions, BeforeRender)

    # Optionally force https://
    settings = config.registry.settings
    should_force_https = asbool(settings.get('hsts.force_https', False))
    if should_force_https:
        config.add_subscriber(hsts_redirect_to_https, NewRequest)
        config.add_subscriber(set_hsts_header, NewResponse)
        config.set_request_property(secure_application_url, 'application_url',
                reify=True)
        config.set_request_property(secure_resource_url, 'resource_url',
                reify=True)
        config.set_request_property(secure_route_url, 'route_url', reify=True)
        config.add_tween('pyramid_weblayer.hsts.secure_redirect_tween')

    # Has been seen flag.
    config.add_subscriber(set_seen_cookie, NewResponse)
    config.set_request_property(get_has_been_seen, 'has_been_seen', reify=True)

    # Session id.
    config.set_request_property(get_session_id, 'session_id', reify=True)

    # Provide ``request.campaign_url``.
    config.set_request_property(get_campaign_url, 'campaign_url', reify=True)

    # Provide ``request.joined_flash``.
    config.set_request_property(get_joined_flash, 'joined_flash', reify=True)

    # Provide ``request.markdown_to_html``.
    config.set_request_property(markdown_to_html, 'markdown_to_html', reify=True)

    # Provide ``request.serve_spec``.
    config.set_request_property(get_serve_spec, 'serve_spec', reify=True)

    # Provide ``request.track_event``.
    config.set_request_property(get_track_event, 'track_event', reify=True)
    config.set_request_property(get_track_page, 'track_page', reify=True)

    # Favicon and robots.txt.
    config.add_route('favicon_ico', 'favicon.ico')
    config.add_route('robots_txt', 'robots.txt')
    config.scan('pyramid_weblayer')

<|endoftext|>