<commit_before>"""
Akshay Tambe
Pachter and Doudna groups
"""

import subprocess
import sys
import json
import tempfile
import numpy as np
np.random.seed(0)
import redis
import itertools

def get_kmers(sequence, k):
	"""
	Args:
		sequence (string)
		k (int)
	Returns list
		all subsequences of sequence of length ks
	"""
	kmers = []
	for i in range(len(sequence) - k + 1):
		kmers.append(sequence[i:i+k])
	return kmers

def get_cyclic_kmers(read, k, start, end, indel=True):
	"""
	Args
		read (list)
			a fastq entry as a list of lines
		k (int)
			size of kmer
		start (int)
			start site of barcode within read
		end (int)
			end site of barcode within read
	Returns list
			list of tuples (string kmer, string qual)
			the input read is circularized
	"""
	seq = read[1]
	seq = seq[:start] + '$' + seq[start:]
	qual = read[3]
	qual = qual[:start] + '$' + qual[start:]

	alphabet=['A', 'C', 'G', 'T']
	if(end >= len(seq)):#this doesn't typically happen for drop seq
		end = len(seq)
		seq += np.random.choice(alphabet)#add something random for insertions

	#cyclicized, correct length
	cyclic =				lambda s: s[start:end] + s[start:start + k - 1]
	#cyclicized, one nucleotide truncated
	cyclic_truncate =	lambda s: s[start:end - 1] + s[start:start + k - 1]
	#cyclicized, one nucleotide extended
	cyclic_extend =	lambda s: s[start:end + 1] + s[start:start + k - 1]

	kmers = 	get_kmers(cyclic(seq), k)
	quals = 	get_kmers(cyclic(qual), k)
	if(indel == True):
		kmers += get_kmers(cyclic_truncate(seq), k) + \
			get_kmers(cyclic_extend(seq), k)
		quals += get_kmers(cyclic_truncate(qual), k) + \
			get_kmers(cyclic_extend(qual), k)

	ret = [tup for tup in zip(kmers, quals)]
	return ret

def unzip(gzipped_lst):
	"""
	Args
		gzipped (string)
	Returns
		unzipped (string) as a tempfile
	"""
	with open(sys.path[0] + '/params.json', 'r') as r:
		params = json.load(r)

	out_file = tempfile.NamedTemporaryFile(delete=False)
	out_file_name = out_file.name
	out_file.seek(0)
	zcat = subprocess.Popen(
		[params['zcat'],  gzipped_lst[0]],
		stdout = out_file)
	zcat.communicate()
	out_file.close()
	return out_file.name

def get_read_chunks(barcodes, reads, lines=None, random_subset = 1):
	"""
	Args:
		barcodes (unzipped fq file)
		reads (unzipped fq file)
		lines (list)
	yields
		data_buffer (list of tuples)

	simultaneously reads two fastq files (line-by-line) and stores data in a list
	yields this list (chunks of the datasets)
	"""

	BUFFER_SIZE = 100000#number of reads in each chunk
	#size is set to be quite large because updating kmer_idx database is slow
	if(lines == None):
		reads_iter = read_fastq_sequential(reads)
		barcodes_iter = read_fastq_sequential(barcodes)
	else:
		reads_iter = read_fastq_random(reads)
		barcodes_iter = read_fastq_random(barcodes)

	break_outer = False
	while(not break_outer):
		data_buffer = []#list of tuples(barcodes_data, barcodes_offset, reads_offset)

		buffer_count = 0
		while(True):
			try:
				reads_data, reads_offset = next(reads_iter)
				barcodes_data, barcodes_offset = next(barcodes_iter)
			except StopIteration:
				break_outer = True
				break

			if(buffer_count % BUFFER_SIZE == 0 and buffer_count > 0):
				#break_inner = True
				break

			if(np.random.uniform(0, 1) <= random_subset):
				data_buffer.append(
					(reads_data,
					reads_offset,
					barcodes_data,
					barcodes_offset))
				buffer_count += 1

		yield data_buffer

def read_fastq_sequential(fq_file, gzip=False):
	"""
	Args:
		fq_file (path to a fastq format file)
		gzip:
			binary, whether fq_files is gzipped or not
	Yields
		tuple (lines, offset)
			lines: list of 4 lines from fastq file, in order in file
			offset: character offset from beginning of the file for this fq read
	"""
	with open(sys.path[0] + '/params.json', 'r') as r:
		params = json.load(r)

	line_num = 0
	offset = 0
	if(gzip):
		cat = subprocess.Popen(
			[params['zcat'], fq_file],
			stdout = subprocess.PIPE)
	else:
		cat = subprocess.Popen(
			['cat', fq_file],
			stdout = subprocess.PIPE)
	for lines in grouper(cat.stdout, 4):
		read_len = sum([len(line) for line in lines])#length of read in characters
		lines = [(l.rstrip()).decode('utf-8') for l in list(lines)]
		yield(lines, offset)
		line_num += 1
		offset += read_len

def read_multiple_fastq_sequential(fq_file_lst, gzip=False):
	for fq_file in fq_file_lst:
		fq_file_iter = read_fastq_sequential(fq_file, gzip=gzip)
		for (lines, offset) in fq_file_iter:
			yield (lines, offset)

def read_fastq_random(fq_file, offsets):
	"""
	Args:
		fq_file (path to a fastq format file), unzipped
		offsets (list)
	Yields
		tuple (lines, offset)
			lines: list of 4 lines from fastq file, ordered by offets
			offset: character offset from beginning of the file for this fq read
	"""
	fq_reader = open(fq_file, 'rb')
	for (i, offset) in enumerate(offsets):
		fq_reader.seek(offset, 0)
		lines = [fq_reader.readline().rstrip().decode('utf-8') for i in range(4)]
		yield (lines, offset)
	fq_reader.close()

def merge_barcodefiles_10x(reads1, index1):
	""""
	10x genomics to dropseq

	3 fastq.gz files from each file
		I1: read index, UMI
		R1: cell barcode(?)
		R2: RNAseq read

	I1
	@ST-K00126:307:HFM3NBBXX:1:1101:3772:1244 1:N:0:NTCGCCCT
	NTCGCCCT
	+
	#AAAFJ-J

	R1
	@ST-K00126:307:HFM3NBBXX:1:1101:3772:1244 1:N:0:NTCGCCCT
	NCATTTGAGTAACCCTGATGTCATAA
	+
	#AAFFJJJJJJJJJJJJJJJFJJJJJ

	R2
	@ST-K00126:307:HFM3NBBXX:1:1101:3772:1244 2:N:0:NTCGCCCT
	NAAGCCAGTTGTGAATCATGCACATCAGCTCCTTCTGAAATGTGTTTATGGCCTAGGACACAGGGACCCTGGAGACTATGGTGCTGCAGTGCATTATG
	+
	#<<A<FJJJFJFJJJJJJJJJFJFJJJJJJJJJJJJJJJFJJFFJJJJJAFJJFJF7JJJJFJAJJJ<J<7-A<FFFFJ-F<FJJJJJJJJJJ7FJJA
	"""

	out_file = tempfile.NamedTemporaryFile(delete=False)
	out_file_name = out_file.name
	out_file.seek(0)

	writer = open(out_file, 'wb')
	reads_iter = read_multiple_fastq_sequential(
		reads1, gzip=True)
	index_iter = read_multiple_fastq_sequential(
		indexq, gzip=True)

	while(True):
		try:
			reads, _ = next(reads_iter)
			index, _ = next(index_iter)
		except StopIteration:
			break
		get_prefix = lambda r: r[0].split(' ')[0]
		assert(get_prefix(reads) == get_prefix(index)), \
			'Reads are not in order\n%s\n%s' % \
			('\n'.join(reads), '\n'.join(index))
		combined_seq = reads[1] + index[1]
		combined_phred = reads[2] + index[3]

		output = [
			reads[0],
			combined_seq,
			reads[2],
			combined_phred]
		writer.write( ('\n'.join(output) + '\n').encode('utf-8'))
	writer.close()
	return out_file.name

def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    args = [iter(iterable)] * n
    return zip(*args)

def save_paths_text(output_dir, paths, prefix=''):
	paths_file = '%s/%s_paths.txt' % (output_dir, prefix)
	with open(paths_file, 'w') as writer:
		for tup in paths:
			writer.write('%s\t%i\t%i\t%s\n' %
				(tup[0], tup[1], tup[2], ','.join(tup[3])))
	return paths_file

def initialize_redis_pipeline(db=0):
	redis_db = redis.StrictRedis(host="localhost", port=6379, db=db)#redis

	redis_db.flushall()
	redis_pipe = redis_db.pipeline()
	return redis_db, redis_pipe

def get_from_db(kmer_idx_pipe, keys):
	for key in keys:
		kmer_idx_pipe.get(key)
	pipe_out = kmer_idx_pipe.execute()
	entries = []
	for entry in pipe_out:
		#entry is a comma separated bytestring of ints. return just the list
		if(entry != None):
			offsets = [int(i) for i in entry.decode('utf-8').split(',')[0:-1]]
			entries.append(offsets)
	return entries

def get_digital_expression(tsv_file, cells_file):
	num_cells = get_num_cells(cells)
	nonzero_ec, ec_to_index = get_nonzero_ec(tsv_file)

	dge = np.zeros((num_cells, len(nonzero_ec)))
	cells_iter = read_tsv_by_cell(tsv_file)
	for (i, (cell, total_counts, ec_counts)) in enumerate(cells_iter):
		for (eq_class, count) in ec_counts:
			dge[i, ec_to_idx[eq_class]] = count / total_counts
	#print('Shape of DGE: %i %i' % (np.shape(dge)))
	return dge

def get_nonzero_ec(tsv_file):
	nonzero_ec = set()
	with open(tsv_file, 'rb') as inf:
		for line in inf:
			[eq_class, cell, count] = \
				line.strip().decode('utf-8').split('\t')
			nonzero_ec.add(eq_class)

	ec_to_index = {}
	for (i, ec) in enumerate(nonzero_ec):
		ec_to_index[int(ec)] = i
	num_ec = i + 1
	print('Total nonzero ECs: %i ' % num_ec)

	return nonzero_ec, ec_to_index

def get_num_cells(cells_file):
	cells = set()
	with open(cells_file, 'rb') as inf:
		for (i, line) in enumerate(inf):
			pass
	print('Total number of cells: %i' % i)
	return i

def read_tsv_by_cell(tsv_file):
	tsv_iter = open(tsv_file, 'rb')
	ec_counts = []
	prev_cell = ''
	total_counts = 0
	while(True):
		while(True):
			try:
				tsv_line = next(tsv_iter)
			except StopIteration:
				tsv_iter.close()
				raise StopIteration
				break
			[eq_class, cell, count] = [ \
				int(i) for i in tsv_line.decode('utf-8').strip().split('\t')]
			tsv_data = (eq_class, count)
			if(len(ec_counts) == 0):
				ec_counts.append(tsv_data)
				total_counts = count
				prev_cell = cell
			elif(prev_cell == cell):
				ec_counts.append(tsv_data)
				total_counts += count
			else:
				yield cell, total_counts, ec_counts
				ec_counts = [tsv_data]
				prev_cell = cell
				total_counts = count

class Logger(object):
	"""
	Copypasta from stackoverflow forums. All credit to user Eric Leschinsky
	http://stackoverflow.com/questions/14906764/

	Writes all stdout to file as well as printing to stdout
	Initialize with sys.stdout = Logger()
	"""

	def __init__(self, fname):
		self.terminal = sys.stdout
		self.log = open(fname, 'w')
		self.pos = 0

	def write(self, message):
		self.pos = self.log.tell()
		self.terminal.write(message)
		self.log.write(message)

	def flush(self):
		self.log.seek(self.pos)

<commit_msg>Fix Typos in merge_barcodefiles_10x()<commit_after>"""
Akshay Tambe
Pachter and Doudna groups
"""

import subprocess
import sys
import json
import tempfile
import numpy as np
np.random.seed(0)
import redis
import itertools

def get_kmers(sequence, k):
	"""
	Args:
		sequence (string)
		k (int)
	Returns list
		all subsequences of sequence of length ks
	"""
	kmers = []
	for i in range(len(sequence) - k + 1):
		kmers.append(sequence[i:i+k])
	return kmers

def get_cyclic_kmers(read, k, start, end, indel=True):
	"""
	Args
		read (list)
			a fastq entry as a list of lines
		k (int)
			size of kmer
		start (int)
			start site of barcode within read
		end (int)
			end site of barcode within read
	Returns list
			list of tuples (string kmer, string qual)
			the input read is circularized
	"""
	seq = read[1]
	seq = seq[:start] + '$' + seq[start:]
	qual = read[3]
	qual = qual[:start] + '$' + qual[start:]

	alphabet=['A', 'C', 'G', 'T']
	if(end >= len(seq)):#this doesn't typically happen for drop seq
		end = len(seq)
		seq += np.random.choice(alphabet)#add something random for insertions

	#cyclicized, correct length
	cyclic =				lambda s: s[start:end] + s[start:start + k - 1]
	#cyclicized, one nucleotide truncated
	cyclic_truncate =	lambda s: s[start:end - 1] + s[start:start + k - 1]
	#cyclicized, one nucleotide extended
	cyclic_extend =	lambda s: s[start:end + 1] + s[start:start + k - 1]

	kmers = 	get_kmers(cyclic(seq), k)
	quals = 	get_kmers(cyclic(qual), k)
	if(indel == True):
		kmers += get_kmers(cyclic_truncate(seq), k) + \
			get_kmers(cyclic_extend(seq), k)
		quals += get_kmers(cyclic_truncate(qual), k) + \
			get_kmers(cyclic_extend(qual), k)

	ret = [tup for tup in zip(kmers, quals)]
	return ret

def unzip(gzipped_lst):
	"""
	Args
		gzipped (string)
	Returns
		unzipped (string) as a tempfile
	"""
	with open(sys.path[0] + '/params.json', 'r') as r:
		params = json.load(r)

	out_file = tempfile.NamedTemporaryFile(delete=False)
	out_file_name = out_file.name
	out_file.seek(0)
	zcat = subprocess.Popen(
		[params['zcat'],  gzipped_lst[0]],
		stdout = out_file)
	zcat.communicate()
	out_file.close()
	return out_file.name

def get_read_chunks(barcodes, reads, lines=None, random_subset = 1):
	"""
	Args:
		barcodes (unzipped fq file)
		reads (unzipped fq file)
		lines (list)
	yields
		data_buffer (list of tuples)

	simultaneously reads two fastq files (line-by-line) and stores data in a list
	yields this list (chunks of the datasets)
	"""

	BUFFER_SIZE = 100000#number of reads in each chunk
	#size is set to be quite large because updating kmer_idx database is slow
	if(lines == None):
		reads_iter = read_fastq_sequential(reads)
		barcodes_iter = read_fastq_sequential(barcodes)
	else:
		reads_iter = read_fastq_random(reads)
		barcodes_iter = read_fastq_random(barcodes)

	break_outer = False
	while(not break_outer):
		data_buffer = []#list of tuples(barcodes_data, barcodes_offset, reads_offset)

		buffer_count = 0
		while(True):
			try:
				reads_data, reads_offset = next(reads_iter)
				barcodes_data, barcodes_offset = next(barcodes_iter)
			except StopIteration:
				break_outer = True
				break

			if(buffer_count % BUFFER_SIZE == 0 and buffer_count > 0):
				#break_inner = True
				break

			if(np.random.uniform(0, 1) <= random_subset):
				data_buffer.append(
					(reads_data,
					reads_offset,
					barcodes_data,
					barcodes_offset))
				buffer_count += 1

		yield data_buffer

def read_fastq_sequential(fq_file, gzip=False):
	"""
	Args:
		fq_file (path to a fastq format file)
		gzip:
			binary, whether fq_files is gzipped or not
	Yields
		tuple (lines, offset)
			lines: list of 4 lines from fastq file, in order in file
			offset: character offset from beginning of the file for this fq read
	"""
	with open(sys.path[0] + '/params.json', 'r') as r:
		params = json.load(r)

	line_num = 0
	offset = 0
	if(gzip):
		cat = subprocess.Popen(
			[params['zcat'], fq_file],
			stdout = subprocess.PIPE)
	else:
		cat = subprocess.Popen(
			['cat', fq_file],
			stdout = subprocess.PIPE)
	for lines in grouper(cat.stdout, 4):
		read_len = sum([len(line) for line in lines])#length of read in characters
		lines = [(l.rstrip()).decode('utf-8') for l in list(lines)]
		yield(lines, offset)
		line_num += 1
		offset += read_len

def read_multiple_fastq_sequential(fq_file_lst, gzip=False):
	for fq_file in fq_file_lst:
		fq_file_iter = read_fastq_sequential(fq_file, gzip=gzip)
		for (lines, offset) in fq_file_iter:
			yield (lines, offset)

def read_fastq_random(fq_file, offsets):
	"""
	Args:
		fq_file (path to a fastq format file), unzipped
		offsets (list)
	Yields
		tuple (lines, offset)
			lines: list of 4 lines from fastq file, ordered by offets
			offset: character offset from beginning of the file for this fq read
	"""
	fq_reader = open(fq_file, 'rb')
	for (i, offset) in enumerate(offsets):
		fq_reader.seek(offset, 0)
		lines = [fq_reader.readline().rstrip().decode('utf-8') for i in range(4)]
		yield (lines, offset)
	fq_reader.close()

def merge_barcodefiles_10x(reads1, index1):
	""""
	10x genomics to dropseq

	3 fastq.gz files from each file
		I1: read index, UMI
		R1: cell barcode(?)
		R2: RNAseq read

	I1
	@ST-K00126:307:HFM3NBBXX:1:1101:3772:1244 1:N:0:NTCGCCCT
	NTCGCCCT
	+
	#AAAFJ-J

	R1
	@ST-K00126:307:HFM3NBBXX:1:1101:3772:1244 1:N:0:NTCGCCCT
	NCATTTGAGTAACCCTGATGTCATAA
	+
	#AAFFJJJJJJJJJJJJJJJFJJJJJ

	R2
	@ST-K00126:307:HFM3NBBXX:1:1101:3772:1244 2:N:0:NTCGCCCT
	NAAGCCAGTTGTGAATCATGCACATCAGCTCCTTCTGAAATGTGTTTATGGCCTAGGACACAGGGACCCTGGAGACTATGGTGCTGCAGTGCATTATG
	+
	#<<A<FJJJFJFJJJJJJJJJFJFJJJJJJJJJJJJJJJFJJFFJJJJJAFJJFJF7JJJJFJAJJJ<J<7-A<FFFFJ-F<FJJJJJJJJJJ7FJJA
	"""

	out_file = tempfile.NamedTemporaryFile(delete=False)
	out_file_name = out_file.name
	out_file.seek(0)

	writer = open(out_file_name, 'wb')
	reads_iter = read_multiple_fastq_sequential(
		reads1, gzip=True)
	index_iter = read_multiple_fastq_sequential(
		index1, gzip=True)

	while(True):
		try:
			reads, _ = next(reads_iter)
			index, _ = next(index_iter)
		except StopIteration:
			break
		get_prefix = lambda r: r[0].split(' ')[0]
		assert(get_prefix(reads) == get_prefix(index)), \
			'Reads are not in order\n%s\n%s' % \
			('\n'.join(reads), '\n'.join(index))
		combined_seq = reads[1] + index[1]
		combined_phred = reads[2] + index[3]

		output = [
			reads[0],
			combined_seq,
			reads[2],
			combined_phred]
		writer.write( ('\n'.join(output) + '\n').encode('utf-8'))
	writer.close()
	return out_file.name

def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    args = [iter(iterable)] * n
    return zip(*args)

def save_paths_text(output_dir, paths, prefix=''):
	paths_file = '%s/%s_paths.txt' % (output_dir, prefix)
	with open(paths_file, 'w') as writer:
		for tup in paths:
			writer.write('%s\t%i\t%i\t%s\n' %
				(tup[0], tup[1], tup[2], ','.join(tup[3])))
	return paths_file

def initialize_redis_pipeline(db=0):
	redis_db = redis.StrictRedis(host="localhost", port=6379, db=db)#redis

	redis_db.flushall()
	redis_pipe = redis_db.pipeline()
	return redis_db, redis_pipe

def get_from_db(kmer_idx_pipe, keys):
	for key in keys:
		kmer_idx_pipe.get(key)
	pipe_out = kmer_idx_pipe.execute()
	entries = []
	for entry in pipe_out:
		#entry is a comma separated bytestring of ints. return just the list
		if(entry != None):
			offsets = [int(i) for i in entry.decode('utf-8').split(',')[0:-1]]
			entries.append(offsets)
	return entries

def get_digital_expression(tsv_file, cells_file):
	num_cells = get_num_cells(cells)
	nonzero_ec, ec_to_index = get_nonzero_ec(tsv_file)

	dge = np.zeros((num_cells, len(nonzero_ec)))
	cells_iter = read_tsv_by_cell(tsv_file)
	for (i, (cell, total_counts, ec_counts)) in enumerate(cells_iter):
		for (eq_class, count) in ec_counts:
			dge[i, ec_to_idx[eq_class]] = count / total_counts
	#print('Shape of DGE: %i %i' % (np.shape(dge)))
	return dge

def get_nonzero_ec(tsv_file):
	nonzero_ec = set()
	with open(tsv_file, 'rb') as inf:
		for line in inf:
			[eq_class, cell, count] = \
				line.strip().decode('utf-8').split('\t')
			nonzero_ec.add(eq_class)

	ec_to_index = {}
	for (i, ec) in enumerate(nonzero_ec):
		ec_to_index[int(ec)] = i
	num_ec = i + 1
	print('Total nonzero ECs: %i ' % num_ec)

	return nonzero_ec, ec_to_index

def get_num_cells(cells_file):
	cells = set()
	with open(cells_file, 'rb') as inf:
		for (i, line) in enumerate(inf):
			pass
	print('Total number of cells: %i' % i)
	return i

def read_tsv_by_cell(tsv_file):
	tsv_iter = open(tsv_file, 'rb')
	ec_counts = []
	prev_cell = ''
	total_counts = 0
	while(True):
		while(True):
			try:
				tsv_line = next(tsv_iter)
			except StopIteration:
				tsv_iter.close()
				raise StopIteration
				break
			[eq_class, cell, count] = [ \
				int(i) for i in tsv_line.decode('utf-8').strip().split('\t')]
			tsv_data = (eq_class, count)
			if(len(ec_counts) == 0):
				ec_counts.append(tsv_data)
				total_counts = count
				prev_cell = cell
			elif(prev_cell == cell):
				ec_counts.append(tsv_data)
				total_counts += count
			else:
				yield cell, total_counts, ec_counts
				ec_counts = [tsv_data]
				prev_cell = cell
				total_counts = count

class Logger(object):
	"""
	Copypasta from stackoverflow forums. All credit to user Eric Leschinsky
	http://stackoverflow.com/questions/14906764/

	Writes all stdout to file as well as printing to stdout
	Initialize with sys.stdout = Logger()
	"""

	def __init__(self, fname):
		self.terminal = sys.stdout
		self.log = open(fname, 'w')
		self.pos = 0

	def write(self, message):
		self.pos = self.log.tell()
		self.terminal.write(message)
		self.log.write(message)

	def flush(self):
		self.log.seek(self.pos)

<|endoftext|>


<commit_before># Licensed under an MIT open source license - see LICENSE

import numpy as np
import warnings
from numpy.fft import fftshift
import astropy.units as u

from ..rfft_to_fft import rfft_to_fft
from slice_thickness import change_slice_thickness
from ..base_pspec2 import StatisticBase_PSpec2D
from ..base_statistic import BaseStatisticMixIn
from ...io import common_types, threed_types
from ..fitting_utils import check_fit_limits

class VCA(BaseStatisticMixIn, StatisticBase_PSpec2D):

    '''
    The VCA technique (Lazarian & Pogosyan, 2004).

    Parameters
    ----------
    cube : %(dtypes)s
        Data cube.
    header : FITS header, optional
        Corresponding FITS header.
    slice_sizes : float or int, optional
        Slices to degrade the cube to.
    '''

    __doc__ %= {"dtypes": " or ".join(common_types + threed_types)}

    def __init__(self, cube, header=None, slice_size=None):
        super(VCA, self).__init__()

        self.input_data_header(cube, header)

        if np.isnan(self.data).any():
            self.data[np.isnan(self.data)] = 0

        if slice_size is None:
            self.slice_size = 1.0

        if slice_size != 1.0:
            self.data = \
                change_slice_thickness(self.data,
                                       slice_thickness=self.slice_size)

        self._ps1D_stddev = None

    def compute_pspec(self):
        '''
        Compute the 2D power spectrum.
        '''

        vca_fft = fftshift(rfft_to_fft(self.data))

        self._ps2D = np.power(vca_fft, 2.).sum(axis=0)

    def run(self, verbose=False, brk=None, return_stddev=True,
            logspacing=False, low_cut=None, high_cut=None,
            ang_units=False, unit=u.deg, use_wavenumber=False):
        '''
        Full computation of VCA.

        Parameters
        ----------
        verbose : bool, optional
            Enables plotting.
        brk : float, optional
            Initial guess for the break point.
        return_stddev : bool, optional
            Return the standard deviation in the 1D bins.
        logspacing : bool, optional
            Return logarithmically spaced bins for the lags.
        ang_units : bool, optional
            Convert frequencies to angular units using the given header.
        unit : u.Unit, optional
            Choose the angular unit to convert to when ang_units is enabled.
        use_wavenumber : bool, optional
            Plot the x-axis as the wavenumber rather than spatial frequency.
        '''

        self.compute_pspec()
        self.compute_radial_pspec(return_stddev=return_stddev,
                                  logspacing=logspacing)
        self.fit_pspec(brk=brk, low_cut=low_cut, high_cut=high_cut,
                       large_scale=0.5)

        if verbose:

            print(self.fit.summary())

            self.plot_fit(show=True, show_2D=True, ang_units=ang_units,
                          unit=unit, use_wavenumber=use_wavenumber)

        return self

class VCA_Distance(object):

    '''
    Calculate the distance between two cubes using VCA. The 1D power spectrum
    is modeled by a linear model. The distance is the t-statistic of the
    interaction between the two slopes.

    Parameters
    ----------
    cube1 : %(dtypes)s
        Data cube.
    cube2 : %(dtypes)s
        Data cube.
    slice_size : float, optional
        Slice to degrade the cube to.
    breaks : float, list or array, optional
        Specify where the break point is. If None, attempts to find using
        spline. If not specified, no break point will be used.
    fiducial_model : VCA
        Computed VCA object. use to avoid recomputing.
    '''

    __doc__ %= {"dtypes": " or ".join(common_types + threed_types)}

    def __init__(self, cube1, cube2, slice_size=1.0, breaks=None,
                 fiducial_model=None, logspacing=False, low_cut=None,
                 high_cut=None):
        super(VCA_Distance, self).__init__()

        if not isinstance(slice_size, float):
            raise TypeError("slice_size must be a float.")

        low_cut, high_cut = check_fit_limits(low_cut, high_cut)

        if not isinstance(breaks, list) and not isinstance(breaks, np.ndarray):
            breaks = [breaks] * 2

        if fiducial_model is not None:
            self.vca1 = fiducial_model
        else:
            self.vca1 = \
                VCA(cube1, slice_size=slice_size)
            self.vca1.run(brk=breaks[0], low_cut=low_cut[0],
                          high_cut=high_cut[0], logspacing=logspacing)

        self.vca2 = \
            VCA(cube2, slice_size=slice_size)
        self.vca2.run(brk=breaks[1], low_cut=low_cut[1], high_cut=high_cut[1],
                      logspacing=logspacing)

    def distance_metric(self, verbose=False, label1=None, label2=None,
                        ang_units=False, unit=u.deg, use_wavenumber=False):
        '''

        Implements the distance metric for 2 VCA transforms, each with the
        same channel width. We fit the linear portion of the transform to
        represent the powerlaw.

        Parameters
        ----------
        verbose : bool, optional
            Enables plotting.
        label1 : str, optional
            Object or region name for cube1
        label2 : str, optional
            Object or region name for cube2
        ang_units : bool, optional
            Convert frequencies to angular units using the given header.
        unit : u.Unit, optional
            Choose the angular unit to convert to when ang_units is enabled.
        use_wavenumber : bool, optional
            Plot the x-axis as the wavenumber rather than spatial frequency.
        '''

        # Construct t-statistic
        self.distance = \
            np.abs((self.vca1.slope - self.vca2.slope) /
                   np.sqrt(self.vca1.slope_err**2 +
                           self.vca2.slope_err**2))

        if verbose:

            print("Fit to %s" % (label1))
            print(self.vca1.fit.summary())
            print("Fit to %s" % (label2))
            print(self.vca2.fit.summary())

            import matplotlib.pyplot as p
            self.vca1.plot_fit(show=False, color='b', label=label1, symbol='D',
                               ang_units=ang_units, unit=unit)
            self.vca2.plot_fit(show=False, color='g', label=label2, symbol='o',
                               ang_units=ang_units, unit=unit)
            p.legend(loc='upper right')
            p.show()

        return self
<commit_msg>Pass wavenumber through<commit_after># Licensed under an MIT open source license - see LICENSE

import numpy as np
import warnings
from numpy.fft import fftshift
import astropy.units as u

from ..rfft_to_fft import rfft_to_fft
from slice_thickness import change_slice_thickness
from ..base_pspec2 import StatisticBase_PSpec2D
from ..base_statistic import BaseStatisticMixIn
from ...io import common_types, threed_types
from ..fitting_utils import check_fit_limits

class VCA(BaseStatisticMixIn, StatisticBase_PSpec2D):

    '''
    The VCA technique (Lazarian & Pogosyan, 2004).

    Parameters
    ----------
    cube : %(dtypes)s
        Data cube.
    header : FITS header, optional
        Corresponding FITS header.
    slice_sizes : float or int, optional
        Slices to degrade the cube to.
    '''

    __doc__ %= {"dtypes": " or ".join(common_types + threed_types)}

    def __init__(self, cube, header=None, slice_size=None):
        super(VCA, self).__init__()

        self.input_data_header(cube, header)

        if np.isnan(self.data).any():
            self.data[np.isnan(self.data)] = 0

        if slice_size is None:
            self.slice_size = 1.0

        if slice_size != 1.0:
            self.data = \
                change_slice_thickness(self.data,
                                       slice_thickness=self.slice_size)

        self._ps1D_stddev = None

    def compute_pspec(self):
        '''
        Compute the 2D power spectrum.
        '''

        vca_fft = fftshift(rfft_to_fft(self.data))

        self._ps2D = np.power(vca_fft, 2.).sum(axis=0)

    def run(self, verbose=False, brk=None, return_stddev=True,
            logspacing=False, low_cut=None, high_cut=None,
            ang_units=False, unit=u.deg, use_wavenumber=False):
        '''
        Full computation of VCA.

        Parameters
        ----------
        verbose : bool, optional
            Enables plotting.
        brk : float, optional
            Initial guess for the break point.
        return_stddev : bool, optional
            Return the standard deviation in the 1D bins.
        logspacing : bool, optional
            Return logarithmically spaced bins for the lags.
        ang_units : bool, optional
            Convert frequencies to angular units using the given header.
        unit : u.Unit, optional
            Choose the angular unit to convert to when ang_units is enabled.
        use_wavenumber : bool, optional
            Plot the x-axis as the wavenumber rather than spatial frequency.
        '''

        self.compute_pspec()
        self.compute_radial_pspec(return_stddev=return_stddev,
                                  logspacing=logspacing)
        self.fit_pspec(brk=brk, low_cut=low_cut, high_cut=high_cut,
                       large_scale=0.5)

        if verbose:

            print(self.fit.summary())

            self.plot_fit(show=True, show_2D=True, ang_units=ang_units,
                          unit=unit, use_wavenumber=use_wavenumber)

        return self

class VCA_Distance(object):

    '''
    Calculate the distance between two cubes using VCA. The 1D power spectrum
    is modeled by a linear model. The distance is the t-statistic of the
    interaction between the two slopes.

    Parameters
    ----------
    cube1 : %(dtypes)s
        Data cube.
    cube2 : %(dtypes)s
        Data cube.
    slice_size : float, optional
        Slice to degrade the cube to.
    breaks : float, list or array, optional
        Specify where the break point is. If None, attempts to find using
        spline. If not specified, no break point will be used.
    fiducial_model : VCA
        Computed VCA object. use to avoid recomputing.
    '''

    __doc__ %= {"dtypes": " or ".join(common_types + threed_types)}

    def __init__(self, cube1, cube2, slice_size=1.0, breaks=None,
                 fiducial_model=None, logspacing=False, low_cut=None,
                 high_cut=None):
        super(VCA_Distance, self).__init__()

        if not isinstance(slice_size, float):
            raise TypeError("slice_size must be a float.")

        low_cut, high_cut = check_fit_limits(low_cut, high_cut)

        if not isinstance(breaks, list) and not isinstance(breaks, np.ndarray):
            breaks = [breaks] * 2

        if fiducial_model is not None:
            self.vca1 = fiducial_model
        else:
            self.vca1 = \
                VCA(cube1, slice_size=slice_size)
            self.vca1.run(brk=breaks[0], low_cut=low_cut[0],
                          high_cut=high_cut[0], logspacing=logspacing)

        self.vca2 = \
            VCA(cube2, slice_size=slice_size)
        self.vca2.run(brk=breaks[1], low_cut=low_cut[1], high_cut=high_cut[1],
                      logspacing=logspacing)

    def distance_metric(self, verbose=False, label1=None, label2=None,
                        ang_units=False, unit=u.deg, use_wavenumber=False):
        '''

        Implements the distance metric for 2 VCA transforms, each with the
        same channel width. We fit the linear portion of the transform to
        represent the powerlaw.

        Parameters
        ----------
        verbose : bool, optional
            Enables plotting.
        label1 : str, optional
            Object or region name for cube1
        label2 : str, optional
            Object or region name for cube2
        ang_units : bool, optional
            Convert frequencies to angular units using the given header.
        unit : u.Unit, optional
            Choose the angular unit to convert to when ang_units is enabled.
        use_wavenumber : bool, optional
            Plot the x-axis as the wavenumber rather than spatial frequency.
        '''

        # Construct t-statistic
        self.distance = \
            np.abs((self.vca1.slope - self.vca2.slope) /
                   np.sqrt(self.vca1.slope_err**2 +
                           self.vca2.slope_err**2))

        if verbose:

            print("Fit to %s" % (label1))
            print(self.vca1.fit.summary())
            print("Fit to %s" % (label2))
            print(self.vca2.fit.summary())

            import matplotlib.pyplot as p
            self.vca1.plot_fit(show=False, color='b', label=label1, symbol='D',
                               ang_units=ang_units, unit=unit,
                               use_wavenumber=use_wavenumber)
            self.vca2.plot_fit(show=False, color='g', label=label2, symbol='o',
                               ang_units=ang_units, unit=unit,
                               use_wavenumber=use_wavenumber)
            p.legend(loc='upper right')
            p.show()

        return self
<|endoftext|>


<commit_before>"""
One analysis done with four different approaches:
    1. Monte Carlo integration
    2. Polynomial chaos expansion w/Pseudo-spectral method
    2. Polynomial chaos expansion w/Point collocation method
    2. Polynomial chaos expansion w/Intrusive Galerkin method
"""
import chaospy as cp
import numpy as np
import odespy

# The model solver
def u(x, a, I):
    return I*np.exp(-a*x)
x = np.linspace(0, 10, 1000)

# Defining the random distributions:
a = cp.Uniform(0, 0.1)
I = cp.Uniform(8, 10)
dist = cp.J(a, I)

## Monte Carlo integration
samples = dist.sample(10**5)
u_mc = [U_hat(*s) for s in samples.T]

mean = np.mean(u_mc, 1)
var = np.var(u_mc, 1)

## Polynomial chaos expansion
## using Pseudo-spectral method and Gaussian Quadrature
order = 5
P, norms = cp.orth_ttr(order, dist, retall=True)
nodes, weights = cp.generate_quadrature(order+1, dist, rule="G")
solves = [u(x, s[0], s[1]) for s in nodes.T]
U_hat = cp.fit_quadrature(P, nodes, weights, solves, norms=norms)

mean = cp.E(U_hat, dist)
var = cp.Var(U_hat, dist)

## Polynomial chaos expansion
## using Point collocation method and quasi-random samples
order = 5
P = cp.orth_ttr(order, dist)
nodes = dist.sample(2*len(P), "M")
solves = [u(x, s[0], s[1]) for s in nodes.T]
U_hat = cp.fit_regression(P, nodes, solves, rule="T")

mean = cp.E(U_hat, dist)
var = cp.Var(U_hat, dist)

## Polynomial chaos expansion
## using Intrusive Gallerkin method
order = 5
P, norm = cp.orth_ttr(order, dist, retall=True, normed=True)

# support structures
q0, q1 = cp.variable(2)
P_nk = cp.outer(P, P)
E_ank = cp.E(q0*P_nk, dist)
E_ik = cp.E(q1*P, dist)
sE_ank = cp.sum(E_ank, 0)

# Right hand side of the ODE
def f(c_k, x):
    return -cp.sum(c_k*E_ank, -1)/norm

solver = odespy.RK4(f)
c_0 = E_ik/norm
solver.set_initial_condition(c_0)
c_n, x = solver.solve(x)
U_hat = cp.sum(P*c_n, -1)

mean = cp.E(U_hat, dist)
var = cp.Var(U_hat, dist)
<commit_msg>Adding math to Galerkin description<commit_after>"""
One analysis done with four different approaches:
    1. Monte Carlo integration
    2. Polynomial chaos expansion w/Pseudo-spectral method
    2. Polynomial chaos expansion w/Point collocation method
    2. Polynomial chaos expansion w/Intrusive Galerkin method
"""
import chaospy as cp
import numpy as np
import odespy

# The model solver
def u(x, a, I):
    return I*np.exp(-a*x)
x = np.linspace(0, 10, 1000)

# Defining the random distributions:
a = cp.Uniform(0, 0.1)
I = cp.Uniform(8, 10)
dist = cp.J(a, I)

## Monte Carlo integration
samples = dist.sample(10**5)
u_mc = [u(x, *s) for s in samples.T]

mean = np.mean(u_mc, 1)
var = np.var(u_mc, 1)

## Polynomial chaos expansion
## using Pseudo-spectral method and Gaussian Quadrature
order = 5
P, norms = cp.orth_ttr(order, dist, retall=True)
nodes, weights = cp.generate_quadrature(order+1, dist, rule="G")
solves = [u(x, s[0], s[1]) for s in nodes.T]
U_hat = cp.fit_quadrature(P, nodes, weights, solves, norms=norms)

mean = cp.E(U_hat, dist)
var = cp.Var(U_hat, dist)

## Polynomial chaos expansion
## using Point collocation method and quasi-random samples
order = 5
P = cp.orth_ttr(order, dist)
nodes = dist.sample(2*len(P), "M")
solves = [u(x, s[0], s[1]) for s in nodes.T]
U_hat = cp.fit_regression(P, nodes, solves, rule="T")

mean = cp.E(U_hat, dist)
var = cp.Var(U_hat, dist)

## Polynomial chaos expansion
## using Intrusive Gallerkin method
# :math:
# u' = -a*u
# d/dx sum(c*P) = -a*sum(c*P)
# <d/dx sum(c*P),P[k]> = <-a*sum(c*P), P[k]>
# d/dx c[k]*<P[k],P[k]> = -sum(c*<a*P,P[k]>)
# d/dx c = -E( outer(a*P,P) ) / E( P*P )
#
# u(0) = I
# <sum(c(0)*P), P[k]> = <I, P[k]>
# c[k](0) <P[k],P[k]> = <I, P[k]>
# c(0) = E( I*P ) / E( P*P )
order = 5
P, norm = cp.orth_ttr(order, dist, retall=True, normed=True)

# support structures
q0, q1 = cp.variable(2)
P_nk = cp.outer(P, P)
E_ank = cp.E(q0*P_nk, dist)
E_ik = cp.E(q1*P, dist)
sE_ank = cp.sum(E_ank, 0)

# Right hand side of the ODE
def f(c_k, x):
    return -cp.sum(c_k*E_ank, -1)/norm

solver = odespy.RK4(f)
c_0 = E_ik/norm
solver.set_initial_condition(c_0)
c_n, x = solver.solve(x)
U_hat = cp.sum(P*c_n, -1)

mean = cp.E(U_hat, dist)
var = cp.Var(U_hat, dist)
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""
.. _tune_relay_x86:

Auto-tuning a Convolutional Network for x86 CPU
===============================================
**Author**: `Yao Wang <https://github.com/kevinthesun>`_, `Eddie Yan <https://github.com/eqy>`_

This is a tutorial about how to tune convolution neural network
for x86 CPU.

Note that this tutorial will not run on Windows or recent versions of macOS. To
get it to run, you will need to wrap the body of this tutorial in a :code:`if
__name__ == "__main__":` block.
"""
import os
import numpy as np

import tvm
from tvm import relay, autotvm
from tvm.relay import testing
from tvm.autotvm.tuner import XGBTuner, GATuner, RandomTuner, GridSearchTuner
from tvm.autotvm.graph_tuner import DPTuner, PBQPTuner
import tvm.contrib.graph_executor as runtime

#################################################################
# Define network
# --------------
# First we need to define the network in relay frontend API.
# We can either load some pre-defined network from :code:`relay.testing`
# or building :any:`relay.testing.resnet` with relay.
# We can also load models from MXNet, ONNX and TensorFlow.
#
# In this tutorial, we choose resnet-18 as tuning example.

def get_network(name, batch_size):
    """Get the symbol definition and random weight of a network"""
    input_shape = (batch_size, 3, 224, 224)
    output_shape = (batch_size, 1000)

    if "resnet" in name:
        n_layer = int(name.split("-")[1])
        mod, params = relay.testing.resnet.get_workload(
            num_layers=n_layer, batch_size=batch_size, dtype=dtype
        )
    elif "vgg" in name:
        n_layer = int(name.split("-")[1])
        mod, params = relay.testing.vgg.get_workload(
            num_layers=n_layer, batch_size=batch_size, dtype=dtype
        )
    elif name == "mobilenet":
        mod, params = relay.testing.mobilenet.get_workload(batch_size=batch_size, dtype=dtype)
    elif name == "squeezenet_v1.1":
        mod, params = relay.testing.squeezenet.get_workload(
            batch_size=batch_size, version="1.1", dtype=dtype
        )
    elif name == "inception_v3":
        input_shape = (batch_size, 3, 299, 299)
        mod, params = relay.testing.inception_v3.get_workload(batch_size=batch_size, dtype=dtype)
    elif name == "mxnet":
        # an example for mxnet model
        from mxnet.gluon.model_zoo.vision import get_model

        block = get_model("resnet18_v1", pretrained=True)
        mod, params = relay.frontend.from_mxnet(block, shape={input_name: input_shape}, dtype=dtype)
        net = mod["main"]
        net = relay.Function(
            net.params, relay.nn.softmax(net.body), None, net.type_params, net.attrs
        )
        mod = tvm.IRModule.from_expr(net)
    else:
        raise ValueError("Unsupported network: " + name)

    return mod, params, input_shape, output_shape

# Replace "llvm" with the correct target of your CPU.
# For example, for AWS EC2 c5 instance with Intel Xeon
# Platinum 8000 series, the target should be "llvm -mcpu=skylake-avx512".
# For AWS EC2 c4 instance with Intel Xeon E5-2666 v3, it should be
# "llvm -mcpu=core-avx2".
target = "llvm"

batch_size = 1
dtype = "float32"
model_name = "resnet-18"
log_file = "%s.log" % model_name
graph_opt_sch_file = "%s_graph_opt.log" % model_name

# Set the input name of the graph
# For ONNX models, it is typically "0".
input_name = "data"

# Set number of threads used for tuning based on the number of
# physical CPU cores on your machine.
num_threads = 1
os.environ["TVM_NUM_THREADS"] = str(num_threads)

#################################################################
# Configure tensor tuning settings and create tasks
# -------------------------------------------------
# To get better kernel execution performance on x86 CPU,
# we need to change data layout of convolution kernel from
# "NCHW" to "NCHWc". To deal with this situation, we define
# conv2d_NCHWc operator in topi. We will tune this operator
# instead of plain conv2d.
#
# We will use local mode for tuning configuration. RPC tracker
# mode can be setup similarly to the approach in
# :ref:`tune_relay_arm` tutorial.
#
# To perform a precise measurement, we should repeat the measurement several
# times and use the average of results. In addition, we need to flush the cache
# for the weight tensors between repeated measurements. This can make the measured
# latency of one operator closer to its actual latency during end-to-end inference.

tuning_option = {
    "log_filename": log_file,
    "tuner": "random",
    "early_stopping": None,
    "measure_option": autotvm.measure_option(
        builder=autotvm.LocalBuilder(),
        runner=autotvm.LocalRunner(
            number=1, repeat=10, min_repeat_ms=0, enable_cpu_cache_flush=True
        ),
    ),
}

# You can skip the implementation of this function for this tutorial.
def tune_kernels(
    tasks, measure_option, tuner="gridsearch", early_stopping=None, log_filename="tuning.log"
):

    for i, task in enumerate(tasks):
        prefix = "[Task %2d/%2d] " % (i + 1, len(tasks))

        # create tuner
        if tuner == "xgb" or tuner == "xgb-rank":
            tuner_obj = XGBTuner(task, loss_type="rank")
        elif tuner == "ga":
            tuner_obj = GATuner(task, pop_size=50)
        elif tuner == "random":
            tuner_obj = RandomTuner(task)
        elif tuner == "gridsearch":
            tuner_obj = GridSearchTuner(task)
        else:
            raise ValueError("Invalid tuner: " + tuner)

        # do tuning
        n_trial = len(task.config_space)
        tuner_obj.tune(
            n_trial=n_trial,
            early_stopping=early_stopping,
            measure_option=measure_option,
            callbacks=[
                autotvm.callback.progress_bar(n_trial, prefix=prefix),
                autotvm.callback.log_to_file(log_filename),
            ],
        )

# Use graph tuner to achieve graph level optimal schedules
# Set use_DP=False if it takes too long to finish.
def tune_graph(graph, dshape, records, opt_sch_file, use_DP=True):
    target_op = [
        relay.op.get("nn.conv2d"),
    ]
    Tuner = DPTuner if use_DP else PBQPTuner
    executor = Tuner(graph, {input_name: dshape}, records, target_op, target)
    executor.benchmark_layout_transform(min_exec_num=2000)
    executor.run()
    executor.write_opt_sch2record_file(opt_sch_file)

########################################################################
# Finally, we launch tuning jobs and evaluate the end-to-end performance.

def tune_and_evaluate(tuning_opt):
    # extract workloads from relay program
    print("Extract tasks...")
    mod, params, data_shape, out_shape = get_network(model_name, batch_size)
    tasks = autotvm.task.extract_from_program(
        mod["main"], target=target, params=params, ops=(relay.op.get("nn.conv2d"),)
    )

    # run tuning tasks
    tune_kernels(tasks, **tuning_opt)
    tune_graph(mod["main"], data_shape, log_file, graph_opt_sch_file)

    # compile kernels with graph-level best records
    with autotvm.apply_graph_best(graph_opt_sch_file):
        print("Compile...")
        with tvm.transform.PassContext(opt_level=3):
            lib = relay.build_module.build(mod, target=target, params=params)

        # upload parameters to device
        dev = tvm.cpu()
        data_tvm = tvm.nd.array((np.random.uniform(size=data_shape)).astype(dtype))
        module = runtime.GraphModule(lib["default"](dev))
        module.set_input(input_name, data_tvm)

        # evaluate
        print("Evaluate inference time cost...")
        ftimer = module.module.time_evaluator("run", dev, number=100, repeat=3)
        prof_res = np.array(ftimer().results) * 1000  # convert to millisecond
        print(
            "Mean inference time (std dev): %.2f ms (%.2f ms)"
            % (np.mean(prof_res), np.std(prof_res))
        )

# We do not run the tuning in our webpage server since it takes too long.
# Uncomment the following line to run it by yourself.

# tune_and_evaluate(tuning_option)

######################################################################
# Sample Output
# -------------
# The tuning needs to compile many programs and extract feature from them.
# So a high performance CPU is recommended.
# One sample output is listed below.
#
# .. code-block:: bash
#
#    Extract tasks...
#    Tuning...
#    [Task  1/12]  Current/Best:  598.05/2497.63 GFLOPS | Progress: (252/252) | 1357.95 s Done.
#    [Task  2/12]  Current/Best:  522.63/2279.24 GFLOPS | Progress: (784/784) | 3989.60 s Done.
#    [Task  3/12]  Current/Best:  447.33/1927.69 GFLOPS | Progress: (784/784) | 3869.14 s Done.
#    [Task  4/12]  Current/Best:  481.11/1912.34 GFLOPS | Progress: (672/672) | 3274.25 s Done.
#    [Task  5/12]  Current/Best:  414.09/1598.45 GFLOPS | Progress: (672/672) | 2720.78 s Done.
#    [Task  6/12]  Current/Best:  508.96/2273.20 GFLOPS | Progress: (768/768) | 3718.75 s Done.
#    [Task  7/12]  Current/Best:  469.14/1955.79 GFLOPS | Progress: (576/576) | 2665.67 s Done.
#    [Task  8/12]  Current/Best:  230.91/1658.97 GFLOPS | Progress: (576/576) | 2435.01 s Done.
#    [Task  9/12]  Current/Best:  487.75/2295.19 GFLOPS | Progress: (648/648) | 3009.95 s Done.
#    [Task 10/12]  Current/Best:  182.33/1734.45 GFLOPS | Progress: (360/360) | 1755.06 s Done.
#    [Task 11/12]  Current/Best:  372.18/1745.15 GFLOPS | Progress: (360/360) | 1684.50 s Done.
#    [Task 12/12]  Current/Best:  215.34/2271.11 GFLOPS | Progress: (400/400) | 2128.74 s Done.
#    Compile...
#    Evaluate inference time cost...
#    Mean inference time (std dev): 3.16 ms (0.03 ms)
<commit_msg>Extend tune_relay_x86 tutorial to measure default and kernel level tune (#8794)<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""
.. _tune_relay_x86:

Auto-tuning a Convolutional Network for x86 CPU
===============================================
**Author**: `Yao Wang <https://github.com/kevinthesun>`_, `Eddie Yan <https://github.com/eqy>`_

This is a tutorial about how to tune convolution neural network
for x86 CPU.

Note that this tutorial will not run on Windows or recent versions of macOS. To
get it to run, you will need to wrap the body of this tutorial in a :code:`if
__name__ == "__main__":` block.
"""
import os
import numpy as np

import tvm
from tvm import relay, autotvm
from tvm.relay import testing
from tvm.autotvm.tuner import XGBTuner, GATuner, RandomTuner, GridSearchTuner
from tvm.autotvm.graph_tuner import DPTuner, PBQPTuner
import tvm.contrib.graph_executor as runtime

#################################################################
# Define network
# --------------
# First we need to define the network in relay frontend API.
# We can either load some pre-defined network from :code:`relay.testing`
# or building :any:`relay.testing.resnet` with relay.
# We can also load models from MXNet, ONNX and TensorFlow.
#
# In this tutorial, we choose resnet-18 as tuning example.

def get_network(name, batch_size):
    """Get the symbol definition and random weight of a network"""
    input_shape = (batch_size, 3, 224, 224)
    output_shape = (batch_size, 1000)

    if "resnet" in name:
        n_layer = int(name.split("-")[1])
        mod, params = relay.testing.resnet.get_workload(
            num_layers=n_layer, batch_size=batch_size, dtype=dtype
        )
    elif "vgg" in name:
        n_layer = int(name.split("-")[1])
        mod, params = relay.testing.vgg.get_workload(
            num_layers=n_layer, batch_size=batch_size, dtype=dtype
        )
    elif name == "mobilenet":
        mod, params = relay.testing.mobilenet.get_workload(batch_size=batch_size, dtype=dtype)
    elif name == "squeezenet_v1.1":
        mod, params = relay.testing.squeezenet.get_workload(
            batch_size=batch_size, version="1.1", dtype=dtype
        )
    elif name == "inception_v3":
        input_shape = (batch_size, 3, 299, 299)
        mod, params = relay.testing.inception_v3.get_workload(batch_size=batch_size, dtype=dtype)
    elif name == "mxnet":
        # an example for mxnet model
        from mxnet.gluon.model_zoo.vision import get_model

        block = get_model("resnet18_v1", pretrained=True)
        mod, params = relay.frontend.from_mxnet(block, shape={input_name: input_shape}, dtype=dtype)
        net = mod["main"]
        net = relay.Function(
            net.params, relay.nn.softmax(net.body), None, net.type_params, net.attrs
        )
        mod = tvm.IRModule.from_expr(net)
    else:
        raise ValueError("Unsupported network: " + name)

    return mod, params, input_shape, output_shape

# Replace "llvm" with the correct target of your CPU.
# For example, for AWS EC2 c5 instance with Intel Xeon
# Platinum 8000 series, the target should be "llvm -mcpu=skylake-avx512".
# For AWS EC2 c4 instance with Intel Xeon E5-2666 v3, it should be
# "llvm -mcpu=core-avx2".
target = "llvm"

batch_size = 1
dtype = "float32"
model_name = "resnet-18"
log_file = "%s.log" % model_name
graph_opt_sch_file = "%s_graph_opt.log" % model_name

# Set the input name of the graph
# For ONNX models, it is typically "0".
input_name = "data"

# Set number of threads used for tuning based on the number of
# physical CPU cores on your machine.
num_threads = 1
os.environ["TVM_NUM_THREADS"] = str(num_threads)

#################################################################
# Configure tensor tuning settings and create tasks
# -------------------------------------------------
# To get better kernel execution performance on x86 CPU,
# we need to change data layout of convolution kernel from
# "NCHW" to "NCHWc". To deal with this situation, we define
# conv2d_NCHWc operator in topi. We will tune this operator
# instead of plain conv2d.
#
# We will use local mode for tuning configuration. RPC tracker
# mode can be setup similarly to the approach in
# :ref:`tune_relay_arm` tutorial.
#
# To perform a precise measurement, we should repeat the measurement several
# times and use the average of results. In addition, we need to flush the cache
# for the weight tensors between repeated measurements. This can make the measured
# latency of one operator closer to its actual latency during end-to-end inference.

tuning_option = {
    "log_filename": log_file,
    "tuner": "random",
    "early_stopping": None,
    "measure_option": autotvm.measure_option(
        builder=autotvm.LocalBuilder(),
        runner=autotvm.LocalRunner(
            number=1, repeat=10, min_repeat_ms=0, enable_cpu_cache_flush=True
        ),
    ),
}

# You can skip the implementation of this function for this tutorial.
def tune_kernels(
    tasks, measure_option, tuner="gridsearch", early_stopping=None, log_filename="tuning.log"
):

    for i, task in enumerate(tasks):
        prefix = "[Task %2d/%2d] " % (i + 1, len(tasks))

        # create tuner
        if tuner == "xgb" or tuner == "xgb-rank":
            tuner_obj = XGBTuner(task, loss_type="rank")
        elif tuner == "ga":
            tuner_obj = GATuner(task, pop_size=50)
        elif tuner == "random":
            tuner_obj = RandomTuner(task)
        elif tuner == "gridsearch":
            tuner_obj = GridSearchTuner(task)
        else:
            raise ValueError("Invalid tuner: " + tuner)

        # do tuning
        n_trial = len(task.config_space)
        tuner_obj.tune(
            n_trial=n_trial,
            early_stopping=early_stopping,
            measure_option=measure_option,
            callbacks=[
                autotvm.callback.progress_bar(n_trial, prefix=prefix),
                autotvm.callback.log_to_file(log_filename),
            ],
        )

# Use graph tuner to achieve graph level optimal schedules
# Set use_DP=False if it takes too long to finish.
def tune_graph(graph, dshape, records, opt_sch_file, use_DP=True):
    target_op = [
        relay.op.get("nn.conv2d"),
    ]
    Tuner = DPTuner if use_DP else PBQPTuner
    executor = Tuner(graph, {input_name: dshape}, records, target_op, target)
    executor.benchmark_layout_transform(min_exec_num=2000)
    executor.run()
    executor.write_opt_sch2record_file(opt_sch_file)

########################################################################
# Finally, we launch tuning jobs and evaluate the end-to-end performance.

def evaluate_performance(lib, data_shape):
    # upload parameters to device
    dev = tvm.cpu()
    data_tvm = tvm.nd.array((np.random.uniform(size=data_shape)).astype(dtype))
    module = runtime.GraphModule(lib["default"](dev))
    module.set_input(input_name, data_tvm)

    # evaluate
    print("Evaluate inference time cost...")
    ftimer = module.module.time_evaluator("run", dev, number=100, repeat=3)
    prof_res = np.array(ftimer().results) * 1000  # convert to millisecond
    print(
        "Mean inference time (std dev): %.2f ms (%.2f ms)" % (np.mean(prof_res), np.std(prof_res))
    )

def tune_and_evaluate(tuning_opt):
    # extract workloads from relay program
    print("Extract tasks...")
    mod, params, data_shape, out_shape = get_network(model_name, batch_size)
    tasks = autotvm.task.extract_from_program(
        mod["main"], target=target, params=params, ops=(relay.op.get("nn.conv2d"),)
    )

    # run tuning tasks
    tune_kernels(tasks, **tuning_opt)
    tune_graph(mod["main"], data_shape, log_file, graph_opt_sch_file)

    # compile kernels in default mode
    print("Evaluation of the network compiled in 'default' mode without auto tune:")
    with tvm.transform.PassContext(opt_level=3):
        print("Compile...")
        lib = relay.build(mod, target=target, params=params)
        evaluate_performance(lib, data_shape)

    # compile kernels in kernel tuned only mode
    print("\nEvaluation of the network been tuned on kernel level:")
    with autotvm.apply_history_best(log_file):
        print("Compile...")
        with tvm.transform.PassContext(opt_level=3):
            lib = relay.build(mod, target=target, params=params)
        evaluate_performance(lib, data_shape)

    # compile kernels with graph-level best records
    print("\nEvaluation of the network been tuned on graph level:")
    with autotvm.apply_graph_best(graph_opt_sch_file):
        print("Compile...")
        with tvm.transform.PassContext(opt_level=3):
            lib = relay.build_module.build(mod, target=target, params=params)
        evaluate_performance(lib, data_shape)

# We do not run the tuning in our webpage server since it takes too long.
# Uncomment the following line to run it by yourself.

# tune_and_evaluate(tuning_option)

######################################################################
# Sample Output
# -------------
# The tuning needs to compile many programs and extract feature from them.
# So a high performance CPU is recommended.
# One sample output is listed below.
#
# .. code-block:: bash
#
#    Extract tasks...
#    Tuning...
#    [Task  1/12]  Current/Best:  598.05/2497.63 GFLOPS | Progress: (252/252) | 1357.95 s Done.
#    [Task  2/12]  Current/Best:  522.63/2279.24 GFLOPS | Progress: (784/784) | 3989.60 s Done.
#    [Task  3/12]  Current/Best:  447.33/1927.69 GFLOPS | Progress: (784/784) | 3869.14 s Done.
#    [Task  4/12]  Current/Best:  481.11/1912.34 GFLOPS | Progress: (672/672) | 3274.25 s Done.
#    [Task  5/12]  Current/Best:  414.09/1598.45 GFLOPS | Progress: (672/672) | 2720.78 s Done.
#    [Task  6/12]  Current/Best:  508.96/2273.20 GFLOPS | Progress: (768/768) | 3718.75 s Done.
#    [Task  7/12]  Current/Best:  469.14/1955.79 GFLOPS | Progress: (576/576) | 2665.67 s Done.
#    [Task  8/12]  Current/Best:  230.91/1658.97 GFLOPS | Progress: (576/576) | 2435.01 s Done.
#    [Task  9/12]  Current/Best:  487.75/2295.19 GFLOPS | Progress: (648/648) | 3009.95 s Done.
#    [Task 10/12]  Current/Best:  182.33/1734.45 GFLOPS | Progress: (360/360) | 1755.06 s Done.
#    [Task 11/12]  Current/Best:  372.18/1745.15 GFLOPS | Progress: (360/360) | 1684.50 s Done.
#    [Task 12/12]  Current/Best:  215.34/2271.11 GFLOPS | Progress: (400/400) | 2128.74 s Done.
#    INFO Start to benchmark layout transformation...
#    INFO Benchmarking layout transformation successful.
#    INFO Start to run dynamic programming algorithm...
#    INFO Start forward pass...
#    INFO Finished forward pass.
#    INFO Start backward pass...
#    INFO Finished backward pass...
#    INFO Finished DPExecutor run.
#    INFO Writing optimal schedules to resnet-18_graph_opt.log successfully.
#
#    Evaluation of the network compiled in 'default' mode without auto tune:
#    Compile...
#    Evaluate inference time cost...
#    Mean inference time (std dev): 4.5 ms (0.03 ms)
#
#    Evaluation of the network been tuned on kernel level:
#    Compile...
#    Evaluate inference time cost...
#    Mean inference time (std dev): 3.2 ms (0.03 ms)
#
#    Evaluation of the network been tuned on graph level:
#    Compile...
#    Config for target=llvm -keys=cpu -link-params=0, workload=('dense_nopack.x86', ('TENSOR', (1, 512), 'float32'), ('TENSOR', (1000, 512), 'float32'), None, 'float32') is missing in ApplyGraphBest context. A fallback configuration is used, which may bring great performance regression.
#    Config for target=llvm -keys=cpu -link-params=0, workload=('dense_pack.x86', ('TENSOR', (1, 512), 'float32'), ('TENSOR', (1000, 512), 'float32'), None, 'float32') is missing in ApplyGraphBest context. A fallback configuration is used, which may bring great performance regression.
#    Evaluate inference time cost...
#    Mean inference time (std dev): 3.16 ms (0.03 ms)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013, 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
    invenio.modules.documents.api
    -----------------------------

    Documents API

    Following example shows how to handle documents metadata::

        >>> from flask import g
        >>> from invenio.base.factory import create_app
        >>> app = create_app()
        >>> ctx = app.test_request_context()
        >>> ctx.push()
        >>> from invenio.modules.documents import api
        >>> from invenio.modules.jsonalchemy.jsonext.engines import memory
        >>> app.config['DOCUMENTS_ENGINE'] = \
        "invenio.modules.jsonalchemy.jsonext.engines.memory:MemoryStorage"
        >>> d = api.Document.create({'title': 'Title 1'})
        >>> d['title']
        'Title 1'
        >>> d['creator']
        0
        >>> d['title'] = 'New Title 1'
        >>> d = d.update()
        >>> api.Document.get_document(d['_id'])['title']
        'New Title 1'
        >>> ctx.pop()
"""

import fs
import six

from datetime import datetime
from fs.opener import opener

from invenio.modules.jsonalchemy.wrappers import SmartJson
from invenio.modules.jsonalchemy.reader import Reader

from . import signals, errors

class Document(SmartJson):
    """Document"""

    __storagename__ = 'documents'

    @classmethod
    def create(cls, data, model='document_base', master_format='json',
               **kwargs):
        document = Reader.translate(data, cls, master_format=master_format,
                                    model=model, namespace='documentext',
                                    **kwargs)
        cls.storage_engine.save_one(document.dumps())
        signals.document_created.send(document)
        return document

    @classmethod
    def get_document(cls, uuid, include_deleted=False):
        """Returns document instance identified by UUID.

        Find existing document::

            >>> from flask import g
            >>> from invenio.base.factory import create_app
            >>> app = create_app()
            >>> ctx = app.test_request_context()
            >>> ctx.push()
            >>> from invenio.modules.documents import api
            >>> from invenio.modules.jsonalchemy.jsonext.engines import memory
            >>> app.config['DOCUMENTS_ENGINE'] = \
            "invenio.modules.jsonalchemy.jsonext.engines.memory:MemoryStorage"
            >>> d = api.Document.create({'title': 'Title 1'})
            >>> e = api.Document.get_document(d['_id'])

        If you try to find deleted document you will get an exception::

            >>> e.delete()
            >>> api.Document.get_document(d['_id'])
            Traceback (most recent call last):
             ...
            DeletedDocument

        and also if you try to find not existing document::

            >>> import uuid
            >>> api.Document.get_document(str(uuid.uuid4()))
            Traceback (most recent call last):
             ...
            DocumentNotFound
            >>> ctx.pop()

        :returns: a :class:`Document` instance.
        :raises: :class:`~.invenio.modules.documents.errors.DocumentNotFound`
            or :class:`~invenio.modules.documents.errors.DeletedDocument`
        """
        try:
            document = cls(cls.storage_engine.get_one(uuid))
        except:
            raise errors.DocumentNotFound

        if not include_deleted and document['deleted']:
            raise errors.DeletedDocument
        return document

    def _save(self):
        try:
            return self.__class__.storage_engine.update_one(self.dumps(),
                                                            id=self['_id'])
        except:
            return self.__class__.storage_engine.save_one(self.dumps(),
                                                          id=self['_id'])

    def update(self):
        """Update document object."""
        #FIXME This should be probably done in model dump.
        self['modification_date'] = datetime.now()
        return self._save()

    def setcontents(self, source, name, chunk_size=65536):
        """A convenience method to create a new file from a string or file-like
        object.

        :note: All paths has to be absolute or specified in full URI format.

        :param data: .
        :param name: File URI or filename generator taking `self` as argument.
        """

        if isinstance(source, six.string_types):
            self['source'] = source
            f = opener.open(source, 'rb')
        else:
            f = source

        if callable(name):
            name = name(self)
        else:
            name = fs.path.abspath(name)

        signals.document_before_content_set.send(self, name=name)

        data = f.read()
        _fs, filename = opener.parse(name)
        _fs.setcontents(filename, data, chunk_size)
        _fs.close()

        signals.document_after_content_set.send(self, name=name)

        if hasattr(f, 'close'):
            f.close()

        self['uri'] = name
        self._save()

    def open(self, mode='r', **kwargs):
        """Open a the 'uri' as a file-like object."""
        _fs, filename = opener.parse(self['uri'])
        return _fs.open(filename, mode=mode, **kwargs)

    def delete(self, force=False):
        """Deletes the instance of document.

        :param force: If it is True then the document is deleted including
            attached files and metadata.
        """

        self['deleted'] = True

        if force and self.get('uri') is not None:
            signals.document_before_file_delete.send(self)
            fs, filename = opener.parse(self['uri'])
            fs.remove(filename)
            self['uri'] = None

        self._save()
<commit_msg>documents: checker of source and uri addition<commit_after># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2013, 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""
    invenio.modules.documents.api
    -----------------------------

    Documents API

    Following example shows how to handle documents metadata::

        >>> from flask import g
        >>> from invenio.base.factory import create_app
        >>> app = create_app()
        >>> ctx = app.test_request_context()
        >>> ctx.push()
        >>> from invenio.modules.documents import api
        >>> from invenio.modules.jsonalchemy.jsonext.engines import memory
        >>> app.config['DOCUMENTS_ENGINE'] = \
        "invenio.modules.jsonalchemy.jsonext.engines.memory:MemoryStorage"
        >>> d = api.Document.create({'title': 'Title 1'})
        >>> d['title']
        'Title 1'
        >>> d['creator']
        0
        >>> d['title'] = 'New Title 1'
        >>> d = d.update()
        >>> api.Document.get_document(d['_id'])['title']
        'New Title 1'
        >>> ctx.pop()
"""

import fs
import six

from datetime import datetime
from fs.opener import opener

from invenio.modules.jsonalchemy.wrappers import SmartJson
from invenio.modules.jsonalchemy.reader import Reader

from . import signals, errors

class Document(SmartJson):
    """Document"""

    __storagename__ = 'documents'

    @classmethod
    def create(cls, data, model='document_base', master_format='json',
               **kwargs):
        document = Reader.translate(data, cls, master_format=master_format,
                                    model=model, namespace='documentext',
                                    **kwargs)
        cls.storage_engine.save_one(document.dumps())
        signals.document_created.send(document)
        return document

    @classmethod
    def get_document(cls, uuid, include_deleted=False):
        """Returns document instance identified by UUID.

        Find existing document::

            >>> from flask import g
            >>> from invenio.base.factory import create_app
            >>> app = create_app()
            >>> ctx = app.test_request_context()
            >>> ctx.push()
            >>> from invenio.modules.documents import api
            >>> from invenio.modules.jsonalchemy.jsonext.engines import memory
            >>> app.config['DOCUMENTS_ENGINE'] = \
            "invenio.modules.jsonalchemy.jsonext.engines.memory:MemoryStorage"
            >>> d = api.Document.create({'title': 'Title 1'})
            >>> e = api.Document.get_document(d['_id'])

        If you try to find deleted document you will get an exception::

            >>> e.delete()
            >>> api.Document.get_document(d['_id'])
            Traceback (most recent call last):
             ...
            DeletedDocument

        and also if you try to find not existing document::

            >>> import uuid
            >>> api.Document.get_document(str(uuid.uuid4()))
            Traceback (most recent call last):
             ...
            DocumentNotFound
            >>> ctx.pop()

        :returns: a :class:`Document` instance.
        :raises: :class:`~.invenio.modules.documents.errors.DocumentNotFound`
            or :class:`~invenio.modules.documents.errors.DeletedDocument`
        """
        try:
            document = cls(cls.storage_engine.get_one(uuid))
        except:
            raise errors.DocumentNotFound

        if not include_deleted and document['deleted']:
            raise errors.DeletedDocument
        return document

    def _save(self):
        try:
            return self.__class__.storage_engine.update_one(self.dumps(),
                                                            id=self['_id'])
        except:
            return self.__class__.storage_engine.save_one(self.dumps(),
                                                          id=self['_id'])

    def update(self):
        """Update document object."""
        #FIXME This should be probably done in model dump.
        self['modification_date'] = datetime.now()
        return self._save()

    def setcontents(self, source, name, chunk_size=65536):
        """A convenience method to create a new file from a string or file-like
        object.

        :note: All paths has to be absolute or specified in full URI format.

        :param data: .
        :param name: File URI or filename generator taking `self` as argument.
        """

        if isinstance(source, six.string_types):
            self['source'] = source
            f = opener.open(source, 'rb')
        else:
            f = source

        if callable(name):
            name = name(self)
        else:
            name = fs.path.abspath(name)

        signals.document_before_content_set.send(self, name=name)

        if self.get('source', '') != name:
            data = f.read()
            _fs, filename = opener.parse(name)
            _fs.setcontents(filename, data, chunk_size)
            _fs.close()

        signals.document_after_content_set.send(self, name=name)

        if hasattr(f, 'close'):
            f.close()

        self['uri'] = name
        self._save()

    def open(self, mode='r', **kwargs):
        """Open a the 'uri' as a file-like object."""
        _fs, filename = opener.parse(self['uri'])
        return _fs.open(filename, mode=mode, **kwargs)

    def delete(self, force=False):
        """Deletes the instance of document.

        :param force: If it is True then the document is deleted including
            attached files and metadata.
        """

        self['deleted'] = True

        if force and self.get('uri') is not None:
            signals.document_before_file_delete.send(self)
            fs, filename = opener.parse(self['uri'])
            fs.remove(filename)
            self['uri'] = None

        self._save()
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-

from django.db import models
from django.contrib.auth.models import User
from datetime import datetime

class Categorie(models.Model):
    id = models.AutoField(primary_key=True, db_column='id')
    name = models.CharField(max_length=45, unique=True, db_column='name')
    description = models.TextField(db_column='description')
    class Meta:
        db_table = 'Categories'
    def __str__(self):
        return self.name

class Challenge(models.Model):
    id = models.AutoField(primary_key=True, db_column='id')
    name = models.CharField(max_length=45, unique=True, db_column='name')
    img = models.CharField(max_length=255, unique=False, db_column='img')
    description = models.TextField(db_column='description')
    points = models.IntegerField(null=False, blank=False, db_column='points')
    url = models.CharField(max_length=255, unique=True, db_column='url')
    flag = models.CharField(max_length=255, unique=True, db_column='flag')
    categorie = models.ForeignKey('Categorie', to_field='id', db_column='categorie')
    seuil = models.IntegerField(default=0, null=False, blank=False, db_column='seuil')
    private = models.BooleanField(default=True, db_column='blocked')
    class Meta:
        db_table = 'Challenges'
    def __str__(self):
        return self.name

class Validation(models.Model):
    id = models.AutoField(primary_key=True, db_column='id')
    user = models.ForeignKey(User)
    chall = models.ForeignKey('Challenge', to_field='id', db_column='chall')
    value = models.IntegerField(null=False, blank=False, db_column='value')
    timestamp = models.DateTimeField(default=lambda :datetime.now(), db_column='timestamp')
    class Meta:
        db_table = 'Validations'
<commit_msg>Supprimer une categorie ne supprime plus les challenges associés<commit_after>#-*- coding: utf-8 -*-

from django.db import models
from django.contrib.auth.models import User
from datetime import datetime

class Categorie(models.Model):
    id = models.AutoField(primary_key=True, db_column='id')
    name = models.CharField(max_length=45, unique=True, db_column='name')
    description = models.TextField(db_column='description')
    class Meta:
        db_table = 'Categories'
    def __str__(self):
        return self.name

class Challenge(models.Model):
    id = models.AutoField(primary_key=True, db_column='id')
    name = models.CharField(max_length=45, unique=True, db_column='name')
    img = models.CharField(max_length=255, unique=False, db_column='img')
    description = models.TextField(db_column='description')
    points = models.IntegerField(null=False, blank=False, db_column='points')
    url = models.CharField(max_length=255, unique=True, db_column='url')
    flag = models.CharField(max_length=255, unique=True, db_column='flag')
    categorie = models.ForeignKey('Categorie', to_field='id', blank=True, null=True, on_delete=models.SET_NULL, db_column='categorie')
    seuil = models.IntegerField(default=0, null=False, blank=False, db_column='seuil')
    private = models.BooleanField(default=True, db_column='blocked')
    class Meta:
        db_table = 'Challenges'
    def __str__(self):
        return self.name

class Validation(models.Model):
    id = models.AutoField(primary_key=True, db_column='id')
    user = models.ForeignKey(User)
    chall = models.ForeignKey('Challenge', to_field='id', db_column='chall')
    value = models.IntegerField(null=False, blank=False, db_column='value')
    timestamp = models.DateTimeField(default=lambda :datetime.now(), db_column='timestamp')
    class Meta:
        db_table = 'Validations'
<|endoftext|>


<commit_before>import numpy as np
import tensorflow as tf

import time
import os
import cPickle
import argparse

from utils import *
from model import Model
import random

import svgwrite
from IPython.display import SVG, display

# main code (not in a main function since I want to run this script in IPython as well).
def in_ipython():
  try:
    __IPYTHON__
  except NameError:
    return False
  else:
    return True

parser = argparse.ArgumentParser()
parser.add_argument('--filename', type=str, default='output',
                   help='filename of .svg file to output, without .svg')
parser.add_argument('--sample_length', type=int, default=600,
                   help='number of strokes to sample')
parser.add_argument('--picture_size', type=float, default=160,
                   help='a centered svg will be generated of this size')
parser.add_argument('--scale_factor', type=float, default=1,
                   help='factor to scale down by for svg output.  smaller means bigger output')
parser.add_argument('--num_picture', type=int, default=20,
                   help='number of pictures to generate')
parser.add_argument('--num_col', type=int, default=5,
                   help='if num_picture > 1, how many pictures per row?')
parser.add_argument('--dataset_name', type=str, default="kanji",
                   help='name of directory containing training data')
parser.add_argument('--color_mode', type=int, default=1,
                   help='set to 0 if you are a black and white sort of person...')
parser.add_argument('--stroke_width', type=float, default=2.0,
                   help='thickness of pen lines')
parser.add_argument('--temperature', type=float, default=0.1,
                   help='sampling temperature')
sample_args = parser.parse_args()

color_mode = True
if sample_args.color_mode == 0:
  color_mode = False

with open(os.path.join('save', sample_args.dataset_name, 'config.pkl')) as f: # future
  saved_args = cPickle.load(f)

model = Model(saved_args, True)
sess = tf.InteractiveSession()
saver = tf.train.Saver(tf.global_variables())

ckpt = tf.train.get_checkpoint_state(os.path.join('save', sample_args.dataset_name))
print "loading model: ",ckpt.model_checkpoint_path

saver.restore(sess, ckpt.model_checkpoint_path)

def draw_sketch_array(strokes_array, svg_only = False):
  draw_stroke_color_array(strokes_array, factor=sample_args.scale_factor, maxcol = sample_args.num_col, svg_filename = sample_args.filename+'.svg', stroke_width = sample_args.stroke_width, block_size = sample_args.picture_size, svg_only = svg_only, color_mode = color_mode)

def sample_sketches(min_size_ratio = 0.0, max_size_ratio = 0.8, min_num_stroke = 4, max_num_stroke=22, svg_only = True):
  N = sample_args.num_picture
  frame_size = float(sample_args.picture_size)
  max_size = frame_size * max_size_ratio
  min_size = frame_size * min_size_ratio
  count = 0
  sketch_list = []
  param_list = []

  temp_mixture = sample_args.temperature
  temp_pen = sample_args.temperature

  while count < N:
    #print "attempting to generate picture #", count
    print '.',
    [strokes, params] = model.sample(sess, sample_args.sample_length, temp_mixture, temp_pen, stop_if_eoc = True)
    [sx, sy, num_stroke, num_char, _] = strokes.sum(0)
    if num_stroke < min_num_stroke or num_char == 0 or num_stroke > max_num_stroke:
      #print "num_stroke ", num_stroke, " num_char ", num_char
      continue
    [sx, sy, sizex, sizey] = calculate_start_point(strokes)
    if sizex > max_size or sizey > max_size:
      #print "sizex ", sizex, " sizey ", sizey
      continue
    if sizex < min_size or sizey < min_size:
      #print "sizex ", sizex, " sizey ", sizey
      continue
    # success
    print count+1,"/",N
    count += 1
    sketch_list.append(strokes)
    param_list.append(params)
  # draw the pics
  draw_sketch_array(sketch_list, svg_only = svg_only)
  return sketch_list, param_list

if __name__ == '__main__':
  ipython_mode = in_ipython()
  if ipython_mode:
    print "IPython detected"
  else:
    print "Console mode"
  [strokes, params] = sample_sketches(svg_only = not ipython_mode)

<commit_msg>removing sample2.py<commit_after><|endoftext|>


<commit_before>"""
The MIT License (MIT)

Copyright (c) <2016> <Larry McCaig (aka: Larz60+ aka: Larz60p)>

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

"""
import ReadRecord as RdRec

class ShowData:
    def __init__(self):
        self.rr = RdRec.ReadRecord('StockData.json')
        self.stock_market_record = self.rr.read_data_file()

def show_data(self):
    stkmktrec = self.stock_market_record
    #  get a list of field names:
    print('Record fields: {}'.format(stkmktrec._fields))

    # List entire record
    print('\nEntire record: {}'.format(stkmktrec))

    # Get individual field
    print('\ndbtabledesc: {}'.format(stkmktrec.dbtabledesc))

    # Show database column entries
    print('\ndatabase column 0: {}'.format(stkmktrec.columns[0]))
    print('database column 1: {}'.format(stkmktrec.columns[1]))
    print('database column 2: {}'.format(stkmktrec.columns[2]))

    # Column data by key:
    for n in range(len(stkmktrec.columns)):
        column = stkmktrec.columns[n]
        print('\nColumn {} all: {}'.format(n, column))
        print('Column data {} field_name: {}'.format(n, column.field_name))
        print('Column data {} db_column_name: {}'.format(n, column.db_column_name))
        print('Column data {} db_column_desc: {}'.format(n, column.db_column_desc))
        print('Column data {} db_column_type: {}'.format(n, column.db_column_type))

    # Using get_field_item
    print('\nUsing get_field_item - Column 1, db_column_desc: {}'
          .format(self.rr.get_field_item(1, itemname='db_column_desc')))
    # same with bad data
    print('With bad data you get: {}'
          .format(self.rr.get_field_item(1, itemname='donkykong')))

if __name__ == '__main__':
    sd = ShowData()
    sd.show_data()
<commit_msg>Delete ShowData.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import socket
import dns.resolver

if len(sys.argv) < 2:
    sys.stderr.write("Usage: %s domain\n" % sys.argv[0])
    sys.exit(1)

domain=sys.argv[1]

answers = dns.resolver.query(domain, 'TXT')
for rdata in answers:
  for txt_string in rdata.strings:
    if txt_string.startswith('v=spf1'):
      full_spf=txt_string.replace('v=spf1','')
      print 'Initial SPF string : ', full_spf

while (full_spf.find('include:') > 0):
  for item in full_spf.split(' '):
    if item.startswith('include:'):
      sec_domain=item.replace('include:','')
      answers = dns.resolver.query(sec_domain, 'TXT')
      for rdata in answers:
        for txt_string in rdata.strings:
          if txt_string.startswith('v=spf'):
            txt_string=txt_string.replace('v=spf1','')
            print "SPF entry of %s : %s" % (sec_domain, txt_string)
            full_spf=full_spf.replace(item,txt_string)

print 'Full SPF : ', full_spf

<commit_msg>Refactoring<commit_after>#!/usr/bin/python

import sys
import socket
import dns.resolver
import re

if len(sys.argv) < 2:
    sys.stderr.write("Usage: %s domain\n" % sys.argv[0])
    sys.exit(1)

domain=sys.argv[1]

def getspf (domain):
   answers = dns.resolver.query(domain, 'TXT')
   for rdata in answers:
     for txt_string in rdata.strings:
       if txt_string.startswith('v=spf1'):
         return txt_string.replace('v=spf1','')

full_spf=getspf(domain)
print 'Initial SPF string : ', full_spf

while (full_spf.find('include:') > 0):
  for item in full_spf.split(' '):
    if item.startswith('include:'):
      sec_domain=item.replace('include:','')
      sec_spf=getspf(sec_domain)
      sec_spf=re.sub(' .all','',sec_spf)
      full_spf=full_spf.replace(item,sec_spf)
      print "SPF entry of %s : %s" % (sec_domain, sec_spf)

print 'Full SPF : ', full_spf

<|endoftext|>


<commit_before>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse
import time

def make_request(url, params, auth=None):
    """
    Prepares a request from a url, params, and optionally authentication.
    """
    req = urllib2.Request(url + urllib.urlencode(params))

    if auth:
        req.add_header('AUTHORIZATION', 'Basic ' + auth)

    return urllib2.urlopen(req)

def beginTransaction(url, auth):
    print('Starting transaction...')
    params = {'output_format': 'JSON'}
    request = make_request(url=url+'beginTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("BeginTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on beginTransaction: {0}".format(response['response']['error']))

    print('Transaction started')
    transactionId = response['response']['Transaction']['ID']

    return transactionId;

def endTransaction(url, auth, cancel, transactionId):
    print('Ending transaction...')
    params = {'output_format': 'JSON', 'cancel': cancel, 'transactionId': transactionId}
    request = make_request(url=url+'endTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("EndTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on endTransaction: {0}".format(response['response']['error']))

def getTaskStatus(url, auth, taskID, printStatus):
    print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json', params=params, auth=auth)

    print request.getcode()
    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "RUNNING":
            #print response
            #taskAmount = response['task']['amount']
            taskAmount = "#" #Amount value isn't showing up in response.
            print "++Task "+str(taskID)+" is running and "+taskAmount+" percentage complete."
        elif taskStatus == "FAILED":
            errorMessage = response['task']['error']['message']
            print "++Task "+str(taskID)+" failed with error message: "+errorMessage+"."
        else:
            print "++Task "+str(taskID)+" is "+taskStatus+"."

    return taskID;

def waitOnTask(url, auth, taskID):
    print "Waiting for task "+str(taskID)+"..."

    maxTime = 20
    timeSlept = 0
    sleepCycle = 2

    while timeSlept < maxTime and getTaskStatus(url, auth, taskID, True) in ['WAITING','RUNNING']:
        time.sleep(sleepCycle)
        timeSlept += sleepCycle

    print "Task "+str(taskID)+" is done"

def downloadFromOSM(url, auth, transactionId):
    print('Downloading from OpenStreetMap ...')
    params = {'output_format': 'JSON', 'update': 'true'}
    request = make_request(url=url+'osm/download.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("OSM Download failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskID = response['task']['id']

    #print response
    if response['task']['status'] == 'FAILED':
        raise Exception("An error occurred when pulling new data from OSM: {0}".format(response['task']['status']))

    if response['task']['status'] == 'WAITING':
        print('Download from OpenStreetMap is waiting to be processed.  Task ID is '+str(taskID)+'.')

    if response['task']['status'] == 'RUNNING':
        print('Download from OpenStreetMap is being processed.  Task ID is '+str(taskID)+'.')

    return taskID;

def parse_url(url):

    if (url is None) or len(url) == 0:
        return None

    index = url.rfind('/')

    if index != (len(url)-1):
        url += '/'

    return url

def run(args):
    #url = parse_url(args)
    #if url is None:
    #    print "Please specify a url"
    #    return 1

    geoserver = parse_url(args.geoserver)
    repo = args.repo
    url_repo = geoserver+'geogig/'+repo+'/'
    url_tasks = geoserver+'geogig/tasks'

    authorname = args.authorname
    authoremail = args.authoremail
    auth = None
    if args.username and args.password:
      auth = b64encode('{0}:{1}'.format(args.username, args.password))

    print "=================================="
    print "#==#"
    print "CyberGIS Script / geogig_sync_osm.py"
    print "Downloading Updates from OpenStreetMap"
    print "#==#"

    transID = -1
    try:
        transID = beginTransaction(url_repo, auth)
    except Exception:
        transID = -1
        raise

    if transID != -1:
        taskID = -1
        try:
            taskID = downloadFromOSM(url_repo, auth, transID)
        except Exception:
            taskID = -1
            endTransaction(url_repo, auth, True, transID)
            raise

        if taskID != -1:
            waitOnTask(url_tasks, auth, taskID)

    try:
        endTransaction(url_repo, auth, False, transID)
    except Exception:
        pass

    print "=================================="

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')
parser.add_argument("--geoserver", help="The url of the GeoServer servicing the GeoGig repository.")
parser.add_argument("--repo", help="The GeoServer id of the GeoGig repository you want to sync.")
parser.add_argument("--username", help="The username to use for basic auth requests.")
parser.add_argument("--password", help="The password to use for basic auth requests.")
parser.add_argument("--authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("--authoremail", help="The author email to use when merging non-conflicting branches.")

args = parser.parse_args()
run(args)
<commit_msg>Update geogig_sync_osm.py<commit_after>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse
import time

def make_request(url, params, auth=None):
    """
    Prepares a request from a url, params, and optionally authentication.
    """
    req = urllib2.Request(url + urllib.urlencode(params))

    if auth:
        req.add_header('AUTHORIZATION', 'Basic ' + auth)

    return urllib2.urlopen(req)

def beginTransaction(url, auth):
    print('Starting transaction...')
    params = {'output_format': 'JSON'}
    request = make_request(url=url+'beginTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("BeginTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on beginTransaction: {0}".format(response['response']['error']))

    print('Transaction started')
    transactionId = response['response']['Transaction']['ID']

    return transactionId;

def endTransaction(url, auth, cancel, transactionId):
    print('Ending transaction...')
    params = {'output_format': 'JSON', 'cancel': cancel, 'transactionId': transactionId}
    request = make_request(url=url+'endTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("EndTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on endTransaction: {0}".format(response['response']['error']))

def getTaskStatus(url, auth, taskID, printStatus):
    print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json', params=params, auth=auth)

    print request.getcode()
    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "RUNNING":
            #print response
            #taskAmount = response['task']['amount']
            taskAmount = "#" #Amount value isn't showing up in response.
            print "++Task "+str(taskID)+" is running and "+taskAmount+" percentage complete."
        elif taskStatus == "FAILED":
            errorMessage = response['task']['error']['message']
            print "++Task "+str(taskID)+" failed with error message: "+errorMessage+"."
        else:
            print "++Task "+str(taskID)+" is "+taskStatus+"."

    return taskID;

def waitOnTask(url, auth, taskID):
    print "Waiting for task "+str(taskID)+"..."

    maxTime = 60
    timeSlept = 0
    sleepCycle = 2

    while timeSlept < maxTime and getTaskStatus(url, auth, taskID, True) in ['WAITING','RUNNING']:
        print "Time Slept: "+str(timeSlept)
        time.sleep(sleepCycle)
        timeSlept += sleepCycle

    print "Task "+str(taskID)+" is done"

def downloadFromOSM(url, auth, transactionId):
    print('Downloading from OpenStreetMap ...')
    params = {'output_format': 'JSON', 'update': 'true'}
    request = make_request(url=url+'osm/download.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("OSM Download failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskID = response['task']['id']

    #print response
    if response['task']['status'] == 'FAILED':
        raise Exception("An error occurred when pulling new data from OSM: {0}".format(response['task']['status']))

    if response['task']['status'] == 'WAITING':
        print('Download from OpenStreetMap is waiting to be processed.  Task ID is '+str(taskID)+'.')

    if response['task']['status'] == 'RUNNING':
        print('Download from OpenStreetMap is being processed.  Task ID is '+str(taskID)+'.')

    return taskID;

def parse_url(url):

    if (url is None) or len(url) == 0:
        return None

    index = url.rfind('/')

    if index != (len(url)-1):
        url += '/'

    return url

def run(args):
    #url = parse_url(args)
    #if url is None:
    #    print "Please specify a url"
    #    return 1

    geoserver = parse_url(args.geoserver)
    repo = args.repo
    url_repo = geoserver+'geogig/'+repo+'/'
    url_tasks = geoserver+'geogig/tasks'

    authorname = args.authorname
    authoremail = args.authoremail
    auth = None
    if args.username and args.password:
      auth = b64encode('{0}:{1}'.format(args.username, args.password))

    print "=================================="
    print "#==#"
    print "CyberGIS Script / geogig_sync_osm.py"
    print "Downloading Updates from OpenStreetMap"
    print "#==#"

    transID = -1
    try:
        transID = beginTransaction(url_repo, auth)
    except Exception:
        transID = -1
        raise

    if transID != -1:
        taskID = -1
        try:
            taskID = downloadFromOSM(url_repo, auth, transID)
        except Exception:
            taskID = -1
            endTransaction(url_repo, auth, True, transID)
            raise

        if taskID != -1:
            waitOnTask(url_tasks, auth, taskID)

    try:
        endTransaction(url_repo, auth, False, transID)
    except Exception:
        pass

    print "=================================="

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')
parser.add_argument("--geoserver", help="The url of the GeoServer servicing the GeoGig repository.")
parser.add_argument("--repo", help="The GeoServer id of the GeoGig repository you want to sync.")
parser.add_argument("--username", help="The username to use for basic auth requests.")
parser.add_argument("--password", help="The password to use for basic auth requests.")
parser.add_argument("--authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("--authoremail", help="The author email to use when merging non-conflicting branches.")

args = parser.parse_args()
run(args)
<|endoftext|>