<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# ------------------------------------------------------------------------------
#
# Author: Gabriele Girelli
# Email: gigi.ga90@gmail.com
# Version: 1.2.0
# Date: 20170628
# Project: RNA FISH oligo design
# Description:	filter BLASTN output based on:
# 					- homology (#PM/k),
# 					- number of OT, and
# 					- common (saturated) OTs.
#
# Notes:
# 		PM: Perfect Match.
# 		OT: Off Target.
# 		The script should be run on single-gene outputs.
#
# Changelog:
# 		1.0.0: first implementation.
# 		1.1.0: added argparser support.
# 		1.2.0: changed saturation OT filter.
#
# ------------------------------------------------------------------------------

# DEPENDENCIES =================================================================

import argparse
import numpy as np

# PARAMETERS ===================================================================

# Add script description
parser = argparse.ArgumentParser(
	description = 'Filter BLASTN output.'
)

# Add mandatory arguments
parser.add_argument('blastInput', type = str, nargs = 1,
	help = 'Path to BLAST input fasta file.')
parser.add_argument('blastOutput', type = str, nargs = 1,
	help = 'Path to BLAST output with outfmt 6.')
parser.add_argument('geneTranscriptTable', type = str, nargs = 1,
	help = """Path to table with TRANSCRIPT_ID:GENE_SYMBOL
	tabulation-separated columns.""", default = [30])
parser.add_argument('output', type = str, nargs = 1,
	help = 'Path to output fasta file, after filtering.')

# Add arguments with default value
parser.add_argument('-k', type = int, nargs = 1,
	metavar = 'k', help = """Oligonucleotide length in nt.
	Default: 30.""", default = [30])
parser.add_argument('-t', '--homology-thr', type = float, nargs = 1,
	metavar = 'ht', help = """Threshold on maximum homology, as fraction of k.
	Accepts float values from 0 to 1.
	Default: .85""", default = [.85])
parser.add_argument('-e', '--gene-thr', type = int, nargs = 1,
	metavar = 'gt', help = """Threshold on the number of off-targets gene,
	for a single oligo.
	Default: 20""", default = [20])
parser.add_argument('-s', '--saturation-thr', type = int, nargs = 1,
	metavar = 'st', help = """
	Threshold on the number of oligos off-targeting a gene,
	for the selections of 'saturated' off-target genes. Oligos targeting
	a saturated off-target are filtered out.
	Default: 5""", default = [5])

# Parse arguments
args = parser.parse_args()

# Assign to in-script variables
blast_input = args.blastInput[0]
blast_output = args.blastOutput[0]
gene_transcript_table = args.geneTranscriptTable[0]
output_file = args.output[0]
k = args.k[0]
homThr = args.homology_thr[0]
gene_ot_thr = args.gene_thr[0]
oligo_ot_thr = args.saturation_thr[0]

# Log to screen the settings
print("""
Settings:
              BLASTN input : %s
             BLASTN output : %s
     Gene-Transcript table : %s
               Output file : %s
                         K : %d
        Homology threshold : %f
       Gene Off-Target thr : %d
 Off-Target saturation thr : %d

""" % (blast_input, blast_output, gene_transcript_table, output_file,
	k, homThr, gene_ot_thr, oligo_ot_thr))

# FUNCTIONS ====================================================================

# RUN ==========================================================================
print("Run:")

# Build TRANSCRIPT_ID:GENE_SYMBOL dictionary -----------------------------------
print(" · Building TRANSCRIPT_ID:GENE_SYMBOL dictionary...")

# Initialize empty TRANSCRIPT_ID:GENE_SYMBOL dictionary
trn_gene_dict = {}

# Read table line by line
with open(gene_transcript_table) as gttf:
	for line in gttf:

		# Split every line in two fields
		tmp = line.strip().split('\t')

		# Store key,value couples in the dictionary
		trn_gene_dict[tmp[0]] = tmp[1]

# Work on BLASTN output --------------------------------------------------------

# Filter based on max homology
# ----------------------------
print(" · Filtering based on maximum homology...")

# Initalize empty maximum homology dictionary
max_homology = {}

# Read table line by line
with open(blast_output) as bof:
	for line in bof:

		# Split every line by column
		tmp = line.strip().split('\t')

		# Identify oligomer ID
		OID = tmp[0].split(':')[0]

		# Identify target gene
		target = tmp[0].split('_')[0]

		# Identify transcript ID
		transcript = tmp[1].split('.')[0]

		# Calculate homology
		homology = (int(tmp[3]) - int(tmp[4])) / float(k)

		# If it is an off-target
		if target != trn_gene_dict[transcript]:
			if not OID in max_homology.keys():
				max_homology[OID] = homology
			else:
				# With homology higher than the threshold
				if homology > max_homology[OID]:
					max_homology[OID] = homology

# Identify oligos that pass the threshold
pass_homology = []
for OID in max_homology.keys():
	if max_homology[OID] < homThr:
		pass_homology.append(OID)
# Log
print(" >>> %d oligos do not have any off-targets." % (len(pass_homology),))
print(" >>> %d oligos have off-targets." % (len(max_homology),))
print(" >>> Saving off-target free oligos. Analyzing further the rest.")

# Check off target location
# -------------------------
print(" · Identifying off-target locations...")

# Initialize empty off-targets dictionary
# OligoID:OTgene:OTtranscript
ot_dict = {}

# Read table line by line
with open(blast_output) as bof:
	for line in bof:

		# Split every line by column
		tmp = line.strip().split('\t')

		# Identify oligomer ID
		OID = tmp[0].split(':')[0]

		# Work only on oligos with off-targets
		# that do not pass the homology filter
		if OID in pass_homology:
			continue

		# Identify target gene
		target = tmp[0].split('_')[0]

		# Identify transcript ID
		ot_trans = tmp[1].split('.')[0]

		# Calculate homology
		homology = (int(tmp[3]) - int(tmp[4])) / float(k)

		# Identify the off-target
		ot_gene = trn_gene_dict[ot_trans]

		# If it is an off-target
		if target != ot_gene:

			# With homology higher than the threshold
			if homology >= homThr:

				if not OID in ot_dict.keys():
					ot_dict[OID] = {}
					ot_dict[OID][ot_gene] = {}
					ot_dict[OID][ot_gene][ot_trans] = 1
				else:
					if not ot_gene in ot_dict[OID].keys():
						ot_dict[OID][ot_gene] = {}
						ot_dict[OID][ot_gene][ot_trans] = 1
					else:
						if not ot_trans in ot_dict[OID][ot_gene].keys():
							ot_dict[OID][ot_gene][ot_trans] = 1
						else:
							ot_dict[OID][ot_gene][ot_trans] += 1

# Calculate number of off-targets
# -------------------------------
print(" · Filtering based on number of off-target genes...")

ot_gene_count = {}
for OID in ot_dict.keys():
	ot_gene_count[OID] = len(ot_dict[OID].keys())
print(""" >>> OT counts summary:
             min : %f
      1st Quart. : %f
          median : %f
            mean : %f
      2nd Quart. : %f
             max : %f
 >>> Current threshold at the %d-ith percentile.""" % (
	np.percentile(ot_gene_count.values(), 0),
	np.percentile(ot_gene_count.values(), 25),
	np.percentile(ot_gene_count.values(), 50),
	np.mean(ot_gene_count.values()),
	np.percentile(ot_gene_count.values(), 75),
	np.percentile(ot_gene_count.values(), 100),
	int(sum(np.array(ot_gene_count.values()) < gene_ot_thr)
		/ float(len(ot_gene_count.values())) * 100)
))

# Filter based on number of OTs per oligo
pass_oligo_ot_count = []
for OID in ot_gene_count.keys():
	if ot_gene_count[OID] < gene_ot_thr:
		pass_oligo_ot_count.append(OID)
print(" >>> %d oligos pass the OT count filter" % (len(pass_oligo_ot_count),))

# Calculate number of common off-targets
# --------------------------------------

pass_gene_ot_count = pass_oligo_ot_count

if 1 == len(pass_oligo_ot_count):
	print(" · Skipping saturated off-target transcript filter...")
else:
	print(" · Filtering based on saturated off-target transcripts...")

	gene_ot_count = {}
	for OID in pass_oligo_ot_count:
		for gene in ot_dict[OID].keys():
			if not gene in gene_ot_count.keys():
				gene_ot_count[gene] = ot_dict[OID][gene]
			else:
				for trans in ot_dict[OID][gene].keys():
					if not trans in gene_ot_count[gene].keys():
						gene_ot_count[gene][trans] = ot_dict[OID][gene][trans]
					else:
						gene_ot_count[gene][trans] += ot_dict[OID][gene][trans]

	# Filter based on number of OTs per transcript
	for gene in gene_ot_count.keys():
		for trans in gene_ot_count[gene].keys():
			# Discard oligos that off-target a transcript
			# shared by too many oligos
			if gene_ot_count[gene][trans] >= oligo_ot_thr:
				for OID in pass_gene_ot_count:
					if gene in ot_dict[OID].keys():
						if trans in ot_dict[OID][gene].keys():
							pass_gene_ot_count.remove(OID)

	print(" >>> %d oligos pass the saturation OT filter"
		% (len(pass_gene_ot_count),))

# Merge list of filtered oligos
# -----------------------------

output_list = pass_homology
output_list.extend(pass_gene_ot_count)
output_list.sort()

# Log
print(" · %d oligos can be used for further screening." % (len(output_list),))

# Prepare output ---------------------------------------------------------------
print(" · Generating FASTA output...")

# Prepare output string
s = ''

# Variable to keep the non-header lines
keep = False

# Read Fasta line by line
with open(blast_input) as bif:
	for line in bif:

		# If header line
		if '>' == line[0]:
			# Check if the sequence should be kept
			if line[1:].split(':')[0] in output_list:
				keep = True
				s += line
			else:
				keep = False
		# Otherwise keep it if it passed the filters
		elif keep:
			s += line

# Write output
f = open(output_file, 'w')
f.write(s)
f.close()

# END ==========================================================================

print("""
DONE!
""")

################################################################################
<commit_msg>Implemented default max homology of 0 for oligos with no off-targets. Fixed log.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# ------------------------------------------------------------------------------
#
# Author: Gabriele Girelli
# Email: gigi.ga90@gmail.com
# Version: 1.2.0
# Date: 20170628
# Project: RNA FISH oligo design
# Description:	filter BLASTN output based on:
# 					- homology (#PM/k),
# 					- number of OT, and
# 					- common (saturated) OTs.
#
# Notes:
# 		PM: Perfect Match.
# 		OT: Off Target.
# 		The script should be run on single-gene outputs.
#
# Changelog:
# 		1.0.0: first implementation.
# 		1.1.0: added argparser support.
# 		1.2.0: changed saturation OT filter.
#
# ------------------------------------------------------------------------------

# DEPENDENCIES =================================================================

import argparse
import numpy as np

# PARAMETERS ===================================================================

# Add script description
parser = argparse.ArgumentParser(
	description = 'Filter BLASTN output.'
)

# Add mandatory arguments
parser.add_argument('blastInput', type = str, nargs = 1,
	help = 'Path to BLAST input fasta file.')
parser.add_argument('blastOutput', type = str, nargs = 1,
	help = 'Path to BLAST output with outfmt 6.')
parser.add_argument('geneTranscriptTable', type = str, nargs = 1,
	help = """Path to table with TRANSCRIPT_ID:GENE_SYMBOL
	tabulation-separated columns.""", default = [30])
parser.add_argument('output', type = str, nargs = 1,
	help = 'Path to output fasta file, after filtering.')

# Add arguments with default value
parser.add_argument('-k', type = int, nargs = 1,
	metavar = 'k', help = """Oligonucleotide length in nt.
	Default: 30.""", default = [30])
parser.add_argument('-t', '--homology-thr', type = float, nargs = 1,
	metavar = 'ht', help = """Threshold on maximum homology, as fraction of k.
	Accepts float values from 0 to 1.
	Default: .85""", default = [.85])
parser.add_argument('-e', '--gene-thr', type = int, nargs = 1,
	metavar = 'gt', help = """Threshold on the number of off-targets gene,
	for a single oligo.
	Default: 20""", default = [20])
parser.add_argument('-s', '--saturation-thr', type = int, nargs = 1,
	metavar = 'st', help = """
	Threshold on the number of oligos off-targeting a gene,
	for the selections of 'saturated' off-target genes. Oligos targeting
	a saturated off-target are filtered out.
	Default: 5""", default = [5])

# Parse arguments
args = parser.parse_args()

# Assign to in-script variables
blast_input = args.blastInput[0]
blast_output = args.blastOutput[0]
gene_transcript_table = args.geneTranscriptTable[0]
output_file = args.output[0]
k = args.k[0]
homThr = args.homology_thr[0]
gene_ot_thr = args.gene_thr[0]
oligo_ot_thr = args.saturation_thr[0]

# Log to screen the settings
print("""
Settings:
              BLASTN input : %s
             BLASTN output : %s
     Gene-Transcript table : %s
               Output file : %s
                         K : %d
        Homology threshold : %f
       Gene Off-Target thr : %d
 Off-Target saturation thr : %d

""" % (blast_input, blast_output, gene_transcript_table, output_file,
	k, homThr, gene_ot_thr, oligo_ot_thr))

# FUNCTIONS ====================================================================

# RUN ==========================================================================
print("Run:")

# Build TRANSCRIPT_ID:GENE_SYMBOL dictionary -----------------------------------
print(" · Building TRANSCRIPT_ID:GENE_SYMBOL dictionary...")

# Initialize empty TRANSCRIPT_ID:GENE_SYMBOL dictionary
trn_gene_dict = {}

# Read table line by line
with open(gene_transcript_table) as gttf:
	for line in gttf:

		# Split every line in two fields
		tmp = line.strip().split('\t')

		# Store key,value couples in the dictionary
		trn_gene_dict[tmp[0]] = tmp[1]

# Work on BLASTN output --------------------------------------------------------

# Filter based on max homology
# ----------------------------
print(" · Filtering based on maximum homology...")

# Initalize empty maximum homology dictionary
max_homology = {}

# Read table line by line
with open(blast_output) as bof:
	for line in bof:

		# Split every line by column
		tmp = line.strip().split('\t')

		# Identify oligomer ID
		OID = tmp[0].split(':')[0]

		# Identify target gene
		target = tmp[0].split('_')[0]

		# Identify transcript ID
		transcript = tmp[1].split('.')[0]

		# Calculate homology
		homology = (int(tmp[3]) - int(tmp[4])) / float(k)

		# If it is an off-target
		if target != trn_gene_dict[transcript]:
			if not OID in max_homology.keys():
				max_homology[OID] = homology
			else:
				# With homology higher than the threshold
				if homology > max_homology[OID]:
					max_homology[OID] = homology
		elif not OID in max_homology.keys():
			max_homology[OID] = 0

# Identify oligos that pass the threshold
pass_homology = []
for OID in max_homology.keys():
	if max_homology[OID] < homThr:
		pass_homology.append(OID)

# Log
print(" >>> %d oligos do not have any off-targets." % (len(pass_homology),))
print(" >>> %d oligos have off-targets." % (len(max_homology)-len(pass_homology),))
print(" >>> Saving off-target free oligos. Analyzing further the rest.")

# Check off target location
# -------------------------
print(" · Identifying off-target locations...")

# Initialize empty off-targets dictionary
# OligoID:OTgene:OTtranscript
ot_dict = {}

# Read table line by line
with open(blast_output) as bof:
	for line in bof:

		# Split every line by column
		tmp = line.strip().split('\t')

		# Identify oligomer ID
		OID = tmp[0].split(':')[0]

		# Work only on oligos with off-targets
		# that do not pass the homology filter
		if OID in pass_homology:
			continue

		# Identify target gene
		target = tmp[0].split('_')[0]

		# Identify transcript ID
		ot_trans = tmp[1].split('.')[0]

		# Calculate homology
		homology = (int(tmp[3]) - int(tmp[4])) / float(k)

		# Identify the off-target
		ot_gene = trn_gene_dict[ot_trans]

		# If it is an off-target
		if target != ot_gene:

			# With homology higher than the threshold
			if homology >= homThr:

				if not OID in ot_dict.keys():
					ot_dict[OID] = {}
					ot_dict[OID][ot_gene] = {}
					ot_dict[OID][ot_gene][ot_trans] = 1
				else:
					if not ot_gene in ot_dict[OID].keys():
						ot_dict[OID][ot_gene] = {}
						ot_dict[OID][ot_gene][ot_trans] = 1
					else:
						if not ot_trans in ot_dict[OID][ot_gene].keys():
							ot_dict[OID][ot_gene][ot_trans] = 1
						else:
							ot_dict[OID][ot_gene][ot_trans] += 1

# Calculate number of off-targets
# -------------------------------
print(" · Filtering based on number of off-target genes...")

ot_gene_count = {}
for OID in ot_dict.keys():
	ot_gene_count[OID] = len(ot_dict[OID].keys())
print(""" >>> OT counts summary:
             min : %f
      1st Quart. : %f
          median : %f
            mean : %f
      2nd Quart. : %f
             max : %f
 >>> Current threshold at the %d-ith percentile.""" % (
	np.percentile(ot_gene_count.values(), 0),
	np.percentile(ot_gene_count.values(), 25),
	np.percentile(ot_gene_count.values(), 50),
	np.mean(ot_gene_count.values()),
	np.percentile(ot_gene_count.values(), 75),
	np.percentile(ot_gene_count.values(), 100),
	int(sum(np.array(ot_gene_count.values()) < gene_ot_thr)
		/ float(len(ot_gene_count.values())) * 100)
))

# Filter based on number of OTs per oligo
pass_oligo_ot_count = []
for OID in ot_gene_count.keys():
	if ot_gene_count[OID] < gene_ot_thr:
		pass_oligo_ot_count.append(OID)
print(" >>> %d oligos pass the OT count filter" % (len(pass_oligo_ot_count),))

# Calculate number of common off-targets
# --------------------------------------

pass_gene_ot_count = pass_oligo_ot_count

if 1 == len(pass_oligo_ot_count):
	print(" · Skipping saturated off-target transcript filter...")
else:
	print(" · Filtering based on saturated off-target transcripts...")

	gene_ot_count = {}
	for OID in pass_oligo_ot_count:
		for gene in ot_dict[OID].keys():
			if not gene in gene_ot_count.keys():
				gene_ot_count[gene] = ot_dict[OID][gene]
			else:
				for trans in ot_dict[OID][gene].keys():
					if not trans in gene_ot_count[gene].keys():
						gene_ot_count[gene][trans] = ot_dict[OID][gene][trans]
					else:
						gene_ot_count[gene][trans] += ot_dict[OID][gene][trans]

	# Filter based on number of OTs per transcript
	for gene in gene_ot_count.keys():
		for trans in gene_ot_count[gene].keys():
			# Discard oligos that off-target a transcript
			# shared by too many oligos
			if gene_ot_count[gene][trans] >= oligo_ot_thr:
				for OID in pass_gene_ot_count:
					if gene in ot_dict[OID].keys():
						if trans in ot_dict[OID][gene].keys():
							pass_gene_ot_count.remove(OID)

	print(" >>> %d oligos pass the saturation OT filter"
		% (len(pass_gene_ot_count),))

# Merge list of filtered oligos
# -----------------------------

output_list = pass_homology
output_list.extend(pass_gene_ot_count)
output_list.sort()

# Log
print(" · %d oligos can be used for further screening." % (len(output_list),))

# Prepare output ---------------------------------------------------------------
print(" · Generating FASTA output...")

# Prepare output string
s = ''

# Variable to keep the non-header lines
keep = False

# Read Fasta line by line
with open(blast_input) as bif:
	for line in bif:

		# If header line
		if '>' == line[0]:
			# Check if the sequence should be kept
			if line[1:].split(':')[0] in output_list:
				keep = True
				s += line
			else:
				keep = False
		# Otherwise keep it if it passed the filters
		elif keep:
			s += line

# Write output
f = open(output_file, 'w')
f.write(s)
f.close()

# END ==========================================================================

print("""
DONE!
""")

################################################################################
<|endoftext|>


<commit_before>import unittest

from dynd import nd
import numpy as np
from datashape import dshape

from blaze.api import into, discover
import blaze

def skip(test_foo):
    return

def skip_if_not(x):
    def maybe_a_test_function(test_foo):
        if not x:
            return
        else:
            return test_foo
    return maybe_a_test_function

class Test_into(unittest.TestCase):
    def test_containers(self):
        self.assertEqual(into([], (1, 2, 3)),
                                  [1, 2, 3])
        self.assertEqual(into((), (1, 2, 3)),
                                  (1, 2, 3))
        self.assertEqual(into({}, [(1, 2), (3, 4)]),
                                  {1: 2, 3: 4})
        self.assertEqual(into((), {1: 2, 3: 4}),
                                  ((1, 2), (3, 4)))
        self.assertEqual(into((), {'cat': 2, 'dog': 4}),
                                  (('cat', 2), ('dog', 4)))

    def test_dynd(self):
        self.assertEqual(nd.as_py(into(nd.array(), (1, 2, 3))),
                         nd.as_py(nd.array([1, 2, 3])))
        self.assertEqual(into([], nd.array([1, 2])),
                                  [1, 2])
        self.assertEqual(into([], nd.array([[1, 2], [3, 4]])),
                                  [[1, 2], [3, 4]])

    def test_numpy(self):
        assert (into(np.array(0), [1, 2]) == np.array([1, 2])).all()
        self.assertEqual(into([], np.array([1, 2])),
                         [1, 2])

    def test_type(self):
        self.assertEqual(into(list, (1, 2, 3)),
                         into([], (1, 2, 3)))
        self.assertEqual(str(into(np.ndarray, (1, 2, 3))),
                         str(into(np.ndarray(()), (1, 2, 3))))

try:
    from pandas import DataFrame
except ImportError:
    DataFrame = None

try:
    from blaze.data.python import Python
except ImportError:
    Python = None

@skip_if_not(DataFrame and Python)
def test_pandas_data_descriptor():
    data = [['Alice', 100], ['Bob', 200]]
    schema='{name: string, amount: int}'
    dd = Python(data, schema=schema)
    result = into(DataFrame, dd)
    expected = DataFrame(data, columns=['name', 'amount'])
    print(result)
    print(expected)

    assert str(result) == str(expected)

@skip_if_not(DataFrame and nd.array)
def test_pandas_dynd():
    data = [['Alice', 100], ['Bob', 200]]
    schema='{name: string, amount: int}'

    arr = nd.array(data, dtype=schema)

    result = into(DataFrame, arr)
    expected = DataFrame(data, columns=['name', 'amount'])
    print(result)
    print(expected)

    assert str(result) == str(expected)

@skip_if_not(DataFrame)
def test_discover_pandas():
    data = [['Alice', 100], ['Bob', 200]]
    df = DataFrame(data, columns=['name', 'balance'])

    print(discover(df))
    assert discover(df).subshape[0] == dshape('{name: string, balance: int64}')

@skip_if_not(DataFrame and nd.array)
def test_discover_pandas():
    data = [('Alice', 100), ('Bob', 200)]
    df = DataFrame(data, columns=['name', 'balance'])

    result = into(nd.array, df)

    assert nd.as_py(result, tuple=True) == data
<commit_msg>fully explicit import for into<commit_after>import unittest

from dynd import nd
import numpy as np
from datashape import dshape

from blaze.api.into import into, discover
import blaze

def skip(test_foo):
    return

def skip_if_not(x):
    def maybe_a_test_function(test_foo):
        if not x:
            return
        else:
            return test_foo
    return maybe_a_test_function

class Test_into(unittest.TestCase):
    def test_containers(self):
        self.assertEqual(into([], (1, 2, 3)),
                                  [1, 2, 3])
        self.assertEqual(into((), (1, 2, 3)),
                                  (1, 2, 3))
        self.assertEqual(into({}, [(1, 2), (3, 4)]),
                                  {1: 2, 3: 4})
        self.assertEqual(into((), {1: 2, 3: 4}),
                                  ((1, 2), (3, 4)))
        self.assertEqual(into((), {'cat': 2, 'dog': 4}),
                                  (('cat', 2), ('dog', 4)))

    def test_dynd(self):
        self.assertEqual(nd.as_py(into(nd.array(), (1, 2, 3))),
                         nd.as_py(nd.array([1, 2, 3])))
        self.assertEqual(into([], nd.array([1, 2])),
                                  [1, 2])
        self.assertEqual(into([], nd.array([[1, 2], [3, 4]])),
                                  [[1, 2], [3, 4]])

    def test_numpy(self):
        assert (into(np.array(0), [1, 2]) == np.array([1, 2])).all()
        self.assertEqual(into([], np.array([1, 2])),
                         [1, 2])

    def test_type(self):
        self.assertEqual(into(list, (1, 2, 3)),
                         into([], (1, 2, 3)))
        self.assertEqual(str(into(np.ndarray, (1, 2, 3))),
                         str(into(np.ndarray(()), (1, 2, 3))))

try:
    from pandas import DataFrame
except ImportError:
    DataFrame = None

try:
    from blaze.data.python import Python
except ImportError:
    Python = None

@skip_if_not(DataFrame and Python)
def test_pandas_data_descriptor():
    data = [['Alice', 100], ['Bob', 200]]
    schema='{name: string, amount: int}'
    dd = Python(data, schema=schema)
    result = into(DataFrame, dd)
    expected = DataFrame(data, columns=['name', 'amount'])
    print(result)
    print(expected)

    assert str(result) == str(expected)

@skip_if_not(DataFrame and nd.array)
def test_pandas_dynd():
    data = [['Alice', 100], ['Bob', 200]]
    schema='{name: string, amount: int}'

    arr = nd.array(data, dtype=schema)

    result = into(DataFrame, arr)
    expected = DataFrame(data, columns=['name', 'amount'])
    print(result)
    print(expected)

    assert str(result) == str(expected)

@skip_if_not(DataFrame)
def test_discover_pandas():
    data = [['Alice', 100], ['Bob', 200]]
    df = DataFrame(data, columns=['name', 'balance'])

    print(discover(df))
    assert discover(df).subshape[0] == dshape('{name: string, balance: int64}')

@skip_if_not(DataFrame and nd.array)
def test_discover_pandas():
    data = [('Alice', 100), ('Bob', 200)]
    df = DataFrame(data, columns=['name', 'balance'])

    result = into(nd.array, df)

    assert nd.as_py(result, tuple=True) == data
<|endoftext|>


<commit_before>import unittest

from dynd import nd
import numpy as np
from datashape import dshape
from datetime import datetime
import tempfile
import os

from blaze.api.into import into, discover
import blaze
from blaze import Table
from blaze.utils import tmpfile
import pytest

def skip(test_foo):
    return

def skip_if_not(x):
    def maybe_a_test_function(test_foo):
        if not x:
            return
        else:
            return test_foo
    return maybe_a_test_function

class Test_into(unittest.TestCase):
    def test_containers(self):
        self.assertEqual(into([], (1, 2, 3)),
                                  [1, 2, 3])
        self.assertEqual(into([], iter((1, 2, 3))),
                                  [1, 2, 3])
        self.assertEqual(into((), (1, 2, 3)),
                                  (1, 2, 3))
        self.assertEqual(into({}, [(1, 2), (3, 4)]),
                                  {1: 2, 3: 4})
        self.assertEqual(into((), {1: 2, 3: 4}),
                                  ((1, 2), (3, 4)))
        self.assertEqual(into((), {'cat': 2, 'dog': 4}),
                                  (('cat', 2), ('dog', 4)))

    def test_dynd(self):
        self.assertEqual(nd.as_py(into(nd.array(), (1, 2, 3))),
                         nd.as_py(nd.array([1, 2, 3])))
        self.assertEqual(into([], nd.array([1, 2])),
                                  [1, 2])
        self.assertEqual(into([], nd.array([[1, 2], [3, 4]])),
                                  [[1, 2], [3, 4]])

    def test_numpy(self):
        assert (into(np.array(0), [1, 2]) == np.array([1, 2])).all()
        assert (into(np.array(0), iter([1, 2])) == np.array([1, 2])).all()
        self.assertEqual(into([], np.array([1, 2])),
                         [1, 2])

    def test_numpy_datetime(self):
        assert isinstance(into(np.ndarray(0), [datetime(2014, 1, 1)])[0],
                          np.datetime64)

    def test_type(self):
        self.assertEqual(into(list, (1, 2, 3)),
                         into([], (1, 2, 3)))
        self.assertEqual(str(into(np.ndarray, (1, 2, 3))),
                         str(into(np.ndarray(()), (1, 2, 3))))

try:
    from pandas import DataFrame, read_csv
except ImportError:
    DataFrame = None

try:
    from blaze.data import Python, CSV
except ImportError:
    Python = None
    CSV = None

try:
    from bokeh.objects import ColumnDataSource
except ImportError:
    ColumnDataSource = None

try:
    from tables import (Table as PyTables, open_file, UInt8Col, StringCol,
        IsDescription)
except ImportError:
    PyTables = None

@pytest.yield_fixture
def h5():
    class Test(IsDescription):
        posted_dow = UInt8Col()
        jobtype  = UInt8Col()
        location  = UInt8Col()
        date  = StringCol(20)
        country  = StringCol(2)

    with tmpfile('.h5') as filename:
        h5file = open_file(filename, mode = "w", title = "Test file")
        group = h5file.create_group("/", 'test', 'Info')
        tab = h5file.create_table(group, 'sample', Test, "Example")
        arow = tab.row
        arow['country'] = 'ab'
        arow['date'] = '20121105'
        arow['location'] = 0
        arow['jobtype']  = 1
        arow['posted_dow']  = 3
        # Insert a new record
        arow.append()
        tab.flush()
        yield h5file
        # Close (and flush) the file
        h5file.close()

@pytest.yield_fixture
def good_csv():

    with tempfile.NamedTemporaryFile(mode='w', suffix=".csv") as f:
        badfile = open(f.name, mode="w")
        # Insert a new record
        badfile.write("userid,text,country\n")
        badfile.write("1,Alice,az\n")
        badfile.write("2,Bob,bl\n")
        badfile.write("3,Charlie,cz\n")
        badfile.flush()
        yield badfile
        # Close (and flush) the file
        badfile.close()

@pytest.yield_fixture
def bad_csv_df():

    with tempfile.NamedTemporaryFile(mode='w', suffix=".csv") as f:
        badfile = open(f.name, mode="w")
        # Insert a new record
        badfile.write("userid,text,country\n")
        badfile.write("1,Alice,az\n")
        badfile.write("2,Bob,bl\n")
        for i in range(0,100):
            badfile.write(str(i) + ",badguy,zz\n")
        badfile.write("4,Dan,gb,extra,extra\n")
        badfile.flush()
        yield badfile
        # Close (and flush) the file
        badfile.close()

@pytest.yield_fixture
def out_hdf5():
    with tempfile.NamedTemporaryFile(mode='w', suffix=".h5") as f:
        yield f

@skip_if_not(PyTables and DataFrame)
def test_into_pytables_dataframe(h5):
    samp = h5.root.test.sample
    final = into(DataFrame, samp)
    assert len(final) == 1

@skip_if_not(DataFrame and Python)
def test_pandas_data_descriptor():
    data = [['Alice', 100], ['Bob', 200]]
    schema='{name: string, amount: int}'
    dd = Python(data, schema=schema)
    result = into(DataFrame, dd)
    expected = DataFrame(data, columns=['name', 'amount'])
    print(result)
    print(expected)

    assert str(result) == str(expected)

@skip_if_not(DataFrame and nd.array)
def test_pandas_dynd():
    data = [['Alice', 100], ['Bob', 200]]
    schema='{name: string, amount: int}'

    arr = nd.array(data, dtype=schema)

    result = into(DataFrame, arr)
    expected = DataFrame(data, columns=['name', 'amount'])
    print(result)
    print(expected)

    assert str(result) == str(expected)

@skip_if_not(DataFrame)
def test_pandas_numpy():
    data = [('Alice', 100), ('Bob', 200)]
    dtype=[('name', 'O'), ('amount', int)]

    x = np.array(data, dtype=dtype)

    result = into(DataFrame(), x)
    expected = DataFrame(data, columns=['name', 'amount'])
    assert str(result) == str(expected)

    result = into(DataFrame(columns=['name', 'amount']), x)
    expected = DataFrame(data, columns=['name', 'amount'])
    print(result)
    print(expected)
    assert str(result) == str(expected)

@skip_if_not(DataFrame)
def test_pandas_seq():
    assert str(into(DataFrame, [1, 2])) == \
            str(DataFrame([1, 2]))
    assert str(into(DataFrame, (1, 2))) == \
            str(DataFrame([1, 2]))
    assert str(into(DataFrame(columns=['a', 'b']), [(1, 2), (3, 4)])) == \
            str(DataFrame([[1, 2], [3, 4]], columns=['a', 'b']))

@skip_if_not(DataFrame)
def test_pandas_pandas():
    data = [('Alice', 100), ('Bob', 200)]
    df = DataFrame(data, columns=['name', 'balance'])
    new_df = into(DataFrame, df)
    # Data must be the same
    assert np.all(new_df == df)
    # new_df should be a copy of df
    assert id(new_df) != id(df)

@skip_if_not(DataFrame)
def test_DataFrame_Series():
    data = [('Alice', 100), ('Bob', 200)]
    df = DataFrame(data, columns=['name', 'balance'])

    new_df = into(DataFrame, df['name'])

    assert np.all(new_df == DataFrame([['Alice'], ['Bob']], columns=['name']))

    # new_df should be a copy of df
    assert id(new_df) != id(df['name'])

    assert isinstance(new_df, DataFrame)

def test_discover_ndarray():
    data = [['Alice', 100], ['Bob', 200]]
    schema='{name: string, balance: int32}'
    arr = nd.array(data, dtype=schema)
    assert discover(arr) == 2 * dshape(schema)

@skip_if_not(DataFrame)
def test_discover_pandas():
    data = [['Alice', 100], ['Bob', 200]]
    df = DataFrame(data, columns=['name', 'balance'])

    print(discover(df))
    assert discover(df).subshape[0] == dshape('{name: string, balance: int64}')

@skip_if_not(DataFrame and nd.array)
def test_discover_pandas():
    data = [('Alice', 100), ('Bob', 200)]
    df = DataFrame(data, columns=['name', 'balance'])

    result = into(nd.array, df)

    assert nd.as_py(result, tuple=True) == data

@skip_if_not(Table and DataFrame)
def test_into_table_dataframe():
    data = [['Alice', 100], ['Bob', 200]]
    t = Table(data, columns=['name', 'amount'])

    assert list(into(DataFrame(), t).columns) == list(t.columns)
    assert into([], into(DataFrame(), t)) == list(map(tuple, data))

@skip_if_not(Table and ColumnDataSource)
def test_Column_data_source():
    data = [('Alice', 100), ('Bob', 200)]
    t = Table(data, columns=['name', 'balance'])

    cds = into(ColumnDataSource(), t)

    assert isinstance(cds, ColumnDataSource)
    assert set(cds.column_names) == set(t.columns)

def test_numpy_list():
    data = [('Alice', 100), ('Bob', 200)]

    dtype = into(np.ndarray, data).dtype
    assert np.issubdtype(dtype[0], str)
    assert np.issubdtype(dtype[1], int)

    assert into([], into(np.ndarray, data)) == data

@skip_if_not(Table)
def test_numpy_tableExpr():
    data = [('Alice', 100), ('Bob', 200)]
    t = Table(data, '{name: string, amount: int64}')

    assert into(np.ndarray, t).dtype == \
            np.dtype([('name', 'O'), ('amount', 'i8')])

@skip_if_not(DataFrame and CSV)
def test_DataFrame_CSV():
    with filetext('1,2\n3,4\n') as fn:
        csv = CSV(fn, schema='{a: int64, b: float64}')
        df = into(DataFrame, csv)

        expected = DataFrame([[1, 2.0], [3, 4.0]],
                             columns=['a', 'b'])

        assert str(df) == str(expected)
        assert list(df.dtypes) == [np.int64, np.float64]

@skip_if_not(PyTables and DataFrame)
def test_into_tables_path(good_csv, out_hdf5):
    tble = into(PyTables, good_csv.name, filename=out_hdf5.name, datapath='foo')
    assert len(tble) == 3

@skip_if_not(PyTables and DataFrame)
def test_into_tables_path_bad_csv(bad_csv_df, out_hdf5):
    tble = into(PyTables, bad_csv_df.name,
                          filename=out_hdf5.name,
                          datapath='foo',
                          error_bad_lines=False)
    df_from_tbl = into(DataFrame, tble)
    #Check that it's the same as straight from the CSV
    df_from_csv = into(DataFrame, bad_csv_df.name, error_bad_lines=False)
    assert len(df_from_csv) == len(df_from_tbl)
    assert list(df_from_csv.columns) == list(df_from_tbl.columns)
    assert (df_from_csv == df_from_tbl).all().all()

def test_numpy_datetimes():
    L = [datetime(2000, 12, 1), datetime(2000, 1, 1, 1, 1, 1)]
    assert into([], np.array(L, dtype='M8[us]')) == L
    assert into([], np.array(L, dtype='M8[ns]')) == L

def test_numpy_python3_bytes_to_string_conversion():
    x = np.array(['a','b'], dtype='S1')
    assert all(isinstance(s, str) for s in into(list, x))
    x = np.array([(1, 'a'), (2, 'b')], dtype=[('id', 'i4'), ('letter', 'S1')])
    assert isinstance(into(list, x)[0][1], str)
<commit_msg>replace NamedTemporaryFile with tmpfile<commit_after>import unittest

from dynd import nd
import numpy as np
from datashape import dshape
from datetime import datetime
import tempfile
import os

from blaze.api.into import into, discover
import blaze
from blaze import Table
from blaze.utils import tmpfile
import pytest

def skip(test_foo):
    return

def skip_if_not(x):
    def maybe_a_test_function(test_foo):
        if not x:
            return
        else:
            return test_foo
    return maybe_a_test_function

class Test_into(unittest.TestCase):
    def test_containers(self):
        self.assertEqual(into([], (1, 2, 3)),
                                  [1, 2, 3])
        self.assertEqual(into([], iter((1, 2, 3))),
                                  [1, 2, 3])
        self.assertEqual(into((), (1, 2, 3)),
                                  (1, 2, 3))
        self.assertEqual(into({}, [(1, 2), (3, 4)]),
                                  {1: 2, 3: 4})
        self.assertEqual(into((), {1: 2, 3: 4}),
                                  ((1, 2), (3, 4)))
        self.assertEqual(into((), {'cat': 2, 'dog': 4}),
                                  (('cat', 2), ('dog', 4)))

    def test_dynd(self):
        self.assertEqual(nd.as_py(into(nd.array(), (1, 2, 3))),
                         nd.as_py(nd.array([1, 2, 3])))
        self.assertEqual(into([], nd.array([1, 2])),
                                  [1, 2])
        self.assertEqual(into([], nd.array([[1, 2], [3, 4]])),
                                  [[1, 2], [3, 4]])

    def test_numpy(self):
        assert (into(np.array(0), [1, 2]) == np.array([1, 2])).all()
        assert (into(np.array(0), iter([1, 2])) == np.array([1, 2])).all()
        self.assertEqual(into([], np.array([1, 2])),
                         [1, 2])

    def test_numpy_datetime(self):
        assert isinstance(into(np.ndarray(0), [datetime(2014, 1, 1)])[0],
                          np.datetime64)

    def test_type(self):
        self.assertEqual(into(list, (1, 2, 3)),
                         into([], (1, 2, 3)))
        self.assertEqual(str(into(np.ndarray, (1, 2, 3))),
                         str(into(np.ndarray(()), (1, 2, 3))))

try:
    from pandas import DataFrame, read_csv
except ImportError:
    DataFrame = None

try:
    from blaze.data import Python, CSV
except ImportError:
    Python = None
    CSV = None

try:
    from bokeh.objects import ColumnDataSource
except ImportError:
    ColumnDataSource = None

try:
    from tables import (Table as PyTables, open_file, UInt8Col, StringCol,
        IsDescription)
except ImportError:
    PyTables = None

@pytest.yield_fixture
def h5():
    class Test(IsDescription):
        posted_dow = UInt8Col()
        jobtype  = UInt8Col()
        location  = UInt8Col()
        date  = StringCol(20)
        country  = StringCol(2)

    with tmpfile('.h5') as filename:
        h5file = open_file(filename, mode = "w", title = "Test file")
        group = h5file.create_group("/", 'test', 'Info')
        tab = h5file.create_table(group, 'sample', Test, "Example")
        arow = tab.row
        arow['country'] = 'ab'
        arow['date'] = '20121105'
        arow['location'] = 0
        arow['jobtype']  = 1
        arow['posted_dow']  = 3
        # Insert a new record
        arow.append()
        tab.flush()
        yield h5file
        # Close (and flush) the file
        h5file.close()

@pytest.yield_fixture
def good_csv():

    with tmpfile(".csv") as filename:
        badfile = open(filename, mode="w")
        # Insert a new record
        badfile.write("userid,text,country\n")
        badfile.write("1,Alice,az\n")
        badfile.write("2,Bob,bl\n")
        badfile.write("3,Charlie,cz\n")
        badfile.flush()
        yield badfile
        # Close (and flush) the file
        badfile.close()

@pytest.yield_fixture
def bad_csv_df():

    with tmpfile(".csv") as filename:
        badfile = open(filename, mode="w")
        # Insert a new record
        badfile.write("userid,text,country\n")
        badfile.write("1,Alice,az\n")
        badfile.write("2,Bob,bl\n")
        for i in range(0,100):
            badfile.write(str(i) + ",badguy,zz\n")
        badfile.write("4,Dan,gb,extra,extra\n")
        badfile.flush()
        yield badfile
        # Close (and flush) the file
        badfile.close()

@pytest.yield_fixture
def out_hdf5():
    with tmpfile(".h5") as filename:
        yield filename

@skip_if_not(PyTables and DataFrame)
def test_into_pytables_dataframe(h5):
    samp = h5.root.test.sample
    final = into(DataFrame, samp)
    assert len(final) == 1

@skip_if_not(DataFrame and Python)
def test_pandas_data_descriptor():
    data = [['Alice', 100], ['Bob', 200]]
    schema='{name: string, amount: int}'
    dd = Python(data, schema=schema)
    result = into(DataFrame, dd)
    expected = DataFrame(data, columns=['name', 'amount'])
    print(result)
    print(expected)

    assert str(result) == str(expected)

@skip_if_not(DataFrame and nd.array)
def test_pandas_dynd():
    data = [['Alice', 100], ['Bob', 200]]
    schema='{name: string, amount: int}'

    arr = nd.array(data, dtype=schema)

    result = into(DataFrame, arr)
    expected = DataFrame(data, columns=['name', 'amount'])
    print(result)
    print(expected)

    assert str(result) == str(expected)

@skip_if_not(DataFrame)
def test_pandas_numpy():
    data = [('Alice', 100), ('Bob', 200)]
    dtype=[('name', 'O'), ('amount', int)]

    x = np.array(data, dtype=dtype)

    result = into(DataFrame(), x)
    expected = DataFrame(data, columns=['name', 'amount'])
    assert str(result) == str(expected)

    result = into(DataFrame(columns=['name', 'amount']), x)
    expected = DataFrame(data, columns=['name', 'amount'])
    print(result)
    print(expected)
    assert str(result) == str(expected)

@skip_if_not(DataFrame)
def test_pandas_seq():
    assert str(into(DataFrame, [1, 2])) == \
            str(DataFrame([1, 2]))
    assert str(into(DataFrame, (1, 2))) == \
            str(DataFrame([1, 2]))
    assert str(into(DataFrame(columns=['a', 'b']), [(1, 2), (3, 4)])) == \
            str(DataFrame([[1, 2], [3, 4]], columns=['a', 'b']))

@skip_if_not(DataFrame)
def test_pandas_pandas():
    data = [('Alice', 100), ('Bob', 200)]
    df = DataFrame(data, columns=['name', 'balance'])
    new_df = into(DataFrame, df)
    # Data must be the same
    assert np.all(new_df == df)
    # new_df should be a copy of df
    assert id(new_df) != id(df)

@skip_if_not(DataFrame)
def test_DataFrame_Series():
    data = [('Alice', 100), ('Bob', 200)]
    df = DataFrame(data, columns=['name', 'balance'])

    new_df = into(DataFrame, df['name'])

    assert np.all(new_df == DataFrame([['Alice'], ['Bob']], columns=['name']))

    # new_df should be a copy of df
    assert id(new_df) != id(df['name'])

    assert isinstance(new_df, DataFrame)

def test_discover_ndarray():
    data = [['Alice', 100], ['Bob', 200]]
    schema='{name: string, balance: int32}'
    arr = nd.array(data, dtype=schema)
    assert discover(arr) == 2 * dshape(schema)

@skip_if_not(DataFrame)
def test_discover_pandas():
    data = [['Alice', 100], ['Bob', 200]]
    df = DataFrame(data, columns=['name', 'balance'])

    print(discover(df))
    assert discover(df).subshape[0] == dshape('{name: string, balance: int64}')

@skip_if_not(DataFrame and nd.array)
def test_discover_pandas():
    data = [('Alice', 100), ('Bob', 200)]
    df = DataFrame(data, columns=['name', 'balance'])

    result = into(nd.array, df)

    assert nd.as_py(result, tuple=True) == data

@skip_if_not(Table and DataFrame)
def test_into_table_dataframe():
    data = [['Alice', 100], ['Bob', 200]]
    t = Table(data, columns=['name', 'amount'])

    assert list(into(DataFrame(), t).columns) == list(t.columns)
    assert into([], into(DataFrame(), t)) == list(map(tuple, data))

@skip_if_not(Table and ColumnDataSource)
def test_Column_data_source():
    data = [('Alice', 100), ('Bob', 200)]
    t = Table(data, columns=['name', 'balance'])

    cds = into(ColumnDataSource(), t)

    assert isinstance(cds, ColumnDataSource)
    assert set(cds.column_names) == set(t.columns)

def test_numpy_list():
    data = [('Alice', 100), ('Bob', 200)]

    dtype = into(np.ndarray, data).dtype
    assert np.issubdtype(dtype[0], str)
    assert np.issubdtype(dtype[1], int)

    assert into([], into(np.ndarray, data)) == data

@skip_if_not(Table)
def test_numpy_tableExpr():
    data = [('Alice', 100), ('Bob', 200)]
    t = Table(data, '{name: string, amount: int64}')

    assert into(np.ndarray, t).dtype == \
            np.dtype([('name', 'O'), ('amount', 'i8')])

@skip_if_not(DataFrame and CSV)
def test_DataFrame_CSV():
    with filetext('1,2\n3,4\n') as fn:
        csv = CSV(fn, schema='{a: int64, b: float64}')
        df = into(DataFrame, csv)

        expected = DataFrame([[1, 2.0], [3, 4.0]],
                             columns=['a', 'b'])

        assert str(df) == str(expected)
        assert list(df.dtypes) == [np.int64, np.float64]

@skip_if_not(PyTables and DataFrame)
def test_into_tables_path(good_csv, out_hdf5):
    tble = into(PyTables, good_csv.name, filename=out_hdf5, datapath='foo')
    assert len(tble) == 3

@skip_if_not(PyTables and DataFrame)
def test_into_tables_path_bad_csv(bad_csv_df, out_hdf5):
    tble = into(PyTables, bad_csv_df.name,
                          filename=out_hdf5,
                          datapath='foo',
                          error_bad_lines=False)
    df_from_tbl = into(DataFrame, tble)
    #Check that it's the same as straight from the CSV
    df_from_csv = into(DataFrame, bad_csv_df.name, error_bad_lines=False)
    assert len(df_from_csv) == len(df_from_tbl)
    assert list(df_from_csv.columns) == list(df_from_tbl.columns)
    assert (df_from_csv == df_from_tbl).all().all()

def test_numpy_datetimes():
    L = [datetime(2000, 12, 1), datetime(2000, 1, 1, 1, 1, 1)]
    assert into([], np.array(L, dtype='M8[us]')) == L
    assert into([], np.array(L, dtype='M8[ns]')) == L

def test_numpy_python3_bytes_to_string_conversion():
    x = np.array(['a','b'], dtype='S1')
    assert all(isinstance(s, str) for s in into(list, x))
    x = np.array([(1, 'a'), (2, 'b')], dtype=[('id', 'i4'), ('letter', 'S1')])
    assert isinstance(into(list, x)[0][1], str)
<|endoftext|>


<commit_before>"""Generates constants for use in blinkpy."""

import os

MAJOR_VERSION = 0
MINOR_VERSION = 11
PATCH_VERSION = '0.dev'

__version__ = '{}.{}.{}'.format(MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION)

REQUIRED_PYTHON_VER = (3, 5, 3)

PROJECT_NAME = 'blinkpy'
PROJECT_PACKAGE_NAME = 'blinkpy'
PROJECT_LICENSE = 'MIT'
PROJECT_AUTHOR = 'Kevin Fronczak'
PROJECT_COPYRIGHT = ' 2017, {}'.format(PROJECT_AUTHOR)
PROJECT_URL = 'https://github.com/fronzbot/blinkpy'
PROJECT_EMAIL = 'kfronczak@gmail.com'
PROJECT_DESCRIPTION = ('A Blink camera Python library '
                       'running on Python 3.')
PROJECT_LONG_DESCRIPTION = ('blinkpy is an open-source '
                            'unofficial API for the Blink Camera '
                            'system with the intention for easy '
                            'integration into various home '
                            'automation platforms.')
if os.path.exists('README.rst'):
    PROJECT_LONG_DESCRIPTION = open('README.rst').read()
PROJECT_CLASSIFIERS = [
    'Intended Audience :: Developers',
    'License :: OSI Approved :: MIT License',
    'Operating System :: OS Independent',
    'Programming Language :: Python :: 3.5',
    'Programming Language :: Python :: 3.6',
    'Programming Language :: Python :: 3.7',
    'Topic :: Home Automation'
]

PROJECT_GITHUB_USERNAME = 'fronzbot'
PROJECT_GITHUB_REPOSITORY = 'blinkpy'

PYPI_URL = 'https://pypi.python.org/pypi/{}'.format(PROJECT_PACKAGE_NAME)

'''
URLS
'''
BLINK_URL = 'immedia-semi.com'
LOGIN_URL = 'https://prod.' + BLINK_URL + '/login'
LOGIN_BACKUP_URL = 'https://rest.piri/' + BLINK_URL + '/login'
BASE_URL = 'https://prod.' + BLINK_URL
DEFAULT_URL = 'prod.' + BLINK_URL

'''
Dictionaries
'''
ONLINE = {'online': True, 'offline': False}
<commit_msg>version bump<commit_after>"""Generates constants for use in blinkpy."""

import os

MAJOR_VERSION = 0
MINOR_VERSION = 11
PATCH_VERSION = 0

__version__ = '{}.{}.{}'.format(MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION)

REQUIRED_PYTHON_VER = (3, 5, 3)

PROJECT_NAME = 'blinkpy'
PROJECT_PACKAGE_NAME = 'blinkpy'
PROJECT_LICENSE = 'MIT'
PROJECT_AUTHOR = 'Kevin Fronczak'
PROJECT_COPYRIGHT = ' 2017, {}'.format(PROJECT_AUTHOR)
PROJECT_URL = 'https://github.com/fronzbot/blinkpy'
PROJECT_EMAIL = 'kfronczak@gmail.com'
PROJECT_DESCRIPTION = ('A Blink camera Python library '
                       'running on Python 3.')
PROJECT_LONG_DESCRIPTION = ('blinkpy is an open-source '
                            'unofficial API for the Blink Camera '
                            'system with the intention for easy '
                            'integration into various home '
                            'automation platforms.')
if os.path.exists('README.rst'):
    PROJECT_LONG_DESCRIPTION = open('README.rst').read()
PROJECT_CLASSIFIERS = [
    'Intended Audience :: Developers',
    'License :: OSI Approved :: MIT License',
    'Operating System :: OS Independent',
    'Programming Language :: Python :: 3.5',
    'Programming Language :: Python :: 3.6',
    'Programming Language :: Python :: 3.7',
    'Topic :: Home Automation'
]

PROJECT_GITHUB_USERNAME = 'fronzbot'
PROJECT_GITHUB_REPOSITORY = 'blinkpy'

PYPI_URL = 'https://pypi.python.org/pypi/{}'.format(PROJECT_PACKAGE_NAME)

'''
URLS
'''
BLINK_URL = 'immedia-semi.com'
LOGIN_URL = 'https://prod.' + BLINK_URL + '/login'
LOGIN_BACKUP_URL = 'https://rest.piri/' + BLINK_URL + '/login'
BASE_URL = 'https://prod.' + BLINK_URL
DEFAULT_URL = 'prod.' + BLINK_URL

'''
Dictionaries
'''
ONLINE = {'online': True, 'offline': False}
<|endoftext|>


<commit_before>"""Generates constants for use in blinkpy."""

import os

MAJOR_VERSION = 0
MINOR_VERSION = 11
PATCH_VERSION = 2

__version__ = '{}.{}.{}'.format(MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION)

REQUIRED_PYTHON_VER = (3, 5, 3)

PROJECT_NAME = 'blinkpy'
PROJECT_PACKAGE_NAME = 'blinkpy'
PROJECT_LICENSE = 'MIT'
PROJECT_AUTHOR = 'Kevin Fronczak'
PROJECT_COPYRIGHT = ' 2017, {}'.format(PROJECT_AUTHOR)
PROJECT_URL = 'https://github.com/fronzbot/blinkpy'
PROJECT_EMAIL = 'kfronczak@gmail.com'
PROJECT_DESCRIPTION = ('A Blink camera Python library '
                       'running on Python 3.')
PROJECT_LONG_DESCRIPTION = ('blinkpy is an open-source '
                            'unofficial API for the Blink Camera '
                            'system with the intention for easy '
                            'integration into various home '
                            'automation platforms.')
if os.path.exists('README.rst'):
    PROJECT_LONG_DESCRIPTION = open('README.rst').read()
PROJECT_CLASSIFIERS = [
    'Intended Audience :: Developers',
    'License :: OSI Approved :: MIT License',
    'Operating System :: OS Independent',
    'Programming Language :: Python :: 3.5',
    'Programming Language :: Python :: 3.6',
    'Programming Language :: Python :: 3.7',
    'Topic :: Home Automation'
]

PROJECT_GITHUB_USERNAME = 'fronzbot'
PROJECT_GITHUB_REPOSITORY = 'blinkpy'

PYPI_URL = 'https://pypi.python.org/pypi/{}'.format(PROJECT_PACKAGE_NAME)

'''
URLS
'''
BLINK_URL = 'immedia-semi.com'
DEFAULT_URL = "{}.{}".format('prod', BLINK_URL)
BASE_URL = "https://{}".format(DEFAULT_URL)
LOGIN_URL = "{}/login".format(BASE_URL)
LOGIN_BACKUP_URL = "https://{}.{}/login".format('rest.piri', BLINK_URL)

'''
Dictionaries
'''
ONLINE = {'online': True, 'offline': False}
<commit_msg>Dev version bump<commit_after>"""Generates constants for use in blinkpy."""

import os

MAJOR_VERSION = 0
MINOR_VERSION = 12
PATCH_VERSION = '0.dev'

__version__ = '{}.{}.{}'.format(MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION)

REQUIRED_PYTHON_VER = (3, 5, 3)

PROJECT_NAME = 'blinkpy'
PROJECT_PACKAGE_NAME = 'blinkpy'
PROJECT_LICENSE = 'MIT'
PROJECT_AUTHOR = 'Kevin Fronczak'
PROJECT_COPYRIGHT = ' 2017, {}'.format(PROJECT_AUTHOR)
PROJECT_URL = 'https://github.com/fronzbot/blinkpy'
PROJECT_EMAIL = 'kfronczak@gmail.com'
PROJECT_DESCRIPTION = ('A Blink camera Python library '
                       'running on Python 3.')
PROJECT_LONG_DESCRIPTION = ('blinkpy is an open-source '
                            'unofficial API for the Blink Camera '
                            'system with the intention for easy '
                            'integration into various home '
                            'automation platforms.')
if os.path.exists('README.rst'):
    PROJECT_LONG_DESCRIPTION = open('README.rst').read()
PROJECT_CLASSIFIERS = [
    'Intended Audience :: Developers',
    'License :: OSI Approved :: MIT License',
    'Operating System :: OS Independent',
    'Programming Language :: Python :: 3.5',
    'Programming Language :: Python :: 3.6',
    'Programming Language :: Python :: 3.7',
    'Topic :: Home Automation'
]

PROJECT_GITHUB_USERNAME = 'fronzbot'
PROJECT_GITHUB_REPOSITORY = 'blinkpy'

PYPI_URL = 'https://pypi.python.org/pypi/{}'.format(PROJECT_PACKAGE_NAME)

'''
URLS
'''
BLINK_URL = 'immedia-semi.com'
DEFAULT_URL = "{}.{}".format('prod', BLINK_URL)
BASE_URL = "https://{}".format(DEFAULT_URL)
LOGIN_URL = "{}/login".format(BASE_URL)
LOGIN_BACKUP_URL = "https://{}.{}/login".format('rest.piri', BLINK_URL)

'''
Dictionaries
'''
ONLINE = {'online': True, 'offline': False}
<|endoftext|>


<commit_before>from datetime import date

from bluebottle.activities.states import OrganizerStateMachine
from bluebottle.activities.triggers import (
    ActivityTriggers, ContributorTriggers
)
from bluebottle.deeds.models import Deed, DeedParticipant
from bluebottle.deeds.states import (
    DeedStateMachine, ParticipantStateMachine
)
from bluebottle.fsm.effects import RelatedTransitionEffect, TransitionEffect
from bluebottle.fsm.triggers import (
    register, TransitionTrigger, ModelChangedTrigger
)

def is_started(effect):
    """
    has started
    """
    return (
        effect.instance.start and
        effect.instance.start < date.today()
    )

def is_not_started(effect):
    """
    hasn't started yet
    """
    return not is_started(effect)

def is_finished(effect):
    """
    has finished
    """
    return (
        effect.instance.end and
        effect.instance.end < date.today()
    )

def is_not_finished(effect):
    """
    hasn't finished yet
    """
    return not is_finished(effect)

def has_participants(effect):
    """ has participants"""
    return len(effect.instance.participants) > 0

def has_no_participants(effect):
    """ has accepted participants"""
    return not has_participants(effect)

def has_no_start_date(effect):
    """ has accepted participants"""
    return not effect.instance.start

@register(Deed)
class DeedTriggers(ActivityTriggers):
    triggers = ActivityTriggers.triggers + [
        ModelChangedTrigger('start', effects=[TransitionEffect(DeedStateMachine.start)]),
        ModelChangedTrigger(
            'end',
            effects=[
                TransitionEffect(DeedStateMachine.restart, conditions=[is_started]),
                TransitionEffect(DeedStateMachine.reopen, conditions=[is_not_started]),
                TransitionEffect(DeedStateMachine.succeed, conditions=[is_finished, has_participants]),
                TransitionEffect(DeedStateMachine.expire, conditions=[is_finished, has_no_participants]),
            ]
        ),

        ModelChangedTrigger(
            'start',
            effects=[
                TransitionEffect(DeedStateMachine.start, conditions=[is_started]),
                TransitionEffect(DeedStateMachine.reopen, conditions=[is_not_started]),
            ]
        ),

        TransitionTrigger(
            DeedStateMachine.expire,
            effects=[
                RelatedTransitionEffect('organizer', OrganizerStateMachine.fail),
            ]
        ),

    ]

def activity_is_finished(effect):
    """activity is finished"""
    return (
        effect.instance.activity.end and
        effect.instance.activity.end < date.today()
    )

def activity_will_be_empty(effect):
    """activity will be empty"""
    return len(effect.instance.activity.participants) == 1

@register(DeedParticipant)
class DeedParticipantTriggers(ContributorTriggers):
    triggers = ContributorTriggers.triggers + [
        TransitionTrigger(
            ParticipantStateMachine.remove,
            effects=[
                RelatedTransitionEffect(
                    'activity',
                    DeedStateMachine.expire,
                    conditions=[activity_is_finished, activity_will_be_empty]
                ),
            ]
        ),
        TransitionTrigger(
            ParticipantStateMachine.accept,
            effects=[
                RelatedTransitionEffect(
                    'activity',
                    DeedStateMachine.succeed,
                    conditions=[activity_is_finished]
                ),
            ]
        ),

    ]
<commit_msg>Even more consistent<commit_after>from datetime import date

from bluebottle.activities.states import OrganizerStateMachine
from bluebottle.activities.triggers import (
    ActivityTriggers, ContributorTriggers
)
from bluebottle.deeds.models import Deed, DeedParticipant
from bluebottle.deeds.states import (
    DeedStateMachine, DeedParticipantStateMachine
)
from bluebottle.fsm.effects import RelatedTransitionEffect, TransitionEffect
from bluebottle.fsm.triggers import (
    register, TransitionTrigger, ModelChangedTrigger
)

def is_started(effect):
    """
    has started
    """
    return (
        effect.instance.start and
        effect.instance.start < date.today()
    )

def is_not_started(effect):
    """
    hasn't started yet
    """
    return not is_started(effect)

def is_finished(effect):
    """
    has finished
    """
    return (
        effect.instance.end and
        effect.instance.end < date.today()
    )

def is_not_finished(effect):
    """
    hasn't finished yet
    """
    return not is_finished(effect)

def has_participants(effect):
    """ has participants"""
    return len(effect.instance.participants) > 0

def has_no_participants(effect):
    """ has accepted participants"""
    return not has_participants(effect)

def has_no_start_date(effect):
    """ has accepted participants"""
    return not effect.instance.start

@register(Deed)
class DeedTriggers(ActivityTriggers):
    triggers = ActivityTriggers.triggers + [
        ModelChangedTrigger('start', effects=[TransitionEffect(DeedStateMachine.start)]),
        ModelChangedTrigger(
            'end',
            effects=[
                TransitionEffect(DeedStateMachine.restart, conditions=[is_started]),
                TransitionEffect(DeedStateMachine.reopen, conditions=[is_not_started]),
                TransitionEffect(DeedStateMachine.succeed, conditions=[is_finished, has_participants]),
                TransitionEffect(DeedStateMachine.expire, conditions=[is_finished, has_no_participants]),
            ]
        ),

        ModelChangedTrigger(
            'start',
            effects=[
                TransitionEffect(DeedStateMachine.start, conditions=[is_started]),
                TransitionEffect(DeedStateMachine.reopen, conditions=[is_not_started]),
            ]
        ),

        TransitionTrigger(
            DeedStateMachine.expire,
            effects=[
                RelatedTransitionEffect('organizer', OrganizerStateMachine.fail),
            ]
        ),

    ]

def activity_is_finished(effect):
    """activity is finished"""
    return (
        effect.instance.activity.end and
        effect.instance.activity.end < date.today()
    )

def activity_will_be_empty(effect):
    """activity will be empty"""
    return len(effect.instance.activity.participants) == 1

@register(DeedParticipant)
class DeedParticipantTriggers(ContributorTriggers):
    triggers = ContributorTriggers.triggers + [
        TransitionTrigger(
            DeedParticipantStateMachine.remove,
            effects=[
                RelatedTransitionEffect(
                    'activity',
                    DeedStateMachine.expire,
                    conditions=[activity_is_finished, activity_will_be_empty]
                ),
            ]
        ),
        TransitionTrigger(
            DeedParticipantStateMachine.accept,
            effects=[
                RelatedTransitionEffect(
                    'activity',
                    DeedStateMachine.succeed,
                    conditions=[activity_is_finished]
                ),
            ]
        ),

    ]
<|endoftext|>


<commit_before>#!/bin/false
# This file is part of Espruino, a JavaScript interpreter for Microcontrollers
#
# Copyright (C) 2013 Gordon Williams <gw@pur3.co.uk>
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# ----------------------------------------------------------------------------------------
# This file contains information for a specific board - the available pins, and where LEDs,
# Buttons, and other in-built peripherals are. It is used to build documentation as well
# as various source and header files for Espruino.
# ----------------------------------------------------------------------------------------

import pinutils;
info = {
 'name' : "Espruino Board rev 1.0",
 'link' : [ "http://www.espruino.com/kick" ],
 'variables' : 2000,
};
chip = {
  'part' : "STM32F103RGT6",
  'family' : "STM32F1",
  'package' : "LQFP64",
  'ram' : 96,
  'flash' : 1024,
  'speed' : 72,
  'usart' : 5,
  'spi' : 3,
  'i2c' : 2,
  'adc' : 3,
  'dac' : 2,
};
# left-right, or top-bottom order
board = {
  'top' : [ 'D2', 'C11', 'C10', 'A14', 'A14', 'A13', 'A10', 'A9', 'A8', 'C9', 'C8', 'C7', 'C6', 'B15', 'B14', 'B13', 'B12', '3.3', 'VBAT', 'GND' ],
  'bottom' : [ 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B9', 'C13', 'C14', 'C15', 'C0', 'C1', 'C2', 'C3', 'A0', 'A1', 'A2', '3.3', 'VBAT', 'GND' ],
  'mid' : ['B2', 'B1', 'B0', 'C5', 'C4', 'A7', 'A6', 'A5', 'A4', 'A3' ]
};

devices = {
  'OSC' : { 'pin_in' :  'D0',
            'pin_out' : 'D1' },
  'LED1' : { 'pin' : 'B7' },
  'LED2' : { 'pin' : 'B8' },
  'LED3' : { 'pin' : 'B9' },
  'BTN1' : { 'pin' : 'A3' },
  'USB' : { 'pin_disc' :  'C12',
            'pin_dm' : 'A11',
            'pin_bp' : 'A12' },
  'SD' :  { 'pin_cs' :  'D2',
            'pin_di' :  'B15',
            'pin_do' :  'B14',
            'pin_clk' :  'B13' },
  'BLUETOOTH' : { 'pin_tx' : 'A9',
                  'pin_rx' : 'A10' },
};

board_css = """
#board {
  width: 585px;
  height: 431px;
  top: 280px;
  background-image: url(img/ESPRUINOBOARD_R1_0.jpg);
}
#boardcontainer {
  height: 1000px;
}
#top {
  top: -20px;
  left: 43px;
}

#bottom  {
  top: 431px;
  left: 43px;
}
#mid {
  top: 48px;
  left: 280px;
}
.midpin {
  padding: 3px;
}
""";

def get_pins():
  pins = pinutils.scan_pin_file([], 'stm32f103xe.csv', 6, 10, 11)
  return pinutils.only_from_package(pinutils.fill_gaps_in_pin_list(pins), chip["package"])
<commit_msg>fix espruino 1v0 flash image<commit_after>#!/bin/false
# This file is part of Espruino, a JavaScript interpreter for Microcontrollers
#
# Copyright (C) 2013 Gordon Williams <gw@pur3.co.uk>
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# ----------------------------------------------------------------------------------------
# This file contains information for a specific board - the available pins, and where LEDs,
# Buttons, and other in-built peripherals are. It is used to build documentation as well
# as various source and header files for Espruino.
# ----------------------------------------------------------------------------------------

import pinutils;
info = {
 'name' : "Espruino Board rev 1.0",
 'link' : [ "http://www.espruino.com/kick" ],
 'variables' : 2000,
};
chip = {
  'part' : "STM32F103RGT6",
  'family' : "STM32F1",
  'package' : "LQFP64",
  'ram' : 96,
  'flash' : 512, # 1024, but in 2 banks
  'speed' : 72,
  'usart' : 5,
  'spi' : 3,
  'i2c' : 2,
  'adc' : 3,
  'dac' : 2,
};
# left-right, or top-bottom order
board = {
  'top' : [ 'D2', 'C11', 'C10', 'A14', 'A14', 'A13', 'A10', 'A9', 'A8', 'C9', 'C8', 'C7', 'C6', 'B15', 'B14', 'B13', 'B12', '3.3', 'VBAT', 'GND' ],
  'bottom' : [ 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B9', 'C13', 'C14', 'C15', 'C0', 'C1', 'C2', 'C3', 'A0', 'A1', 'A2', '3.3', 'VBAT', 'GND' ],
  'mid' : ['B2', 'B1', 'B0', 'C5', 'C4', 'A7', 'A6', 'A5', 'A4', 'A3' ]
};

devices = {
  'OSC' : { 'pin_in' :  'D0',
            'pin_out' : 'D1' },
  'LED1' : { 'pin' : 'B7' },
  'LED2' : { 'pin' : 'B8' },
  'LED3' : { 'pin' : 'B9' },
  'BTN1' : { 'pin' : 'A3' },
  'USB' : { 'pin_disc' :  'C12',
            'pin_dm' : 'A11',
            'pin_bp' : 'A12' },
  'SD' :  { 'pin_cs' :  'D2',
            'pin_di' :  'B15',
            'pin_do' :  'B14',
            'pin_clk' :  'B13' },
  'BLUETOOTH' : { 'pin_tx' : 'A9',
                  'pin_rx' : 'A10' },
};

board_css = """
#board {
  width: 585px;
  height: 431px;
  top: 280px;
  background-image: url(img/ESPRUINOBOARD_R1_0.jpg);
}
#boardcontainer {
  height: 1000px;
}
#top {
  top: -20px;
  left: 43px;
}

#bottom  {
  top: 431px;
  left: 43px;
}
#mid {
  top: 48px;
  left: 280px;
}
.midpin {
  padding: 3px;
}
""";

def get_pins():
  pins = pinutils.scan_pin_file([], 'stm32f103xe.csv', 6, 10, 11)
  return pinutils.only_from_package(pinutils.fill_gaps_in_pin_list(pins), chip["package"])
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function

from os import mkdir, remove
from os.path import exists, expanduser, isdir, join, splitext
from sys import stdout
from zipfile import ZipFile
from six.moves.urllib.request import urlopen

def _bokeh_dir(create=False):
    bokeh_dir = expanduser("~/.bokeh")
    if not exists(bokeh_dir):
        if not create: return bokeh_dir
        print("Creating ~/.bokeh directory")
        try:
            mkdir(bokeh_dir)
        except OSError:
            raise RuntimeError("could not create bokeh config directory at %s" % bokeh_dir)
    else:
        if not isdir(bokeh_dir):
            raise RuntimeError("%s exists but is not a directory" % bokeh_dir)
    return bokeh_dir

def _data_dir(file_name=None, create=False):
    try:
        import yaml
    except ImportError:
        raise RuntimeError("'yaml' and 'pyyaml' are required to use bokeh.sampledata functions")
    bokeh_dir = _bokeh_dir(create=create)
    data_dir = join(bokeh_dir, "data")
    try:
        config = yaml.load(open(join(bokeh_dir, 'config')))
        data_dir = expanduser(config['sampledata_dir'])
    except (IOError, TypeError):
        pass
    if not exists(data_dir):
        if not create:
            raise RuntimeError('bokeh sample data directory does not exist, please execute bokeh.sampledata.download()')
        print("Creating %s directory" % data_dir)
        try:
            mkdir(data_dir)
        except OSError:
            raise RuntimeError("could not create bokeh data directory at %s" % data_dir)
    else:
        if not isdir(data_dir):
            raise RuntimeError("%s exists but is not a directory" % data_dir)
    if file_name is not None:
        return join(data_dir, file_name)
    else:
        return data_dir

def download(progress=True):
    '''
    Download larger data sets for various Bokeh examples.
    '''
    data_dir = _data_dir(create=True)
    print("Using data directory: %s" % data_dir)

    s3 = 'https://s3.amazonaws.com/bokeh_data/'
    files = [
        (s3, 'CGM.csv'),
        (s3, 'US_Counties.zip'),
        (s3, 'us_cities.json'),
        (s3, 'unemployment09.csv'),
        (s3, 'AAPL.csv'),
        (s3, 'FB.csv'),
        (s3, 'GOOG.csv'),
        (s3, 'IBM.csv'),
        (s3, 'MSFT.csv'),
        (s3, 'WPP2012_SA_DB03_POPULATION_QUINQUENNIAL.zip'),
        (s3, 'gapminder_fertility.csv'),
        (s3, 'gapminder_population.csv'),
        (s3, 'gapminder_life_expectancy.csv'),
        (s3, 'gapminder_regions.csv'),
        (s3, 'world_cities.zip'),
        (s3, 'airports.json'),
        (s3, 'movies.db.zip'),
    ]

    for base_url, file_name in files:
        _getfile(base_url, file_name, data_dir, progress=progress)

def _getfile(base_url, file_name, data_dir, progress=True):
    file_url = join(base_url, file_name)
    file_path = join(data_dir, file_name)

    url = urlopen(file_url)

    with open(file_path, 'wb') as file:
        file_size = int(url.headers["Content-Length"])
        print("Downloading: %s (%d bytes)" % (file_name, file_size))

        fetch_size = 0
        block_size = 16384

        while True:
            data = url.read(block_size)
            if not data:
                break

            fetch_size += len(data)
            file.write(data)

            if progress:
                status = "\r%10d [%6.2f%%]" % (fetch_size, fetch_size*100.0/file_size)
                stdout.write(status)
                stdout.flush()

    if progress:
        print()

    real_name, ext = splitext(file_name)

    if ext == '.zip':
        if not splitext(real_name)[1]:
            real_name += ".csv"

        print("Unpacking: %s" % real_name)

        with ZipFile(file_path, 'r') as zip_file:
            zip_file.extract(real_name, data_dir)

        remove(file_path)
<commit_msg>Fix default bokeh dir on Windows<commit_after>from __future__ import absolute_import, print_function

from os import mkdir, remove
from os.path import exists, expanduser, isdir, join, splitext
from sys import stdout
from zipfile import ZipFile
from six.moves.urllib.request import urlopen

def _bokeh_dir(create=False):
    bokeh_dir = join(expanduser("~"), ".bokeh")
    if not exists(bokeh_dir):
        if not create: return bokeh_dir
        print("Creating %s directory" % bokeh_dir)
        try:
            mkdir(bokeh_dir)
        except OSError:
            raise RuntimeError("could not create bokeh config directory at %s" % bokeh_dir)
    else:
        if not isdir(bokeh_dir):
            raise RuntimeError("%s exists but is not a directory" % bokeh_dir)
    return bokeh_dir

def _data_dir(file_name=None, create=False):
    try:
        import yaml
    except ImportError:
        raise RuntimeError("'yaml' and 'pyyaml' are required to use bokeh.sampledata functions")
    bokeh_dir = _bokeh_dir(create=create)
    data_dir = join(bokeh_dir, "data")
    try:
        config = yaml.load(open(join(bokeh_dir, 'config')))
        data_dir = expanduser(config['sampledata_dir'])
    except (IOError, TypeError):
        pass
    if not exists(data_dir):
        if not create:
            raise RuntimeError('bokeh sample data directory does not exist, please execute bokeh.sampledata.download()')
        print("Creating %s directory" % data_dir)
        try:
            mkdir(data_dir)
        except OSError:
            raise RuntimeError("could not create bokeh data directory at %s" % data_dir)
    else:
        if not isdir(data_dir):
            raise RuntimeError("%s exists but is not a directory" % data_dir)
    if file_name is not None:
        return join(data_dir, file_name)
    else:
        return data_dir

def download(progress=True):
    '''
    Download larger data sets for various Bokeh examples.
    '''
    data_dir = _data_dir(create=True)
    print("Using data directory: %s" % data_dir)

    s3 = 'https://s3.amazonaws.com/bokeh_data/'
    files = [
        (s3, 'CGM.csv'),
        (s3, 'US_Counties.zip'),
        (s3, 'us_cities.json'),
        (s3, 'unemployment09.csv'),
        (s3, 'AAPL.csv'),
        (s3, 'FB.csv'),
        (s3, 'GOOG.csv'),
        (s3, 'IBM.csv'),
        (s3, 'MSFT.csv'),
        (s3, 'WPP2012_SA_DB03_POPULATION_QUINQUENNIAL.zip'),
        (s3, 'gapminder_fertility.csv'),
        (s3, 'gapminder_population.csv'),
        (s3, 'gapminder_life_expectancy.csv'),
        (s3, 'gapminder_regions.csv'),
        (s3, 'world_cities.zip'),
        (s3, 'airports.json'),
        (s3, 'movies.db.zip'),
    ]

    for base_url, file_name in files:
        _getfile(base_url, file_name, data_dir, progress=progress)

def _getfile(base_url, file_name, data_dir, progress=True):
    file_url = join(base_url, file_name)
    file_path = join(data_dir, file_name)

    url = urlopen(file_url)

    with open(file_path, 'wb') as file:
        file_size = int(url.headers["Content-Length"])
        print("Downloading: %s (%d bytes)" % (file_name, file_size))

        fetch_size = 0
        block_size = 16384

        while True:
            data = url.read(block_size)
            if not data:
                break

            fetch_size += len(data)
            file.write(data)

            if progress:
                status = "\r%10d [%6.2f%%]" % (fetch_size, fetch_size*100.0/file_size)
                stdout.write(status)
                stdout.flush()

    if progress:
        print()

    real_name, ext = splitext(file_name)

    if ext == '.zip':
        if not splitext(real_name)[1]:
            real_name += ".csv"

        print("Unpacking: %s" % real_name)

        with ZipFile(file_path, 'r') as zip_file:
            zip_file.extract(real_name, data_dir)

        remove(file_path)
<|endoftext|>


<commit_before>from django.contrib.sites.models import Site
from django.conf import settings
import types

def site(request):
    """
    Adds site-related context variables to the context.
    """
    current_site = Site.objects.get_current()

    return {
        'SITE_NAME': current_site.name,
        'SITE_DOMAIN': current_site.domain,
        'SITE_URL': "http://www.%s" % (current_site.domain),
    }

# see "css_classes" below
CSS_CLASSES = {
    # for "blueprint css", full width
    'blueprint': {
        'content_main_2col':    'span-13 prepend-1 colborder',
        'content_related':      'span-8 last',
        'content_main':         'span-22 prepend-1 append-1',
        'nav':                  'span-24 last',
        'public_vcard_content': 'span-10',
        'public_vcard_map':     'span-10',
        'overview_avatar':      'span-3',
        'overview_details':     'span-9 last',
        'overview_location':    'span-12 last',
    },
    # for "960gs" with 12 columns, full width
    '960gs-12': {
        'content_main_2col':    'grid_8',
        'content_related':      'grid_4',
        'content_main':         'grid_12',
        'nav':                  'grid_12',
        'public_vcard_content': 'grid_6 alpha',
        'public_vcard_map':     'grid_6 omega',
        'overview_avatar':      'grid_2 alpha',
        'overview_details':     'grid_6 omega',
        'overview_location':    'grid_8 alpha omega',
    },
    # for "960gs" with 16 columns, full width
    '960gs-16': {
        'content_main_2col':    'grid_12',
        'content_related':      'grid_4',
        'content_main':         'grid_16',
        'nav':                  'grid_16',
        'public_vcard_content': 'grid_8 alpha',
        'public_vcard_map':     'grid_8 omega',
        'overview_avatar':      'grid_3 alpha',
        'overview_details':     'grid_9 omega',
        'overview_location':    'grid_12 alpha omega',
    },
    # for "960gs" with 12 columns, inside a width of 9 columns (as example)
    '960gs-12-in-9': {
        'content_main_2col':    'grid_6 alpha omega',
        'content_related':      'grid_3 omega',
        'content_main':         'grid_9 alpha',
        'nav':                  'grid_9 alpha omega',
        'public_vcard_content': 'grid_4 alpha',
        'public_vcard_map':     'grid_5 omega',
        'overview_avatar':      'grid_2 alpha',
        'overview_details':     'grid_6 omega',
        'overview_location':    'grid_6 alpha omega',
    },
}

def css_classes(request):
    """
    If USERPROFILE_CSS_CLASSES  is in the settings, it should be a string
    (a key for the above dict "CSS_CLASSES", or a dict, with the same
    keys as "CSS_CLASSE"
    In templates, you should use
        class="{{css_classes.key}}"
    with key one of keys described above.
    """
    try:
        if isinstance(settings.USERPROFILE_CSS_CLASSES, types.StringTypes) \
            and settings.USERPROFILE_CSS_CLASSES in CSS_CLASSES:
            css_dict = CSS_CLASSES[settings.USERPROFILE_CSS_CLASSES]
        else:
            css_dict = settings.USERPROFILE_CSS_CLASSES
    except:
        css_dict = CSS_CLASSES['blueprint']

    return { 'css_classes': css_dict }
<commit_msg>update css classes for 960gs (all block must be included in a grid_12 or grid_16)<commit_after>from django.contrib.sites.models import Site
from django.conf import settings
import types

def site(request):
    """
    Adds site-related context variables to the context.
    """
    current_site = Site.objects.get_current()

    return {
        'SITE_NAME': current_site.name,
        'SITE_DOMAIN': current_site.domain,
        'SITE_URL': "http://www.%s" % (current_site.domain),
    }

# see "css_classes" below
CSS_CLASSES = {
    # for "blueprint css", full width
    'blueprint': {
        'content_main_2col':    'span-13 prepend-1 colborder',
        'content_related':      'span-8 last',
        'content_main':         'span-22 prepend-1 append-1',
        'nav':                  'span-24 last',
        'public_vcard_content': 'span-10',
        'public_vcard_map':     'span-10',
        'overview_avatar':      'span-3',
        'overview_details':     'span-9 last',
        'overview_location':    'span-12 last',
    },
    # for "960gs" with 12 columns, full width
    '960gs-12': {
        'content_main_2col':    'grid_8 alpha',
        'content_related':      'grid_4 omega',
        'content_main':         'grid_12 alpha omega',
        'nav':                  'grid_12 alpha omega',
        'public_vcard_content': 'grid_6 alpha',
        'public_vcard_map':     'grid_6 omega',
        'overview_avatar':      'grid_2 alpha',
        'overview_details':     'grid_6 omega',
        'overview_location':    'grid_8 alpha omega',
    },
    # for "960gs" with 16 columns, full width
    '960gs-16': {
        'content_main_2col':    'grid_12 alpha',
        'content_related':      'grid_4 omega',
        'content_main':         'grid_16 alpha omega',
        'nav':                  'grid_16 alpha omega',
        'public_vcard_content': 'grid_8 alpha',
        'public_vcard_map':     'grid_8 omega',
        'overview_avatar':      'grid_3 alpha',
        'overview_details':     'grid_9 omega',
        'overview_location':    'grid_12 alpha omega',
    },
    # for "960gs" with 12 columns, inside a width of 9 columns (as example)
    '960gs-12-in-9': {
        'content_main_2col':    'grid_6 alpha omega',
        'content_related':      'grid_3 omega',
        'content_main':         'grid_9 alpha',
        'nav':                  'grid_9 alpha omega',
        'public_vcard_content': 'grid_4 alpha',
        'public_vcard_map':     'grid_5 omega',
        'overview_avatar':      'grid_2 alpha',
        'overview_details':     'grid_6 omega',
        'overview_location':    'grid_6 alpha omega',
    },
}

def css_classes(request):
    """
    If USERPROFILE_CSS_CLASSES  is in the settings, it should be a string
    (a key for the above dict "CSS_CLASSES", or a dict, with the same
    keys as "CSS_CLASSE"
    In templates, you should use
        class="{{css_classes.key}}"
    with key one of keys described above.
    """
    try:
        if isinstance(settings.USERPROFILE_CSS_CLASSES, types.StringTypes) \
            and settings.USERPROFILE_CSS_CLASSES in CSS_CLASSES:
            css_dict = CSS_CLASSES[settings.USERPROFILE_CSS_CLASSES]
        else:
            css_dict = settings.USERPROFILE_CSS_CLASSES
    except:
        css_dict = CSS_CLASSES['blueprint']

    return { 'css_classes': css_dict }
<|endoftext|>


<commit_before>from functools import lru_cache
import os
import glob

import numpy as np
import pymultinest as pmn

from . import model
from . import photometry
from . import spectrum
from . import filter
from .utils import SdfError
from . import config as cfg

# these are for getting info to pymultinest
global_obs = ()
global_mod = ()
global_p_rng = ()

@lru_cache(maxsize=2)
def concat_obs(o):
    """Concatenate observations

    Concatenate the observations (filters and spectra), not
    shifting the normalisation of the spectra, but noting how
    many observations there are in each component (as the models
    will probably contain more due to colours/indices).

    """

    obs_wav = np.array([],dtype=float)
    obs_filt = np.array([],dtype=object)
    obs_fnu = np.array([],dtype=float)
    obs_e_fnu = np.array([],dtype=float)
    obs_uplim = np.array([],dtype=bool)
    obs_ignore = np.array([],dtype=bool)
    obs_bibcode = np.array([],dtype=str)
    obs_nel = np.array([],dtype=int)
    ispec = -2 # start one extra from end, we will put -1 there for phot
    obs_ispec = np.array([],dtype=int)
    for obs in o:
        if isinstance(obs,photometry.Photometry):
            obs_wav = np.append(obs_wav,obs.mean_wavelength())
            obs_filt = np.append(obs_fnu,obs.filters)
            obs_fnu = np.append(obs_fnu,obs.fnujy)
            obs_e_fnu = np.append(obs_e_fnu,obs.e_fnujy)
            obs_ispec = np.append(obs_ispec,np.repeat(-1,len(obs.filters)))
            obs_uplim = np.append(obs_uplim,obs.upperlim)
            obs_ignore = np.append(obs_ignore,obs.ignore)
            obs_bibcode = np.append(obs_bibcode,obs.bibcode)
        elif isinstance(obs,spectrum.ObsSpectrum):
            n = len(obs.wavelength)
            obs_wav = np.append(obs_wav,obs.wavelength)
            obs_filt = np.append(obs_filt,np.repeat(None,n))
            obs_fnu = np.append(obs_fnu,obs.fnujy)
            obs_e_fnu = np.append(obs_e_fnu,obs.e_fnujy)
            obs_ispec = np.append(obs_ispec,np.repeat(ispec,n))
            ispec -= 1
            obs_uplim = np.append(obs_uplim,np.zeros(n,dtype=bool))
            obs_ignore = np.append(obs_ignore,np.zeros(n,dtype=bool))
            obs_bibcode = np.append(obs_bibcode,np.repeat(obs.bibcode,n))
        obs_nel = np.append(obs_nel,len(obs.fnujy))

    return (obs_fnu,obs_e_fnu,obs_uplim,obs_ignore,obs_ispec,
           obs_nel,obs_wav,obs_filt,obs_bibcode)

def residual(param,*args):
    """Return residuals for a model compared to observation.

    The structure of the observation and models needs to
    match, and the filters and/or wavelengths present within
    each of the correspondong observation and model obects
    must match.

    For pure photometry the args would be:
    (Photometry,),(PhotModel,)

    Where there are spectra, the parameters for a given set of
    (PhotModel,SpecModel) are the same, and the args would be:
    (Photometry,ObsSpectrum),((PhotModel,SpecModel),)

    For combining multiple models the args would be:
    (Ph,ObsSp),((PhMod,SpMod),(PhMod,SpMod))

    The parameters are ordered similarly, with each tuple of
    models sharing the same parameters, and with the normalisations
    for observed spectra last, in the same order as the spectra.

    To deal with colours/indices, which need the model components
    to be added first, and then derived, PhotModels have extra
    elements with the colour/index base filters, which are used
    to derive the colours/indices as we go.

    """

    o,m = args # observations and models

    # concatenate observations
    obs_fnu,obs_e_fnu,obs_uplim,obs_ignore,obs_ispec,\
            obs_nel,obs_wav,obs_filt,_ = concat_obs(o)

    # multiply spectra by appropriate normalisation, ispec starts at
    # -2 so we can add 1.0 for photometry at the end of the params
    spec_norm = np.take(np.append(param,1.0),obs_ispec)
    obs_fnu = obs_fnu * spec_norm
    obs_e_fnu = obs_e_fnu * spec_norm

    # get model fluxes, including filling of colours/indices
    mod_fnu,_ = model.model_fluxes(m,param,obs_nel)

    # residuals in significance units, setting zero where (photometry)
    # is to be ignored, and for upper limits (but amended below)
    resid = np.zeros(len(obs_fnu))
    ok = np.invert( np.any([obs_uplim,obs_ignore],axis=0) )
    resid[ok] = (obs_fnu[ok] - mod_fnu[ok]) / obs_e_fnu[ok]

    # set residual if any upper limits exceeded, otherwise zero
    for i,lim in enumerate(obs_uplim):
        if lim:
            if mod_fnu[i] > obs_fnu[i]:
                resid[i] = (obs_fnu[i]-mod_fnu[i])/(obs_fnu[i]/3.)

    return resid,obs_wav,obs_filt

def residual_phot(param,*args):
    """Return residuals for a model compared to observation.

    This is an unused version for only photometry.
    """

    p,m = args # photometry and model

    # get model fluxes
    if isinstance(m,(tuple,list)):
        model_fnujy = np.zeros(p.nused)
        i0 = 0
        for mod in m:
            nparam = len(mod.parameters)+1
            flux = mod.fnujy(param[i0:i0+nparam])
            model_fnujy = model_fnujy + flux
            i0 += nparam
    else:
        model_fnujy = m.fnujy(param)

    # residuals in significance units
    resid = np.zeros(p.nused)
    ok = np.invert(p.upperlim)
    resid[ok] = (p.fnujy[ok] - model_fnujy[ok]) / p.e_fnujy[ok]

    # set residual if any upper limits exceeded, otherwise zero
    # (e_fnujy may be zero so resid may be nan)
    if np.any(p.upperlim):
        lim = p.upperlim
        if model_fnujy[lim] > p.fnujy[lim]:
            resid[lim] = (p.fnujy[lim]-model_fnujy[lim])/(p.fnujy[lim]/3.)
        else:
            resid[lim] = np.zeros(np.sum(p.upperlim))

    print(resid)
    return resid

def chisq(param,*args):
    """Return sum of squared residuals."""

    res,_,_ = residual(param,*args)
    return np.sum( np.square( res ) )

def lnlike(param,*args):
    """Return log likelihood."""

    chi2 = chisq(param,*args)
    if np.isfinite(chi2):
        return -0.5 * chi2
    else:
        return -np.inf

def multinest_prior(cube,ndim,nparam):
    """Prior for pymultinest, turns values given in each element
    of cube from range 0-1 to the range for that parameter
    """

    # get parameter ranges
    global global_p_rng
    pars = global_p_rng

    for i in range(ndim):
        cube[i] = pars[i][0] + cube[i] * (pars[i][1]-pars[i][0])

def multinest_lnlike(cube,ndim,nparam):
    """Return log likelihood."""

    global global_obs,global_mod
    o,m = (global_obs,global_mod)
    param = np.array([])
    for i in range(ndim):
        param = np.append(param,cube[i])
    return lnlike(param,o,m)

def multinest(o,m,dir):
    """Run pymultinest to fit model(s) to photometry."""

    dir = dir.rstrip('/')

    m_info = model.models_info(m)
    pmn_out = dir+'/'+m_info['name']+cfg.fitting['pmn_model_suffix']
    global global_obs,global_mod,global_p_rng
    global_obs = o
    global_mod = m
    global_p_rng = m_info['p_rng']

    pmn.run(multinest_lnlike,multinest_prior,m_info['ndim'],
            n_live_points=cfg.fitting['n_live'],
            n_iter_before_update=cfg.fitting['n_update'],
            multimodal=True,sampling_efficiency=0.3,
            verbose=cfg.fitting['verb'],
            outputfiles_basename=pmn_out)

def pmn_models(dir):
    """Return the models fitted by multinest."""

    fs = glob.glob( dir + '/*' + cfg.fitting['pmn_model_suffix'] + '.txt' )
    models = ()
    mbase = []
    for f in fs:
        pmn_out = f.rstrip('.txt')
        mbase.append(pmn_out)
        tmp = pmn_out.rstrip(cfg.fitting['pmn_model_suffix'])
        tmp = os.path.basename(tmp)
        models += (tmp.split(cfg.fitting['model_join']),)

    return mbase,models

def sort_evidence(ev_in,ndim):
    """Return argsort for models using evidence.

    For a model with more parameters to be preferred, the log evidence
    must be more than ev_threshold higher (just higher if the number
    of dimensions is the same). Method is to initially sort models by
    evidence, and then go through and change things based on these
    criteria we are done.
    """

    if len(ev_in) != len(ndim):
        raise SdfError("length of ev_in ({}) and ndim ({}) not equal".\
                       format(ev_in,ndim))
    if isinstance(ev_in,list):
        ev_in = np.array(ev_in)
    if isinstance(ndim,list):
        ndim = np.array(ndim)

#    print(ev_in,ndim)
    srt = np.argsort(ndim)
    ev_srt = ev_in[srt]
    dim_srt = ndim[srt]
    order = np.array(srt)
    last = np.zeros(len(order))
#    print(ev_srt,order)

    while not np.all( np.equal(order,last) ):
        last = np.array(order)
        for i in range(len(ndim)-1):
            if ev_in[order][i+1] > ev_in[order][i]+cfg.fitting['ev_threshold']\
               or ndim[order][i+1] == ndim[order][i]\
                  and ev_in[order][i+1] > ev_in[order][i]:
                tmp = order[i+1]
                order[i+1] = order[i]
                order[i] = tmp
#            print(i,ev_in[order])

#    print(order)
    return order
<commit_msg>change upper limit chi calculation<commit_after>from functools import lru_cache
import os
import glob

import numpy as np
import pymultinest as pmn

from . import model
from . import photometry
from . import spectrum
from . import filter
from .utils import SdfError
from . import config as cfg

# these are for getting info to pymultinest
global_obs = ()
global_mod = ()
global_p_rng = ()

@lru_cache(maxsize=2)
def concat_obs(o):
    """Concatenate observations

    Concatenate the observations (filters and spectra), not
    shifting the normalisation of the spectra, but noting how
    many observations there are in each component (as the models
    will probably contain more due to colours/indices).

    """

    obs_wav = np.array([],dtype=float)
    obs_filt = np.array([],dtype=object)
    obs_fnu = np.array([],dtype=float)
    obs_e_fnu = np.array([],dtype=float)
    obs_uplim = np.array([],dtype=bool)
    obs_ignore = np.array([],dtype=bool)
    obs_bibcode = np.array([],dtype=str)
    obs_nel = np.array([],dtype=int)
    ispec = -2 # start one extra from end, we will put -1 there for phot
    obs_ispec = np.array([],dtype=int)
    for obs in o:
        if isinstance(obs,photometry.Photometry):
            obs_wav = np.append(obs_wav,obs.mean_wavelength())
            obs_filt = np.append(obs_fnu,obs.filters)
            obs_fnu = np.append(obs_fnu,obs.fnujy)
            obs_e_fnu = np.append(obs_e_fnu,obs.e_fnujy)
            obs_ispec = np.append(obs_ispec,np.repeat(-1,len(obs.filters)))
            obs_uplim = np.append(obs_uplim,obs.upperlim)
            obs_ignore = np.append(obs_ignore,obs.ignore)
            obs_bibcode = np.append(obs_bibcode,obs.bibcode)
        elif isinstance(obs,spectrum.ObsSpectrum):
            n = len(obs.wavelength)
            obs_wav = np.append(obs_wav,obs.wavelength)
            obs_filt = np.append(obs_filt,np.repeat(None,n))
            obs_fnu = np.append(obs_fnu,obs.fnujy)
            obs_e_fnu = np.append(obs_e_fnu,obs.e_fnujy)
            obs_ispec = np.append(obs_ispec,np.repeat(ispec,n))
            ispec -= 1
            obs_uplim = np.append(obs_uplim,np.zeros(n,dtype=bool))
            obs_ignore = np.append(obs_ignore,np.zeros(n,dtype=bool))
            obs_bibcode = np.append(obs_bibcode,np.repeat(obs.bibcode,n))
        obs_nel = np.append(obs_nel,len(obs.fnujy))

    return (obs_fnu,obs_e_fnu,obs_uplim,obs_ignore,obs_ispec,
           obs_nel,obs_wav,obs_filt,obs_bibcode)

def residual(param,*args):
    """Return residuals for a model compared to observation.

    The structure of the observation and models needs to
    match, and the filters and/or wavelengths present within
    each of the correspondong observation and model obects
    must match.

    For pure photometry the args would be:
    (Photometry,),(PhotModel,)

    Where there are spectra, the parameters for a given set of
    (PhotModel,SpecModel) are the same, and the args would be:
    (Photometry,ObsSpectrum),((PhotModel,SpecModel),)

    For combining multiple models the args would be:
    (Ph,ObsSp),((PhMod,SpMod),(PhMod,SpMod))

    The parameters are ordered similarly, with each tuple of
    models sharing the same parameters, and with the normalisations
    for observed spectra last, in the same order as the spectra.

    To deal with colours/indices, which need the model components
    to be added first, and then derived, PhotModels have extra
    elements with the colour/index base filters, which are used
    to derive the colours/indices as we go.

    """

    o,m = args # observations and models

    # concatenate observations
    obs_fnu,obs_e_fnu,obs_uplim,obs_ignore,obs_ispec,\
            obs_nel,obs_wav,obs_filt,_ = concat_obs(o)

    # multiply spectra by appropriate normalisation, ispec starts at
    # -2 so we can add 1.0 for photometry at the end of the params
    spec_norm = np.take(np.append(param,1.0),obs_ispec)
    obs_fnu = obs_fnu * spec_norm
    obs_e_fnu = obs_e_fnu * spec_norm

    # get model fluxes, including filling of colours/indices
    mod_fnu,_ = model.model_fluxes(m,param,obs_nel)

    # residuals in significance units, setting zero where (photometry)
    # is to be ignored, and for upper limits (but amended below)
    resid = np.zeros(len(obs_fnu))
    ok = np.invert( np.any([obs_uplim,obs_ignore],axis=0) )
    resid[ok] = (obs_fnu[ok] - mod_fnu[ok]) / obs_e_fnu[ok]

    # set residual if any 3sigma upper limits exceeded at the 1sigma
    # level, otherwise zero as set above
    for i,lim in enumerate(obs_uplim):
        if lim:
            if mod_fnu[i] > obs_fnu[i]/3.:
                resid[i] = mod_fnu[i] / (obs_fnu[i]/3.)

    return resid,obs_wav,obs_filt

def residual_phot(param,*args):
    """Return residuals for a model compared to observation.

    This is an unused version for only photometry.
    """

    p,m = args # photometry and model

    # get model fluxes
    if isinstance(m,(tuple,list)):
        model_fnujy = np.zeros(p.nused)
        i0 = 0
        for mod in m:
            nparam = len(mod.parameters)+1
            flux = mod.fnujy(param[i0:i0+nparam])
            model_fnujy = model_fnujy + flux
            i0 += nparam
    else:
        model_fnujy = m.fnujy(param)

    # residuals in significance units
    resid = np.zeros(p.nused)
    ok = np.invert(p.upperlim)
    resid[ok] = (p.fnujy[ok] - model_fnujy[ok]) / p.e_fnujy[ok]

    # set residual if any upper limits exceeded, otherwise zero
    # (e_fnujy may be zero so resid may be nan)
    if np.any(p.upperlim):
        lim = p.upperlim
        if model_fnujy[lim] > p.fnujy[lim]:
            resid[lim] = (p.fnujy[lim]-model_fnujy[lim])/(p.fnujy[lim]/3.)
        else:
            resid[lim] = np.zeros(np.sum(p.upperlim))

    print(resid)
    return resid

def chisq(param,*args):
    """Return sum of squared residuals."""

    res,_,_ = residual(param,*args)
    return np.sum( np.square( res ) )

def lnlike(param,*args):
    """Return log likelihood."""

    chi2 = chisq(param,*args)
    if np.isfinite(chi2):
        return -0.5 * chi2
    else:
        return -np.inf

def multinest_prior(cube,ndim,nparam):
    """Prior for pymultinest, turns values given in each element
    of cube from range 0-1 to the range for that parameter
    """

    # get parameter ranges
    global global_p_rng
    pars = global_p_rng

    for i in range(ndim):
        cube[i] = pars[i][0] + cube[i] * (pars[i][1]-pars[i][0])

def multinest_lnlike(cube,ndim,nparam):
    """Return log likelihood."""

    global global_obs,global_mod
    o,m = (global_obs,global_mod)
    param = np.array([])
    for i in range(ndim):
        param = np.append(param,cube[i])
    return lnlike(param,o,m)

def multinest(o,m,dir):
    """Run pymultinest to fit model(s) to photometry."""

    dir = dir.rstrip('/')

    m_info = model.models_info(m)
    pmn_out = dir+'/'+m_info['name']+cfg.fitting['pmn_model_suffix']
    global global_obs,global_mod,global_p_rng
    global_obs = o
    global_mod = m
    global_p_rng = m_info['p_rng']

    pmn.run(multinest_lnlike,multinest_prior,m_info['ndim'],
            n_live_points=cfg.fitting['n_live'],
            n_iter_before_update=cfg.fitting['n_update'],
            multimodal=True,sampling_efficiency=0.3,
            verbose=cfg.fitting['verb'],
            outputfiles_basename=pmn_out)

def pmn_models(dir):
    """Return the models fitted by multinest."""

    fs = glob.glob( dir + '/*' + cfg.fitting['pmn_model_suffix'] + '.txt' )
    models = ()
    mbase = []
    for f in fs:
        pmn_out = f.rstrip('.txt')
        mbase.append(pmn_out)
        tmp = pmn_out.rstrip(cfg.fitting['pmn_model_suffix'])
        tmp = os.path.basename(tmp)
        models += (tmp.split(cfg.fitting['model_join']),)

    return mbase,models

def sort_evidence(ev_in,ndim):
    """Return argsort for models using evidence.

    For a model with more parameters to be preferred, the log evidence
    must be more than ev_threshold higher (just higher if the number
    of dimensions is the same). Method is to initially sort models by
    evidence, and then go through and change things based on these
    criteria we are done.
    """

    if len(ev_in) != len(ndim):
        raise SdfError("length of ev_in ({}) and ndim ({}) not equal".\
                       format(ev_in,ndim))
    if isinstance(ev_in,list):
        ev_in = np.array(ev_in)
    if isinstance(ndim,list):
        ndim = np.array(ndim)

#    print(ev_in,ndim)
    srt = np.argsort(ndim)
    ev_srt = ev_in[srt]
    dim_srt = ndim[srt]
    order = np.array(srt)
    last = np.zeros(len(order))
#    print(ev_srt,order)

    while not np.all( np.equal(order,last) ):
        last = np.array(order)
        for i in range(len(ndim)-1):
            if ev_in[order][i+1] > ev_in[order][i]+cfg.fitting['ev_threshold']\
               or ndim[order][i+1] == ndim[order][i]\
                  and ev_in[order][i+1] > ev_in[order][i]:
                tmp = order[i+1]
                order[i+1] = order[i]
                order[i] = tmp
#            print(i,ev_in[order])

#    print(order)
    return order
<|endoftext|>