<commit_before>##!/usr/local/bin/ python3.5
# coding: utf-8
# myoscore.py aim to calculate a unique score based on prediction tools in order
# to prioritize variants with a unique score

# ==============================================================================
# IMPORT LIBRARIES

import vcf		# read vcf => PyVCF :https://pyvcf.readthedocs.io/en/latest/
import sys		# system command
import csv		# read, write csv
import re		# regex
import argparse	# for options
import os		# for options
import pprint	# pretty print

# ==============================================================================

def parse(args):
	"""Parse arguments
    Keyword arguments:
	    args -- command line arguments
	Return :
		args.input	-- input file
		argd.output	-- output file
    """
	# Initialise parser
	parser = argparse.ArgumentParser(
		description='Calculating the meta score on annotated vcf')

	# Add group for mandatory arguments
	mandatoryArgs = parser.add_argument_group('mandatory arguments')
	## input file
	mandatoryArgs.add_argument('-i','--input', metavar='file.vcf',
		type=lambda arg: is_valid_path(parser, arg), help='vcf file annotated',
		required=True)
	## output file
	mandatoryArgs.add_argument('-o','--output', metavar='file.csv',
		help='generated .csv file with variants and score', required=True)

	args = parser.parse_args()
	return(
		args.input,
		args.output,
	)
# ==============================================================================

def score(exonicFunction = ".", ref = ".", alt = ".", function = ".",
		ADAscore = ".", RFscore = ".", Zscore = ".", SIFT = ".", HDIV = ".",
		HVAR = ".", LRT = ".", MutationTaster = ".", FATHMM = ".",
		PROVEAN = ".", MKL = ".", SVM = ".", LR = ".", CLINSIG = ".",):

	"""
	Calculate a score based on pathogenicity

    Keyword arguments:
	    exonicFunction	-- ExonicFunc_refGene (default ".")
		ref				-- ref (default ".")
		alt				-- alt (default ".")
		function		-- Func_refGene (default ".")
		ADAscore		-- SNV_ADA_SCORE (default ".")
		RFscore			-- dbscSNV_RF_SCORE (default ".")
		Zscore			-- dpsi_zscore (default ".")
		SIFT			-- SIFT_pred (default ".")
		HDIV			-- Polyphen2_HDIV_pred (default ".")
		HVAR			-- Polyphen2_HVAR_pred (default ".")
		LRT				-- LRT_pred (default ".")
		MutationTaster	-- MutationTaster_pred (default ".")
		FATHMM			-- FATHMM_pred (default ".")
		PROVEAN			-- PROVEAN_pred (default ".")
		MKL				-- fathmm_MKL_coding_pred (default ".")
		SVM				-- MetaSVM_pred (default ".")
		LR				-- MetaLR_pred (default ".")

	Return tuple 'score','level' (score max = 10, min = 0):
		'10db','clinvar'	: deleterious impact (10) reported as so in ClinVar
		'10sfs','stop'		: deleterious impact (10) by stop gain or loss
		'10sfs','frameshift': deleterious impact (10) by frameshift indel
		'10sp','splice'		: deleterious impact (10) on splicing
		'u','na'			: unknown exonic function
		'na','0'			: cannot predict impact ; no scores prediction
		'3.3','3/9'			: score of impact (3.3) ; 3 deleterious score on 9
    """

	# Create a dictionnary of impact score tools used
	scores_impact = {"SIFT" : SIFT, "HDIV" : HDIV, "HVAR" : HVAR, "LRT" : LRT,
		"MutationTaster" : MutationTaster, "FATHMM" : FATHMM,
		"PROVEAN" : PROVEAN, "MKL" : MKL, "SVM" : SVM, "LR" : LR}
# for record in vcf_reader:
# 	diff = ((len(record.REF)) - (len(str(record.ALT[0]))))
#
# 	if diff %3 != 0 and record.INFO['Func.refGene'][0] == "exonic" :
# 		print ("diff : ", diff)
# 		print ("Func.refGene : ", record.INFO['Func.refGene'][0])
# 		print ("ExonicFunc.refGene : ", record.INFO['ExonicFunc.refGene']),
# 		print ("longueur REF :", len(record.REF), "longueur ALT :", len(str((record.ALT[0]))))
# 		print ("(lg.REF - lg.ALT) :", diff)
# 		print ("REF : ", record.REF, "ALT : ", record.ALT)
# 		print (str(record.INFO['Gene.refGene'][0]))
	# Test and calculate the score.
	deleterious = 0
	available = 0
	score_adjusted = 0

	for score, impact in scores_impact.items():
		if(impact == "D"):
			deleterious += 1
			available += 1
		elif(impact != "."):
			available += 1

	# Return meta score and available tools
	if available > 0:
		score_adjusted = deleterious/available * 10
	# Select variant described by ClinVar database entries as "P/pathogenic"
	# and not as "B/benign"
	if ("Pathogenic" in str(CLINSIG) or "pathogenic" in str(CLINSIG)) and \
	((not "Benign" in str(CLINSIG)) and (not "benign" in str(CLINSIG))):
		return(score_adjusted, "10db", available, "clinvar",1)
	# Test impact on stop codon
	if (exonicFunction =="stopgain" or exonicFunction =="stoploss"
	or exonicFunction == "frameshift_deletion" or exonicFunction =="frameshift_insertion"):
		return(score_adjusted, "10sfs", available, "stop",2)

	if ((len(str(ref))) - (len(str(alt)))) % 3 !=0 and exonicFunction == "exonic":
		return(score_adjusted, "10sfs", available, "frameshift",2)
	# Test impact on splice function
	if(str(ADAscore) != "."):
		if(float(ADAscore) >= 0.6):
			return(score_adjusted, "10spADA",available, "splice",4)
	if(str(RFscore) != "."):
		if(float(RFscore) >= 0.6):
			return(score_adjusted, "10spRF",available, "splice",3)
	if(str(Zscore) != "."):
		if(float(Zscore) < -2):
			if (ADAscore == ".") and (RFscore == ".") :
				return(score_adjusted, "10sp", available, "splice",5)
	if (((ref == "-") or (alt == "-")) and re.match("splicing",function)):
		return(score_adjusted, "10sp_indel",available, "splice",6)
	# Test if variant maps to multiple location, return "u" (UNKNOWN) if it does
	if(exonicFunction == "unknown"):
		return(score_adjusted, "u",available, "u",8)
	#return 'na' if no tools score are available
	#return deleterious/available * 10, "%s/%s" % (deleterious, available)
	else:
		return (score_adjusted, score_adjusted, available, "na",7)

# ==============================================================================
# ==============================================================================

def is_valid_path(parser, tested_path):
    '''
    Test if the tested_file is a valid file.
    A valid file is an existing file.
    :param parser: The argument parser
    :type parser: ArgumentParser
    :param tested_file: Tested file path
    :type tested_file: str
    :returns: tested_file
    :rtype: str
    '''
    if not os.path.exists(tested_path):
        parser.error("The path %s does not exist!" % tested_path)
    else:
        return tested_path  # return the path

# ==============================================================================

# Assigning command line arguments

(inFile, outFile) = parse(sys.argv)

try :
	inFile_h = open(inFile, 'r')
	vcf_reader = vcf.Reader(inFile_h)
except UnicodeDecodeError :
	print ("\n")
	print("###*** The input file format is invalid. Please use an \
	an annotated VCF ***###")
	print ("\n")
	sys.exit()

try :
	f = open(outFile, 'w')
except UnicodeDecodeError :
	print ("\n")
	print("###*** The output file format is invalid")
	print ("\n")
	sys.exit()

# ==============================================================================

# Writing the header
head = "Rank\tScore\tGene.refGene\t"

# Writing sample name in the header
for elt in vcf_reader.samples:
	head = head + elt + "\t"
# Writing the header (second part)

head = head + "ExAc\tClinSig\tFunc.refGene\t\
ExonicFunc.refGene\tAAChange.refGene\t#CHROM\tPOS\tREF\tALT\tADA\tRF\t\
Spidex\tScore\tNumber of tools"

# ==============================================================================

def test_vcf_annotation(record, key, error):

	try:
		record.INFO[key] = ["." if elt is None else elt for elt in record.INFO[key]]
	except KeyError:
		print(
			"The input VCF file provided lacks the annotation by " +
			str(key)
		)
		error = True
	return error

# ==============================================================================

#Create a list of the keys used in PyVCF and required for the scoring
vcf_key = [
	'ExonicFunc.refGene',
	'FATHMM_pred',
	'Func.refGene',
	'dbscSNV_ADA_SCORE',
	'dbscSNV_RF_SCORE',
	'dpsi_zscore',
	'SIFT_pred',
	'Polyphen2_HDIV_pred',
	'Polyphen2_HVAR_pred',
	'LRT_pred',
	'MutationTaster_pred',
	'FATHMM_pred',
	'PROVEAN_pred',
	'fathmm-MKL_coding_pred',
	'MetaSVM_pred',
	'MetaLR_pred',
	'CLINSIG'
	]

# Iterating through the input VCF
for record in vcf_reader:

# =======																========
# Testing if the input vcf is correctly annotated

	error = False
	try:
		record.REF 	= ["." if record.REF is None else record.REF]
	except KeyError:
		print("The input VCF file provided lacks the annotation by REF.")
		error=True

	try:
		record.ALT 	= ["." if elt is None else elt for elt in record.ALT]
	except KeyError:
		print("The input VCF file provided lacks the annotation by ALT.")
		error=True

	for elt in vcf_key:
		error = test_vcf_annotation(record, elt, error)

	if error==True :
		sys.exit("\n" + "Please refer yourself to the documentation in order to\
 process VCF in a valid format" + "\n")

# =======                                                              =========

# call score function, instanciate it to "s"

	s = score(
		record.INFO['ExonicFunc.refGene'][0],
		record.REF,record.ALT[0],
		record.INFO['Func.refGene'][0],
		record.INFO['dbscSNV_ADA_SCORE'][0],
		record.INFO['dbscSNV_RF_SCORE'][0],
		record.INFO['dpsi_zscore'][0],
		record.INFO['SIFT_pred'][0],
		record.INFO['Polyphen2_HDIV_pred'][0],
		record.INFO['Polyphen2_HVAR_pred'][0],
		record.INFO['LRT_pred'][0],
		record.INFO['MutationTaster_pred'][0],
		record.INFO['FATHMM_pred'][0],
		record.INFO['PROVEAN_pred'][0],
		record.INFO['fathmm-MKL_coding_pred'][0],
		record.INFO['MetaSVM_pred'][0],
		record.INFO['MetaLR_pred'][0],
		record.INFO['CLINSIG']
	)

# ==============================================================================

# Writing output csv
	geno	= ""
	for sample in record.samples:
			geno += ("'" + sample['GT'] + "\t")
	f.write(
		str(s[4]) + "\t" +
		str(s[1]) + "\t" +
		str(record.INFO['Gene.refGene'][0]) +"\t" +
		str(geno) +
		str(record.INFO['ExAC_ALL'][0]) +"\t" +
		str(record.INFO['CLINSIG'][0]) +"\t" +
		str(record.INFO['Func.refGene'][0]) +"\t" +
		str(record.INFO['ExonicFunc.refGene'][0]) + "\t" +
		str(record.INFO['AAChange.refGene']) +"\t" +
		str(record.CHROM) + "\t" +
		str(record.POS) + "\t" +
		str(record.REF) + "\t" +
		str(record.ALT) + "\t" +
		str(record.INFO['dbscSNV_ADA_SCORE'][0]) + "\t" +
		str(record.INFO['dbscSNV_RF_SCORE'][0]) + "\t" +
		str(record.INFO['dpsi_zscore'][0]) + "\t" +
		str(s[0]) + "\t" +
		str(s[2]) + "\n"
	)
f.close()

os.system("sort -k1,1 -k2rn "+ outFile +  "> t.xls")
os.system("echo \""+ head + "\" > " + outFile)
os.system("cat t.xls >> " + outFile + " && rm t.xls")
################################################################################
<commit_msg>Write a proper VCF annotated with MPA<commit_after>##GNOMAD_genome
##Argument
##Ajouter xrf
##Doc MPA
##mettre qual
##mettre sequencage
##!/usr/local/bin/ python3.5
# coding: utf-8
# MPA.py aim to calculate a unique score based on prediction tools in order
# to prioritize variants with a unique score

# ==============================================================================
# IMPORT LIBRARIES

import vcf		# read vcf => PyVCF :https://pyvcf.readthedocs.io/en/latest/
import sys		# system command
import csv		# read, write csv
import re		# regex
import argparse	# for options
import os		# for options
import pprint	# pretty print

# ==============================================================================

def parse(args):
	"""Parse arguments
    Keyword arguments:
	    args -- command line arguments
	Return :
		args.input	-- input file
		argd.output	-- output file
    """
	# Initialise parser
	parser = argparse.ArgumentParser(
		description='Calculating the meta score on annotated vcf')

	# Add group for mandatory arguments
	mandatoryArgs = parser.add_argument_group('mandatory arguments')
	## input file
	mandatoryArgs.add_argument('-i','--input', metavar='file.vcf',
		type=lambda arg: is_valid_path(parser, arg), help='vcf file annotated',
		required=True)
	## output file
	mandatoryArgs.add_argument('-o','--output', metavar='file.csv',
		help='generated .csv file with variants and score', required=True)
	## output vcf
	# mandatoryArgs.add_argument('-v','--vcf_output', metavar='file.vcf',
	# 	help='generated .vcf file with MPA annotation', required=True)

	args = parser.parse_args()
	return(
		args.input,
		args.output,
		# args.vcf_output
	)
# ==============================================================================

def score(exonicFunction = ".", ref = ".", alt = ".", function = ".",
		ADAscore = ".", RFscore = ".", Zscore = ".", SIFT = ".", HDIV = ".",
		HVAR = ".", LRT = ".", MutationTaster = ".", FATHMM = ".",
		PROVEAN = ".", MKL = ".", SVM = ".", LR = ".", CLINSIG = ".",):

	"""
	Calculate a score based on pathogenicity

    Keyword arguments:
	    exonicFunction	-- ExonicFunc_refGene (default ".")
		ref				-- ref (default ".")
		alt				-- alt (default ".")
		function		-- Func_refGene (default ".")
		ADAscore		-- SNV_ADA_SCORE (default ".")
		RFscore			-- dbscSNV_RF_SCORE (default ".")
		Zscore			-- dpsi_zscore (default ".")
		SIFT			-- SIFT_pred (default ".")
		HDIV			-- Polyphen2_HDIV_pred (default ".")
		HVAR			-- Polyphen2_HVAR_pred (default ".")
		LRT				-- LRT_pred (default ".")
		MutationTaster	-- MutationTaster_pred (default ".")
		FATHMM			-- FATHMM_pred (default ".")
		PROVEAN			-- PROVEAN_pred (default ".")
		MKL				-- fathmm_MKL_coding_pred (default ".")
		SVM				-- MetaSVM_pred (default ".")
		LR				-- MetaLR_pred (default ".")

	Return tuple 'score','level' (score max = 10, min = 0):
		'10db','clinvar'	: deleterious impact (10) reported as so in ClinVar
		'10sfs','stop'		: deleterious impact (10) by stop gain or loss
		'10sfs','frameshift': deleterious impact (10) by frameshift indel
		'10sp','splice'		: deleterious impact (10) on splicing
		'u','na'			: unknown exonic function
		'na','0'			: cannot predict impact ; no scores prediction
		'3.3','3/9'			: score of impact (3.3) ; 3 deleterious score on 9
    """

	# Create a dictionnary of impact score tools used
	scores_impact = {"SIFT" : SIFT, "HDIV" : HDIV, "HVAR" : HVAR, "LRT" : LRT,
		"MutationTaster" : MutationTaster, "FATHMM" : FATHMM,
		"PROVEAN" : PROVEAN, "MKL" : MKL, "SVM" : SVM, "LR" : LR}
# for record in vcf_reader:
# 	diff = ((len(record.REF)) - (len(str(record.ALT[0]))))
#
# 	if diff %3 != 0 and record.INFO['Func.refGene'][0] == "exonic" :
# 		print ("diff : ", diff)
# 		print ("Func.refGene : ", record.INFO['Func.refGene'][0])
# 		print ("ExonicFunc.refGene : ", record.INFO['ExonicFunc.refGene']),
# 		print ("longueur REF :", len(record.REF), "longueur ALT :", len(str((record.ALT[0]))))
# 		print ("(lg.REF - lg.ALT) :", diff)
# 		print ("REF : ", record.REF, "ALT : ", record.ALT)
# 		print (str(record.INFO['Gene.refGene'][0]))
	# Test and calculate the score.
	deleterious = 0
	available = 0
	score_adjusted = 0

	for score, impact in scores_impact.items():
		if(impact == "D"):
			deleterious += 1
			available += 1
		elif(impact != "."):
			available += 1

	# Return meta score and available tools
	if available > 0:
		score_adjusted = deleterious/available * 10
	# Select variant described by ClinVar database entries as "P/pathogenic"
	# and not as "B/benign"
	if ("Pathogenic" in str(CLINSIG) or "pathogenic" in str(CLINSIG)) and \
	((not "Benign" in str(CLINSIG)) and (not "benign" in str(CLINSIG))):
		return(score_adjusted, "10db", available, "clinvar",1)
	# Test impact on stop codon
	if (exonicFunction =="stopgain" or exonicFunction =="stoploss"
	or exonicFunction == "frameshift_deletion" or exonicFunction =="frameshift_insertion"):
		return(score_adjusted, "10sfs", available, "stop",2)

	if ((len(str(ref))) - (len(str(alt)))) % 3 !=0 and exonicFunction == "exonic":
		return(score_adjusted, "10sfs", available, "frameshift",2)
	# Test impact on splice function
	if(str(ADAscore) != "."):
		if(float(ADAscore) >= 0.6):
			return(score_adjusted, "10spADA",available, "splice",4)
	if(str(RFscore) != "."):
		if(float(RFscore) >= 0.6):
			return(score_adjusted, "10spRF",available, "splice",3)
	if(str(Zscore) != "."):
		if(float(Zscore) < -2):
			if (ADAscore == ".") and (RFscore == ".") :
				return(score_adjusted, "10sp", available, "splice",5)
	if (((ref == "-") or (alt == "-")) and re.match("splicing",function)):
		return(score_adjusted, "10sp_indel",available, "splice",6)
	# Test if variant maps to multiple location, return "u" (UNKNOWN) if it does
	if(exonicFunction == "unknown"):
		return(score_adjusted, "u",available, "u",8)
	#return 'na' if no tools score are available
	#return deleterious/available * 10, "%s/%s" % (deleterious, available)
	else:
		return (score_adjusted, score_adjusted, available, "na",7)

# ==============================================================================
# ==============================================================================

def is_valid_path(parser, tested_path):
    '''
    Test if the tested_file is a valid file.
    A valid file is an existing file.
    :param parser: The argument parser
    :type parser: ArgumentParser
    :param tested_file: Tested file path
    :type tested_file: str
    :returns: tested_file
    :rtype: str
    '''
    if not os.path.exists(tested_path):
        parser.error("The path %s does not exist!" % tested_path)
    else:
        return tested_path  # return the path

# ==============================================================================

# Assigning command line arguments

(inFile, outFile) = parse(sys.argv)

try :
	inFile_h = open(inFile, 'r')
	vcf_reader = vcf.Reader(inFile_h)
except UnicodeDecodeError :
	print ("\n")
	print("###*** The input file format is invalid. Please use an \
	an annotated VCF ***###")
	print ("\n")
	sys.exit()

try :
	f = open(outFile, 'w')
except UnicodeDecodeError :
	print ("\n")
	print("###*** The output file format is invalid")
	print ("\n")
	sys.exit()

vcf_writer = vcf.Writer(open('testttttt.vcf', 'w'), vcf_reader)

# ==============================================================================

# Writing the header
head = "Rank\tScore\tGene.refGene\t"

# Writing sample name in the header
for elt in vcf_reader.samples:
	head = head + elt + "\t"
# Writing the header (second part)

head = head + "ExAc\tGnomAD_genome\tClinSig\tFunc.refGene\t\
ExonicFunc.refGene\tAAChange.refGene\t#CHROM\tPOS\tREF\tALT\tADA\tRF\t\
Spidex\tScore\tNumber of tools"

# ==============================================================================

def test_vcf_annotation(record, key, error):

	try:
		record.INFO[key] = ["." if elt is None else elt for elt in record.INFO[key]]
	except KeyError:
		print(
			"The input VCF file provided lacks the annotation by " +
			str(key)
		)
		error = True
	return error

# ==============================================================================

#Create a list of the keys used in PyVCF and required for the scoring
vcf_key = [
	'ExonicFunc.refGene',
	'FATHMM_pred',
	'Func.refGene',
	'dbscSNV_ADA_SCORE',
	'dbscSNV_RF_SCORE',
	'dpsi_zscore',
	'SIFT_pred',
	'Polyphen2_HDIV_pred',
	'Polyphen2_HVAR_pred',
	'LRT_pred',
	'MutationTaster_pred',
	'FATHMM_pred',
	'PROVEAN_pred',
	'fathmm-MKL_coding_pred',
	'MetaSVM_pred',
	'MetaLR_pred',
	'CLINSIG'
	]

# Iterating through the input VCF
for record in vcf_reader:

# 	print (record.FORMAT)
# 	print ("\n")
# 	print (record.INFO)
# 	print ("\n")
# # =======																========
# Testing if the input vcf is correctly annotated

	error = False
	try:
		record.REF 	= ["." if record.REF is None else record.REF]
	except KeyError:
		print("The input VCF file provided lacks the annotation by REF.")
		error=True

	try:
		record.ALT 	= ["." if elt is None else elt for elt in record.ALT]
	except KeyError:
		print("The input VCF file provided lacks the annotation by ALT.")
		error=True

	for elt in vcf_key:
		error = test_vcf_annotation(record, elt, error)

	if error==True :
		sys.exit("\n" + "Please refer yourself to the documentation in order to\
 process VCF in a valid format" + "\n")

# =======                                                              =========

# call score function, instanciate it to "s"

	s = score(
		record.INFO['ExonicFunc.refGene'][0],
		record.REF,record.ALT[0],
		record.INFO['Func.refGene'][0],
		record.INFO['dbscSNV_ADA_SCORE'][0],
		record.INFO['dbscSNV_RF_SCORE'][0],
		record.INFO['dpsi_zscore'][0],
		record.INFO['SIFT_pred'][0],
		record.INFO['Polyphen2_HDIV_pred'][0],
		record.INFO['Polyphen2_HVAR_pred'][0],
		record.INFO['LRT_pred'][0],
		record.INFO['MutationTaster_pred'][0],
		record.INFO['FATHMM_pred'][0],
		record.INFO['PROVEAN_pred'][0],
		record.INFO['fathmm-MKL_coding_pred'][0],
		record.INFO['MetaSVM_pred'][0],
		record.INFO['MetaLR_pred'][0],
		record.INFO['CLINSIG']
	)

# ==============================================================================

# Writing output csv
	geno	= ""
	for sample in record.samples:
			geno += ("'" + sample['GT'] + "\t")
	f.write(
		str(s[4]) + "\t" +
		str(s[1]) + "\t" +
		str(record.INFO['Gene.refGene'][0]) +"\t" +
		str(geno) +
		str(record.INFO['ExAC_ALL'][0]) +"\t" +
		# str(record.INFO['G'][0]) +"\t" +
		str(record.INFO['CLINSIG'][0]) +"\t" +
		str(record.INFO['Func.refGene'][0]) +"\t" +
		str(record.INFO['ExonicFunc.refGene'][0]) + "\t" +
		str(record.INFO['AAChange.refGene']) +"\t" +
		str(record.CHROM) + "\t" +
		str(record.POS) + "\t" +
		str(record.REF) + "\t" +
		str(record.ALT) + "\t" +
		str(record.INFO['dbscSNV_ADA_SCORE'][0]) + "\t" +
		str(record.INFO['dbscSNV_RF_SCORE'][0]) + "\t" +
		str(record.INFO['dpsi_zscore'][0]) + "\t" +
		str(s[0]) + "\t" +
		str(s[2]) + "\n"
	)

	print(record.FORMAT)
	record.INFO['MPA_number_tools'] = [str(s[2])]
	record.INFO['MPA_score'] = [str(s[1])]
	record.INFO['MPA_rank'] = [str(s[4])]
	vcf_writer.write_record(record)
f.close()

os.system("sort -k1,1 -k2rn "+ outFile +  "> t.xls")
os.system("echo \""+ head + "\" > " + outFile)
os.system("cat t.xls >> " + outFile + " && rm t.xls")
################################################################################
<|endoftext|>


<commit_before>
import json
import requests
import time
import urllib2

def getTemp(turbine):
    url ='https://turbine-farm.run.aws-usw02-pr.ice.predix.io/api/turbines/' + str(turbine) + '/sensors/temperature'
    ##grab value and assign it to temperature
    temperature = urllib2.urlopen(url).read()
    myArray = temperature.split(':')
    myValue = myArray[3].split('}')
    return float(myValue[0])

def getVoltage(turbine):
    str(turbine)
    url ='https://turbine-farm.run.aws-usw02-pr.ice.predix.io/api/turbines/' + str(turbine) + '/sensors/voltage'
    ##grab value and assign it to voltage
    voltage = urllib2.urlopen(url).read()
    myArray = voltage.split(':')
    myValue = myArray[3].split('}')
    return float(myValue[0])

resp = requests.get('https://turbine-farm.run.aws-usw02-pr.ice.predix.io/api/turbines/1/sensors/temperature')
while (resp.status_code == 200):
    #print voltage of each turbine
    for i in [1, 2, 3]:
        print('Voltage for turbine {0} is {1}.'.format(i, getVoltage(i)))
    #print temperature of each turbine
    for i in [1, 2, 3]:
        print('Temperature for turbine {0} is {1}.'.format(i, getTemp(i)))
    ## wait 2 seconds before printing again
    time.sleep(2)
    resp = requests.get('https://turbine-farm.run.aws-usw02-pr.ice.predix.io/api')

print('Thanks for playing')

<commit_msg>changed to grab temp and voltage at the same time<commit_after>
import json
import requests
import time
import urllib2

def getTemp(turbine):
    url ='https://turbine-farm.run.aws-usw02-pr.ice.predix.io/api/turbines/' + str(turbine) + '/sensors/temperature'
    ##grab value and assign it to temperature
    temperature = urllib2.urlopen(url).read()
    myArray = temperature.split(':')
    myValue = myArray[3].split('}')
    return float(myValue[0])

def getVoltage(turbine):
    str(turbine)
    url ='https://turbine-farm.run.aws-usw02-pr.ice.predix.io/api/turbines/' + str(turbine) + '/sensors/voltage'
    ##grab value and assign it to voltage
    voltage = urllib2.urlopen(url).read()
    myArray = voltage.split(':')
    myValue = myArray[3].split('}')
    return float(myValue[0])

resp = requests.get('https://turbine-farm.run.aws-usw02-pr.ice.predix.io/api/turbines/1/sensors/temperature')
while (1):
    #print voltage of each turbine
    for i in [1, 2, 3]:
        print('turbine {0}: Voltage->{1} -- Temperature->{2}.'.format(i, getVoltage(i), getTemp(i)))
    ## wait 2 seconds before printing again
    time.sleep(2)
    resp = requests.get('https://turbine-farm.run.aws-usw02-pr.ice.predix.io/api')

print('Thanks for playing')

<|endoftext|>


<commit_before>import re
import sre_constants
from itertools import chain
import os

from PyQt4 import QtGui
from PyQt4.QtCore import *
from PyQt4.QtGui import *

from Orange.widgets import gui, widget
from Orange.widgets.settings import Setting, ContextSetting
from Orange.widgets.widget import OWWidget, Msg
from Orange.data import Table
from Orange.data.domain import filter_visible
from orangecontrib.text.corpus import Corpus

class IO:
    DATA = 'Data'
    MATCHED = "Matching Docs"
    UNMATCHED = "Other Docs"

class OWCorpusViewer(OWWidget):
    name = "Corpus Viewer"
    description = "Display corpus contents."
    icon = "icons/CorpusViewer.svg"
    priority = 70

    inputs = [(IO.DATA, Table, 'set_data')]
    outputs = [(IO.MATCHED, Corpus, widget.Default), (IO.UNMATCHED, Corpus)]

    search_indices = ContextSetting([0])   # features included in search
    display_indices = ContextSetting([0])  # features for display
    show_tokens = Setting(False)
    autocommit = Setting(True)

    class Warning(OWWidget.Warning):
        no_feats_search = Msg('No features included in search.')
        no_feats_display = Msg('No features selected for display.')

    def __init__(self):
        super().__init__()

        self.corpus = None              # Corpus
        self.corpus_docs = None         # Documents generated from Corpus
        self.output_mask = []           # Output corpus indices
        self.doc_webview = None         # WebView for showing content
        self.search_features = []       # two copies are needed since Display allows drag & drop
        self.display_features = []

        # Info attributes
        self.update_info()
        info_box = gui.widgetBox(self.controlArea, 'Info')
        gui.label(info_box, self, 'Documents: %(n_documents)s')
        gui.label(info_box, self, 'Preprocessed: %(is_preprocessed)s')
        gui.label(info_box, self, '  ◦ Tokens: %(n_tokens)s')
        gui.label(info_box, self, '  ◦ Types: %(n_types)s')
        gui.label(info_box, self, 'POS tagged: %(is_pos_tagged)s')
        gui.label(info_box, self, 'N-grams range: %(ngram_range)s')
        gui.label(info_box, self, 'Matching: %(n_matching)s')

        # Search features
        self.search_listbox = gui.listBox(
            self.controlArea, self, 'search_indices', 'search_features',
            selectionMode=QtGui.QListView.ExtendedSelection,
            box='Search features', callback=self.regenerate_docs,)

        # Display features
        display_box = gui.widgetBox(self.controlArea, 'Display features')
        self.display_listbox = gui.listBox(
            display_box, self, 'display_indices', 'display_features',
            selectionMode=QtGui.QListView.ExtendedSelection,
            callback=self.show_docs, enableDragDrop=True)
        self.show_tokens_checkbox = gui.checkBox(display_box, self, 'show_tokens',
                                                 'Show Tokens && Tags', callback=self.show_docs)

        # Auto-commit box
        gui.auto_commit(self.controlArea, self, 'autocommit', 'Send data', 'Auto send is on')

        # Search
        self.filter_input = gui.lineEdit(self.mainArea, self, '',
                                         orientation=Qt.Horizontal,
                                         label='RegExp Filter:')
        self.filter_input.textChanged.connect(self.refresh_search)

        h_box = gui.widgetBox(self.mainArea, orientation=Qt.Horizontal, addSpace=True)
        h_box.layout().setSpacing(0)

        # Document list
        self.doc_list = QTableView()
        self.doc_list.setSelectionBehavior(QTableView.SelectRows)
        self.doc_list.setSelectionMode(QTableView.ExtendedSelection)
        self.doc_list.setEditTriggers(QtGui.QAbstractItemView.NoEditTriggers)
        self.doc_list.horizontalHeader().setResizeMode(QtGui.QHeaderView.Stretch)
        self.doc_list.horizontalHeader().setVisible(False)
        h_box.layout().addWidget(self.doc_list)

        self.doc_list_model = QStandardItemModel(self)

        self.doc_list.setModel(self.doc_list_model)
        self.doc_list.setFixedWidth(200)
        self.doc_list.selectionModel().selectionChanged.connect(self.show_docs)

        # Document contents
        self.doc_webview = gui.WebviewWidget(h_box, self, debug=True)

    def set_data(self, data=None):
        self.reset_widget()
        self.corpus = data
        if data is not None:
            if not isinstance(data, Corpus):
                self.corpus = Corpus.from_table(data.domain, data)
            self.load_features()
            self.regenerate_docs()
        self.commit()

    def reset_widget(self):
        # Corpus
        self.corpus = None
        self.corpus_docs = None
        self.output_mask = []
        # Widgets
        self.search_listbox.clear()
        self.display_listbox.clear()
        self.filter_input.clear()
        self.update_info()
        # Models/vars
        self.search_features.clear()
        self.display_features.clear()
        self.search_indices.clear()
        self.display_indices.clear()
        self.doc_list_model.clear()
        # Warnings
        self.Warning.clear()
        # WebView
        self.doc_webview.setHtml('')

    def load_features(self):
        self.search_indices = []
        self.display_indices = []
        if self.corpus is not None:
            domain = self.corpus.domain
            self.search_features = list(filter_visible(chain(domain.variables, domain.metas)))
            self.display_features = list(filter_visible(chain(domain.variables, domain.metas)))
            # FIXME: Select features based on ContextSetting
            self.search_indices = list(range(len(self.search_features)))
            self.display_indices = list(range(len(self.display_features)))

            # Enable/disable tokens checkbox
            if not self.corpus.has_tokens():
                self.show_tokens_checkbox.setCheckState(False)
            self.show_tokens_checkbox.setEnabled(self.corpus.has_tokens())

    def list_docs(self):
        """ List documents into the left scrolling area """
        search_keyword = self.filter_input.text().strip('|')
        try:
            reg = re.compile(search_keyword, re.IGNORECASE)
        except sre_constants.error:
            return

        def is_match(x):
            return not bool(search_keyword) or reg.search(x)

        self.output_mask.clear()
        self.doc_list_model.clear()

        for i, (doc, content) in enumerate(zip(self.corpus, self.corpus_docs)):
            if is_match(content):
                item = QStandardItem()
                item.setData('Document {}'.format(i+1), Qt.DisplayRole)
                item.setData(doc, Qt.UserRole)
                self.doc_list_model.appendRow(item)
                self.output_mask.append(i)

        if self.doc_list_model.rowCount() > 0:
            self.doc_list.selectRow(0)          # Select the first document
        else:
            self.doc_webview.setHtml('')
        self.commit()

    def show_docs(self):
        """ Show the selected documents in the right area """
        HTML = '''
        <!doctype html>

        <meta charset='utf-8'>

        {}

        '''
        if self.corpus is None:
            return

        self.Warning.no_feats_display.clear()
        if self.corpus is not None and len(self.display_indices) == 0:
            self.Warning.no_feats_display()

        documents = []
        if self.show_tokens:
            tokens = list(self.corpus.ngrams_iterator(include_postags=True))

        for index in self.doc_list.selectionModel().selectedRows():
            html = '
'

            row_ind = index.data(Qt.UserRole).row_index
            for ind in self.display_indices:
                feature = self.display_features[ind]
                mark = 'class="mark-area"' if ind in self.search_indices else ''
                value = index.data(Qt.UserRole)[feature.name]
                html += '
{}:	' \
                        '{}
'.format(
                    feature.name, mark, value)

            if self.show_tokens:
                html += '
Tokens & Tags:	' \
                        '{}
'.format(''.join('{}'.format(
                                                      token) for token in tokens[row_ind]))

            html += '
'
            documents.append(html)

        self.doc_webview.setHtml(HTML.format('
'.join(documents)))
        if documents:
            self.load_js()
            self.highlight_docs()

    def load_js(self):
        resources = os.path.join(os.path.dirname(__file__), 'resources')
        for script in ('jquery-2.1.4.min.js', 'jquery.mark.min.js', 'highlighter.js', ):
            self.doc_webview.evalJS(open(os.path.join(resources, script), encoding='utf-8').read())

    def regenerate_docs(self):
        self.corpus_docs = None
        self.Warning.no_feats_search.clear()
        if self.corpus is not None:
            feats = [self.search_features[i] for i in self.search_indices]
            if len(feats) == 0:
                self.Warning.no_feats_search()
            self.corpus_docs = self.corpus.documents_from_features(feats)
            self.refresh_search()

    def refresh_search(self):
        if self.corpus:
            self.list_docs()
            self.update_info()

    def highlight_docs(self):
        search_keyword = self.filter_input.text().\
            strip('|').replace('\\', '\\\\')    # escape one \ to  two for mark.js
        if search_keyword:
            self.doc_webview.evalJS('mark("{}");'.format(search_keyword))

    def update_info(self):
        if self.corpus is not None:
            self.n_documents = len(self.corpus)
            self.n_matching = '{}/{}'.format(self.doc_list_model.rowCount(), self.n_documents)
            self.n_tokens = sum(map(len, self.corpus.tokens)) if self.corpus.has_tokens() else 'n/a'
            self.n_types = len(self.corpus.dictionary) if self.corpus.has_tokens() else 'n/a'
            self.is_preprocessed = self.corpus.has_tokens()
            self.is_pos_tagged = self.corpus.pos_tags is not None
            self.ngram_range = '{}-{}'.format(*self.corpus.ngram_range)
        else:
            self.n_documents = ''
            self.n_matching = ''
            self.n_tokens = ''
            self.n_types = ''
            self.is_preprocessed = ''
            self.is_pos_tagged = ''
            self.ngram_range = ''

    def commit(self):
        if self.corpus is not None:
            matched = self.corpus[self.output_mask]
            unmatched_mask = [i for i in range(len(self.corpus)) if i not in self.output_mask]
            unmatched = self.corpus[unmatched_mask]
            self.send(IO.MATCHED, matched)
            self.send(IO.UNMATCHED, unmatched)
        else:
            self.send(IO.MATCHED, None)
            self.send(IO.UNMATCHED, None)

if __name__ == '__main__':
    from orangecontrib.text.tag import pos_tagger
    app = QApplication([])
    widget = OWCorpusViewer()
    widget.show()
    corpus = Corpus.from_file('bookexcerpts')
    corpus = corpus[:3]
    corpus = pos_tagger.tag_corpus(corpus)
    corpus.ngram_range = (1, 2)
    widget.set_data(corpus)
    app.exec()
<commit_msg>CorpusViewer: Fix highlighting when reordering<commit_after>import re
import sre_constants
from itertools import chain
import os

from PyQt4 import QtGui
from PyQt4.QtCore import *
from PyQt4.QtGui import *

from Orange.widgets import gui, widget
from Orange.widgets.settings import Setting, ContextSetting
from Orange.widgets.widget import OWWidget, Msg
from Orange.data import Table
from Orange.data.domain import filter_visible
from orangecontrib.text.corpus import Corpus

class IO:
    DATA = 'Data'
    MATCHED = "Matching Docs"
    UNMATCHED = "Other Docs"

class OWCorpusViewer(OWWidget):
    name = "Corpus Viewer"
    description = "Display corpus contents."
    icon = "icons/CorpusViewer.svg"
    priority = 70

    inputs = [(IO.DATA, Table, 'set_data')]
    outputs = [(IO.MATCHED, Corpus, widget.Default), (IO.UNMATCHED, Corpus)]

    search_indices = ContextSetting([0])   # features included in search
    display_indices = ContextSetting([0])  # features for display
    show_tokens = Setting(False)
    autocommit = Setting(True)

    class Warning(OWWidget.Warning):
        no_feats_search = Msg('No features included in search.')
        no_feats_display = Msg('No features selected for display.')

    def __init__(self):
        super().__init__()

        self.corpus = None              # Corpus
        self.corpus_docs = None         # Documents generated from Corpus
        self.output_mask = []           # Output corpus indices
        self.doc_webview = None         # WebView for showing content
        self.search_features = []       # two copies are needed since Display allows drag & drop
        self.display_features = []

        # Info attributes
        self.update_info()
        info_box = gui.widgetBox(self.controlArea, 'Info')
        gui.label(info_box, self, 'Documents: %(n_documents)s')
        gui.label(info_box, self, 'Preprocessed: %(is_preprocessed)s')
        gui.label(info_box, self, '  ◦ Tokens: %(n_tokens)s')
        gui.label(info_box, self, '  ◦ Types: %(n_types)s')
        gui.label(info_box, self, 'POS tagged: %(is_pos_tagged)s')
        gui.label(info_box, self, 'N-grams range: %(ngram_range)s')
        gui.label(info_box, self, 'Matching: %(n_matching)s')

        # Search features
        self.search_listbox = gui.listBox(
            self.controlArea, self, 'search_indices', 'search_features',
            selectionMode=QtGui.QListView.ExtendedSelection,
            box='Search features', callback=self.regenerate_docs,)

        # Display features
        display_box = gui.widgetBox(self.controlArea, 'Display features')
        self.display_listbox = gui.listBox(
            display_box, self, 'display_indices', 'display_features',
            selectionMode=QtGui.QListView.ExtendedSelection,
            callback=self.show_docs, enableDragDrop=True)
        self.show_tokens_checkbox = gui.checkBox(display_box, self, 'show_tokens',
                                                 'Show Tokens && Tags', callback=self.show_docs)

        # Auto-commit box
        gui.auto_commit(self.controlArea, self, 'autocommit', 'Send data', 'Auto send is on')

        # Search
        self.filter_input = gui.lineEdit(self.mainArea, self, '',
                                         orientation=Qt.Horizontal,
                                         label='RegExp Filter:')
        self.filter_input.textChanged.connect(self.refresh_search)

        h_box = gui.widgetBox(self.mainArea, orientation=Qt.Horizontal, addSpace=True)
        h_box.layout().setSpacing(0)

        # Document list
        self.doc_list = QTableView()
        self.doc_list.setSelectionBehavior(QTableView.SelectRows)
        self.doc_list.setSelectionMode(QTableView.ExtendedSelection)
        self.doc_list.setEditTriggers(QtGui.QAbstractItemView.NoEditTriggers)
        self.doc_list.horizontalHeader().setResizeMode(QtGui.QHeaderView.Stretch)
        self.doc_list.horizontalHeader().setVisible(False)
        h_box.layout().addWidget(self.doc_list)

        self.doc_list_model = QStandardItemModel(self)

        self.doc_list.setModel(self.doc_list_model)
        self.doc_list.setFixedWidth(200)
        self.doc_list.selectionModel().selectionChanged.connect(self.show_docs)

        # Document contents
        self.doc_webview = gui.WebviewWidget(h_box, self, debug=True)

    def set_data(self, data=None):
        self.reset_widget()
        self.corpus = data
        if data is not None:
            if not isinstance(data, Corpus):
                self.corpus = Corpus.from_table(data.domain, data)
            self.load_features()
            self.regenerate_docs()
        self.commit()

    def reset_widget(self):
        # Corpus
        self.corpus = None
        self.corpus_docs = None
        self.output_mask = []
        # Widgets
        self.search_listbox.clear()
        self.display_listbox.clear()
        self.filter_input.clear()
        self.update_info()
        # Models/vars
        self.search_features.clear()
        self.display_features.clear()
        self.search_indices.clear()
        self.display_indices.clear()
        self.doc_list_model.clear()
        # Warnings
        self.Warning.clear()
        # WebView
        self.doc_webview.setHtml('')

    def load_features(self):
        self.search_indices = []
        self.display_indices = []
        if self.corpus is not None:
            domain = self.corpus.domain
            self.search_features = list(filter_visible(chain(domain.variables, domain.metas)))
            self.display_features = list(filter_visible(chain(domain.variables, domain.metas)))
            # FIXME: Select features based on ContextSetting
            self.search_indices = list(range(len(self.search_features)))
            self.display_indices = list(range(len(self.display_features)))

            # Enable/disable tokens checkbox
            if not self.corpus.has_tokens():
                self.show_tokens_checkbox.setCheckState(False)
            self.show_tokens_checkbox.setEnabled(self.corpus.has_tokens())

    def list_docs(self):
        """ List documents into the left scrolling area """
        search_keyword = self.filter_input.text().strip('|')
        try:
            reg = re.compile(search_keyword, re.IGNORECASE)
        except sre_constants.error:
            return

        def is_match(x):
            return not bool(search_keyword) or reg.search(x)

        self.output_mask.clear()
        self.doc_list_model.clear()

        for i, (doc, content) in enumerate(zip(self.corpus, self.corpus_docs)):
            if is_match(content):
                item = QStandardItem()
                item.setData('Document {}'.format(i+1), Qt.DisplayRole)
                item.setData(doc, Qt.UserRole)
                self.doc_list_model.appendRow(item)
                self.output_mask.append(i)

        if self.doc_list_model.rowCount() > 0:
            self.doc_list.selectRow(0)          # Select the first document
        else:
            self.doc_webview.setHtml('')
        self.commit()

    def show_docs(self):
        """ Show the selected documents in the right area """
        HTML = '''
        <!doctype html>

        <meta charset='utf-8'>

        {}

        '''
        if self.corpus is None:
            return

        self.Warning.no_feats_display.clear()
        if len(self.display_indices) == 0:
            self.Warning.no_feats_display()

        documents = []
        if self.show_tokens:
            tokens = list(self.corpus.ngrams_iterator(include_postags=True))

        marked_search_features = [f for i, f in enumerate(self.search_features)
                                  if i in self.search_indices]

        for index in self.doc_list.selectionModel().selectedRows():
            html = '
'

            row_ind = index.data(Qt.UserRole).row_index
            for ind in self.display_indices:
                feature = self.display_features[ind]
                mark = 'class="mark-area"' if feature in marked_search_features else ''
                value = index.data(Qt.UserRole)[feature.name]
                html += '
{}:	' \
                        '{}
'.format(
                    feature.name, mark, value)

            if self.show_tokens:
                html += '
Tokens & Tags:	' \
                        '{}
'.format(''.join('{}'.format(
                                                      token) for token in tokens[row_ind]))

            html += '
'
            documents.append(html)

        self.doc_webview.setHtml(HTML.format('
'.join(documents)))
        if documents:
            self.load_js()
            self.highlight_docs()

    def load_js(self):
        resources = os.path.join(os.path.dirname(__file__), 'resources')
        for script in ('jquery-2.1.4.min.js', 'jquery.mark.min.js', 'highlighter.js', ):
            self.doc_webview.evalJS(open(os.path.join(resources, script), encoding='utf-8').read())

    def regenerate_docs(self):
        self.corpus_docs = None
        self.Warning.no_feats_search.clear()
        if self.corpus is not None:
            feats = [self.search_features[i] for i in self.search_indices]
            if len(feats) == 0:
                self.Warning.no_feats_search()
            self.corpus_docs = self.corpus.documents_from_features(feats)
            self.refresh_search()

    def refresh_search(self):
        if self.corpus:
            self.list_docs()
            self.update_info()

    def highlight_docs(self):
        search_keyword = self.filter_input.text().\
            strip('|').replace('\\', '\\\\')    # escape one \ to  two for mark.js
        if search_keyword:
            self.doc_webview.evalJS('mark("{}");'.format(search_keyword))

    def update_info(self):
        if self.corpus is not None:
            self.n_documents = len(self.corpus)
            self.n_matching = '{}/{}'.format(self.doc_list_model.rowCount(), self.n_documents)
            self.n_tokens = sum(map(len, self.corpus.tokens)) if self.corpus.has_tokens() else 'n/a'
            self.n_types = len(self.corpus.dictionary) if self.corpus.has_tokens() else 'n/a'
            self.is_preprocessed = self.corpus.has_tokens()
            self.is_pos_tagged = self.corpus.pos_tags is not None
            self.ngram_range = '{}-{}'.format(*self.corpus.ngram_range)
        else:
            self.n_documents = ''
            self.n_matching = ''
            self.n_tokens = ''
            self.n_types = ''
            self.is_preprocessed = ''
            self.is_pos_tagged = ''
            self.ngram_range = ''

    def commit(self):
        if self.corpus is not None:
            matched = self.corpus[self.output_mask]
            unmatched_mask = [i for i in range(len(self.corpus)) if i not in self.output_mask]
            unmatched = self.corpus[unmatched_mask]
            self.send(IO.MATCHED, matched)
            self.send(IO.UNMATCHED, unmatched)
        else:
            self.send(IO.MATCHED, None)
            self.send(IO.UNMATCHED, None)

if __name__ == '__main__':
    from orangecontrib.text.tag import pos_tagger
    app = QApplication([])
    widget = OWCorpusViewer()
    widget.show()
    corpus = Corpus.from_file('bookexcerpts')
    corpus = corpus[:3]
    corpus = pos_tagger.tag_corpus(corpus)
    corpus.ngram_range = (1, 2)
    widget.set_data(corpus)
    app.exec()
<|endoftext|>


<commit_before>'''
Implements the RTSRoot class.

This file is part of RTSLib.
Copyright (c) 2011-2013 by Datera, Inc
Copyright (c) 2011-2014 by Red Hat, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
'''

import re
import os
import stat
import json

from .node import CFSNode
from .target import Target
from .fabric import FabricModule
from .tcm import so_mapping, StorageObject
from .utils import RTSLibError, RTSLibBrokenLink, modprobe, mount_configfs
from .utils import dict_remove, set_attributes

default_save_file = "/etc/target/saveconfig.json"

class RTSRoot(CFSNode):
    '''
    This is an interface to the root of the configFS object tree.
    Is allows one to start browsing Target and StorageObjects,
    as well as helper methods to return arbitrary objects from the
    configFS tree.

    >>> import rtslib.root as root
    >>> rtsroot = root.RTSRoot()
    >>> rtsroot.path
    '/sys/kernel/config/target'
    >>> rtsroot.exists
    True
    >>> rtsroot.targets # doctest: +ELLIPSIS
    [...]
    >>> rtsroot.tpgs # doctest: +ELLIPSIS
    [...]
    >>> rtsroot.storage_objects # doctest: +ELLIPSIS
    [...]
    >>> rtsroot.network_portals # doctest: +ELLIPSIS
    [...]

    '''

    # RTSRoot private stuff
    def __init__(self):
        '''
        Instantiate an RTSRoot object. Basically checks for configfs setup and
        base kernel modules (tcm)
        '''
        super(RTSRoot, self).__init__()
        modprobe('configfs')
        mount_configfs()
        modprobe('target_core_mod')
        self._create_in_cfs_ine('any')

    def _list_targets(self):
        self._check_self()
        for fabric_module in self.fabric_modules:
            for target in fabric_module.targets:
                yield target

    def _list_storage_objects(self):
        self._check_self()
        for so in StorageObject.all():
            yield so

    def _list_tpgs(self):
        self._check_self()
        for t in self.targets:
            for tpg in t.tpgs:
                yield tpg

    def _list_node_acls(self):
        self._check_self()
        for t in self.tpgs:
            for node_acl in t.node_acls:
                yield node_acl

    def _list_node_acl_groups(self):
        self._check_self()
        for t in self.tpgs:
            for nag in t.node_acl_groups:
                yield nag

    def _list_mapped_luns(self):
        self._check_self()
        for na in self.node_acls:
            for mlun in na.mapped_luns:
                yield mlun

    def _list_mapped_lun_groups(self):
        self._check_self()
        for nag in self.node_acl_groups:
            for mlg in na.mapped_lun_groups:
                yield mlg

    def _list_network_portals(self):
        self._check_self()
        for t in self.tpgs:
            for p in t.network_portals:
                yield p

    def _list_luns(self):
        self._check_self()
        for t in self.tpgs:
            for lun in t.luns:
                yield lun

    def _list_sessions(self):
        self._check_self()
        for na in self.node_acls:
            if na.session:
                yield na.session

    def _list_fabric_modules(self):
        self._check_self()
        for mod in FabricModule.all():
            yield mod

    def __str__(self):
        return "rtslib"

    # RTSRoot public stuff

    def dump(self):
        '''
        Returns a dict representing the complete state of the target
        config, suitable for serialization/deserialization, and then
        handing to restore().
        '''
        d = super(RTSRoot, self).dump()
        d['storage_objects'] = [so.dump() for so in self.storage_objects]
        d['targets'] = [t.dump() for t in self.targets]
        d['fabric_modules'] = [f.dump() for f in self.fabric_modules
                               if f.has_feature("discovery_auth")
                               if f.discovery_enable_auth]
        return d

    def clear_existing(self, confirm=False):
        '''
        Remove entire current configuration.
        '''
        if not confirm:
            raise RTSLibError("As a precaution, confirm=True needs to be set")

        # Targets depend on storage objects, delete them first.
        for t in self.targets:
            t.delete()
        for fm in (f for f in self.fabric_modules if f.has_feature("discovery_auth")):
            fm.clear_discovery_auth_settings()
        for so in self.storage_objects:
            so.delete()

    def restore(self, config, clear_existing=False, abort_on_error=False):
        '''
        Takes a dict generated by dump() and reconfigures the target to match.
        Returns list of non-fatal errors that were encountered.
        Will refuse to restore over an existing configuration unless clear_existing
            is True.
        '''
        if clear_existing:
            self.clear_existing(confirm=True)
        elif any(self.storage_objects) or any(self.targets):
            raise RTSLibError("storageobjects or targets present, not restoring")

        errors = []

        if abort_on_error:
            def err_func(err_str):
                raise RTSLibError(err_str)
        else:
            def err_func(err_str):
                errors.append(err_str + ", skipped")

        for index, so in enumerate(config.get('storage_objects', [])):
            if 'name' not in so:
                err_func("'name' not defined in storage object %d" % index)
                continue
            try:
                so_cls = so_mapping[so['plugin']]
            except KeyError:
                err_func("'plugin' not defined or invalid in storageobject %s" % so['name'])
                continue
            kwargs = so.copy()
            dict_remove(kwargs, ('exists', 'attributes', 'plugin', 'buffered_mode'))
            try:
                so_obj = so_cls(**kwargs)
            except Exception as e:
                err_func("Could not create StorageObject %s: %s" % (so['name'], e))
                continue

            # Custom err func to include block name
            def so_err_func(x):
                return err_func("Storage Object %s/%s: %s" % (so['plugin'], so['name'], x))

            set_attributes(so_obj, so.get('attributes', {}), so_err_func)

        # Don't need to create fabric modules
        for index, fm in enumerate(config.get('fabric_modules', [])):
            if 'name' not in fm:
                err_func("'name' not defined in fabricmodule %d" % index)
                continue
            for fm_obj in self.fabric_modules:
                if fm['name'] == fm_obj.name:
                    fm_obj.setup(fm, err_func)
                    break

        for index, t in enumerate(config.get('targets', [])):
            if 'wwn' not in t:
                err_func("'wwn' not defined in target %d" % index)
                continue
            if 'fabric' not in t:
                err_func("target %s missing 'fabric' field" % t['wwn'])
                continue
            if t['fabric'] not in (f.name for f in self.fabric_modules):
                err_func("Unknown fabric '%s'" % t['fabric'])
                continue

            fm_obj = FabricModule(t['fabric'])

            # Instantiate target
            Target.setup(fm_obj, t, err_func)

        return errors

    def save_to_file(self, save_file=None):
        '''
        Write the configuration in json format to a file.
        Save file defaults to '/etc/targets/saveconfig.json'.
        '''
        if not save_file:
            save_file = default_save_file

        with open(save_file+".temp", "w+") as f:
            os.fchmod(f.fileno(), stat.S_IRUSR | stat.S_IWUSR)
            f.write(json.dumps(self.dump(), sort_keys=True, indent=2))
            f.write("\n")
            os.fsync(f.fileno())

        os.rename(save_file+".temp", save_file)

    def restore_from_file(self, restore_file=None, clear_existing=True, abort_on_error=False):
        '''
        Restore the configuration from a file in json format.
        Restore file defaults to '/etc/targets/saveconfig.json'.
        Returns a list of non-fatal errors. If abort_on_error is set,
          it will raise the exception instead of continuing.
        '''
        if not restore_file:
            restore_file = default_save_file

        with open(restore_file, "r") as f:
            config = json.loads(f.read())
            return self.restore(config, clear_existing=clear_existing,
                                abort_on_error=abort_on_error)

    targets = property(_list_targets,
            doc="Get the list of Target objects.")
    tpgs = property(_list_tpgs,
            doc="Get the list of all the existing TPG objects.")
    node_acls = property(_list_node_acls,
            doc="Get the list of all the existing NodeACL objects.")
    node_acl_groups = property(_list_node_acl_groups,
            doc="Get the list of all the existing NodeACLGroup objects.")
    mapped_luns = property(_list_mapped_luns,
            doc="Get the list of all the existing MappedLUN objects.")
    mapped_lun_groups = property(_list_mapped_lun_groups,
            doc="Get the list of all the existing MappedLUNGroup objects.")
    sessions = property(_list_sessions,
            doc="Get the list of all the existing sessions.")
    network_portals = property(_list_network_portals,
            doc="Get the list of all the existing Network Portal objects.")
    storage_objects = property(_list_storage_objects,
            doc="Get the list of all the existing Storage objects.")
    luns = property(_list_luns,
            doc="Get the list of all existing LUN objects.")
    fabric_modules = property(_list_fabric_modules,
            doc="Get the list of all FabricModule objects.")

def _test():
    '''Run the doctests.'''
    import doctest
    doctest.testmod()

if __name__ == "__main__":
    _test()
<commit_msg>Fix an undefined variable error caused by a typo<commit_after>'''
Implements the RTSRoot class.

This file is part of RTSLib.
Copyright (c) 2011-2013 by Datera, Inc
Copyright (c) 2011-2014 by Red Hat, Inc.

Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
'''

import re
import os
import stat
import json

from .node import CFSNode
from .target import Target
from .fabric import FabricModule
from .tcm import so_mapping, StorageObject
from .utils import RTSLibError, RTSLibBrokenLink, modprobe, mount_configfs
from .utils import dict_remove, set_attributes

default_save_file = "/etc/target/saveconfig.json"

class RTSRoot(CFSNode):
    '''
    This is an interface to the root of the configFS object tree.
    Is allows one to start browsing Target and StorageObjects,
    as well as helper methods to return arbitrary objects from the
    configFS tree.

    >>> import rtslib.root as root
    >>> rtsroot = root.RTSRoot()
    >>> rtsroot.path
    '/sys/kernel/config/target'
    >>> rtsroot.exists
    True
    >>> rtsroot.targets # doctest: +ELLIPSIS
    [...]
    >>> rtsroot.tpgs # doctest: +ELLIPSIS
    [...]
    >>> rtsroot.storage_objects # doctest: +ELLIPSIS
    [...]
    >>> rtsroot.network_portals # doctest: +ELLIPSIS
    [...]

    '''

    # RTSRoot private stuff
    def __init__(self):
        '''
        Instantiate an RTSRoot object. Basically checks for configfs setup and
        base kernel modules (tcm)
        '''
        super(RTSRoot, self).__init__()
        modprobe('configfs')
        mount_configfs()
        modprobe('target_core_mod')
        self._create_in_cfs_ine('any')

    def _list_targets(self):
        self._check_self()
        for fabric_module in self.fabric_modules:
            for target in fabric_module.targets:
                yield target

    def _list_storage_objects(self):
        self._check_self()
        for so in StorageObject.all():
            yield so

    def _list_tpgs(self):
        self._check_self()
        for t in self.targets:
            for tpg in t.tpgs:
                yield tpg

    def _list_node_acls(self):
        self._check_self()
        for t in self.tpgs:
            for node_acl in t.node_acls:
                yield node_acl

    def _list_node_acl_groups(self):
        self._check_self()
        for t in self.tpgs:
            for nag in t.node_acl_groups:
                yield nag

    def _list_mapped_luns(self):
        self._check_self()
        for na in self.node_acls:
            for mlun in na.mapped_luns:
                yield mlun

    def _list_mapped_lun_groups(self):
        self._check_self()
        for nag in self.node_acl_groups:
            for mlg in nag.mapped_lun_groups:
                yield mlg

    def _list_network_portals(self):
        self._check_self()
        for t in self.tpgs:
            for p in t.network_portals:
                yield p

    def _list_luns(self):
        self._check_self()
        for t in self.tpgs:
            for lun in t.luns:
                yield lun

    def _list_sessions(self):
        self._check_self()
        for na in self.node_acls:
            if na.session:
                yield na.session

    def _list_fabric_modules(self):
        self._check_self()
        for mod in FabricModule.all():
            yield mod

    def __str__(self):
        return "rtslib"

    # RTSRoot public stuff

    def dump(self):
        '''
        Returns a dict representing the complete state of the target
        config, suitable for serialization/deserialization, and then
        handing to restore().
        '''
        d = super(RTSRoot, self).dump()
        d['storage_objects'] = [so.dump() for so in self.storage_objects]
        d['targets'] = [t.dump() for t in self.targets]
        d['fabric_modules'] = [f.dump() for f in self.fabric_modules
                               if f.has_feature("discovery_auth")
                               if f.discovery_enable_auth]
        return d

    def clear_existing(self, confirm=False):
        '''
        Remove entire current configuration.
        '''
        if not confirm:
            raise RTSLibError("As a precaution, confirm=True needs to be set")

        # Targets depend on storage objects, delete them first.
        for t in self.targets:
            t.delete()
        for fm in (f for f in self.fabric_modules if f.has_feature("discovery_auth")):
            fm.clear_discovery_auth_settings()
        for so in self.storage_objects:
            so.delete()

    def restore(self, config, clear_existing=False, abort_on_error=False):
        '''
        Takes a dict generated by dump() and reconfigures the target to match.
        Returns list of non-fatal errors that were encountered.
        Will refuse to restore over an existing configuration unless clear_existing
            is True.
        '''
        if clear_existing:
            self.clear_existing(confirm=True)
        elif any(self.storage_objects) or any(self.targets):
            raise RTSLibError("storageobjects or targets present, not restoring")

        errors = []

        if abort_on_error:
            def err_func(err_str):
                raise RTSLibError(err_str)
        else:
            def err_func(err_str):
                errors.append(err_str + ", skipped")

        for index, so in enumerate(config.get('storage_objects', [])):
            if 'name' not in so:
                err_func("'name' not defined in storage object %d" % index)
                continue
            try:
                so_cls = so_mapping[so['plugin']]
            except KeyError:
                err_func("'plugin' not defined or invalid in storageobject %s" % so['name'])
                continue
            kwargs = so.copy()
            dict_remove(kwargs, ('exists', 'attributes', 'plugin', 'buffered_mode'))
            try:
                so_obj = so_cls(**kwargs)
            except Exception as e:
                err_func("Could not create StorageObject %s: %s" % (so['name'], e))
                continue

            # Custom err func to include block name
            def so_err_func(x):
                return err_func("Storage Object %s/%s: %s" % (so['plugin'], so['name'], x))

            set_attributes(so_obj, so.get('attributes', {}), so_err_func)

        # Don't need to create fabric modules
        for index, fm in enumerate(config.get('fabric_modules', [])):
            if 'name' not in fm:
                err_func("'name' not defined in fabricmodule %d" % index)
                continue
            for fm_obj in self.fabric_modules:
                if fm['name'] == fm_obj.name:
                    fm_obj.setup(fm, err_func)
                    break

        for index, t in enumerate(config.get('targets', [])):
            if 'wwn' not in t:
                err_func("'wwn' not defined in target %d" % index)
                continue
            if 'fabric' not in t:
                err_func("target %s missing 'fabric' field" % t['wwn'])
                continue
            if t['fabric'] not in (f.name for f in self.fabric_modules):
                err_func("Unknown fabric '%s'" % t['fabric'])
                continue

            fm_obj = FabricModule(t['fabric'])

            # Instantiate target
            Target.setup(fm_obj, t, err_func)

        return errors

    def save_to_file(self, save_file=None):
        '''
        Write the configuration in json format to a file.
        Save file defaults to '/etc/targets/saveconfig.json'.
        '''
        if not save_file:
            save_file = default_save_file

        with open(save_file+".temp", "w+") as f:
            os.fchmod(f.fileno(), stat.S_IRUSR | stat.S_IWUSR)
            f.write(json.dumps(self.dump(), sort_keys=True, indent=2))
            f.write("\n")
            os.fsync(f.fileno())

        os.rename(save_file+".temp", save_file)

    def restore_from_file(self, restore_file=None, clear_existing=True, abort_on_error=False):
        '''
        Restore the configuration from a file in json format.
        Restore file defaults to '/etc/targets/saveconfig.json'.
        Returns a list of non-fatal errors. If abort_on_error is set,
          it will raise the exception instead of continuing.
        '''
        if not restore_file:
            restore_file = default_save_file

        with open(restore_file, "r") as f:
            config = json.loads(f.read())
            return self.restore(config, clear_existing=clear_existing,
                                abort_on_error=abort_on_error)

    targets = property(_list_targets,
            doc="Get the list of Target objects.")
    tpgs = property(_list_tpgs,
            doc="Get the list of all the existing TPG objects.")
    node_acls = property(_list_node_acls,
            doc="Get the list of all the existing NodeACL objects.")
    node_acl_groups = property(_list_node_acl_groups,
            doc="Get the list of all the existing NodeACLGroup objects.")
    mapped_luns = property(_list_mapped_luns,
            doc="Get the list of all the existing MappedLUN objects.")
    mapped_lun_groups = property(_list_mapped_lun_groups,
            doc="Get the list of all the existing MappedLUNGroup objects.")
    sessions = property(_list_sessions,
            doc="Get the list of all the existing sessions.")
    network_portals = property(_list_network_portals,
            doc="Get the list of all the existing Network Portal objects.")
    storage_objects = property(_list_storage_objects,
            doc="Get the list of all the existing Storage objects.")
    luns = property(_list_luns,
            doc="Get the list of all existing LUN objects.")
    fabric_modules = property(_list_fabric_modules,
            doc="Get the list of all FabricModule objects.")

def _test():
    '''Run the doctests.'''
    import doctest
    doctest.testmod()

if __name__ == "__main__":
    _test()
<|endoftext|>


<commit_before>import sys
import atexit

# todo: make written-to stream passable

class Plan(object):
  def __init__(self, plan, param=None):
    self.counter = 0
    self.expected_tests = None
    self.ended = False

    if isinstance(plan, int):
      self.expected_tests = plan
      print "1..%u" % self.expected_tests
    elif plan == "no_plan": 1
    elif plan == "skip_all":
      print "1..0 # skip %s" % param
      raise SystemExit(0) # ??? this is what T::B does, but sucks
    else:
      raise TestBadPlan(plan)

  def increment_counter(self):
    self.counter += 1

  def __del__(self):
    if self.ended: return
    self.ended = True
    if self.expected_tests is None:
      print "1..%u" % self.counter
    elif self.counter != self.expected_tests:
      print "# Looks like you planned %u tests but ran %u." % (self.expected_tests, self.counter)

class Builder(object):
  global_defaults = {
    "_plan":    None,
    "current":  1,
    "has_plan": False,
  }
  global_test_builder = global_defaults.copy()

  def __init__(self, plan=None, plan_param=None):
    self.__dict__ = self.global_test_builder
    if plan: self.set_plan(plan, plan_param)

  @classmethod
  def create(cls, plan=None, plan_param=None):
    # self = new.instance(cls) # ? this sucks, too
    self = Builder()
    self.__dict__  = self.global_defaults.copy()
    if plan: self.set_plan(plan, plan_param)
    return self

  def set_plan(self, plan, plan_param=None):
    if self.get_plan(): raise TestPlannedAlready(plan, plan_param)
    self._plan = Plan(plan, plan_param)
    atexit.register(self._plan.__del__)

  def get_plan(self): return self._plan

  def ok(self, is_ok, desc=None, skip=None, todo=None):
    self.get_plan().increment_counter()
    if is_ok: report = "ok"
    else:     report = "not ok"

    sys.stdout.write("%s %u" % (report, self.current))

    if desc: sys.stdout.write(" - %s" % desc)
    if skip: sys.stdout.write(" # SKIP %s" % skip)
    if todo: sys.stdout.write(" # TODO %s" % todo)

    print

    self.current += 1

  def reset(self):
    self.__dict__.clear()
    for key in self.global_defaults.iterkeys():
      self.__dict__[key] = self.global_defaults[key]

class TestPlannedAlready(Exception):
  def __init__(self, plan, param=None):
    self.plan  = plan
    self.param = param
  def __str__(self):
    return "tried to plan twice; second plan: %s, %s" % self.plan, self.param

class TestWithoutPlan(Exception):
  def __str__(self):
    return "tried running tests without a plan"

class TestBadPlan(Exception):
  def __init__(self, plan):
    self.plan = plan
  def __str__(self):
    return "didn't understand plan '%s'" % self.plan
<commit_msg> r20601@knave:  rjbs | 2006-04-03 17:36:33 -0400  long lines<commit_after>import sys
import atexit

# todo: make written-to stream passable

class Plan(object):
  def __init__(self, plan, param=None):
    self.counter = 0
    self.expected_tests = None
    self.ended = False

    if isinstance(plan, int):
      self.expected_tests = plan
      print "1..%u" % self.expected_tests
    elif plan == "no_plan": 1
    elif plan == "skip_all":
      print "1..0 # skip %s" % param
      raise SystemExit(0) # ??? this is what T::B does, but sucks
    else:
      raise TestBadPlan(plan)

  def increment_counter(self):
    self.counter += 1

  def __del__(self):
    if self.ended: return
    self.ended = True
    if self.expected_tests is None:
      print "1..%u" % self.counter
    elif self.counter != self.expected_tests:
      print "# Looks like you planned %u tests but ran %u." \
        % (self.expected_tests, self.counter)

class Builder(object):
  global_defaults = {
    "_plan":    None,
    "current":  1,
    "has_plan": False,
  }
  global_test_builder = global_defaults.copy()

  def __init__(self, plan=None, plan_param=None):
    self.__dict__ = self.global_test_builder
    if plan: self.set_plan(plan, plan_param)

  @classmethod
  def create(cls, plan=None, plan_param=None):
    # self = new.instance(cls) # ? this sucks, too
    self = Builder()
    self.__dict__  = self.global_defaults.copy()
    if plan: self.set_plan(plan, plan_param)
    return self

  def set_plan(self, plan, plan_param=None):
    if self.get_plan(): raise TestPlannedAlready(plan, plan_param)
    self._plan = Plan(plan, plan_param)
    atexit.register(self._plan.__del__)

  def get_plan(self): return self._plan

  def ok(self, is_ok, desc=None, skip=None, todo=None):
    self.get_plan().increment_counter()
    if is_ok: report = "ok"
    else:     report = "not ok"

    sys.stdout.write("%s %u" % (report, self.current))

    if desc: sys.stdout.write(" - %s" % desc)
    if skip: sys.stdout.write(" # SKIP %s" % skip)
    if todo: sys.stdout.write(" # TODO %s" % todo)

    print

    self.current += 1

  def reset(self):
    self.__dict__.clear()
    for key in self.global_defaults.iterkeys():
      self.__dict__[key] = self.global_defaults[key]

class TestPlannedAlready(Exception):
  def __init__(self, plan, param=None):
    self.plan  = plan
    self.param = param
  def __str__(self):
    return "tried to plan twice; second plan: %s, %s" % self.plan, self.param

class TestWithoutPlan(Exception):
  def __str__(self):
    return "tried running tests without a plan"

class TestBadPlan(Exception):
  def __init__(self, plan):
    self.plan = plan
  def __str__(self):
    return "didn't understand plan '%s'" % self.plan
<|endoftext|>


<commit_before>from sys import argv
from urlgrab import Cache
from codecs import open
import re
from common import *
from urlparse import urljoin

cache = Cache()
url = argv[1]

id = re.search("/works/(\d+)", url)
id = id.groups()[0]

navigate = "http://archiveofourown.org/works/%s/navigate"%id
print navigate

data = cache.get(navigate).read()

info = re.search("
Chapter Index for ([^<]+) by ([^<]+)
", data)
(title, author) = info.groups()

titlePattern = re.compile("
\s+(.*?)\s+
")
summary = re.compile("
]+?class=\"summary module\"[^>]*?>(.+?)
", re.DOTALL|re.MULTILINE)
notes = re.compile("
]*>(.+?)
", re.DOTALL|re.MULTILINE)
mainContent = re.compile("
Chapter Text
(.*?)", re.DOTALL|re.MULTILINE)
volumePattern = re.compile("
(\d+). ([^<]+)")

volumes = sorted(volumePattern.findall(data))

print volumes
volumes = dict([(int(x[1]), (x[0],x[2])) for x in volumes])
print volumes

toc = tocStart(title)

for volumeUrl, volumeTitle in volumes.values():
	print volumeUrl, volumeTitle
	chapterPage = cache.get(urljoin(navigate, volumeUrl) + "?view_adult=true", max_age = -1).read()
	items = [summary, notes, mainContent]
	items = [x.search(chapterPage) for x in items]
	items = [x.groups()[0] for x in items if x != None]
	content = "\n".join(items)
	generatePage(volumeUrl, volumeTitle, content, title, toc)
tocEnd(toc)
makeMobi(title, author)<commit_msg>UTF-8 of title and content for AO3<commit_after>from sys import argv
from urlgrab import Cache
from codecs import open
import re
from common import *
from urlparse import urljoin

cache = Cache()
url = argv[1]

id = re.search("/works/(\d+)", url)
id = id.groups()[0]

navigate = "http://archiveofourown.org/works/%s/navigate"%id
print navigate

data = cache.get(navigate).read()
info = re.search("
Chapter Index for ([^<]+) by ([^<]+)
", data)
data = data.decode("utf-8")
(title, author) = info.groups()

titlePattern = re.compile("
\s+(.*?)\s+
")
summary = re.compile("
]+?class=\"summary module\"[^>]*?>(.+?)
", re.DOTALL|re.MULTILINE)
notes = re.compile("
]*>(.+?)
", re.DOTALL|re.MULTILINE)
mainContent = re.compile("
Chapter Text
(.*?)", re.DOTALL|re.MULTILINE)
volumePattern = re.compile("
(\d+). ([^<]+)")

volumes = sorted(volumePattern.findall(data))

print volumes
volumes = dict([(int(x[1]), (x[0],x[2])) for x in volumes])
print volumes

toc = tocStart(title)

for volumeUrl, volumeTitle in volumes.values():
	print volumeUrl, volumeTitle
	chapterPage = cache.get(urljoin(navigate, volumeUrl) + "?view_adult=true", max_age = -1).read()
	chapterPage = chapterPage.decode("utf-8")
	items = [summary, notes, mainContent]
	items = [x.search(chapterPage) for x in items]
	items = [x.groups()[0] for x in items if x != None]
	content = "\n".join(items)
	generatePage(volumeUrl, volumeTitle, content, title, toc)
tocEnd(toc)
makeMobi(title, author)<|endoftext|>