<commit_before># Copyright 2016 The Johns Hopkins University Applied Physics Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Create the cachedb configuration which consists of

  * An cachmanager server to run three daemons for
        * cache-write
        * cache-miss
        * cache-delayed write
  * Lambdas
  * SNS topics
  * SQS queues

This will most likely be merged into production once it is finished.

"""

import configuration
import library as lib
import hosts
import json
import scalyr
import uuid
from update_lambda_fcn import load_lambdas_on_s3

# Region production is created in.  Later versions of boto3 should allow us to
# extract this from the session variable.  Hard coding for now.
PRODUCTION_REGION = 'us-east-1'

INCOMING_SUBNET = "52.3.13.189/32"  # microns-bastion elastic IP

CACHE_MANAGER_TYPE = {
    "development": "t2.micro",
    "production": "t2.medium",
}

def create_config(session, domain, keypair=None, user_data=None):
    """
    Create the CloudFormationConfiguration object.
    Args:
        session: amazon session object
        domain: domain of the stack being created
        keypair: keypair used to by instances being created
        user_data: information used by the endpoint instance and vault
        db_config: information needed by rds

    Returns: the config for the Cloud Formation stack

    """
    config = configuration.CloudFormationConfiguration(domain, PRODUCTION_REGION)

    # do a couple of verification checks
    if config.subnet_domain is not None:
        raise Exception("Invalid VPC domain name")

    vpc_id = lib.vpc_id_lookup(session, domain)
    if session is not None and vpc_id is None:
        raise Exception("VPC does not exists, exiting...")

    # Lookup the VPC, External Subnet, Internal Security Group IDs that are
    # needed by other resources
    config.add_arg(configuration.Arg.VPC("VPC", vpc_id,
                                         "ID of VPC to create resources in"))

    external_subnet_id = lib.subnet_id_lookup(session, "external." + domain)
    config.add_arg(configuration.Arg.Subnet("ExternalSubnet",
                                            external_subnet_id,
                                            "ID of External Subnet to create resources in"))

    internal_subnet_id = lib.subnet_id_lookup(session, "internal." + domain)
    config.add_arg(configuration.Arg.Subnet("InternalSubnet",
                                            internal_subnet_id,
                                            "ID of Internal Subnet to create resources in"))

    internal_sg_id = lib.sg_lookup(session, vpc_id, "internal." + domain)
    config.add_arg(configuration.Arg.SecurityGroup("InternalSecurityGroup",
                                                   internal_sg_id,
                                                   "ID of internal Security Group"))

    role = lib.role_arn_lookup(session, "lambda_cache_execution")
    config.add_arg(configuration.Arg.String("LambdaCacheExecutionRole", role,
                                            "IAM role for multilambda." + domain))

    index_bucket_name = "cuboids." + domain
    if not lib.s3_bucket_exists(session, index_bucket_name):
        config.add_s3_bucket("cuboidBucket", index_bucket_name)
    config.add_s3_bucket_policy(
        "cuboidBucketPolicy", index_bucket_name,
        ['s3:GetObject', 's3:PutObject'],
        { 'AWS': role})

    config.add_ec2_instance("CacheManager",
                                "cachemanager." + domain,
                                lib.ami_lookup(session, "cachemanager.boss"),
                                keypair,
                                subnet="InternalSubnet",
                                public_ip=False,
                                type_=CACHE_MANAGER_TYPE,
                                security_groups=["InternalSecurityGroup"],
                                user_data=user_data,
                                role="cachemanager") # arn:aws:iam::256215146792:instance-profile/cachemanager"

    lambda_sec_group = lib.sg_lookup(session, vpc_id, 'internal.' + domain)
    filter_by_host_name = ([{
        'Name': 'tag:Name',
        'Values': ['*internal.' + domain]
    }])
    lambda_subnets = lib.multi_subnet_id_lookup(session, filter_by_host_name)

    multi_lambda_name = 'multiLambda-' + domain.replace('.', '-')
    config.add_lambda("MultiLambda",
                      multi_lambda_name,
                      "LambdaCacheExecutionRole",
                      s3=("boss-lambda-env",
                          "multilambda.{}.zip".format(domain),
                          "local/lib/python3.4/site-packages/lambda/lambda_loader.handler"),
                      timeout=60,
                      memory=1024,
                      security_groups=[lambda_sec_group],
                      subnets=lambda_subnets)

    return config

def generate(folder, domain):
    """Create the configuration and save it to disk"""
    name = lib.domain_to_stackname("cachedb." + domain)
    config = create_config(None, domain)
    config.generate(name, folder)

def create(session, domain):
    """Create the configuration, and launch it"""
    user_data = configuration.UserData()
    user_data["system"]["fqdn"] = "cachemanager." + domain
    user_data["system"]["type"] = "cachemanager"
    user_data["aws"]["cache"] = "cache." + domain
    user_data["aws"]["cache-state"] = "cache-state." + domain
    user_data["aws"]["cache-db"] = '0'  ## cache-db and cache-stat-db need to be in user_data for lambda to get to them.
    user_data["aws"]["cache-state-db"] = '0'

    keypair = lib.keypair_lookup(session)

    try:
        name = lib.domain_to_stackname("cachedb." + domain)
        pre_init(session, domain)

        config = create_config(session, domain, keypair, str(user_data))

        success = config.create(session, name)
        print("finished config.create")
        if not success:
            raise Exception("Create Failed")
        else:
            post_init(session, domain)
    except:
        print("Error detected") # Do we want to revoke if an exception from post_init?
        raise

def pre_init(session, domain):
    """Send spdb, bossutils, lambda, and lambda_utils to the lambda build
    server, build the lambda environment, and upload to S3.
    """
    load_lambdas_on_s3(domain)

def post_init(session, domain):
    print("post_init")

    # Tell Scalyr to get CloudWatch metrics for these instances.
    instances = ["cachemanager." + domain]
    scalyr.add_instances_to_scalyr(
        session, PRODUCTION_REGION, instances)

def delete(session, domain):
    # NOTE: CloudWatch logs for the DNS Lambda are not deleted
    lib.delete_stack(session, domain, "cachedb")
    lib.route53_delete_records(session, domain, "cachemanager." + domain)
<commit_msg>updated cachedb.py to mirror endpoint config for boss.config file<commit_after># Copyright 2016 The Johns Hopkins University Applied Physics Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Create the cachedb configuration which consists of

  * An cachmanager server to run three daemons for
        * cache-write
        * cache-miss
        * cache-delayed write
  * Lambdas
  * SNS topics
  * SQS queues

This will most likely be merged into production once it is finished.

"""

import configuration
import library as lib
import hosts
import json
import scalyr
import uuid
from update_lambda_fcn import load_lambdas_on_s3

# Region production is created in.  Later versions of boto3 should allow us to
# extract this from the session variable.  Hard coding for now.
PRODUCTION_REGION = 'us-east-1'

INCOMING_SUBNET = "52.3.13.189/32"  # microns-bastion elastic IP

CACHE_MANAGER_TYPE = {
    "development": "t2.micro",
    "production": "t2.medium",
}

# Prefixes uses to generate names of SQS queues.
S3FLUSH_QUEUE_PREFIX = 'S3flush.'
DEADLETTER_QUEUE_PREFIX = 'Deadletter.'

def create_config(session, domain, keypair=None, user_data=None):
    """
    Create the CloudFormationConfiguration object.
    Args:
        session: amazon session object
        domain: domain of the stack being created
        keypair: keypair used to by instances being created
        user_data: information used by the endpoint instance and vault
        db_config: information needed by rds

    Returns: the config for the Cloud Formation stack

    """
    config = configuration.CloudFormationConfiguration(domain, PRODUCTION_REGION)

    # do a couple of verification checks
    if config.subnet_domain is not None:
        raise Exception("Invalid VPC domain name")

    vpc_id = lib.vpc_id_lookup(session, domain)
    if session is not None and vpc_id is None:
        raise Exception("VPC does not exists, exiting...")

    # Lookup the VPC, External Subnet, Internal Security Group IDs that are
    # needed by other resources
    config.add_arg(configuration.Arg.VPC("VPC", vpc_id,
                                         "ID of VPC to create resources in"))

    external_subnet_id = lib.subnet_id_lookup(session, "external." + domain)
    config.add_arg(configuration.Arg.Subnet("ExternalSubnet",
                                            external_subnet_id,
                                            "ID of External Subnet to create resources in"))

    internal_subnet_id = lib.subnet_id_lookup(session, "internal." + domain)
    config.add_arg(configuration.Arg.Subnet("InternalSubnet",
                                            internal_subnet_id,
                                            "ID of Internal Subnet to create resources in"))

    internal_sg_id = lib.sg_lookup(session, vpc_id, "internal." + domain)
    config.add_arg(configuration.Arg.SecurityGroup("InternalSecurityGroup",
                                                   internal_sg_id,
                                                   "ID of internal Security Group"))

    role = lib.role_arn_lookup(session, "lambda_cache_execution")
    config.add_arg(configuration.Arg.String("LambdaCacheExecutionRole", role,
                                            "IAM role for multilambda." + domain))

    index_bucket_name = "cuboids." + domain
    if not lib.s3_bucket_exists(session, index_bucket_name):
        config.add_s3_bucket("cuboidBucket", index_bucket_name)
    config.add_s3_bucket_policy(
        "cuboidBucketPolicy", index_bucket_name,
        ['s3:GetObject', 's3:PutObject'],
        { 'AWS': role})

    config.add_ec2_instance("CacheManager",
                                "cachemanager." + domain,
                                lib.ami_lookup(session, "cachemanager.boss"),
                                keypair,
                                subnet="InternalSubnet",
                                public_ip=False,
                                type_=CACHE_MANAGER_TYPE,
                                security_groups=["InternalSecurityGroup"],
                                user_data=user_data,
                                role="cachemanager") # arn:aws:iam::256215146792:instance-profile/cachemanager"

    lambda_sec_group = lib.sg_lookup(session, vpc_id, 'internal.' + domain)
    filter_by_host_name = ([{
        'Name': 'tag:Name',
        'Values': ['*internal.' + domain]
    }])
    lambda_subnets = lib.multi_subnet_id_lookup(session, filter_by_host_name)

    multi_lambda_name = 'multiLambda-' + domain.replace('.', '-')
    config.add_lambda("MultiLambda",
                      multi_lambda_name,
                      "LambdaCacheExecutionRole",
                      s3=("boss-lambda-env",
                          "multilambda.{}.zip".format(domain),
                          "local/lib/python3.4/site-packages/lambda/lambda_loader.handler"),
                      timeout=60,
                      memory=1024,
                      security_groups=[lambda_sec_group],
                      subnets=lambda_subnets)

    return config

def generate(folder, domain):
    """Create the configuration and save it to disk"""
    name = lib.domain_to_stackname("cachedb." + domain)
    config = create_config(None, domain)
    config.generate(name, folder)

def create(session, domain):
    """Create the configuration, and launch it"""
    s3flushqname = lib.domain_to_stackname(S3FLUSH_QUEUE_PREFIX + domain)
    deadqname = lib.domain_to_stackname(DEADLETTER_QUEUE_PREFIX + domain)

    # Configure Vault and create the user data config that the endpoint will
    # use for connecting to Vault and the DB instance
    user_data = configuration.UserData()
    user_data["system"]["fqdn"] = "cachemanager." + domain
    user_data["system"]["type"] = "cachemanager"
    user_data["aws"]["cache"] = "cache." + domain
    user_data["aws"]["cache-state"] = "cache-state." + domain
    user_data["aws"]["cache-db"] = "0"
    user_data["aws"]["cache-state-db"] = "0"

    # Use CloudFormation's Ref function so that queues' URLs are placed into
    # the Boss config file.
    user_data["aws"]["s3-flush-queue"] = '{{"Ref": "{}" }}'.format(s3flushqname)
    user_data["aws"]["s3-flush-deadletter-queue"] = '{{"Ref": "{}" }}'.format(deadqname)
    user_data["aws"]["cuboid_bucket"] = "cuboids." + domain
    user_data["aws"]["s3-index-table"] = "s3index." + domain

    # Lambda names can't have periods.
    sanitized_domain = domain.replace('.', '-')
    user_data["lambda"]["flush_function"] = "multiLambda-" + sanitized_domain
    user_data["lambda"]["page_in_function"] = "multiLambda-" + sanitized_domain

    keypair = lib.keypair_lookup(session)

    try:
        name = lib.domain_to_stackname("cachedb." + domain)
        pre_init(session, domain)

        config = create_config(session, domain, keypair, str(user_data))

        success = config.create(session, name)
        print("finished config.create")
        if not success:
            raise Exception("Create Failed")
        else:
            post_init(session, domain)
    except:
        print("Error detected") # Do we want to revoke if an exception from post_init?
        raise

def pre_init(session, domain):
    """Send spdb, bossutils, lambda, and lambda_utils to the lambda build
    server, build the lambda environment, and upload to S3.
    """
    load_lambdas_on_s3(domain)

def post_init(session, domain):
    print("post_init")

    # Tell Scalyr to get CloudWatch metrics for these instances.
    instances = ["cachemanager." + domain]
    scalyr.add_instances_to_scalyr(
        session, PRODUCTION_REGION, instances)

def delete(session, domain):
    # NOTE: CloudWatch logs for the DNS Lambda are not deleted
    lib.delete_stack(session, domain, "cachedb")
    lib.route53_delete_records(session, domain, "cachemanager." + domain)
<|endoftext|>


<commit_before>#!/usr/bin/env Python

import argparse
import os
import csv
import datetime
from shutil import rmtree
import numpy as np
from collections import defaultdict
from multiprocessing import cpu_count

# Arg parser
def make_arg_parser():
	parser = argparse.ArgumentParser(description='Given the processed shotgun reads, measure coverage of predicted BGCs in metagenome')
	parser.add_argument('-s', '--seqs', help="The combined seqs file (fasta) containing QC'd shotgun reads", required=False)
	parser.add_argument('--bgc_fna', help="The BGC DNA fasta file (all BGCs in one multi-fasta file)", required=False)
	parser.add_argument('--bgc_edx', help="The pre-made BURST database file)", required=False)
	parser.add_argument('--bgc_acx', help="The pre-made BURST accelerator file)", required=False)
	parser.add_argument('--alignment', help='A pre-calculated all-vs-all alignment file in .b6 format', required=False)
	parser.add_argument('--debug', help='Keep the temp directory for debugging', default=False, action='store_true', required=False)
	# parser.add_argument('-b', '--bread', help='Where to find the header for the sequence (default="ref|,|")', default='ref|,|')
	parser.add_argument('-l', '--lengths', help="The table providing length of each BGC in bp", required=True)
	parser.add_argument('-o', '--output', help='Directory in which to save the results (default = cwd)', required=False, default='.')
	# parser.add_argument('-t', '--threshold', help='Coverage threshold at which to accept a cluster alignment (default = 75%)', required=False, default=75, type=float)
	parser.add_argument('-p', '--threads', help='Number of processors to use for alignment (default = all available)', required=False, default='-')
	return parser

# Make a database if necessary
def create_bgc_db(bgc_db, temp_path, burst):
	acc = os.path.join(temp_path, 'bgc_db.acc')
	edb = os.path.join(temp_path, 'bgc_db.edb')
	os.system(' '.join([burst, '-r', bgc_db, '-a', acc, '-o', edb, '-d DNA -s']))
	os.system(' '.join([burst, '-a', acc, '-o /dev/null']))
	os.system(' '.join([burst, '-r', edb, '-o /dev/null']))
	os.remove(acc)
	os.remove(edb)
	acx = os.path.join(temp_path, 'bgc_db.acx')
	edx = os.path.join(temp_path, 'bgc_db.edx')
	if not os.path.isfile(acx) or not os.path.isfile(edx):
		raise ValueError('BURST databases not successfully generated: check input fasta and BURST installation')
	return acx, edx

# Run the BURST alignment
def align_bgcs(temp_path, seqs, acx, edx, cpus):
	alignment = os.path.join(temp_path, 'bgc_alignments.b6')
	if acx != '-':
		os.system(' '.join(['burst15 -m ALLPATHS -fr -hr -q', seqs, '-n -r', edx, '-o', alignment, '-t', cpus, '-i 0.9', '-a', acx]))
	else:
		os.system(' '.join(['burst15 -m ALLPATHS -fr -hr -q', seqs, '-n -r', edx, '-o', alignment, '-t', cpus, '-i 0.9']))
	print("BURST alignment completed")
	return alignment

def get_percent_coverage(tally):
	# Get number of hits
	hits = np.sum(tally > 0)
	# Get number of misses
	misses = np.sum(tally == 0)
	# Coverage
	percent_coverage = hits/(hits+misses)*100
	return percent_coverage

def zero_runs(tally):
	# Stack Overflow:
	# https://stackoverflow.com/questions/24885092/finding-the-consecutive-zeros-in-a-numpy-array
	# Create an array that is 1 where a is 0, and pad each end with an extra 0.
	iszero = np.concatenate(([0], np.equal(tally, 0).view(np.int8), [0]))
	absdiff = np.abs(np.diff(iszero))
	# Runs start and end where absdiff is 1.
	ranges = np.where(absdiff == 1)[0].reshape(-1, 2)
	return ranges

def max_uncovered_region(tally):
	coverages = zero_runs(tally)
	if coverages.size == 0:
		return 0
	else:
		return np.max(coverages[:, 1] - coverages[:, 0])

def expected_coverage(tally, median_read_length):
	number_of_reads = np.sum(tally)/median_read_length
	return 1-np.exp(-(number_of_reads*median_read_length)/tally.shape[0])

def invert_bincount(c):
	return np.repeat(np.arange(c.size), c)

# plt.hist(invert_bincount(sample_dict[gene_cluster_id]), bins=100)

# Run across all samples to get coverage in the meta-metagenome
def meta_coverage(sample_to_gene_cluster_to_row):
	gene_cluster_to_row = dict()
	for key, value in sample_to_gene_cluster_to_row.items():
		for gene_cluster, row in sample_to_gene_cluster_to_row[key].items():
			if gene_cluster not in gene_cluster_to_row:
				gene_cluster_to_row[gene_cluster] = row
			else:
				gene_cluster_to_row[gene_cluster] += row

# tally = gene_cluster_to_row['ncbi_tid|451515|genbank|CP000255.1_cluster003|organism|k__Bacteria;p__Firmicutes;c__Bacilli;o__Bacillales;f__Staphylococcaceae;g__Staphylococcus;s__Staphylococcus_aureus;t__Staphylococcus_aureus_subsp._aureus_USA300_FPR3757']

def main():
	parser = make_arg_parser()
	args = parser.parse_args()
	outpath = args.output
	if args.seqs:
		seqs = args.seqs
	lengths = args.lengths
	if args.threads == '-':
		cpus = int(cpu_count())
	else:
		cpus = int(args.threads)
	# if not args.metrics == '-':
	# 	output_metrics = str(args.metrics)
	# else:
	# 	args.metrics = 'all'
	# threshold = float(args.threshold)
	if not os.path.isdir(outpath):
		os.mkdir(outpath)
		if not os.path.isdir(outpath):
			raise ValueError('Error creating output directory; check given path and try again')
	timestamp = datetime.datetime.now()
	timestamp = timestamp.strftime("%Y%m%d_%H%M")
	tempdir = '_'.join(['tempz', timestamp])
	temp_path = os.path.join(outpath, tempdir)
	os.mkdir(temp_path)
	if args.bgc_fna:
		fna_bytes = int(os.path.getsize(args.bgc_fna))
		if fna_bytes < 1000000000:
			burst = 'burst12'
		else:
			burst = 'burst15'
		bgc_db = args.bgc_fna
		acx, edx = create_bgc_db(bgc_db, temp_path, burst)
	elif args.bgc_edx:
		edx = args.bgc_edx
		if args.bgc_acx:
			acx = args.bgc_acx
		else:
			acx = '-'
	elif args.alignment:
		alignment = args.alignment
	elif not args.bgc_fna and not args.bgc_edx and not args.alignment:
		raise ValueError('No reference sequences or database provided.\nPlease include something to align to or the pre-computed alignment file')
	# Generate the alignment between sequences and BGCs
	if not alignment:
		try:
			os.path.isfile(seqs)
		except ValueError:
			print('Sequences file not found')
		alignment = align_bgcs(temp_path, seqs, acx, edx, cpus)

	# Calculate coverage based on the alignment #
	# Parse the BGC lengths file
	gene_cluster_sizes = defaultdict(int)
	with open(lengths) as inf:
		csv_inf = csv.reader(inf, delimiter='\t')
		for row in csv_inf:
			gene_cluster_sizes[row[0]] = int(row[-1])

	# Parse the alignment file
	sample_to_gene_cluster_to_row = defaultdict(dict)
	sample_names = []
	read_lengths = []
	sample_cluster_hit_counter = defaultdict(dict)
	with open(alignment) as inf:
		csv_inf = csv.reader(inf, delimiter="\t")
		for line in csv_inf:
			sample_id = line[0].split('_')[0]
			sample_names.append(sample_id)
			sample_dict = sample_to_gene_cluster_to_row[sample_id]
			bgc_hit_counter = sample_cluster_hit_counter[sample_id]
			gene_cluster_id = line[1]
			if not gene_cluster_id in sample_dict:
				sample_dict[gene_cluster_id] = np.zeros(gene_cluster_sizes[gene_cluster_id], dtype=int)
				bgc_hit_counter[gene_cluster_id] = 0
			begin = int(line[9])
			end = int(line[8])
			# Reverse complement
			if begin > end:
				begin = int(line[8])
				end = int(line[9])
			read_lengths.append(end-begin)
			sample_dict[gene_cluster_id][begin:end] += 1
			bgc_hit_counter[gene_cluster_id] += 1
	read_lengths_array = np.array(read_lengths, dtype=int)
	median_read_length = np.median(read_lengths_array)

	# Loop through the data to generate the output table
	# coverage_result = defaultdict(dict)
	outfile = os.path.join(outpath, 'coverage_result_%s.txt') % timestamp
	with open(outfile, 'w') as outf:
		outf.write('sample_id\tbgc_id\tbgc_length\tmax_gap\thits_to_bgc\ttotal_base_coverage\tpercent_coverage\texpected_coverage\tratio_covered_to_expected\n')
		sample_names = set(sample_names)
		for sample in sample_names:
			sample_dict_parse = sample_to_gene_cluster_to_row[sample]
			sample_hit_counts = sample_cluster_hit_counter[sample]
			for bgc_id in sample_dict_parse:
				tally = sample_dict_parse[bgc_id]
				hits_to_bgc = sample_hit_counts[bgc_id]
				total_base_coverage = int(np.sum(tally))  # Need this?
				max_gap = max_uncovered_region(tally)
				# print(max_gap)
				percent_coverage = get_percent_coverage(tally)
				# print(percent_coverage)
				predicted_coverage = expected_coverage(tally, median_read_length) * 100
				# print(predicted_coverage)
				coverage_ratio = percent_coverage / predicted_coverage
				# print(coverage_ratio)
				outf.write('%s\t%s\t%d\t%d\t%d\t%d\t%s\t%s\t%s\n' % (sample, bgc_id, gene_cluster_sizes[bgc_id], max_gap, hits_to_bgc, total_base_coverage, percent_coverage, predicted_coverage, coverage_ratio))
	print('Coverage analysis complete.')
	if not args.debug:
		rmtree(temp_path)
	# The calls to get the values #
	# max_gap = max_uncovered_region(sample_dict[gene_cluster_id])
	# percent_coverage = get_percent_coverage(sample_dict[gene_cluster_id])
	# expected_coverage = expected_coverage(sample_dict[gene_cluster_id], median_read_length) * 100
	# coverage_ratio = percent_coverage / expected_coverage

if __name__ == '__main__':
	main()
<commit_msg>Updated argument flag and final print statement<commit_after>#!/usr/bin/env Python

import argparse
import os
import csv
import datetime
from shutil import rmtree
import numpy as np
from collections import defaultdict
from multiprocessing import cpu_count

# Arg parser
def make_arg_parser():
	parser = argparse.ArgumentParser(description='Given the processed shotgun reads, measure coverage of predicted BGCs in metagenome')
	parser.add_argument('-s', '--seqs', help="The combined seqs file (fasta) containing QC'd shotgun reads", required=False)
	parser.add_argument('--bgc_fna', help="The BGC DNA fasta file (all BGCs in one multi-fasta file)", required=False)
	parser.add_argument('--bgc_edx', help="The pre-made BURST database file)", required=False)
	parser.add_argument('--bgc_acx', help="The pre-made BURST accelerator file)", required=False)
	parser.add_argument('-a', '--alignment', help='A pre-calculated all-vs-all alignment file in .b6 format', required=False)
	parser.add_argument('--debug', help='Keep the temp directory for debugging', default=False, action='store_true', required=False)
	# parser.add_argument('-b', '--bread', help='Where to find the header for the sequence (default="ref|,|")', default='ref|,|')
	parser.add_argument('-l', '--lengths', help="The table providing length of each BGC in bp", required=True)
	parser.add_argument('-o', '--output', help='Directory in which to save the results (default = cwd)', required=False, default='.')
	# parser.add_argument('-t', '--threshold', help='Coverage threshold at which to accept a cluster alignment (default = 75%)', required=False, default=75, type=float)
	parser.add_argument('-p', '--threads', help='Number of processors to use for alignment (default = all available)', required=False, default='-')
	return parser

# Make a database if necessary
def create_bgc_db(bgc_db, temp_path, burst):
	acc = os.path.join(temp_path, 'bgc_db.acc')
	edb = os.path.join(temp_path, 'bgc_db.edb')
	os.system(' '.join([burst, '-r', bgc_db, '-a', acc, '-o', edb, '-d DNA -s']))
	os.system(' '.join([burst, '-a', acc, '-o /dev/null']))
	os.system(' '.join([burst, '-r', edb, '-o /dev/null']))
	os.remove(acc)
	os.remove(edb)
	acx = os.path.join(temp_path, 'bgc_db.acx')
	edx = os.path.join(temp_path, 'bgc_db.edx')
	if not os.path.isfile(acx) or not os.path.isfile(edx):
		raise ValueError('BURST databases not successfully generated: check input fasta and BURST installation')
	return acx, edx

# Run the BURST alignment
def align_bgcs(temp_path, seqs, acx, edx, cpus):
	alignment = os.path.join(temp_path, 'bgc_alignments.b6')
	if acx != '-':
		os.system(' '.join(['burst15 -m ALLPATHS -fr -hr -q', seqs, '-n -r', edx, '-o', alignment, '-t', cpus, '-i 0.9', '-a', acx]))
	else:
		os.system(' '.join(['burst15 -m ALLPATHS -fr -hr -q', seqs, '-n -r', edx, '-o', alignment, '-t', cpus, '-i 0.9']))
	print("BURST alignment completed")
	return alignment

def get_percent_coverage(tally):
	# Get number of hits
	hits = np.sum(tally > 0)
	# Get number of misses
	misses = np.sum(tally == 0)
	# Coverage
	percent_coverage = hits/(hits+misses)*100
	return percent_coverage

def zero_runs(tally):
	# Stack Overflow:
	# https://stackoverflow.com/questions/24885092/finding-the-consecutive-zeros-in-a-numpy-array
	# Create an array that is 1 where a is 0, and pad each end with an extra 0.
	iszero = np.concatenate(([0], np.equal(tally, 0).view(np.int8), [0]))
	absdiff = np.abs(np.diff(iszero))
	# Runs start and end where absdiff is 1.
	ranges = np.where(absdiff == 1)[0].reshape(-1, 2)
	return ranges

def max_uncovered_region(tally):
	coverages = zero_runs(tally)
	if coverages.size == 0:
		return 0
	else:
		return np.max(coverages[:, 1] - coverages[:, 0])

def expected_coverage(tally, median_read_length):
	number_of_reads = np.sum(tally)/median_read_length
	return 1-np.exp(-(number_of_reads*median_read_length)/tally.shape[0])

def invert_bincount(c):
	return np.repeat(np.arange(c.size), c)

# plt.hist(invert_bincount(sample_dict[gene_cluster_id]), bins=100)

# Run across all samples to get coverage in the meta-metagenome
def meta_coverage(sample_to_gene_cluster_to_row):
	gene_cluster_to_row = dict()
	for key, value in sample_to_gene_cluster_to_row.items():
		for gene_cluster, row in sample_to_gene_cluster_to_row[key].items():
			if gene_cluster not in gene_cluster_to_row:
				gene_cluster_to_row[gene_cluster] = row
			else:
				gene_cluster_to_row[gene_cluster] += row

# tally = gene_cluster_to_row['ncbi_tid|451515|genbank|CP000255.1_cluster003|organism|k__Bacteria;p__Firmicutes;c__Bacilli;o__Bacillales;f__Staphylococcaceae;g__Staphylococcus;s__Staphylococcus_aureus;t__Staphylococcus_aureus_subsp._aureus_USA300_FPR3757']

def main():
	parser = make_arg_parser()
	args = parser.parse_args()
	outpath = args.output
	if args.seqs:
		seqs = args.seqs
	lengths = args.lengths
	if args.threads == '-':
		cpus = int(cpu_count())
	else:
		cpus = int(args.threads)
	# if not args.metrics == '-':
	# 	output_metrics = str(args.metrics)
	# else:
	# 	args.metrics = 'all'
	# threshold = float(args.threshold)
	if not os.path.isdir(outpath):
		os.mkdir(outpath)
		if not os.path.isdir(outpath):
			raise ValueError('Error creating output directory; check given path and try again')
	timestamp = datetime.datetime.now()
	timestamp = timestamp.strftime("%Y%m%d_%H%M")
	tempdir = '_'.join(['tempz', timestamp])
	temp_path = os.path.join(outpath, tempdir)
	os.mkdir(temp_path)
	if args.bgc_fna:
		fna_bytes = int(os.path.getsize(args.bgc_fna))
		if fna_bytes < 1000000000:
			burst = 'burst12'
		else:
			burst = 'burst15'
		bgc_db = args.bgc_fna
		acx, edx = create_bgc_db(bgc_db, temp_path, burst)
	elif args.bgc_edx:
		edx = args.bgc_edx
		if args.bgc_acx:
			acx = args.bgc_acx
		else:
			acx = '-'
	elif args.alignment:
		alignment = args.alignment
	elif not args.bgc_fna and not args.bgc_edx and not args.alignment:
		raise ValueError('No reference sequences or database provided.\nPlease include something to align to or the pre-computed alignment file')
	# Generate the alignment between sequences and BGCs
	if not alignment:
		try:
			os.path.isfile(seqs)
		except ValueError:
			print('Sequences file not found')
		alignment = align_bgcs(temp_path, seqs, acx, edx, cpus)

	# Calculate coverage based on the alignment #
	# Parse the BGC lengths file
	gene_cluster_sizes = defaultdict(int)
	with open(lengths) as inf:
		csv_inf = csv.reader(inf, delimiter='\t')
		for row in csv_inf:
			gene_cluster_sizes[row[0]] = int(row[-1])

	# Parse the alignment file
	sample_to_gene_cluster_to_row = defaultdict(dict)
	sample_names = []
	read_lengths = []
	sample_cluster_hit_counter = defaultdict(dict)
	with open(alignment) as inf:
		csv_inf = csv.reader(inf, delimiter="\t")
		for line in csv_inf:
			sample_id = line[0].split('_')[0]
			sample_names.append(sample_id)
			sample_dict = sample_to_gene_cluster_to_row[sample_id]
			bgc_hit_counter = sample_cluster_hit_counter[sample_id]
			gene_cluster_id = line[1]
			if not gene_cluster_id in sample_dict:
				sample_dict[gene_cluster_id] = np.zeros(gene_cluster_sizes[gene_cluster_id], dtype=int)
				bgc_hit_counter[gene_cluster_id] = 0
			begin = int(line[9])
			end = int(line[8])
			# Reverse complement
			if begin > end:
				begin = int(line[8])
				end = int(line[9])
			read_lengths.append(end-begin)
			sample_dict[gene_cluster_id][begin:end] += 1
			bgc_hit_counter[gene_cluster_id] += 1
	read_lengths_array = np.array(read_lengths, dtype=int)
	median_read_length = np.median(read_lengths_array)

	# Loop through the data to generate the output table
	# coverage_result = defaultdict(dict)
	outfile = os.path.join(outpath, 'coverage_result_%s.txt') % timestamp
	with open(outfile, 'w') as outf:
		outf.write('sample_id\tbgc_id\tbgc_length\tmax_gap\thits_to_bgc\ttotal_base_coverage\tpercent_coverage\texpected_coverage\tratio_covered_to_expected\n')
		sample_names = set(sample_names)
		for sample in sample_names:
			sample_dict_parse = sample_to_gene_cluster_to_row[sample]
			sample_hit_counts = sample_cluster_hit_counter[sample]
			for bgc_id in sample_dict_parse:
				tally = sample_dict_parse[bgc_id]
				hits_to_bgc = sample_hit_counts[bgc_id]
				total_base_coverage = int(np.sum(tally))  # Need this?
				max_gap = max_uncovered_region(tally)
				# print(max_gap)
				percent_coverage = get_percent_coverage(tally)
				# print(percent_coverage)
				predicted_coverage = expected_coverage(tally, median_read_length) * 100
				# print(predicted_coverage)
				coverage_ratio = percent_coverage / predicted_coverage
				# print(coverage_ratio)
				outf.write('%s\t%s\t%d\t%d\t%d\t%d\t%s\t%s\t%s\n' % (sample, bgc_id, gene_cluster_sizes[bgc_id], max_gap, hits_to_bgc, total_base_coverage, percent_coverage, predicted_coverage, coverage_ratio))
	print('Coverage analysis complete. Find your results at:\n%s' % outfile)
	if not args.debug:
		rmtree(temp_path)
	# The calls to get the values #
	# max_gap = max_uncovered_region(sample_dict[gene_cluster_id])
	# percent_coverage = get_percent_coverage(sample_dict[gene_cluster_id])
	# expected_coverage = expected_coverage(sample_dict[gene_cluster_id], median_read_length) * 100
	# coverage_ratio = percent_coverage / expected_coverage

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>"""Adapter layer to distributed job manager server api
"""
import os
import os.path
import logging
import datetime

from file_locator import FileLocator
import utils
from dist_job_mgr.client import get_local_connection
from dist_job_mgr.version import VERSION
import dist_job_mgr.common as common

class DjmAdapterError(Exception):
    pass

logger = logging.getLogger(__name__)

def get_djm_connection():
    fl = FileLocator()
    return get_local_connection(fl.get_djm_server_dir())

class DjmJob(object):
    def __init__(self, c, job_id, nodes):
        self.c = c
        self.job_id = job_id
        self.nodes = nodes
        self.nodes_by_name = {}
        for node in nodes:
            if node["private_ip"]!=None:
                ip_address = node["private_ip"]
            else:
                ip_address = node["public_ip"]
            if not ip_address:
                raise Exception("Neither public ip address nor private ip address specified for node %s, need to specify at least one" % node["name"])
            node["datablox_ip_address"] = ip_address
            self.nodes_by_name[node["name"]] = node
        self.nodes_except_master = filter(lambda name: name!="master",
                                          [node["name"] for node in self.nodes])

    def has_node(self, node_name):
        return self.nodes_by_name.has_key(node_name)

    def get_node(self, node_name):
        return self.nodes_by_name[node_name]

    def stop_job(self, successful=True, msg=None):
        if len(self.nodes_except_master)>0:
            (s, r) = self.c.run_task_on_node_list(self.job_id, "StopWorker", "stop worker",
                                                  self.nodes_except_master)
            if s!=common.TaskStatus.TASK_SUCCESSFUL:
                logger.warn("Not able to stop DJM worker on all nodes")
        if successful:
            self.c.stop_job(self.job_id,
                            common.JobStatus.JOB_SUCCESSFUL,
                            comment=msg)
        else:
            self.c.stop_job(self.job_id,
                            common.JobStatus.JOB_FAILED,
                            comment=msg)

def _check_task_status(s, r, msg):
    if s!=common.TaskStatus.TASK_SUCCESSFUL:
        bad_nodes = filter(lambda res:
                           res.status!=common.TaskStatus.TASK_SUCCESSFUL,
                           r)
        raise Exception("%s for nodes: %s" %
                        (msg, ', '.join([r.node_name for r in bad_nodes])))

def start_job_and_get_nodes(node_list, config_file_name, total_nodes=None,
                            reuse_existing_installs=True):
    """Given a node list and optional number of nodes, try to get the
    requested nodes and start a job.
    """
    if not total_nodes:
        total_nodes = len(node_list)
    if total_nodes<1:
        raise DjmAdapterError("Must have at least one node")
    c = get_djm_connection()
    # make sure there aren't any dead jobs laying around
    c.cleanup_dead_coordinators()
    pool = None
    for node_name in node_list:
        n = c.find_node_by_name(node_name)
        if not n:
            raise DjmAdapterError("Node '%s' not defined" % node_name)
        if n["pool"]:
            if pool and pool!=n["pool"]:
                raise DjmAdapterError("Cannot take nodes from both pool %s and pool %s"%
                                      (pool, n["pool"]))
            pool = n["pool"]
    start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
    j = c.start_job(config_file_name, common.JobType.ONE_TIME_JOB,
                    total_nodes, "Datablox job started %s" % start_time,
                    node_pool_name=pool, requested_nodes=node_list)
    logger.info("Started DJM job %s" % j)
    try:
        fl = FileLocator()
        allocated_nodes = c.query_nodes(job_id=j)
        djm_job = DjmJob(c, j, allocated_nodes)
        logger.info("Setting up nodes")
        # for all the non-master nodes, we setup the caretaker
        nodes_except_master = djm_job.nodes_except_master
        if len(nodes_except_master)>0:
            (s, r) = c.run_task_on_node_list(j, "StartWorker", "start worker",
                                             nodes_except_master)
            _check_task_status(s, r, "DJM worker start failed")
            dist_path = fl.get_engage_distribution_file()
            logger.info("Copying engage distribution")
            (s, r) = c.run_task_on_node_list(j, "CopyFiles",
                                             "Copy engage distribution",
                                             nodes_except_master,
                                             dist_path,
                                             "~/" + os.path.basename(dist_path))
            _check_task_status(s, r, "Copying of engage distribution failed")
            (s, r) = c.run_task_on_node_list(j, "CopyFiles",
                                             "Copy setup script",
                                             nodes_except_master,
                                             os.path.join(fl.get_sw_packages_dir(),
                                                          "setup_caretaker.sh"),
                                             "~/setup_caretaker.sh")
            _check_task_status(s, r, "Copy of caretaker setup script failed")
            (s, r) = c.run_task_on_node_list(j, "Command",
                                             "Make setup script executable",
                                             nodes_except_master,
                                             ["/bin/chmod 755 ~/setup_caretaker.sh"],
                                             shell=True)
            _check_task_status(s, r, "chmod of caretaker setup script failed")
            caretaker_cmd = ["~/setup_caretaker.sh",]
            if reuse_existing_installs:
                caretaker_cmd.append("--reuse-existing-install")
            (s, r) = c.run_task_on_node_list(j, "Command",
                                             "Setup remote caretaker",
                                             nodes_except_master,
                                             caretaker_cmd,
                                             shell=True)
            _check_task_status(s, r, "Caretaker setup script failed")
        # make sure the master node has the caretaker running
        if djm_job.has_node("master"):
            utils.run_svcctl(fl, ["start", "all"])
        return djm_job
    except KeyboardInterrupt:
        logger.exception("Got keyboard interrupt in node initialization")
        c.stop_job(j, common.JobStatus.JOB_FAILED,
                   comment="Got keyboard interrupt in node initialization")
        raise
    except Exception, e:
        logger.exception("DJM problem in node initialization: %s" % e)
        c.stop_job(j, common.JobStatus.JOB_FAILED,
                   comment="DJM problem in node initialization: %s" % e)
        raise

<commit_msg>Add more debugging to master<commit_after>"""Adapter layer to distributed job manager server api
"""
import os
import os.path
import logging
import datetime

from file_locator import FileLocator
import utils
from dist_job_mgr.client import get_local_connection
from dist_job_mgr.version import VERSION
import dist_job_mgr.common as common

class DjmAdapterError(Exception):
    pass

logger = logging.getLogger(__name__)

def get_djm_connection():
    fl = FileLocator()
    return get_local_connection(fl.get_djm_server_dir())

class DjmJob(object):
    def __init__(self, c, job_id, nodes):
        self.c = c
        self.job_id = job_id
        self.nodes = nodes
        self.nodes_by_name = {}
        for node in nodes:
            if node["private_ip"]!=None:
                ip_address = node["private_ip"]
            else:
                ip_address = node["public_ip"]
            if not ip_address:
                raise Exception("Neither public ip address nor private ip address specified for node %s, need to specify at least one" % node["name"])
            node["datablox_ip_address"] = ip_address
            self.nodes_by_name[node["name"]] = node
        self.nodes_except_master = filter(lambda name: name!="master",
                                          [node["name"] for node in self.nodes])

    def has_node(self, node_name):
        return self.nodes_by_name.has_key(node_name)

    def get_node(self, node_name):
        return self.nodes_by_name[node_name]

    def stop_job(self, successful=True, msg=None):
        if len(self.nodes_except_master)>0:
            (s, r) = self.c.run_task_on_node_list(self.job_id, "StopWorker", "stop worker",
                                                  self.nodes_except_master)
            if s!=common.TaskStatus.TASK_SUCCESSFUL:
                logger.warn("Not able to stop DJM worker on all nodes")
        logger.debug("Stopped DJM workers")
        if successful:
            self.c.stop_job(self.job_id,
                            common.JobStatus.JOB_SUCCESSFUL,
                            comment=msg)
            logger.debug("Stopped job %s, status=JOB_SUCESSFUL" % self.job_id)
        else:
            self.c.stop_job(self.job_id,
                            common.JobStatus.JOB_FAILED,
                            comment=msg)
            logger.debug("Stopped job %s, status=FAILED" % self.job_id)

def _check_task_status(s, r, msg):
    if s!=common.TaskStatus.TASK_SUCCESSFUL:
        bad_nodes = filter(lambda res:
                           res.status!=common.TaskStatus.TASK_SUCCESSFUL,
                           r)
        raise Exception("%s for nodes: %s" %
                        (msg, ', '.join([r.node_name for r in bad_nodes])))

def start_job_and_get_nodes(node_list, config_file_name, total_nodes=None,
                            reuse_existing_installs=True):
    """Given a node list and optional number of nodes, try to get the
    requested nodes and start a job.
    """
    if not total_nodes:
        total_nodes = len(node_list)
    if total_nodes<1:
        raise DjmAdapterError("Must have at least one node")
    c = get_djm_connection()
    # make sure there aren't any dead jobs laying around
    c.cleanup_dead_coordinators()
    pool = None
    for node_name in node_list:
        n = c.find_node_by_name(node_name)
        if not n:
            raise DjmAdapterError("Node '%s' not defined" % node_name)
        if n["pool"]:
            if pool and pool!=n["pool"]:
                raise DjmAdapterError("Cannot take nodes from both pool %s and pool %s"%
                                      (pool, n["pool"]))
            pool = n["pool"]
    start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
    j = c.start_job(config_file_name, common.JobType.ONE_TIME_JOB,
                    total_nodes, "Datablox job started %s" % start_time,
                    node_pool_name=pool, requested_nodes=node_list)
    logger.info("Started DJM job %s" % j)
    try:
        fl = FileLocator()
        allocated_nodes = c.query_nodes(job_id=j)
        djm_job = DjmJob(c, j, allocated_nodes)
        logger.info("Setting up nodes")
        # for all the non-master nodes, we setup the caretaker
        nodes_except_master = djm_job.nodes_except_master
        if len(nodes_except_master)>0:
            (s, r) = c.run_task_on_node_list(j, "StartWorker", "start worker",
                                             nodes_except_master)
            _check_task_status(s, r, "DJM worker start failed")
            dist_path = fl.get_engage_distribution_file()
            logger.info("Copying engage distribution")
            (s, r) = c.run_task_on_node_list(j, "CopyFiles",
                                             "Copy engage distribution",
                                             nodes_except_master,
                                             dist_path,
                                             "~/" + os.path.basename(dist_path))
            _check_task_status(s, r, "Copying of engage distribution failed")
            (s, r) = c.run_task_on_node_list(j, "CopyFiles",
                                             "Copy setup script",
                                             nodes_except_master,
                                             os.path.join(fl.get_sw_packages_dir(),
                                                          "setup_caretaker.sh"),
                                             "~/setup_caretaker.sh")
            _check_task_status(s, r, "Copy of caretaker setup script failed")
            (s, r) = c.run_task_on_node_list(j, "Command",
                                             "Make setup script executable",
                                             nodes_except_master,
                                             ["/bin/chmod 755 ~/setup_caretaker.sh"],
                                             shell=True)
            _check_task_status(s, r, "chmod of caretaker setup script failed")
            caretaker_cmd = ["~/setup_caretaker.sh",]
            if reuse_existing_installs:
                caretaker_cmd.append("--reuse-existing-install")
            (s, r) = c.run_task_on_node_list(j, "Command",
                                             "Setup remote caretaker",
                                             nodes_except_master,
                                             caretaker_cmd,
                                             shell=True)
            _check_task_status(s, r, "Caretaker setup script failed")
        # make sure the master node has the caretaker running
        if djm_job.has_node("master"):
            utils.run_svcctl(fl, ["start", "all"])
        return djm_job
    except KeyboardInterrupt:
        logger.exception("Got keyboard interrupt in node initialization")
        c.stop_job(j, common.JobStatus.JOB_FAILED,
                   comment="Got keyboard interrupt in node initialization")
        raise
    except Exception, e:
        logger.exception("DJM problem in node initialization: %s" % e)
        c.stop_job(j, common.JobStatus.JOB_FAILED,
                   comment="DJM problem in node initialization: %s" % e)
        raise

<|endoftext|>


<commit_before>"""Mixin classes that customize the filestore integration of AreaDetector
FilePlugins.

To be used like so:

    from ophyd.areadetector.detectors import PerkinElmerDetector
    from ophyd.areadetector.plugins import HDF5Plugin
    from ophyd.areadetector.trigger_mixins import SingleTrigger
    from ophyd.areadetector.filestore_mixins import (
        FileStoreIterativeWrite)

    class MyPlugin(HDF5Plugin, FileStoreIterativeWrite):
        pass

    dest = '/tmp'  # in production, use a directory on your system -- not /tmp

    class MyDetector(PerkinElmerDetector, SingleTrigger):  # for example
        file_plugin = MyPlugin(suffix='HDF1:', write_file_path=dest)

    det = MyDetector(...)
"""

from __future__ import print_function
import logging
import uuid
import filestore.api as fs

from datetime import datetime
from collections import defaultdict, OrderedDict
from itertools import count
import os

from ..ophydobj import DeviceStatus
from ..device import GenerateDatumInterface, BlueskyInterface

logger = logging.getLogger(__name__)

def new_uid():
    "uuid4 as a string"
    return str(uuid.uuid4())

def new_short_uid():
    "uuid4, skipping the last stanza because of AD length restrictions."
    return '-'.join(new_uid().split('-')[:-1])

class FileStoreBase(BlueskyInterface, GenerateDatumInterface):
    "Base class for FileStore mixin classes"
    def __init__(self, *args, write_file_path=None, read_file_path=None,
                 **kwargs):
        # TODO Can we make these args? Depends on plugin details.
        if write_file_path is None:
            raise ValueError("write_file_path is required")
        self.write_file_path = write_file_path
        self._read_file_path = read_file_path
        super().__init__(*args, **kwargs)
        self._point_counter = None
        self._locked_key_list = False
        self._datum_uids = defaultdict(list)

    @property
    def read_file_path(self):
        "Returns write_file_path if read_file_path is not set"
        if self._read_file_path is None:
            return write_file_path
        else:
            return self._read_file_path

    @read_file_path.setter
    def read_file_path(self, val):
        self._read_file_path = val

    def stage(self):
        self._point_counter = count()
        self._locked_key_list = False
        self._datum_uids.clear()
        # Make a filename.
        self._filename = new_short_uid()
        date = datetime.now().date()
        # AD requires a trailing slash, hence the [''] here
        path = os.path.join(*(date.isoformat().split('-') + ['']))
        full_write_path = os.path.join(self.write_file_path, path)
        full_read_path = os.path.join(self.read_file_path, path)
        self.file_template.put('%s%s_%6.6d.h5')
        ssigs = [
            (self.enable, 1),
            (self.auto_increment, 1),
            (self.array_counter, 0),
            (self.file_number, 0),
            (self.auto_save, 1),
            (self.num_capture, 0),
            (self.file_write_mode, 1),  # 'capture' mode -- for AD 1.x
            (self.file_path, full_write_path),
            (self.file_name, self._filename),
            (self.capture, 1),
        ]
        self.stage_sigs.extend(ssigs)
        super().stage()

        # AD does this same templating in C, but we can't access it
        # so we do it redundantly here in Python.
        fn = self.file_template.get() % (full_read_path, self._filename,
                                         self.file_number.get())

        if not self.file_path_exists.get():
            raise IOError("Path %s does not exist on IOC.", self.file_path)

        res_kwargs = {'frame_per_point': self.num_captured.get()}
        self._resource = fs.insert_resource('AD_HDF5', fn, res_kwargs)

    def generate_datum(self, key):
        "Generate a uid and cache it with its key for later insertion."
        if self._locked_key_list:
            if key not in self._datum_uids:
                raise RuntimeError("modifying after lock")
        uid = new_uid()
        self._datum_uids[key].append(uid)  # e.g., {'dark': [uid, uid], ...}
        return uid

    def describe(self):
        # One object has been 'described' once, no new keys can be added
        # during this stage/unstage cycle.
        self._locked_key_list = self._staged
        res = super().describe()
        for k in self._datum_uids:
            res[k] = self.parent.make_data_key()  # this is on DetectorBase
        return res

    def read(self):
        # One object has been 'read' once, no new keys can be added
        # during this stage/unstage cycle.
        self._locked_key_list = self._staged
        res = super().read()
        for k, v in self._datum_uids.items():
            res[k] = v[-1]
        return res

    def unstage(self):
        self._locked_key_list = False
        self._resource = None
        return super().unstage()

class FileStoreIterativeWrite(FileStoreBase):
    "Save records to filestore as they are generated."
    def generate_datum(self, key):
        uid = super().generate_datum(key)
        i = next(self._point_counter)
        fs.insert_datum(self._resource, uid, {'point_number': i})
        return uid

class FileStoreBulkEntry(FileStoreBase):
    "Cache records as they are created and save them all at the end."
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._datum_kwargs_map = dict()  # store kwargs for each uid

    def generate_datum(self, key):
        "Stash kwargs for each datum, to be used below by unstage."
        uid = super().generate_datum(key)
        i = next(self._point_counter)
        self._datum_kwargs_map[uid] = {'point_number': i}
        # (don't insert, obviously)
        return uid

    def unstage(self):
        "Insert all datums at the end."
        for uids in self._datum_uids.values():
            for uid in uids:
                kwargs = self._datum_kwargs_maps[uid]
                fs.insert_datum(self._resource, uid, kwargs)
        return super().unstage()
<commit_msg>REF: Split HDF5-specific logic out of FileStoreBase.<commit_after>"""Mixin classes that customize the filestore integration of AreaDetector
FilePlugins.

To be used like so:

    from ophyd.areadetector.detectors import PerkinElmerDetector
    from ophyd.areadetector.plugins import HDF5Plugin
    from ophyd.areadetector.trigger_mixins import SingleTrigger
    from ophyd.areadetector.filestore_mixins import (
        FileStoreIterativeWrite)

    class MyPlugin(HDF5Plugin, FileStoreIterativeWrite):
        pass

    dest = '/tmp'  # in production, use a directory on your system -- not /tmp

    class MyDetector(PerkinElmerDetector, SingleTrigger):  # for example
        file_plugin = MyPlugin(suffix='HDF1:', write_file_path=dest)

    det = MyDetector(...)
"""

from __future__ import print_function
import logging
import uuid
import filestore.api as fs

from datetime import datetime
from collections import defaultdict, OrderedDict
from itertools import count
import os

from ..ophydobj import DeviceStatus
from ..device import GenerateDatumInterface, BlueskyInterface

logger = logging.getLogger(__name__)

def new_uid():
    "uuid4 as a string"
    return str(uuid.uuid4())

def new_short_uid():
    "uuid4, skipping the last stanza because of AD length restrictions."
    return '-'.join(new_uid().split('-')[:-1])

class FileStoreBase(BlueskyInterface, GenerateDatumInterface):
    "Base class for FileStore mixin classes"
    def __init__(self, *args, write_file_path=None, read_file_path=None,
                 **kwargs):
        # TODO Can we make these args? Depends on plugin details.
        if write_file_path is None:
            raise ValueError("write_file_path is required")
        self.write_file_path = write_file_path
        self._read_file_path = read_file_path
        super().__init__(*args, **kwargs)
        self._point_counter = None
        self._locked_key_list = False
        self._datum_uids = defaultdict(list)

    @property
    def read_file_path(self):
        "Returns write_file_path if read_file_path is not set"
        if self._read_file_path is None:
            return write_file_path
        else:
            return self._read_file_path

    @read_file_path.setter
    def read_file_path(self, val):
        self._read_file_path = val

    def stage(self):
        self._point_counter = count()
        self._locked_key_list = False
        self._datum_uids.clear()
        # Make a filename.
        self._filename = new_short_uid()
        date = datetime.now().date()
        # AD requires a trailing slash, hence the [''] here
        path = os.path.join(*(date.isoformat().split('-') + ['']))
        full_write_path = os.path.join(self.write_file_path, path)
        full_read_path = os.path.join(self.read_file_path, path)
        ssigs = [
            (self.enable, 1),
            (self.auto_increment, 1),
            (self.array_counter, 0),
            (self.file_number, 0),
            (self.auto_save, 1),
            (self.num_capture, 0),
            (self.file_write_mode, 1),  # 'capture' mode -- for AD 1.x
            (self.file_path, full_write_path),
            (self.file_name, self._filename),
            (self.capture, 1),
        ]
        self.stage_sigs.extend(ssigs)
        super().stage()

        # AD does this same templating in C, but we can't access it
        # so we do it redundantly here in Python.
        self._fn = self.file_template.get() % (full_read_path,
                                               self._filename,
                                               self.file_number.get())

        if not self.file_path_exists.get():
            raise IOError("Path %s does not exist on IOC.", self.file_path)

    def generate_datum(self, key):
        "Generate a uid and cache it with its key for later insertion."
        if self._locked_key_list:
            if key not in self._datum_uids:
                raise RuntimeError("modifying after lock")
        uid = new_uid()
        self._datum_uids[key].append(uid)  # e.g., {'dark': [uid, uid], ...}
        return uid

    def describe(self):
        # One object has been 'described' once, no new keys can be added
        # during this stage/unstage cycle.
        self._locked_key_list = self._staged
        res = super().describe()
        for k in self._datum_uids:
            res[k] = self.parent.make_data_key()  # this is on DetectorBase
        return res

    def read(self):
        # One object has been 'read' once, no new keys can be added
        # during this stage/unstage cycle.
        self._locked_key_list = self._staged
        res = super().read()
        for k, v in self._datum_uids.items():
            res[k] = v[-1]
        return res

    def unstage(self):
        self._locked_key_list = False
        self._resource = None
        return super().unstage()

class FileStoreHDF5(FileStoreBase):
    def stage(self):
        self.stage_sigs.extend([(self.file_template, '%s%s_%6.6d.h5')])
        super().stage()
        res_kwargs = {'frame_per_point': self.num_captured.get()}
        self._resource = fs.insert_resource('AD_HDF5', self._fn, res_kwargs)

class FileStoreTIFF(FileStoreBase):
    # TODO num_captured above, num_images here -- which is correct?
    def stage(self):
        self.stage_sigs.extend([(self.file_template, '%s%s_%6.6d.tiff')])
        super().stage()
        res_kwargs = {'template': self.file_template.get(),
                      'filename': self.file_name.get(),
                      'frame_per_point': self.num_images.get()}
        self._resource = fs.insert_resource('AD_TIFF', self._fn, res_kwargs)

class FileStoreIterativeWrite(FileStoreBase):
    "Save records to filestore as they are generated."
    def generate_datum(self, key):
        uid = super().generate_datum(key)
        i = next(self._point_counter)
        fs.insert_datum(self._resource, uid, {'point_number': i})
        return uid

class FileStoreBulkWrite(FileStoreBase):
    "Cache records as they are created and save them all at the end."
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._datum_kwargs_map = dict()  # store kwargs for each uid

    def generate_datum(self, key):
        "Stash kwargs for each datum, to be used below by unstage."
        uid = super().generate_datum(key)
        i = next(self._point_counter)
        self._datum_kwargs_map[uid] = {'point_number': i}
        # (don't insert, obviously)
        return uid

    def unstage(self):
        "Insert all datums at the end."
        for uids in self._datum_uids.values():
            for uid in uids:
                kwargs = self._datum_kwargs_maps[uid]
                fs.insert_datum(self._resource, uid, kwargs)
        return super().unstage()

# ready-to-use combinations

class FileStoreHDF5IterativeWrite(FileStoreHDF5, FileStoreIterativeWrite):
    pass

class FileStoreHDF5BulkWrite(FileStoreHDF5, FileStoreBulkWrite):
    pass

class FileStoreTIFFIterativeWrite(FileStoreTIFF, FileStoreIterativeWrite):
    pass

class FileStoreTIFFBulkWrite(FileStoreTIFF, FileStoreBulkWrite):
    pass
<|endoftext|>


<commit_before>#!/usr/bin/python

""" Module to handle attributes related to the pbs jobmanager
configuration """

import ConfigParser, os

from osg_configure.modules import utilities
from osg_configure.modules import configfile
from osg_configure.modules import validation
from osg_configure.modules import exceptions
from osg_configure.modules.jobmanagerbase import JobManagerConfiguration

__all__ = ['PBSConfiguration']

PBS_CONFIG_FILE = '/etc/grid-services/available/jobmanager-pbs'

class PBSConfiguration(JobManagerConfiguration):
  """Class to handle attributes related to pbs job manager configuration"""

  def __init__(self, *args, **kwargs):
    # pylint: disable-msg=W0142
    super(PBSConfiguration, self).__init__(*args, **kwargs)
    self.logger.debug('PBSConfiguration.__init__ started')
    self.__mappings = {'pbs_location': 'OSG_PBS_LOCATION',
                       'job_contact': 'OSG_JOB_CONTACT',
                       'util_contact': 'OSG_UTIL_CONTACT',
                       'accept_limited': 'accept_limited'}
    self.__optional = ['accept_limited']
    self.__defaults = {'accept_limited' : 'False'}

    self.config_section = "PBS"
    self.__set_default = True
    self.logger.debug('PBSConfiguration.__init__ completed')

  def parseConfiguration(self, configuration):
    """Try to get configuration information from ConfigParser or SafeConfigParser object given
    by configuration and write recognized settings to attributes dict
    """
    self.logger.debug('PBSConfiguration.parseConfiguration started')

    self.checkConfig(configuration)

    if not configuration.has_section(self.config_section):
      self.logger.debug('PBS section not found in config file')
      self.logger.debug('PBSConfiguration.parseConfiguration completed')
      return

    if not self.setStatus(configuration):
      self.logger.debug('PBSConfiguration.parseConfiguration completed')
      return True

    self.attributes['OSG_JOB_MANAGER'] = 'PBS'
    for setting in self.__mappings:
      self.logger.debug("Getting value for %s" % setting)
      temp = configfile.get_option(configuration,
                                   self.config_section,
                                   setting,
                                   self.__optional,
                                   self.__defaults)
      self.attributes[self.__mappings[setting]] = temp
      self.logger.debug("Got %s" % temp)

    # set OSG_JOB_MANAGER_HOME
    self.attributes['OSG_JOB_MANAGER_HOME'] = \
      self.attributes[self.__mappings['pbs_location']]

    # check and warn if unknown options found
    temp = utilities.get_set_membership(configuration.options(self.config_section),
                                        self.__mappings,
                                        configuration.defaults().keys())
    for option in temp:
      if option == 'enabled':
        continue
      self.logger.warning("Found unknown option %s in %s section" %
                           (option, self.config_section))

    if (configuration.has_section('Managed Fork') and
        configuration.has_option('Managed Fork', 'enabled') and
        configuration.get('Managed Fork', 'enabled').upper() == 'TRUE'):
      self.__set_default = False

    self.logger.debug('PBSConfiguration.parseConfiguration completed')

  def getAttributes(self):
    """Return settings"""
    self.logger.debug('PBSConfiguration.getAttributes started')
    if not self.enabled:
      self.logger.debug('PBS not enabled, returning empty dictionary')
      self.logger.debug('PBSConfiguration.parseConfiguration completed')
      return {}
    self.logger.debug('PBSConfiguration.parseConfiguration completed')
    return self.attributes

# pylint: disable-msg=W0613
  def checkAttributes(self, attributes):
    """Check attributes currently stored and make sure that they are consistent"""
    self.logger.debug('PBSConfiguration.checkAttributes started')

    attributes_ok = True

    if not self.enabled:
      self.logger.debug('PBS not enabled, returning True')
      self.logger.debug('PBSConfiguration.checkAttributes completed')
      return attributes_ok

    if self.ignored:
      self.logger.debug('Ignored, returning True')
      self.logger.debug('PBSConfiguration.checkAttributes completed')
      return attributes_ok

    # Make sure all settings are present
    for setting in self.__mappings:
      if self.__mappings[setting] not in self.attributes:
        raise exceptions.SettingError("Missing setting for %s in %s section " %
                                      (setting, self.config_section))

    # make sure locations exist
    if not validation.valid_location(self.attributes[self.__mappings['pbs_location']]):
      attributes_ok = False
      self.logger.error("%s points to non-existent location: %s" %
                          ('pbs_location',
                           self.attributes[self.__mappings['pbs_location']]))

    if not self.validContact(self.attributes[self.__mappings['job_contact']],
                             'pbs'):
      attributes_ok = False
      self.logger.error("%s is not a valid job contact: %s" %
                        ('job_contact',
                         self.attributes[self.__mappings['job_contact']]))

    if not self.validContact(self.attributes[self.__mappings['util_contact']],
                             'pbs'):
      attributes_ok = False
      self.logger.error("%s is not a valid util contact: %s" %
                        ('util_contact',
                         self.attributes[self.__mappings['util_contact']]))

    self.logger.debug('PBSConfiguration.checkAttributes completed')
    return attributes_ok

  def configure(self, attributes):
    """Configure installation using attributes"""
    self.logger.debug('PBSConfiguration.configure started')

    if not self.enabled:
      self.logger.debug('PBS not enabled, returning True')
      self.logger.debug('PBSConfiguration.configure completed')
      return True

    if self.ignored:
      self.logger.warning("%s configuration ignored" % self.config_section)
      self.logger.debug('PBSConfiguration.configure completed')
      return True

    # The accept_limited argument was added for Steve Timm.  We are not adding
    # it to the default config.ini template because we do not think it is
    # useful to a wider audience.
    # See VDT RT ticket 7757 for more information.
    if self.attributes[self.__mappings['accept_limited']].upper() == "TRUE":
      if not self.enable_accept_limited(PBS_CONFIG_FILE):
          self.logger.error('Error writing to condor configuration')
          self.logger.debug('PBSConfiguration.configure completed')
          return False
    elif self.attributes[self.__mappings['accept_limited']].upper() == "FALSE":
      if not self.disable_accept_limited(PBS_CONFIG_FILE):
          self.logger.error('Error writing to condor configuration')
          self.logger.debug('PBSConfiguration.configure completed')
          return False

    if self.__set_default:
        self.logger.debug('Configuring gatekeeper to use regular fork service')
        self.set_default_jobmanager('fork')

    self.logger.debug('PBSConfiguration.configure completed')
    return True

  def moduleName(self):
    """Return a string with the name of the module"""
    return "PBS"

  def separatelyConfigurable(self):
    """Return a boolean that indicates whether this module can be configured separately"""
    return True

  def parseSections(self):
    """Returns the sections from the configuration file that this module handles"""
    return [self.config_section]
<commit_msg>Use the correct filename for the globus pbs configuration<commit_after>#!/usr/bin/python

""" Module to handle attributes related to the pbs jobmanager
configuration """

import ConfigParser, os

from osg_configure.modules import utilities
from osg_configure.modules import configfile
from osg_configure.modules import validation
from osg_configure.modules import exceptions
from osg_configure.modules.jobmanagerbase import JobManagerConfiguration

__all__ = ['PBSConfiguration']

PBS_CONFIG_FILE = '/etc/grid-services/available/jobmanager-pbs-seg'

class PBSConfiguration(JobManagerConfiguration):
  """Class to handle attributes related to pbs job manager configuration"""

  def __init__(self, *args, **kwargs):
    # pylint: disable-msg=W0142
    super(PBSConfiguration, self).__init__(*args, **kwargs)
    self.logger.debug('PBSConfiguration.__init__ started')
    self.__mappings = {'pbs_location': 'OSG_PBS_LOCATION',
                       'job_contact': 'OSG_JOB_CONTACT',
                       'util_contact': 'OSG_UTIL_CONTACT',
                       'accept_limited': 'accept_limited'}
    self.__optional = ['accept_limited']
    self.__defaults = {'accept_limited' : 'False'}

    self.config_section = "PBS"
    self.__set_default = True
    self.logger.debug('PBSConfiguration.__init__ completed')

  def parseConfiguration(self, configuration):
    """Try to get configuration information from ConfigParser or SafeConfigParser object given
    by configuration and write recognized settings to attributes dict
    """
    self.logger.debug('PBSConfiguration.parseConfiguration started')

    self.checkConfig(configuration)

    if not configuration.has_section(self.config_section):
      self.logger.debug('PBS section not found in config file')
      self.logger.debug('PBSConfiguration.parseConfiguration completed')
      return

    if not self.setStatus(configuration):
      self.logger.debug('PBSConfiguration.parseConfiguration completed')
      return True

    self.attributes['OSG_JOB_MANAGER'] = 'PBS'
    for setting in self.__mappings:
      self.logger.debug("Getting value for %s" % setting)
      temp = configfile.get_option(configuration,
                                   self.config_section,
                                   setting,
                                   self.__optional,
                                   self.__defaults)
      self.attributes[self.__mappings[setting]] = temp
      self.logger.debug("Got %s" % temp)

    # set OSG_JOB_MANAGER_HOME
    self.attributes['OSG_JOB_MANAGER_HOME'] = \
      self.attributes[self.__mappings['pbs_location']]

    # check and warn if unknown options found
    temp = utilities.get_set_membership(configuration.options(self.config_section),
                                        self.__mappings,
                                        configuration.defaults().keys())
    for option in temp:
      if option == 'enabled':
        continue
      self.logger.warning("Found unknown option %s in %s section" %
                           (option, self.config_section))

    if (configuration.has_section('Managed Fork') and
        configuration.has_option('Managed Fork', 'enabled') and
        configuration.get('Managed Fork', 'enabled').upper() == 'TRUE'):
      self.__set_default = False

    self.logger.debug('PBSConfiguration.parseConfiguration completed')

  def getAttributes(self):
    """Return settings"""
    self.logger.debug('PBSConfiguration.getAttributes started')
    if not self.enabled:
      self.logger.debug('PBS not enabled, returning empty dictionary')
      self.logger.debug('PBSConfiguration.parseConfiguration completed')
      return {}
    self.logger.debug('PBSConfiguration.parseConfiguration completed')
    return self.attributes

# pylint: disable-msg=W0613
  def checkAttributes(self, attributes):
    """Check attributes currently stored and make sure that they are consistent"""
    self.logger.debug('PBSConfiguration.checkAttributes started')

    attributes_ok = True

    if not self.enabled:
      self.logger.debug('PBS not enabled, returning True')
      self.logger.debug('PBSConfiguration.checkAttributes completed')
      return attributes_ok

    if self.ignored:
      self.logger.debug('Ignored, returning True')
      self.logger.debug('PBSConfiguration.checkAttributes completed')
      return attributes_ok

    # Make sure all settings are present
    for setting in self.__mappings:
      if self.__mappings[setting] not in self.attributes:
        raise exceptions.SettingError("Missing setting for %s in %s section " %
                                      (setting, self.config_section))

    # make sure locations exist
    if not validation.valid_location(self.attributes[self.__mappings['pbs_location']]):
      attributes_ok = False
      self.logger.error("%s points to non-existent location: %s" %
                          ('pbs_location',
                           self.attributes[self.__mappings['pbs_location']]))

    if not self.validContact(self.attributes[self.__mappings['job_contact']],
                             'pbs'):
      attributes_ok = False
      self.logger.error("%s is not a valid job contact: %s" %
                        ('job_contact',
                         self.attributes[self.__mappings['job_contact']]))

    if not self.validContact(self.attributes[self.__mappings['util_contact']],
                             'pbs'):
      attributes_ok = False
      self.logger.error("%s is not a valid util contact: %s" %
                        ('util_contact',
                         self.attributes[self.__mappings['util_contact']]))

    self.logger.debug('PBSConfiguration.checkAttributes completed')
    return attributes_ok

  def configure(self, attributes):
    """Configure installation using attributes"""
    self.logger.debug('PBSConfiguration.configure started')

    if not self.enabled:
      self.logger.debug('PBS not enabled, returning True')
      self.logger.debug('PBSConfiguration.configure completed')
      return True

    if self.ignored:
      self.logger.warning("%s configuration ignored" % self.config_section)
      self.logger.debug('PBSConfiguration.configure completed')
      return True

    # The accept_limited argument was added for Steve Timm.  We are not adding
    # it to the default config.ini template because we do not think it is
    # useful to a wider audience.
    # See VDT RT ticket 7757 for more information.
    if self.attributes[self.__mappings['accept_limited']].upper() == "TRUE":
      if not self.enable_accept_limited(PBS_CONFIG_FILE):
          self.logger.error('Error writing to condor configuration')
          self.logger.debug('PBSConfiguration.configure completed')
          return False
    elif self.attributes[self.__mappings['accept_limited']].upper() == "FALSE":
      if not self.disable_accept_limited(PBS_CONFIG_FILE):
          self.logger.error('Error writing to condor configuration')
          self.logger.debug('PBSConfiguration.configure completed')
          return False

    if self.__set_default:
        self.logger.debug('Configuring gatekeeper to use regular fork service')
        self.set_default_jobmanager('fork')

    self.logger.debug('PBSConfiguration.configure completed')
    return True

  def moduleName(self):
    """Return a string with the name of the module"""
    return "PBS"

  def separatelyConfigurable(self):
    """Return a boolean that indicates whether this module can be configured separately"""
    return True

  def parseSections(self):
    """Returns the sections from the configuration file that this module handles"""
    return [self.config_section]
<|endoftext|>


<commit_before># ERPNext - web based ERP (http://erpnext.com)
# Copyright (C) 2012 Web Notes Technologies Pvt Ltd
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

class DocType:
	def __init__(self, d, dl):
		self.doc, self.doclist = d, dl

	def validate(self):
		"""make custom css"""
		from jinja2 import Template
		import os

		with open(os.path.dirname(__file__), 'custom_template.css'), 'r') as f:
			temp = Template(f.read())

		if not self.doc.font_size:
			self.doc.font_size = '13px'

		self.doc.small_font_size = str(int(self.doc.font_size[:-2])-2) + 'px'

		self.doc.custom_css = temp.render(doc = self.doc)
		if self.doc.add_css:
			self.doc.custom_css += '\n\n/* User CSS */\n\n' + self.doc.add_css

		from webnotes.cms.make import make_web_core
		make_web_core()

		from webnotes.session_cache import clear_cache
		clear_cache('Guest')

		del self.doc.fields['small_font_size']

		<commit_msg>cms fixes<commit_after># ERPNext - web based ERP (http://erpnext.com)
# Copyright (C) 2012 Web Notes Technologies Pvt Ltd
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

class DocType:
	def __init__(self, d, dl):
		self.doc, self.doclist = d, dl

	def validate(self):
		"""make custom css"""
		from jinja2 import Template
		import os

		with open(os.path.join(os.path.dirname(__file__), 'custom_template.css'), 'r') as f:
			temp = Template(f.read())

		if not self.doc.font_size:
			self.doc.font_size = '13px'

		self.doc.small_font_size = str(int(self.doc.font_size[:-2])-2) + 'px'

		self.doc.custom_css = temp.render(doc = self.doc)
		if self.doc.add_css:
			self.doc.custom_css += '\n\n/* User CSS */\n\n' + self.doc.add_css

		from webnotes.cms.make import make_web_core
		make_web_core()

		from webnotes.session_cache import clear_cache
		clear_cache('Guest')

		del self.doc.fields['small_font_size']

		<|endoftext|>