<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""MoCA CLI"""
import os
import re
import sys
from time import sleep
import click
from moca import bedoperations, pipeline
from moca.bedoperations.fimo import get_start_stop_intervals
from moca.helpers import filename_extension
from moca.helpers import read_memefile
from moca.helpers.job_executor import safe_makedir
from moca.plotter import create_plot
from moca import version
from tqdm import tqdm
from click_help_colors import HelpColorsGroup, HelpColorsCommand

conservation_wig_keys = ['phylop', 'gerp', 'phastcons']

class ProgressBar(object):
    def __init__(self, msg_list):
        self.bar = tqdm(msg_list,
                        bar_format='{desc}{percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}]',
                        smoothing=0,
                        miniters=1,
                        mininterval=0.01)

    def show_progress(self, msg):
        self.bar.set_description(msg)
        self.bar.update()
        sleep(0.1)

    def close(self):
        self.bar.close()
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])

@click.group(cls=HelpColorsGroup,
             help_headers_color='yellow',
             help_options_color='green')
@click.version_option(version=version.__version__)
def cli():
    """moca: Motif Conservation Analysis"""
    pass

@cli.command('find_motifs', context_settings=CONTEXT_SETTINGS)
@click.option('--bedfile', '-i',
              help='Bed file input',
              required=True)
@click.option('--oc',
              '-o',
              help='Output Directory',
              required=True)
@click.option('--configuration',
              '-c',
              help='Configuration file',
              required=True)
@click.option('-s',
              '--slop-length',
              '--slop_length',
              default=50,
              help='Flanking sequence length')
@click.option('-f',
              '--flank-motif',
              '--flank_motif',
              default=5,
              help='Length of sequence flanking motif')
@click.option('--n-motif',
              default=5,
              help='Number of motifs',
              type=int)
@click.option('--cores',
              '-t',
              default=1,
              help='Number of parallel MEME jobs',
              type=int,
              required=True)
@click.option('--genome-build',
              '-g', '-gb',
              help='Key denoting genome build to use in configuration file',
              required=True)

@click.option('--show-progress',
              help='Print progress',
              is_flag=True)

def find_motifs(bedfile, oc, configuration, slop_length,
                flank_motif, n_motif, cores, genome_build, show_progress):
    """Search motifs and create conservation plots"""
    root_dir = os.path.dirname(os.path.abspath(bedfile))
    if not oc:
        moca_out_dir = os.path.join(os.getcwd(), 'moca_output')
    else:
        moca_out_dir = oc
    moca_pipeline = pipeline.Pipeline(configuration)
    genome_data = moca_pipeline.get_genome_data(genome_build)
    genome_fasta = genome_data['fasta']
    genome_table = genome_data['genome_table']

    wigfiles = {}
    for key in list(conservation_wig_keys):
        try:
            wigfiles[key] = genome_data['{}_wig'.format(key)]
        except KeyError:
            pass
    safe_makedir(moca_out_dir)
    bedfile_fn, _ = filename_extension(bedfile)

    if show_progress:
        msg_list = ['Extracting Fasta',
                    'Running MEME',
                    'Running CENTRIMO']
        msg_list_e = ['Generating random Fasta', 'Running fimo random', 'Running fimo main'] + ['Extracting Scores']*len(wigfiles.keys()) + ['Creating PLot']
        msg_list = msg_list + msg_list_e*n_motif
        progress_bar = ProgressBar(msg_list)

    query_train_fasta = os.path.join(moca_out_dir,
                                     bedfile_fn + '_train_flank_{}.fasta'.format(slop_length))
    query_test_fasta = os.path.join(moca_out_dir,
                                    bedfile_fn + '_test_flank_{}.fasta'.format(slop_length))

    if show_progress:
        progress_bar.show_progress('Extracting Fasta')
    bed_o = bedoperations.Bedfile(bedfile, genome_table, moca_out_dir)
    bed_train, bed_test = bed_o.split_train_test_bed(train_peaks_count=500, test_peaks_count=500)

    bed_train_slopped  = bed_o.slop_bed(bed_train, flank_length=slop_length)
    bed_test_slopped  = bed_o.slop_bed(bed_test, flank_length=slop_length)

    bed_o.extract_fasta(bed_train_slopped, fasta_in=genome_fasta, fasta_out=query_train_fasta)
    bed_o.extract_fasta(bed_test_slopped, fasta_in=genome_fasta, fasta_out=query_test_fasta)

    #memechip_out_dir = os.path.join(moca_out_dir, 'memechip_analysis')
    meme_out_dir = os.path.join(moca_out_dir, 'meme_out')
    memechip_out_dir = meme_out_dir
    meme_params = moca_pipeline.get_meme_default_params
    if cores==1:
        re.sub(r' -p*', '', meme_params)
    else:
        re.sub(r'-p*', '-p {}'.format(cores), meme_params)
    if show_progress:
        progress_bar.show_progress('Running MEME')

    #meme_run_out = moca_pipeline.run_memechip(fasta_in=query_fasta, out_dir=memechip_out_dir)
    #
    meme_run_out = moca_pipeline.run_meme(fasta_in=query_train_fasta,
                                          out_dir=meme_out_dir,
                                          strargs=meme_params)
    if meme_run_out['stderr']!='':
        sys.stdout.write('Error running MEME: {}'.format(meme_run_out['stderr']))
        sys.exit(1)
    meme_file = os.path.join(meme_out_dir, 'meme.txt')
    meme_summary = read_memefile(meme_file)

    if show_progress:
        progress_bar.show_progress('Running CENTRIMO')
    centrimo_main_dir = os.path.join(moca_out_dir, 'centrimo_out')
    centrimo_main = moca_pipeline.run_centrimo(meme_file=meme_file,
                                               fasta_in=query_test_fasta,
                                               out_dir=centrimo_main_dir)
    for motif in range(1, meme_summary['total_motifs']+1):
        fimo_rand_dir = os.path.join(memechip_out_dir, 'fimo_random_{}'.format(motif))
        fimo_main_dir = os.path.join(memechip_out_dir, 'fimo_out_{}'.format(motif))

        safe_makedir(fimo_rand_dir)
        random_fasta = os.path.join(fimo_rand_dir, 'random_{}.fa'.format(motif))
        if show_progress:
            progress_bar.show_progress('Generating Random Fasta: {}'.format(motif))
        moca_pipeline.run_fasta_shuffler(fasta_in=query_train_fasta, fasta_out=random_fasta)

        #Random
        if show_progress:
            progress_bar.show_progress('Running FIMO Random')
        fimo_rand = moca_pipeline.run_fimo(motif_file=meme_file,
                                           motif_num=motif,
                                           sequence_file=random_fasta,
                                           out_dir=fimo_rand_dir)
        #Main
        if show_progress:
            progress_bar.show_progress('Running FIMO Main')
        fimo_main = moca_pipeline.run_fimo(motif_file=meme_file,
                                           motif_num=motif,
                                           sequence_file=query_test_fasta,
                                           out_dir=fimo_main_dir)

        fimo_rand_file = os.path.join(fimo_rand_dir, 'fimo.txt')
        fimo_main_file = os.path.join(fimo_main_dir, 'fimo.txt')

        main_intervals = get_start_stop_intervals(fimo_main_file, flank_length=flank_motif)
        random_intervals = get_start_stop_intervals(fimo_rand_file, flank_length=flank_motif)

        sample_score_files = []
        control_score_files = []
        for key in list(conservation_wig_keys):
            try:
                wigfile = wigfiles[key]
            except KeyError:
                continue
            if show_progress:
                progress_bar.show_progress('Creating plots')
            sample_score_file = moca_pipeline.save_conservation_scores(main_intervals, wigfile,
                                                                       fimo_main_dir, out_prefix=key)
            control_score_file = moca_pipeline.save_conservation_scores(random_intervals, wigfile,
                                                                        fimo_rand_dir, out_prefix=key)
            sample_score_files.append(sample_score_file)
            control_score_files.append(control_score_file)
        if show_progress:
            progress_bar.show_progress('Creating Plot')
        create_plot(meme_file,
                    bedfile_fn,
                    output_dir=moca_out_dir,
                    centrimo_dir=centrimo_main_dir,
                    motif_number=motif,
                    flank_length=flank_motif,
                    sample_score_files=sample_score_files,
                    control_score_files=control_score_files,
                    reg_plot_titles=[key.capitalize() for key in list(conservation_wig_keys)],
                    annotate=None)

    if show_progress:
        progress_bar.close()

@cli.command('plot', context_settings=CONTEXT_SETTINGS)
@click.option('--meme-dir', '--meme_dir', help='MEME output directory', required=True)
@click.option('--centrimo-dir', '--centrimo_dir', help='Centrimo output directory', required=True)
@click.option('--fimo-dir-sample', '--fimo_dir_sample', help='Sample fimo.txt', required=True)
@click.option('--fimo-dir-control', '--fimo_dir_control', help='Control fimo.txt', required=True)
@click.option('--name', help='Plot title')
@click.option('--flank-motif',
              default=5,
              help='Length of sequence flanking motif',
              required=True)
@click.option('--motif',
              default=1,
              help='Motif number',
              type=int)
@click.option('--oc',
              '-o',
              help='Output Directory',
              required=True)
@click.option('--configuration',
              '-c',
              help='Configuration file',
              required=True)
@click.option('--show-progress',
              help='Print progress',
              is_flag=True)
@click.option('--genome-build',
              '-g', '-gb',
              help='Key denoting genome build to use in configuration file',
              required=True)

def plot(meme_dir, centrimo_dir, fimo_dir_sample, fimo_dir_control, name,
         flank_motif, motif, oc, configuration, show_progress, genome_build):
    """Create conservation plots"""
    if not oc:
        moca_out_dir = os.path.join(os.getcwd(), 'moca_output')
    else:
        moca_out_dir = oc
    if show_progress:
        progress_bar = ProgressBar(['Creating Plots']*len(conservation_wig_keys))
    moca_pipeline = pipeline.Pipeline(configuration)
    meme_file = os.path.join(meme_dir, 'meme.txt')
    genome_data = moca_pipeline.get_genome_data(genome_build)

    wigfiles = {}
    for key in list(conservation_wig_keys):
        try:
            wigfiles[key] = genome_data['{}_wig'.format(key)]
        except KeyError:
            pass

    fimo_control= os.path.join(fimo_dir_control, 'fimo.txt')
    fimo_sample = os.path.join(fimo_dir_sample, 'fimo.txt')
    safe_makedir(moca_out_dir)
    main_intervals = get_start_stop_intervals(fimo_sample, flank_length=flank_motif)
    random_intervals = get_start_stop_intervals(fimo_control, flank_length=flank_motif)

    sample_score_files = []
    control_score_files = []
    for key in list(conservation_wig_keys):
        wigfile = wigfiles[key]
        if show_progress:
            progress_bar.show_progress('Creating plots')
        sample_score_file = moca_pipeline.save_conservation_scores(main_intervals, wigfile,
                                                                    fimo_dir_sample, out_prefix=key)
        control_score_file = moca_pipeline.save_conservation_scores(random_intervals, wigfile,
                                                                    fimo_dir_control, out_prefix=key)
        sample_score_files.append(sample_score_file)
        control_score_files.append(control_score_file)
    create_plot(meme_file,
                name,
                output_dir=moca_out_dir,
                centrimo_dir=centrimo_dir,
                motif_number=motif,
                flank_length=flank_motif,
                sample_score_files=[sample_score_file],
                control_score_files=[control_score_file],
                reg_plot_titles=[key.capitalize() for key in list(conservation_wig_keys)],
                annotate=None)

<commit_msg>Skip keys for absent wigfiles<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""MoCA CLI"""
import os
import re
import sys
from time import sleep
import click
from moca import bedoperations, pipeline
from moca.bedoperations.fimo import get_start_stop_intervals
from moca.helpers import filename_extension
from moca.helpers import read_memefile
from moca.helpers.job_executor import safe_makedir
from moca.plotter import create_plot
from moca import version
from tqdm import tqdm
from click_help_colors import HelpColorsGroup, HelpColorsCommand

conservation_wig_keys = ['phylop', 'gerp', 'phastcons']

class ProgressBar(object):
    def __init__(self, msg_list):
        self.bar = tqdm(msg_list,
                        bar_format='{desc}{percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}]',
                        smoothing=0,
                        miniters=1,
                        mininterval=0.01)

    def show_progress(self, msg):
        self.bar.set_description(msg)
        self.bar.update()
        sleep(0.1)

    def close(self):
        self.bar.close()
CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])

@click.group(cls=HelpColorsGroup,
             help_headers_color='yellow',
             help_options_color='green')
@click.version_option(version=version.__version__)
def cli():
    """moca: Motif Conservation Analysis"""
    pass

@cli.command('find_motifs', context_settings=CONTEXT_SETTINGS)
@click.option('--bedfile', '-i',
              help='Bed file input',
              required=True)
@click.option('--oc',
              '-o',
              help='Output Directory',
              required=True)
@click.option('--configuration',
              '-c',
              help='Configuration file',
              required=True)
@click.option('-s',
              '--slop-length',
              '--slop_length',
              default=50,
              help='Flanking sequence length')
@click.option('-f',
              '--flank-motif',
              '--flank_motif',
              default=5,
              help='Length of sequence flanking motif')
@click.option('--n-motif',
              default=5,
              help='Number of motifs',
              type=int)
@click.option('--cores',
              '-t',
              default=1,
              help='Number of parallel MEME jobs',
              type=int,
              required=True)
@click.option('--genome-build',
              '-g', '-gb',
              help='Key denoting genome build to use in configuration file',
              required=True)

@click.option('--show-progress',
              help='Print progress',
              is_flag=True)

def find_motifs(bedfile, oc, configuration, slop_length,
                flank_motif, n_motif, cores, genome_build, show_progress):
    """Search motifs and create conservation plots"""
    root_dir = os.path.dirname(os.path.abspath(bedfile))
    if not oc:
        moca_out_dir = os.path.join(os.getcwd(), 'moca_output')
    else:
        moca_out_dir = oc
    moca_pipeline = pipeline.Pipeline(configuration)
    genome_data = moca_pipeline.get_genome_data(genome_build)
    genome_fasta = genome_data['fasta']
    genome_table = genome_data['genome_table']

    wigfiles = {}
    for key in list(conservation_wig_keys):
        try:
            wigfiles[key] = genome_data['{}_wig'.format(key)]
        except KeyError:
            pass
    safe_makedir(moca_out_dir)
    bedfile_fn, _ = filename_extension(bedfile)

    if show_progress:
        msg_list = ['Extracting Fasta',
                    'Running MEME',
                    'Running CENTRIMO']
        msg_list_e = ['Generating random Fasta', 'Running fimo random', 'Running fimo main'] + ['Extracting Scores']*len(wigfiles.keys()) + ['Creating PLot']
        msg_list = msg_list + msg_list_e*n_motif
        progress_bar = ProgressBar(msg_list)

    query_train_fasta = os.path.join(moca_out_dir,
                                     bedfile_fn + '_train_flank_{}.fasta'.format(slop_length))
    query_test_fasta = os.path.join(moca_out_dir,
                                    bedfile_fn + '_test_flank_{}.fasta'.format(slop_length))

    if show_progress:
        progress_bar.show_progress('Extracting Fasta')
    bed_o = bedoperations.Bedfile(bedfile, genome_table, moca_out_dir)
    bed_train, bed_test = bed_o.split_train_test_bed(train_peaks_count=500, test_peaks_count=500)

    bed_train_slopped  = bed_o.slop_bed(bed_train, flank_length=slop_length)
    bed_test_slopped  = bed_o.slop_bed(bed_test, flank_length=slop_length)

    bed_o.extract_fasta(bed_train_slopped, fasta_in=genome_fasta, fasta_out=query_train_fasta)
    bed_o.extract_fasta(bed_test_slopped, fasta_in=genome_fasta, fasta_out=query_test_fasta)

    #memechip_out_dir = os.path.join(moca_out_dir, 'memechip_analysis')
    meme_out_dir = os.path.join(moca_out_dir, 'meme_out')
    memechip_out_dir = meme_out_dir
    meme_params = moca_pipeline.get_meme_default_params
    if cores==1:
        re.sub(r' -p*', '', meme_params)
    else:
        re.sub(r'-p*', '-p {}'.format(cores), meme_params)
    if show_progress:
        progress_bar.show_progress('Running MEME')

    #meme_run_out = moca_pipeline.run_memechip(fasta_in=query_fasta, out_dir=memechip_out_dir)
    #
    meme_run_out = moca_pipeline.run_meme(fasta_in=query_train_fasta,
                                          out_dir=meme_out_dir,
                                          strargs=meme_params)
    if meme_run_out['stderr']!='':
        sys.stdout.write('Error running MEME: {}'.format(meme_run_out['stderr']))
        sys.exit(1)
    meme_file = os.path.join(meme_out_dir, 'meme.txt')
    meme_summary = read_memefile(meme_file)

    if show_progress:
        progress_bar.show_progress('Running CENTRIMO')
    centrimo_main_dir = os.path.join(moca_out_dir, 'centrimo_out')
    centrimo_main = moca_pipeline.run_centrimo(meme_file=meme_file,
                                               fasta_in=query_test_fasta,
                                               out_dir=centrimo_main_dir)
    for motif in range(1, meme_summary['total_motifs']+1):
        fimo_rand_dir = os.path.join(memechip_out_dir, 'fimo_random_{}'.format(motif))
        fimo_main_dir = os.path.join(memechip_out_dir, 'fimo_out_{}'.format(motif))

        safe_makedir(fimo_rand_dir)
        random_fasta = os.path.join(fimo_rand_dir, 'random_{}.fa'.format(motif))
        if show_progress:
            progress_bar.show_progress('Generating Random Fasta: {}'.format(motif))
        moca_pipeline.run_fasta_shuffler(fasta_in=query_train_fasta, fasta_out=random_fasta)

        #Random
        if show_progress:
            progress_bar.show_progress('Running FIMO Random')
        fimo_rand = moca_pipeline.run_fimo(motif_file=meme_file,
                                           motif_num=motif,
                                           sequence_file=random_fasta,
                                           out_dir=fimo_rand_dir)
        #Main
        if show_progress:
            progress_bar.show_progress('Running FIMO Main')
        fimo_main = moca_pipeline.run_fimo(motif_file=meme_file,
                                           motif_num=motif,
                                           sequence_file=query_test_fasta,
                                           out_dir=fimo_main_dir)

        fimo_rand_file = os.path.join(fimo_rand_dir, 'fimo.txt')
        fimo_main_file = os.path.join(fimo_main_dir, 'fimo.txt')

        main_intervals = get_start_stop_intervals(fimo_main_file, flank_length=flank_motif)
        random_intervals = get_start_stop_intervals(fimo_rand_file, flank_length=flank_motif)

        sample_score_files = []
        control_score_files = []
        for key in list(conservation_wig_keys):
            try:
                wigfile = wigfiles[key]
            except KeyError:
                continue
            if show_progress:
                progress_bar.show_progress('Creating plots')
            sample_score_file = moca_pipeline.save_conservation_scores(main_intervals, wigfile,
                                                                       fimo_main_dir, out_prefix=key)
            control_score_file = moca_pipeline.save_conservation_scores(random_intervals, wigfile,
                                                                        fimo_rand_dir, out_prefix=key)
            sample_score_files.append(sample_score_file)
            control_score_files.append(control_score_file)
        if show_progress:
            progress_bar.show_progress('Creating Plot')
        create_plot(meme_file,
                    bedfile_fn,
                    output_dir=moca_out_dir,
                    centrimo_dir=centrimo_main_dir,
                    motif_number=motif,
                    flank_length=flank_motif,
                    sample_score_files=sample_score_files,
                    control_score_files=control_score_files,
                    reg_plot_titles=[key.capitalize() for key in list(conservation_wig_keys)],
                    annotate=None)

    if show_progress:
        progress_bar.close()

@cli.command('plot', context_settings=CONTEXT_SETTINGS)
@click.option('--meme-dir', '--meme_dir', help='MEME output directory', required=True)
@click.option('--centrimo-dir', '--centrimo_dir', help='Centrimo output directory', required=True)
@click.option('--fimo-dir-sample', '--fimo_dir_sample', help='Sample fimo.txt', required=True)
@click.option('--fimo-dir-control', '--fimo_dir_control', help='Control fimo.txt', required=True)
@click.option('--name', help='Plot title')
@click.option('--flank-motif',
              default=5,
              help='Length of sequence flanking motif',
              required=True)
@click.option('--motif',
              default=1,
              help='Motif number',
              type=int)
@click.option('--oc',
              '-o',
              help='Output Directory',
              required=True)
@click.option('--configuration',
              '-c',
              help='Configuration file',
              required=True)
@click.option('--show-progress',
              help='Print progress',
              is_flag=True)
@click.option('--genome-build',
              '-g', '-gb',
              help='Key denoting genome build to use in configuration file',
              required=True)

def plot(meme_dir, centrimo_dir, fimo_dir_sample, fimo_dir_control, name,
         flank_motif, motif, oc, configuration, show_progress, genome_build):
    """Create conservation plots"""
    if not oc:
        moca_out_dir = os.path.join(os.getcwd(), 'moca_output')
    else:
        moca_out_dir = oc
    if show_progress:
        progress_bar = ProgressBar(['Creating Plots']*len(conservation_wig_keys))
    moca_pipeline = pipeline.Pipeline(configuration)
    meme_file = os.path.join(meme_dir, 'meme.txt')
    genome_data = moca_pipeline.get_genome_data(genome_build)

    wigfiles = {}
    for key in list(conservation_wig_keys):
        try:
            wigfiles[key] = genome_data['{}_wig'.format(key)]
        except KeyError:
            pass

    fimo_control= os.path.join(fimo_dir_control, 'fimo.txt')
    fimo_sample = os.path.join(fimo_dir_sample, 'fimo.txt')
    safe_makedir(moca_out_dir)
    main_intervals = get_start_stop_intervals(fimo_sample, flank_length=flank_motif)
    random_intervals = get_start_stop_intervals(fimo_control, flank_length=flank_motif)

    sample_score_files = []
    control_score_files = []
    for key in list(conservation_wig_keys):
        try:
            wigfile = wigfiles[key]
        except KeyError:
            continue
        if show_progress:
            progress_bar.show_progress('Creating plots')
        sample_score_file = moca_pipeline.save_conservation_scores(main_intervals, wigfile,
                                                                    fimo_dir_sample, out_prefix=key)
        control_score_file = moca_pipeline.save_conservation_scores(random_intervals, wigfile,
                                                                    fimo_dir_control, out_prefix=key)
        sample_score_files.append(sample_score_file)
        control_score_files.append(control_score_file)
    create_plot(meme_file,
                name,
                output_dir=moca_out_dir,
                centrimo_dir=centrimo_dir,
                motif_number=motif,
                flank_length=flank_motif,
                sample_score_files=[sample_score_file],
                control_score_files=[control_score_file],
                reg_plot_titles=[key.capitalize() for key in list(conservation_wig_keys)],
                annotate=None)

<|endoftext|>


<commit_before>"""Module that allows custom Sphinx parameters to be set on the notebook and
on the 'other' object passed into Jinja.  Called prior to Jinja conversion
process.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import print_function, absolute_import

# Stdlib imports
# Used to find Sphinx package location
import sphinx
import os.path

# Used to set the default date to today's date
from datetime import date

# Third-party imports
# Needed for Pygments latex definitions.
from pygments.formatters import LatexFormatter

# Our own imports
# Configurable traitlets
from IPython.utils.traitlets import Unicode, Bool

# Needed to override transformer
from .base import (Transformer)

from IPython.nbconvert.utils import console

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

class SphinxTransformer(Transformer):
    """
    Sphinx utility transformer.

    This transformer is used to set variables needed by the latex to build
    Sphinx stylized templates.
    """

    interactive = Bool(False, config=True, help="""
        Allows you to define whether or not the Sphinx exporter will prompt
        you for input during the conversion process.  If this is set to false,
        the author, version, release, date, and chapter_style traits should
        be set.
        """)

    author = Unicode("Unknown Author", config=True, help="Author name")

    version = Unicode("", config=True, help="""
        Version number
        You can leave this blank if you do not want to render a version number.
        Example: "1.0.0"
        """)

    release = Unicode("", config=True, help="""
        Release name
        You can leave this blank if you do not want to render a release name.
        Example: "Rough Draft"
        """)

    publish_date = Unicode("", config=True, help="""
        Publish date
        This is the date to render on the document as the publish date.
        Leave this blank to default to todays date.
        Example: "June 12, 1990"
        """)

    chapter_style = Unicode("Bjarne", config=True, help="""
        Sphinx chapter style
        This is the style to use for the chapter headers in the document.
        You may choose one of the following:
            "Bjarne"    (default)
            "Lenny"
            "Glenn"
            "Conny"
            "Rejne"
            "Sonny"    (used for international documents)
        """)

    output_style = Unicode("notebook", config=True, help="""
        Nbconvert Ipython
        notebook input/output formatting style.
        You may choose one of the following:
            "simple     (recommended for long code segments)"
            "notebook"  (default)
        """)

    center_output = Bool(False, config=True, help="""
        Optional attempt to center all output.  If this is false, no additional
        formatting is applied.
        """)

    use_headers = Bool(True, config=True, help="""
        Whether not a header should be added to the document.
        """)

    #Allow the user to override the title of the notebook (useful for
    #fancy document titles that the file system doesn't support.)
    overridetitle = Unicode("", config=True, help="")

    def call(self, nb, resources):
        """
        Sphinx transformation to apply on each notebook.

        Parameters
        ----------
        nb : NotebookNode
            Notebook being converted
        resources : dictionary
            Additional resources used in the conversion process.  Allows
            transformers to pass variables into the Jinja engine.
        """

        # TODO: Add versatile method of additional notebook metadata.  Include
        #       handling of multiple files.  For now use a temporay namespace,
        #       '_draft' to signify that this needs to change.
        if not "sphinx" in resources:
            resources["sphinx"] = {}

        if self.interactive:

            # Prompt the user for additional meta data that doesn't exist currently
            # but would be usefull for Sphinx.
            resources["sphinx"]["author"] = self._prompt_author()
            resources["sphinx"]["version"] = self._prompt_version()
            resources["sphinx"]["release"] = self._prompt_release()
            resources["sphinx"]["date"] = self._prompt_date()

            # Prompt the user for the document style.
            resources["sphinx"]["chapterstyle"] = self._prompt_chapter_title_style()
            resources["sphinx"]["outputstyle"] = self._prompt_output_style()

            # Small options
            resources["sphinx"]["centeroutput"] = console.prompt_boolean("Do you want to center the output? (false)", False)
            resources["sphinx"]["header"] = console.prompt_boolean("Should a Sphinx document header be used? (true)", True)
        else:

            # Try to use the traitlets.
            resources["sphinx"]["author"] = self.author
            resources["sphinx"]["version"] = self.version
            resources["sphinx"]["release"] = self.release

            # Use todays date if none is provided.
            if self.publish_date:
                resources["sphinx"]["date"] = self.publish_date
            elif len(resources['metadata']['modified_date'].strip()) == 0:
                resources["sphinx"]["date"] = date.today().strftime("%B %-d, %Y")
            else:
                resources["sphinx"]["date"] = resources['metadata']['modified_date']

            # Sphinx traitlets.
            resources["sphinx"]["chapterstyle"] = self.chapter_style
            resources["sphinx"]["outputstyle"] = self.output_style
            resources["sphinx"]["centeroutput"] = self.center_output
            resources["sphinx"]["header"] = self.use_headers

        # Find and pass in the path to the Sphinx dependencies.
        resources["sphinx"]["texinputs"] = os.path.realpath(os.path.join(sphinx.__file__, "..", "texinputs"))

        # Generate Pygments definitions for Latex
        resources["sphinx"]["pygment_definitions"] = self._generate_pygments_latex_def()

        if not (self.overridetitle == None or len(self.overridetitle.strip()) == 0):
            resources['metadata']['name'] = self.overridetitle

        # End
        return nb, resources

    def _generate_pygments_latex_def(self):
        """
        Generate the pygments latex definitions that allows pygments
        to work in latex.
        """

        return LatexFormatter().get_style_defs()

    def _prompt_author(self):
        """
        Prompt the user to input an Author name
        """
        return  console.input("Author name: ")

    def _prompt_version(self):
        """
        prompt the user to enter a version number
        """
        return  console.input("Version (ie ""1.0.0""): ")

    def _prompt_release(self):
        """
        Prompt the user to input a release name
        """

        return  console.input("Release Name (ie ""Rough draft""): ")

    def _prompt_date(self, resources):
        """
        Prompt the user to enter a date
        """

        if resources['metadata']['modified_date']:
            default_date = resources['metadata']['modified_date']
        else:
            default_date = date.today().strftime("%B %-d, %Y")

        user_date = console.input("Date (deafults to \"" + default_date + "\"): ")
        if len(user_date.strip()) == 0:
            user_date = default_date
        return user_date

    def _prompt_output_style(self):
        """
        Prompts the user to pick an IPython output style.
        """

        # Dictionary of available output styles
        styles = {1: "simple",
                  2: "notebook"}

        #Append comments to the menu when displaying it to the user.
        comments = {1: "(recommended for long code segments)",
                    2: "(default)"}

        return console.prompt_dictionary(styles, default_style=2, menu_comments=comments)

    def _prompt_chapter_title_style(self):
        """
        Prompts the user to pick a Sphinx chapter style
        """

        # Dictionary of available Sphinx styles
        styles = {1: "Bjarne",
                  2: "Lenny",
                  3: "Glenn",
                  4: "Conny",
                  5: "Rejne",
                  6: "Sonny"}

        #Append comments to the menu when displaying it to the user.
        comments = {1: "(default)",
                    6: "(for international documents)"}

        return console.prompt_dictionary(styles, menu_comments=comments)

<commit_msg>use sphinx.package_dir to find texinputs<commit_after>"""Module that allows custom Sphinx parameters to be set on the notebook and
on the 'other' object passed into Jinja.  Called prior to Jinja conversion
process.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import print_function, absolute_import

# Stdlib imports
# Used to find Sphinx package location
import sphinx
import os.path

# Used to set the default date to today's date
from datetime import date

# Third-party imports
# Needed for Pygments latex definitions.
from pygments.formatters import LatexFormatter

# Our own imports
# Configurable traitlets
from IPython.utils.traitlets import Unicode, Bool

# Needed to override transformer
from .base import (Transformer)

from IPython.nbconvert.utils import console

#-----------------------------------------------------------------------------
# Classes and functions
#-----------------------------------------------------------------------------

class SphinxTransformer(Transformer):
    """
    Sphinx utility transformer.

    This transformer is used to set variables needed by the latex to build
    Sphinx stylized templates.
    """

    interactive = Bool(False, config=True, help="""
        Allows you to define whether or not the Sphinx exporter will prompt
        you for input during the conversion process.  If this is set to false,
        the author, version, release, date, and chapter_style traits should
        be set.
        """)

    author = Unicode("Unknown Author", config=True, help="Author name")

    version = Unicode("", config=True, help="""
        Version number
        You can leave this blank if you do not want to render a version number.
        Example: "1.0.0"
        """)

    release = Unicode("", config=True, help="""
        Release name
        You can leave this blank if you do not want to render a release name.
        Example: "Rough Draft"
        """)

    publish_date = Unicode("", config=True, help="""
        Publish date
        This is the date to render on the document as the publish date.
        Leave this blank to default to todays date.
        Example: "June 12, 1990"
        """)

    chapter_style = Unicode("Bjarne", config=True, help="""
        Sphinx chapter style
        This is the style to use for the chapter headers in the document.
        You may choose one of the following:
            "Bjarne"    (default)
            "Lenny"
            "Glenn"
            "Conny"
            "Rejne"
            "Sonny"    (used for international documents)
        """)

    output_style = Unicode("notebook", config=True, help="""
        Nbconvert Ipython
        notebook input/output formatting style.
        You may choose one of the following:
            "simple     (recommended for long code segments)"
            "notebook"  (default)
        """)

    center_output = Bool(False, config=True, help="""
        Optional attempt to center all output.  If this is false, no additional
        formatting is applied.
        """)

    use_headers = Bool(True, config=True, help="""
        Whether not a header should be added to the document.
        """)

    #Allow the user to override the title of the notebook (useful for
    #fancy document titles that the file system doesn't support.)
    overridetitle = Unicode("", config=True, help="")

    def call(self, nb, resources):
        """
        Sphinx transformation to apply on each notebook.

        Parameters
        ----------
        nb : NotebookNode
            Notebook being converted
        resources : dictionary
            Additional resources used in the conversion process.  Allows
            transformers to pass variables into the Jinja engine.
        """

        # TODO: Add versatile method of additional notebook metadata.  Include
        #       handling of multiple files.  For now use a temporay namespace,
        #       '_draft' to signify that this needs to change.
        if not "sphinx" in resources:
            resources["sphinx"] = {}

        if self.interactive:

            # Prompt the user for additional meta data that doesn't exist currently
            # but would be usefull for Sphinx.
            resources["sphinx"]["author"] = self._prompt_author()
            resources["sphinx"]["version"] = self._prompt_version()
            resources["sphinx"]["release"] = self._prompt_release()
            resources["sphinx"]["date"] = self._prompt_date()

            # Prompt the user for the document style.
            resources["sphinx"]["chapterstyle"] = self._prompt_chapter_title_style()
            resources["sphinx"]["outputstyle"] = self._prompt_output_style()

            # Small options
            resources["sphinx"]["centeroutput"] = console.prompt_boolean("Do you want to center the output? (false)", False)
            resources["sphinx"]["header"] = console.prompt_boolean("Should a Sphinx document header be used? (true)", True)
        else:

            # Try to use the traitlets.
            resources["sphinx"]["author"] = self.author
            resources["sphinx"]["version"] = self.version
            resources["sphinx"]["release"] = self.release

            # Use todays date if none is provided.
            if self.publish_date:
                resources["sphinx"]["date"] = self.publish_date
            elif len(resources['metadata']['modified_date'].strip()) == 0:
                resources["sphinx"]["date"] = date.today().strftime("%B %-d, %Y")
            else:
                resources["sphinx"]["date"] = resources['metadata']['modified_date']

            # Sphinx traitlets.
            resources["sphinx"]["chapterstyle"] = self.chapter_style
            resources["sphinx"]["outputstyle"] = self.output_style
            resources["sphinx"]["centeroutput"] = self.center_output
            resources["sphinx"]["header"] = self.use_headers

        # Find and pass in the path to the Sphinx dependencies.
        resources["sphinx"]["texinputs"] = os.path.realpath(os.path.join(sphinx.package_dir, "texinputs"))

        # Generate Pygments definitions for Latex
        resources["sphinx"]["pygment_definitions"] = self._generate_pygments_latex_def()

        if not (self.overridetitle == None or len(self.overridetitle.strip()) == 0):
            resources['metadata']['name'] = self.overridetitle

        # End
        return nb, resources

    def _generate_pygments_latex_def(self):
        """
        Generate the pygments latex definitions that allows pygments
        to work in latex.
        """

        return LatexFormatter().get_style_defs()

    def _prompt_author(self):
        """
        Prompt the user to input an Author name
        """
        return  console.input("Author name: ")

    def _prompt_version(self):
        """
        prompt the user to enter a version number
        """
        return  console.input("Version (ie ""1.0.0""): ")

    def _prompt_release(self):
        """
        Prompt the user to input a release name
        """

        return  console.input("Release Name (ie ""Rough draft""): ")

    def _prompt_date(self, resources):
        """
        Prompt the user to enter a date
        """

        if resources['metadata']['modified_date']:
            default_date = resources['metadata']['modified_date']
        else:
            default_date = date.today().strftime("%B %-d, %Y")

        user_date = console.input("Date (deafults to \"" + default_date + "\"): ")
        if len(user_date.strip()) == 0:
            user_date = default_date
        return user_date

    def _prompt_output_style(self):
        """
        Prompts the user to pick an IPython output style.
        """

        # Dictionary of available output styles
        styles = {1: "simple",
                  2: "notebook"}

        #Append comments to the menu when displaying it to the user.
        comments = {1: "(recommended for long code segments)",
                    2: "(default)"}

        return console.prompt_dictionary(styles, default_style=2, menu_comments=comments)

    def _prompt_chapter_title_style(self):
        """
        Prompts the user to pick a Sphinx chapter style
        """

        # Dictionary of available Sphinx styles
        styles = {1: "Bjarne",
                  2: "Lenny",
                  3: "Glenn",
                  4: "Conny",
                  5: "Rejne",
                  6: "Sonny"}

        #Append comments to the menu when displaying it to the user.
        comments = {1: "(default)",
                    6: "(for international documents)"}

        return console.prompt_dictionary(styles, menu_comments=comments)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
The IPython controller application.

Authors:

* Brian Granger
* MinRK

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import with_statement

import os
import socket
import stat
import sys
import uuid

from multiprocessing import Process

import zmq
from zmq.devices import ProcessMonitoredQueue
from zmq.log.handlers import PUBHandler
from zmq.utils import jsonapi as json

from IPython.config.application import boolean_flag
from IPython.core.profiledir import ProfileDir

from IPython.parallel.apps.baseapp import (
    BaseParallelApplication,
    base_flags
)
from IPython.utils.importstring import import_item
from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict

# from IPython.parallel.controller.controller import ControllerFactory
from IPython.zmq.session import Session
from IPython.parallel.controller.heartmonitor import HeartMonitor
from IPython.parallel.controller.hub import HubFactory
from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
from IPython.parallel.controller.sqlitedb import SQLiteDB

from IPython.parallel.util import signal_children, split_url

# conditional import of MongoDB backend class

try:
    from IPython.parallel.controller.mongodb import MongoDB
except ImportError:
    maybe_mongo = []
else:
    maybe_mongo = [MongoDB]

#-----------------------------------------------------------------------------
# Module level variables
#-----------------------------------------------------------------------------

#: The default config file name for this application
default_config_file_name = u'ipcontroller_config.py'

_description = """Start the IPython controller for parallel computing.

The IPython controller provides a gateway between the IPython engines and
clients. The controller needs to be started before the engines and can be
configured using command line options or using a cluster directory. Cluster
directories contain config, log and security files and are usually located in
your ipython directory and named as "profile_name". See the `profile`
and `profile_dir` options for details.
"""

#-----------------------------------------------------------------------------
# The main application
#-----------------------------------------------------------------------------
flags = {}
flags.update(base_flags)
flags.update({
    'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
                    'Use threads instead of processes for the schedulers'),
    'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
                    'use the SQLiteDB backend'),
    'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
                    'use the MongoDB backend'),
    'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
                    'use the in-memory DictDB backend'),
    'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
                    'reuse existing json connection files')
})

flags.update(boolean_flag('secure', 'IPControllerApp.secure',
    "Use HMAC digests for authentication of messages.",
    "Don't authenticate messages."
))

class IPControllerApp(BaseParallelApplication):

    name = u'ipcontroller'
    description = _description
    config_file_name = Unicode(default_config_file_name)
    classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo

    # change default to True
    auto_create = Bool(True, config=True,
        help="""Whether to create profile dir if it doesn't exist.""")

    reuse_files = Bool(False, config=True,
        help='Whether to reuse existing json connection files.'
    )
    secure = Bool(True, config=True,
        help='Whether to use HMAC digests for extra message authentication.'
    )
    ssh_server = Unicode(u'', config=True,
        help="""ssh url for clients to use when connecting to the Controller
        processes. It should be of the form: [user@]server[:port]. The
        Controller's listening addresses must be accessible from the ssh server""",
    )
    location = Unicode(u'', config=True,
        help="""The external IP or domain name of the Controller, used for disambiguating
        engine and client connections.""",
    )
    import_statements = List([], config=True,
        help="import statements to be run at startup.  Necessary in some environments"
    )

    use_threads = Bool(False, config=True,
        help='Use threads instead of processes for the schedulers',
        )

    # internal
    children = List()
    mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')

    def _use_threads_changed(self, name, old, new):
        self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')

    aliases = Dict(dict(
        log_level = 'IPControllerApp.log_level',
        log_url = 'IPControllerApp.log_url',
        reuse_files = 'IPControllerApp.reuse_files',
        secure = 'IPControllerApp.secure',
        ssh = 'IPControllerApp.ssh_server',
        use_threads = 'IPControllerApp.use_threads',
        import_statements = 'IPControllerApp.import_statements',
        location = 'IPControllerApp.location',

        ident = 'Session.session',
        user = 'Session.username',
        exec_key = 'Session.keyfile',

        url = 'HubFactory.url',
        ip = 'HubFactory.ip',
        transport = 'HubFactory.transport',
        port = 'HubFactory.regport',

        ping = 'HeartMonitor.period',

        scheme = 'TaskScheduler.scheme_name',
        hwm = 'TaskScheduler.hwm',

        profile = "BaseIPythonApplication.profile",
        profile_dir = 'ProfileDir.location',

    ))
    flags = Dict(flags)

    def save_connection_dict(self, fname, cdict):
        """save a connection dict to json file."""
        c = self.config
        url = cdict['url']
        location = cdict['location']
        if not location:
            try:
                proto,ip,port = split_url(url)
            except AssertionError:
                pass
            else:
                location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
            cdict['location'] = location
        fname = os.path.join(self.profile_dir.security_dir, fname)
        with open(fname, 'w') as f:
            f.write(json.dumps(cdict, indent=2))
        os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)

    def load_config_from_json(self):
        """load config from existing json connector files."""
        c = self.config
        # load from engine config
        with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
            cfg = json.loads(f.read())
        key = c.Session.key = cfg['exec_key']
        xport,addr = cfg['url'].split('://')
        c.HubFactory.engine_transport = xport
        ip,ports = addr.split(':')
        c.HubFactory.engine_ip = ip
        c.HubFactory.regport = int(ports)
        self.location = cfg['location']

        # load client config
        with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
            cfg = json.loads(f.read())
        assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
        xport,addr = cfg['url'].split('://')
        c.HubFactory.client_transport = xport
        ip,ports = addr.split(':')
        c.HubFactory.client_ip = ip
        self.ssh_server = cfg['ssh']
        assert int(ports) == c.HubFactory.regport, "regport mismatch"

    def init_hub(self):
        c = self.config

        self.do_import_statements()
        reusing = self.reuse_files
        if reusing:
            try:
                self.load_config_from_json()
            except (AssertionError,IOError):
                reusing=False
        # check again, because reusing may have failed:
        if reusing:
            pass
        elif self.secure:
            key = str(uuid.uuid4())
            # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
            # with open(keyfile, 'w') as f:
            #     f.write(key)
            # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
            c.Session.key = key
        else:
            key = c.Session.key = ''

        try:
            self.factory = HubFactory(config=c, log=self.log)
            # self.start_logging()
            self.factory.init_hub()
        except:
            self.log.error("Couldn't construct the Controller", exc_info=True)
            self.exit(1)

        if not reusing:
            # save to new json config files
            f = self.factory
            cdict = {'exec_key' : key,
                    'ssh' : self.ssh_server,
                    'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
                    'location' : self.location
                    }
            self.save_connection_dict('ipcontroller-client.json', cdict)
            edict = cdict
            edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
            self.save_connection_dict('ipcontroller-engine.json', edict)

    #
    def init_schedulers(self):
        children = self.children
        mq = import_item(str(self.mq_class))

        hub = self.factory
        # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
        # IOPub relay (in a Process)
        q = mq(zmq.PUB, zmq.SUB, zmq.PUB, 'N/A','iopub')
        q.bind_in(hub.client_info['iopub'])
        q.bind_out(hub.engine_info['iopub'])
        q.setsockopt_out(zmq.SUBSCRIBE, '')
        q.connect_mon(hub.monitor_url)
        q.daemon=True
        children.append(q)

        # Multiplexer Queue (in a Process)
        q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'in', 'out')
        q.bind_in(hub.client_info['mux'])
        q.setsockopt_in(zmq.IDENTITY, 'mux')
        q.bind_out(hub.engine_info['mux'])
        q.connect_mon(hub.monitor_url)
        q.daemon=True
        children.append(q)

        # Control Queue (in a Process)
        q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'incontrol', 'outcontrol')
        q.bind_in(hub.client_info['control'])
        q.setsockopt_in(zmq.IDENTITY, 'control')
        q.bind_out(hub.engine_info['control'])
        q.connect_mon(hub.monitor_url)
        q.daemon=True
        children.append(q)
        try:
            scheme = self.config.TaskScheduler.scheme_name
        except AttributeError:
            scheme = TaskScheduler.scheme_name.get_default_value()
        # Task Queue (in a Process)
        if scheme == 'pure':
            self.log.warn("task::using pure XREQ Task scheduler")
            q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, 'intask', 'outtask')
            # q.setsockopt_out(zmq.HWM, hub.hwm)
            q.bind_in(hub.client_info['task'][1])
            q.setsockopt_in(zmq.IDENTITY, 'task')
            q.bind_out(hub.engine_info['task'])
            q.connect_mon(hub.monitor_url)
            q.daemon=True
            children.append(q)
        elif scheme == 'none':
            self.log.warn("task::using no Task scheduler")

        else:
            self.log.info("task::using Python %s Task scheduler"%scheme)
            sargs = (hub.client_info['task'][1], hub.engine_info['task'],
                                hub.monitor_url, hub.client_info['notification'])
            kwargs = dict(logname='scheduler', loglevel=self.log_level,
                            log_url = self.log_url, config=dict(self.config))
            if 'Process' in self.mq_class:
                # run the Python scheduler in a Process
                q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
                q.daemon=True
                children.append(q)
            else:
                # single-threaded Controller
                kwargs['in_thread'] = True
                launch_scheduler(*sargs, **kwargs)

    def save_urls(self):
        """save the registration urls to files."""
        c = self.config

        sec_dir = self.profile_dir.security_dir
        cf = self.factory

        with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
            f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))

        with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
            f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))

    def do_import_statements(self):
        statements = self.import_statements
        for s in statements:
            try:
                self.log.msg("Executing statement: '%s'" % s)
                exec s in globals(), locals()
            except:
                self.log.msg("Error running statement: %s" % s)

    def forward_logging(self):
        if self.log_url:
            self.log.info("Forwarding logging to %s"%self.log_url)
            context = zmq.Context.instance()
            lsock = context.socket(zmq.PUB)
            lsock.connect(self.log_url)
            handler = PUBHandler(lsock)
            self.log.removeHandler(self._log_handler)
            handler.root_topic = 'controller'
            handler.setLevel(self.log_level)
            self.log.addHandler(handler)
            self._log_handler = handler
    # #

    def initialize(self, argv=None):
        super(IPControllerApp, self).initialize(argv)
        self.forward_logging()
        self.init_hub()
        self.init_schedulers()

    def start(self):
        # Start the subprocesses:
        self.factory.start()
        child_procs = []
        for child in self.children:
            child.start()
            if isinstance(child, ProcessMonitoredQueue):
                child_procs.append(child.launcher)
            elif isinstance(child, Process):
                child_procs.append(child)
        if child_procs:
            signal_children(child_procs)

        self.write_pid_file(overwrite=True)

        try:
            self.factory.loop.start()
        except KeyboardInterrupt:
            self.log.critical("Interrupted, Exiting...\n")

def launch_new_instance():
    """Create and run the IPython controller"""
    if sys.platform == 'win32':
        # make sure we don't get called from a multiprocessing subprocess
        # this can result in infinite Controllers being started on Windows
        # which doesn't have a proper fork, so multiprocessing is wonky

        # this only comes up when IPython has been installed using vanilla
        # setuptools, and *not* distribute.
        import inspect
        for record in inspect.stack():
            frame = record[0]
            if frame.f_locals.get('__name__') == '__parents_main__':
                # we are a subprocess, don't start another Controller!
                return
    app = IPControllerApp.instance()
    app.initialize()
    app.start()

if __name__ == '__main__':
    launch_new_instance()
<commit_msg>use multiprocessing instead of inspect to prevent spurious controllers<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
The IPython controller application.

Authors:

* Brian Granger
* MinRK

"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import with_statement

import os
import socket
import stat
import sys
import uuid

from multiprocessing import Process

import zmq
from zmq.devices import ProcessMonitoredQueue
from zmq.log.handlers import PUBHandler
from zmq.utils import jsonapi as json

from IPython.config.application import boolean_flag
from IPython.core.profiledir import ProfileDir

from IPython.parallel.apps.baseapp import (
    BaseParallelApplication,
    base_flags
)
from IPython.utils.importstring import import_item
from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict

# from IPython.parallel.controller.controller import ControllerFactory
from IPython.zmq.session import Session
from IPython.parallel.controller.heartmonitor import HeartMonitor
from IPython.parallel.controller.hub import HubFactory
from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
from IPython.parallel.controller.sqlitedb import SQLiteDB

from IPython.parallel.util import signal_children, split_url

# conditional import of MongoDB backend class

try:
    from IPython.parallel.controller.mongodb import MongoDB
except ImportError:
    maybe_mongo = []
else:
    maybe_mongo = [MongoDB]

#-----------------------------------------------------------------------------
# Module level variables
#-----------------------------------------------------------------------------

#: The default config file name for this application
default_config_file_name = u'ipcontroller_config.py'

_description = """Start the IPython controller for parallel computing.

The IPython controller provides a gateway between the IPython engines and
clients. The controller needs to be started before the engines and can be
configured using command line options or using a cluster directory. Cluster
directories contain config, log and security files and are usually located in
your ipython directory and named as "profile_name". See the `profile`
and `profile_dir` options for details.
"""

#-----------------------------------------------------------------------------
# The main application
#-----------------------------------------------------------------------------
flags = {}
flags.update(base_flags)
flags.update({
    'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
                    'Use threads instead of processes for the schedulers'),
    'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
                    'use the SQLiteDB backend'),
    'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
                    'use the MongoDB backend'),
    'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
                    'use the in-memory DictDB backend'),
    'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
                    'reuse existing json connection files')
})

flags.update(boolean_flag('secure', 'IPControllerApp.secure',
    "Use HMAC digests for authentication of messages.",
    "Don't authenticate messages."
))

class IPControllerApp(BaseParallelApplication):

    name = u'ipcontroller'
    description = _description
    config_file_name = Unicode(default_config_file_name)
    classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo

    # change default to True
    auto_create = Bool(True, config=True,
        help="""Whether to create profile dir if it doesn't exist.""")

    reuse_files = Bool(False, config=True,
        help='Whether to reuse existing json connection files.'
    )
    secure = Bool(True, config=True,
        help='Whether to use HMAC digests for extra message authentication.'
    )
    ssh_server = Unicode(u'', config=True,
        help="""ssh url for clients to use when connecting to the Controller
        processes. It should be of the form: [user@]server[:port]. The
        Controller's listening addresses must be accessible from the ssh server""",
    )
    location = Unicode(u'', config=True,
        help="""The external IP or domain name of the Controller, used for disambiguating
        engine and client connections.""",
    )
    import_statements = List([], config=True,
        help="import statements to be run at startup.  Necessary in some environments"
    )

    use_threads = Bool(False, config=True,
        help='Use threads instead of processes for the schedulers',
        )

    # internal
    children = List()
    mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')

    def _use_threads_changed(self, name, old, new):
        self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')

    aliases = Dict(dict(
        log_level = 'IPControllerApp.log_level',
        log_url = 'IPControllerApp.log_url',
        reuse_files = 'IPControllerApp.reuse_files',
        secure = 'IPControllerApp.secure',
        ssh = 'IPControllerApp.ssh_server',
        use_threads = 'IPControllerApp.use_threads',
        import_statements = 'IPControllerApp.import_statements',
        location = 'IPControllerApp.location',

        ident = 'Session.session',
        user = 'Session.username',
        exec_key = 'Session.keyfile',

        url = 'HubFactory.url',
        ip = 'HubFactory.ip',
        transport = 'HubFactory.transport',
        port = 'HubFactory.regport',

        ping = 'HeartMonitor.period',

        scheme = 'TaskScheduler.scheme_name',
        hwm = 'TaskScheduler.hwm',

        profile = "BaseIPythonApplication.profile",
        profile_dir = 'ProfileDir.location',

    ))
    flags = Dict(flags)

    def save_connection_dict(self, fname, cdict):
        """save a connection dict to json file."""
        c = self.config
        url = cdict['url']
        location = cdict['location']
        if not location:
            try:
                proto,ip,port = split_url(url)
            except AssertionError:
                pass
            else:
                location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
            cdict['location'] = location
        fname = os.path.join(self.profile_dir.security_dir, fname)
        with open(fname, 'w') as f:
            f.write(json.dumps(cdict, indent=2))
        os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)

    def load_config_from_json(self):
        """load config from existing json connector files."""
        c = self.config
        # load from engine config
        with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
            cfg = json.loads(f.read())
        key = c.Session.key = cfg['exec_key']
        xport,addr = cfg['url'].split('://')
        c.HubFactory.engine_transport = xport
        ip,ports = addr.split(':')
        c.HubFactory.engine_ip = ip
        c.HubFactory.regport = int(ports)
        self.location = cfg['location']

        # load client config
        with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
            cfg = json.loads(f.read())
        assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
        xport,addr = cfg['url'].split('://')
        c.HubFactory.client_transport = xport
        ip,ports = addr.split(':')
        c.HubFactory.client_ip = ip
        self.ssh_server = cfg['ssh']
        assert int(ports) == c.HubFactory.regport, "regport mismatch"

    def init_hub(self):
        c = self.config

        self.do_import_statements()
        reusing = self.reuse_files
        if reusing:
            try:
                self.load_config_from_json()
            except (AssertionError,IOError):
                reusing=False
        # check again, because reusing may have failed:
        if reusing:
            pass
        elif self.secure:
            key = str(uuid.uuid4())
            # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
            # with open(keyfile, 'w') as f:
            #     f.write(key)
            # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
            c.Session.key = key
        else:
            key = c.Session.key = ''

        try:
            self.factory = HubFactory(config=c, log=self.log)
            # self.start_logging()
            self.factory.init_hub()
        except:
            self.log.error("Couldn't construct the Controller", exc_info=True)
            self.exit(1)

        if not reusing:
            # save to new json config files
            f = self.factory
            cdict = {'exec_key' : key,
                    'ssh' : self.ssh_server,
                    'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
                    'location' : self.location
                    }
            self.save_connection_dict('ipcontroller-client.json', cdict)
            edict = cdict
            edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
            self.save_connection_dict('ipcontroller-engine.json', edict)

    #
    def init_schedulers(self):
        children = self.children
        mq = import_item(str(self.mq_class))

        hub = self.factory
        # maybe_inproc = 'inproc://monitor' if self.use_threads else self.monitor_url
        # IOPub relay (in a Process)
        q = mq(zmq.PUB, zmq.SUB, zmq.PUB, 'N/A','iopub')
        q.bind_in(hub.client_info['iopub'])
        q.bind_out(hub.engine_info['iopub'])
        q.setsockopt_out(zmq.SUBSCRIBE, '')
        q.connect_mon(hub.monitor_url)
        q.daemon=True
        children.append(q)

        # Multiplexer Queue (in a Process)
        q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'in', 'out')
        q.bind_in(hub.client_info['mux'])
        q.setsockopt_in(zmq.IDENTITY, 'mux')
        q.bind_out(hub.engine_info['mux'])
        q.connect_mon(hub.monitor_url)
        q.daemon=True
        children.append(q)

        # Control Queue (in a Process)
        q = mq(zmq.XREP, zmq.XREP, zmq.PUB, 'incontrol', 'outcontrol')
        q.bind_in(hub.client_info['control'])
        q.setsockopt_in(zmq.IDENTITY, 'control')
        q.bind_out(hub.engine_info['control'])
        q.connect_mon(hub.monitor_url)
        q.daemon=True
        children.append(q)
        try:
            scheme = self.config.TaskScheduler.scheme_name
        except AttributeError:
            scheme = TaskScheduler.scheme_name.get_default_value()
        # Task Queue (in a Process)
        if scheme == 'pure':
            self.log.warn("task::using pure XREQ Task scheduler")
            q = mq(zmq.XREP, zmq.XREQ, zmq.PUB, 'intask', 'outtask')
            # q.setsockopt_out(zmq.HWM, hub.hwm)
            q.bind_in(hub.client_info['task'][1])
            q.setsockopt_in(zmq.IDENTITY, 'task')
            q.bind_out(hub.engine_info['task'])
            q.connect_mon(hub.monitor_url)
            q.daemon=True
            children.append(q)
        elif scheme == 'none':
            self.log.warn("task::using no Task scheduler")

        else:
            self.log.info("task::using Python %s Task scheduler"%scheme)
            sargs = (hub.client_info['task'][1], hub.engine_info['task'],
                                hub.monitor_url, hub.client_info['notification'])
            kwargs = dict(logname='scheduler', loglevel=self.log_level,
                            log_url = self.log_url, config=dict(self.config))
            if 'Process' in self.mq_class:
                # run the Python scheduler in a Process
                q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
                q.daemon=True
                children.append(q)
            else:
                # single-threaded Controller
                kwargs['in_thread'] = True
                launch_scheduler(*sargs, **kwargs)

    def save_urls(self):
        """save the registration urls to files."""
        c = self.config

        sec_dir = self.profile_dir.security_dir
        cf = self.factory

        with open(os.path.join(sec_dir, 'ipcontroller-engine.url'), 'w') as f:
            f.write("%s://%s:%s"%(cf.engine_transport, cf.engine_ip, cf.regport))

        with open(os.path.join(sec_dir, 'ipcontroller-client.url'), 'w') as f:
            f.write("%s://%s:%s"%(cf.client_transport, cf.client_ip, cf.regport))

    def do_import_statements(self):
        statements = self.import_statements
        for s in statements:
            try:
                self.log.msg("Executing statement: '%s'" % s)
                exec s in globals(), locals()
            except:
                self.log.msg("Error running statement: %s" % s)

    def forward_logging(self):
        if self.log_url:
            self.log.info("Forwarding logging to %s"%self.log_url)
            context = zmq.Context.instance()
            lsock = context.socket(zmq.PUB)
            lsock.connect(self.log_url)
            handler = PUBHandler(lsock)
            self.log.removeHandler(self._log_handler)
            handler.root_topic = 'controller'
            handler.setLevel(self.log_level)
            self.log.addHandler(handler)
            self._log_handler = handler
    # #

    def initialize(self, argv=None):
        super(IPControllerApp, self).initialize(argv)
        self.forward_logging()
        self.init_hub()
        self.init_schedulers()

    def start(self):
        # Start the subprocesses:
        self.factory.start()
        child_procs = []
        for child in self.children:
            child.start()
            if isinstance(child, ProcessMonitoredQueue):
                child_procs.append(child.launcher)
            elif isinstance(child, Process):
                child_procs.append(child)
        if child_procs:
            signal_children(child_procs)

        self.write_pid_file(overwrite=True)

        try:
            self.factory.loop.start()
        except KeyboardInterrupt:
            self.log.critical("Interrupted, Exiting...\n")

def launch_new_instance():
    """Create and run the IPython controller"""
    if sys.platform == 'win32':
        # make sure we don't get called from a multiprocessing subprocess
        # this can result in infinite Controllers being started on Windows
        # which doesn't have a proper fork, so multiprocessing is wonky

        # this only comes up when IPython has been installed using vanilla
        # setuptools, and *not* distribute.
        import multiprocessing
        p = multiprocessing.current_process()
        # the main process has name 'MainProcess'
        # subprocesses will have names like 'Process-1'
        if p.name != 'MainProcess':
            # we are a subprocess, don't start another Controller!
            return
    app = IPControllerApp.instance()
    app.initialize()
    app.start()

if __name__ == '__main__':
    launch_new_instance()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import re
from reportlab.platypus import *
import reportlab.lib.colors as colors
import reportlab.lib.units as units
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.lib.fonts import addMapping
from reportlab.lib.styles import *
from reportlab.lib.enums import *
from reportlab.pdfbase import pdfmetrics

import os,sys
from utils import log

try:
    from wordaxe.rl.paragraph import Paragraph
    from wordaxe.rl.styles import ParagraphStyle,getSampleStyleSheet
except:
    log.warning("No hyphenation support, install wordaxe")

# Set these, and then **maybe** think of setting the stylesheets below
# if you want finer control

stdFont       = 'Helvetica'
stdBold       = 'Helvetica-Bold'
stdItalic     = 'Helvetica-Oblique'
stdBoldItalic = 'Helvetica-BoldOblique'
stdMono       = 'Courier'

def adjustUnits(v,total=0):
    '''Takes something like 2cm and returns 2*cm.
       If you use % as a unit, it returns the percentage
       of "total".

       Example::

            >>> adjustUnits('50%',200)
            100
    '''
    _,n,u=re.split('(-?[0-9\.]*)',v)
    if u in units.__dict__:
        return float(n)*units.__dict__[u]
    else:
      if u=='%':
          return float(n)*total/100
      log.error('Unknown unit %s' % u)

TTFSearchPath=[os.path.join(os.path.abspath(os.path.dirname(__file__)), 'fonts'),'.']

def findFont(fn):
    if not os.path.isabs(fn):
        for D in TTFSearchPath:
            tfn = os.path.join(D,fn)
            if os.path.isfile(tfn):
                return str(tfn)
    return str(fn)

# Page width, height
pw=0
ph=0

# Page size [w,h]
ps=None

# Margins (top,bottom,left,right)
tm=0
bm=0
lm=0
rm=0
gm=0

#text width
tw=0

# After what level of section do we break page
page_break_level=0

import reportlab.lib.pagesizes as pagesizes

def getStyleSheet(fname):
    """Returns a stylesheet object"""
    global pw,ph,ps,tm,bm,lm,rm,tw,gm
    stylesheet = StyleSheet1()

    from simplejson import loads

    data=loads(open(fname).read())

    styles=data['styles']
    fonts=data['fontsAlias']
    page=data['pageSetup']
    if page['size']: # A standard size
        if page['size'] in pagesizes.__dict__:
            ps=pagesizes.__dict__[page['size']]
        else:
            log.critical('Unknown page size %s' % page['size'])
            sys.exit(1)
    else: #A custom size
        # The sizes are expressed in some unit.
        # For example, 2cm is 2 centimeters, and we need
        # to do 2*cm (cm comes from reportlab.lib.units)
        ps=[adjustUnits(page['width']),adjustUnits(page['height'])]
    pw,ph=ps
    lm=adjustUnits(page['margin-left'])
    rm=adjustUnits(page['margin-right'])
    tm=adjustUnits(page['margin-top'])
    bm=adjustUnits(page['margin-bottom'])
    gm=adjustUnits(page['margin-gutter'])

    # tw is the text width.
    # We need it to calculate header-footer height
    # and compress literal blocks.
    tw=pw-lm-rm-gm
    embedded=data['embeddedFonts']

    for font in embedded:
        try:
            # Each "font" is a list of four files, which will be used for
            # regular / bold / italic / bold+italic versions of the font.
            # If your font doesn't have one of them, just repeat the regular
            # font.

            # Example, using the Tuffy font from http://tulrich.com/fonts/
            # "embeddedFonts" : [
            #                    ["Tuffy.ttf","Tuffy_Bold.ttf","Tuffy_Italic.ttf","Tuffy_Bold_Italic.ttf"]
            #                   ],

            # The fonts will be registered with the file name, minus the extension.

            for variant in font:
                pdfmetrics.registerFont(TTFont(str(variant.split('.')[0]), findFont(variant)))

                # And map them all together
                regular,bold,italic,bolditalic = [ variant.split('.')[0] for variant in font ]
                addMapping(regular,0,0,regular)
                addMapping(regular,0,1,italic)
                addMapping(regular,1,0,bold)
                addMapping(regular,1,1,bolditalic)
        except:
             try:
                fname=font[0].split('.')[0]
                log.error("Error processing font %s, registering as Helvetica alias",fname)
                fonts[fname]='Helvetica'
             except:
                log.error("Error processing font %s",fname)
                sys.exit(1)

    while True:
        for [skey,style] in data['styles']:
            sdict={}
            if 'parent' in style:
                if not style['parent']:
                    del style['parent']
                else:
                    style['parent']=stylesheet[style['parent']]
            for key in style:
                # Handle font aliases
                if key in ['fontName','bulletFontName'] and style[key] in fonts:
                    style[key]=fonts[style[key]]

                # Handle color references by name
                if key in ['backColor','textColor'] and style[key] in colors.__dict__:
                    style[key]=colors.__dict__[style[key]]

                # Handle alignment constants
                if key == 'alignment':
                    style[key]={'TA_LEFT':0, 'TA_CENTER':1, 'TA_CENTRE':1, 'TA_RIGHT':2, 'TA_JUSTIFY':4}[style[key]]

                # Make keys str instead of unicode (required by reportlab)
                sdict[str(key)]=style[key]
                sdict['name']=skey
            stylesheet.add(ParagraphStyle(**sdict))
        else:
            break
    return stylesheet

# Some table styles used for pieces of the document

tstyles={}

# Used for regular tables

tstyleNorm = [ ('VALIGN',(0,0),(-1,-1),'TOP'),
               ('INNERGRID', (0,0), (-1,-1), 0.25, colors.black),
               ('BOX', (0,0), (-1,-1), 0.25, colors.black),
             ]

# Header row in tables

def tstyleHead(rows=1):
    return ('BACKGROUND',(0,0),(-1,rows-1),colors.yellow)

tstyles['normal']=TableStyle(tstyleNorm)

# Used for field lists

tstyles['field']=TableStyle([ ('VALIGN',(0,0),(-1,-1),'TOP'),
                              ('ALIGNMENT',(0,0),(1,-1),'RIGHT'),
                            ])
fieldlist_lwidth=3*units.cm

# Used for endnotes

tstyles['endnote']=TableStyle([ ('VALIGN',(0,0),(-1,-1),'TOP'),
                                ('ALIGNMENT',(0,0),(1,-1),'RIGHT'),
                              ])
# Left column of the endnote. The content of the note takes the rest of
# the available space
endnote_lwidth=2*units.cm

# Used for sidebars

tstyles['sidebar']=TableStyle([ ('VALIGN',(0,0),(-1,-1),'TOP'),
                             ('BACKGROUND',(0,0),(-1,-1),colors.lightyellow),
                           ])

tstyles['bullet']=TableStyle([ ('VALIGN',(0,0),(-1,-1),'TOP'),
                             ])<commit_msg>Small fix in unit handling<commit_after># -*- coding: utf-8 -*-

import re
from reportlab.platypus import *
import reportlab.lib.colors as colors
import reportlab.lib.units as units
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.lib.fonts import addMapping
from reportlab.lib.styles import *
from reportlab.lib.enums import *
from reportlab.pdfbase import pdfmetrics

import os,sys
from utils import log

try:
    from wordaxe.rl.paragraph import Paragraph
    from wordaxe.rl.styles import ParagraphStyle,getSampleStyleSheet
except:
    log.warning("No hyphenation support, install wordaxe")

# Set these, and then **maybe** think of setting the stylesheets below
# if you want finer control

stdFont       = 'Helvetica'
stdBold       = 'Helvetica-Bold'
stdItalic     = 'Helvetica-Oblique'
stdBoldItalic = 'Helvetica-BoldOblique'
stdMono       = 'Courier'

def adjustUnits(v,total=0):
    '''Takes something like 2cm and returns 2*cm.
       If you use % as a unit, it returns the percentage
       of "total".

       Example::

            >>> adjustUnits('50%',200)
            100
    '''
    _,n,u=re.split('(-?[0-9\.]*)',v)
    if not u:
        return n # assume points
    if u in units.__dict__:
        return float(n)*units.__dict__[u]
    else:
      if u=='%':
          return float(n)*total/100
      log.error('Unknown unit "%s"' % u)
      return n

TTFSearchPath=[os.path.join(os.path.abspath(os.path.dirname(__file__)), 'fonts'),'.']

def findFont(fn):
    if not os.path.isabs(fn):
        for D in TTFSearchPath:
            tfn = os.path.join(D,fn)
            if os.path.isfile(tfn):
                return str(tfn)
    return str(fn)

# Page width, height
pw=0
ph=0

# Page size [w,h]
ps=None

# Margins (top,bottom,left,right)
tm=0
bm=0
lm=0
rm=0
gm=0

#text width
tw=0

# After what level of section do we break page
page_break_level=0

import reportlab.lib.pagesizes as pagesizes

def getStyleSheet(fname):
    """Returns a stylesheet object"""
    global pw,ph,ps,tm,bm,lm,rm,tw,gm
    stylesheet = StyleSheet1()

    from simplejson import loads

    data=loads(open(fname).read())

    styles=data['styles']
    fonts=data['fontsAlias']
    page=data['pageSetup']
    if page['size']: # A standard size
        if page['size'] in pagesizes.__dict__:
            ps=pagesizes.__dict__[page['size']]
        else:
            log.critical('Unknown page size %s' % page['size'])
            sys.exit(1)
    else: #A custom size
        # The sizes are expressed in some unit.
        # For example, 2cm is 2 centimeters, and we need
        # to do 2*cm (cm comes from reportlab.lib.units)
        ps=[adjustUnits(page['width']),adjustUnits(page['height'])]
    pw,ph=ps
    lm=adjustUnits(page['margin-left'])
    rm=adjustUnits(page['margin-right'])
    tm=adjustUnits(page['margin-top'])
    bm=adjustUnits(page['margin-bottom'])
    gm=adjustUnits(page['margin-gutter'])

    # tw is the text width.
    # We need it to calculate header-footer height
    # and compress literal blocks.
    tw=pw-lm-rm-gm
    embedded=data['embeddedFonts']

    for font in embedded:
        try:
            # Each "font" is a list of four files, which will be used for
            # regular / bold / italic / bold+italic versions of the font.
            # If your font doesn't have one of them, just repeat the regular
            # font.

            # Example, using the Tuffy font from http://tulrich.com/fonts/
            # "embeddedFonts" : [
            #                    ["Tuffy.ttf","Tuffy_Bold.ttf","Tuffy_Italic.ttf","Tuffy_Bold_Italic.ttf"]
            #                   ],

            # The fonts will be registered with the file name, minus the extension.

            for variant in font:
                pdfmetrics.registerFont(TTFont(str(variant.split('.')[0]), findFont(variant)))

                # And map them all together
                regular,bold,italic,bolditalic = [ variant.split('.')[0] for variant in font ]
                addMapping(regular,0,0,regular)
                addMapping(regular,0,1,italic)
                addMapping(regular,1,0,bold)
                addMapping(regular,1,1,bolditalic)
        except:
             try:
                fname=font[0].split('.')[0]
                log.error("Error processing font %s, registering as Helvetica alias",fname)
                fonts[fname]='Helvetica'
             except:
                log.error("Error processing font %s",fname)
                sys.exit(1)

    while True:
        for [skey,style] in data['styles']:
            sdict={}
            if 'parent' in style:
                if not style['parent']:
                    del style['parent']
                else:
                    style['parent']=stylesheet[style['parent']]
            for key in style:
                # Handle font aliases
                if key in ['fontName','bulletFontName'] and style[key] in fonts:
                    style[key]=fonts[style[key]]

                # Handle color references by name
                if key in ['backColor','textColor'] and style[key] in colors.__dict__:
                    style[key]=colors.__dict__[style[key]]

                # Handle alignment constants
                if key == 'alignment':
                    style[key]={'TA_LEFT':0, 'TA_CENTER':1, 'TA_CENTRE':1, 'TA_RIGHT':2, 'TA_JUSTIFY':4}[style[key]]

                # Make keys str instead of unicode (required by reportlab)
                sdict[str(key)]=style[key]
                sdict['name']=skey
            stylesheet.add(ParagraphStyle(**sdict))
        else:
            break
    return stylesheet

# Some table styles used for pieces of the document

tstyles={}

# Used for regular tables

tstyleNorm = [ ('VALIGN',(0,0),(-1,-1),'TOP'),
               ('INNERGRID', (0,0), (-1,-1), 0.25, colors.black),
               ('BOX', (0,0), (-1,-1), 0.25, colors.black),
             ]

# Header row in tables

def tstyleHead(rows=1):
    return ('BACKGROUND',(0,0),(-1,rows-1),colors.yellow)

tstyles['normal']=TableStyle(tstyleNorm)

# Used for field lists

tstyles['field']=TableStyle([ ('VALIGN',(0,0),(-1,-1),'TOP'),
                              ('ALIGNMENT',(0,0),(1,-1),'RIGHT'),
                            ])
fieldlist_lwidth=3*units.cm

# Used for endnotes

tstyles['endnote']=TableStyle([ ('VALIGN',(0,0),(-1,-1),'TOP'),
                                ('ALIGNMENT',(0,0),(1,-1),'RIGHT'),
                              ])
# Left column of the endnote. The content of the note takes the rest of
# the available space
endnote_lwidth=2*units.cm

# Used for sidebars

tstyles['sidebar']=TableStyle([ ('VALIGN',(0,0),(-1,-1),'TOP'),
                             ('BACKGROUND',(0,0),(-1,-1),colors.lightyellow),
                           ])

tstyles['bullet']=TableStyle([ ('VALIGN',(0,0),(-1,-1),'TOP'),
                             ])<|endoftext|>