<commit_before>#!/usr/bin/env python
"""NBConvert is a utility for conversion of .ipynb files.

Command-line interface for the NbConvert conversion utility.
"""
#-----------------------------------------------------------------------------
#Copyright (c) 2013, the IPython Development Team.
#
#Distributed under the terms of the Modified BSD License.
#
#The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
#Imports
#-----------------------------------------------------------------------------

# Stdlib imports
from __future__ import print_function
import sys
import os
import glob

# From IPython
from IPython.core.application import BaseIPythonApplication, base_aliases, base_flags
from IPython.config import catch_config_error, Configurable
from IPython.utils.traitlets import (
    Unicode, List, Instance, DottedObjectName, Type, CaselessStrEnum,
)
from IPython.utils.importstring import import_item

from .exporters.export import export_by_name, get_export_names, ExporterNameError
from IPython.nbconvert import exporters, transformers, writers, post_processors
from .utils.base import NbConvertBase
from .utils.exceptions import ConversionException

#-----------------------------------------------------------------------------
#Classes and functions
#-----------------------------------------------------------------------------

class DottedOrNone(DottedObjectName):
    """
    A string holding a valid dotted object name in Python, such as A.b3._c
    Also allows for None type."""

    default_value = u''

    def validate(self, obj, value):
        if value is not None and len(value) > 0:
            return super(DottedOrNone, self).validate(obj, value)
        else:
            return value

nbconvert_aliases = {}
nbconvert_aliases.update(base_aliases)
nbconvert_aliases.update({
    'to' : 'NbConvertApp.export_format',
    'template' : 'Exporter.template_file',
    'notebooks' : 'NbConvertApp.notebooks',
    'writer' : 'NbConvertApp.writer_class',
    'post': 'NbConvertApp.post_processor_class'
})

nbconvert_flags = {}
nbconvert_flags.update(base_flags)
nbconvert_flags.update({
    'stdout' : (
        {'NbConvertApp' : {'writer_class' : "StdoutWriter"}},
        "Write notebook output to stdout instead of files."
        )
})

class NbConvertApp(BaseIPythonApplication):
    """Application used to convert to and from notebook file type (*.ipynb)"""

    name = 'ipython-nbconvert'
    aliases = nbconvert_aliases
    flags = nbconvert_flags

    def _classes_default(self):
        classes = [NbConvertBase]
        for pkg in (exporters, transformers, writers):
            for name in dir(pkg):
                cls = getattr(pkg, name)
                if isinstance(cls, type) and issubclass(cls, Configurable):
                    classes.append(cls)
        return classes

    description = Unicode(
        u"""This application is used to convert notebook files (*.ipynb)
        to various other formats.""")

    examples = Unicode(u"""
        The simplest way to use nbconvert is

        > ipython nbconvert mynotebook.ipynb

        which will convert mynotebook.ipynb to the default format (probably HTML).

        You can specify the export format with `--to`.
        Options include {0}

        > ipython nbconvert --to latex mynotebook.ipnynb

        Both HTML and LaTeX support multiple output templates. LaTeX includes
        'basic', 'book', and 'article'.  HTML includes 'basic' and 'full'.  You
        can specify the flavor of the format used.

        > ipython nbconvert --to html --template reveal mynotebook.ipnynb

        You can also pipe the output to stdout, rather than a file

        > ipython nbconvert mynotebook.ipynb --stdout

        or to a PDF

        > ipython nbconvert mynotebook.ipynb --to latex --pdf

        Multiple notebooks can be given at the command line in a couple of
        different ways:

        > ipython nbconvert notebook*.ipynb
        > ipython nbconvert notebook1.ipynb notebook2.ipynb

        or you can specify the notebooks list in a config file, containing::

            c.NbConvertApp.notebooks = ["my_notebook.ipynb"]

        > ipython nbconvert --config mycfg.py
        """.format(get_export_names()))

    # Writer specific variables
    writer = Instance('IPython.nbconvert.writers.base.WriterBase',
                      help="""Instance of the writer class used to write the
                      results of the conversion.""")
    writer_class = DottedObjectName('FilesWriter', config=True,
                                    help="""Writer class used to write the
                                    results of the conversion""")
    writer_aliases = {'FilesWriter': 'IPython.nbconvert.writers.files.FilesWriter',
                      'PDFWriter': 'IPython.nbconvert.writers.pdf.PDFWriter',
                      'DebugWriter': 'IPython.nbconvert.writers.debug.DebugWriter',
                      'StdoutWriter': 'IPython.nbconvert.writers.stdout.StdoutWriter'}
    writer_factory = Type()

    def _writer_class_changed(self, name, old, new):
        if new in self.writer_aliases:
            new = self.writer_aliases[new]
        self.writer_factory = import_item(new)

    # Post-processor specific variables
    post_processor = Instance('IPython.nbconvert.post_processors.base.PostProcessorBase',
                      help="""Instance of the PostProcessor class used to write the
                      results of the conversion.""")

    post_processor_class = DottedOrNone(config=True,
                                    help="""PostProcessor class used to write the
                                    results of the conversion""")
    post_processor_aliases = {'PDF': 'IPython.nbconvert.post_processors.pdf.PDFPostProcessor'}
    post_processor_factory = Type()

    def _post_processor_class_changed(self, name, old, new):
        if new in self.post_processor_aliases:
            new = self.post_processor_aliases[new]
        if new:
            self.post_processor_factory = import_item(new)

    # Other configurable variables
    export_format = CaselessStrEnum(get_export_names(),
        default_value="html",
        config=True,
        help="""The export format to be used."""
    )

    notebooks = List([], config=True, help="""List of notebooks to convert.
                     Wildcards are supported.
                     Filenames passed positionally will be added to the list.
                     """)

    @catch_config_error
    def initialize(self, argv=None):
        super(NbConvertApp, self).initialize(argv)
        self.init_syspath()
        self.init_notebooks()
        self.init_writer()
        self.init_post_processor()

    def init_syspath(self):
        """
        Add the cwd to the sys.path ($PYTHONPATH)
        """
        sys.path.insert(0, os.getcwd())

    def init_notebooks(self):
        """Construct the list of notebooks.
        If notebooks are passed on the command-line,
        they override notebooks specified in config files.
        Glob each notebook to replace notebook patterns with filenames.
        """

        # Specifying notebooks on the command-line overrides (rather than adds)
        # the notebook list
        if self.extra_args:
            patterns = self.extra_args
        else:
            patterns = self.notebooks

        # Use glob to replace all the notebook patterns with filenames.
        filenames = []
        for pattern in patterns:

            # Use glob to find matching filenames.  Allow the user to convert
            # notebooks without having to type the extension.
            globbed_files = glob.glob(pattern)
            globbed_files.extend(glob.glob(pattern + '.ipynb'))

            for filename in globbed_files:
                if not filename in filenames:
                    filenames.append(filename)
        self.notebooks = filenames

    def init_writer(self):
        """
        Initialize the writer (which is stateless)
        """
        self._writer_class_changed(None, self.writer_class, self.writer_class)
        self.writer = self.writer_factory(parent=self)

    def init_post_processor(self):
        """
        Initialize the post_processor (which is stateless)
        """
        self._post_processor_class_changed(None, self.post_processor_class,
            self.post_processor_class)
        if self.post_processor_factory:
            self.post_processor = self.post_processor_factory(parent=self)

    def start(self):
        """
        Ran after initialization completed
        """
        super(NbConvertApp, self).start()
        self.convert_notebooks()

    def convert_notebooks(self):
        """
        Convert the notebooks in the self.notebook traitlet
        """
        # Export each notebook
        conversion_success = 0
        for notebook_filename in self.notebooks:

            # Get a unique key for the notebook and set it in the resources object.
            basename = os.path.basename(notebook_filename)
            notebook_name = basename[:basename.rfind('.')]
            resources = {}
            resources['unique_key'] = notebook_name
            resources['output_files_dir'] = '%s_files' % notebook_name

            # Try to export
            try:
                output, resources = export_by_name(self.export_format,
                                              notebook_filename,
                                              resources=resources,
                                              config=self.config)
            except ExporterNameError as e:
                print("Error while converting '%s': '%s' exporter not found."
                      %(notebook_filename, self.export_format),
                      file=sys.stderr)
                print("Known exporters are:",
                      "\n\t" + "\n\t".join(get_export_names()),
                      file=sys.stderr)
                self.exit(1)
            except ConversionException as e:
                print("Error while converting '%s': %s" %(notebook_filename, e),
                      file=sys.stderr)
                self.exit(1)
            else:
                write_resultes = self.writer.write(output, resources, notebook_name=notebook_name)

                #Post-process if post processor has been defined.
                if hasattr(self, 'post_processor') and self.post_processor:
                    self.post_processor(write_resultes)
                conversion_success += 1

        # If nothing was converted successfully, help the user.
        if conversion_success == 0:
            self.print_help()
            sys.exit(-1)

#-----------------------------------------------------------------------------
# Main entry point
#-----------------------------------------------------------------------------

launch_new_instance = NbConvertApp.launch_instance
<commit_msg>Updated example string for nbconvertapp.py<commit_after>#!/usr/bin/env python
"""NBConvert is a utility for conversion of .ipynb files.

Command-line interface for the NbConvert conversion utility.
"""
#-----------------------------------------------------------------------------
#Copyright (c) 2013, the IPython Development Team.
#
#Distributed under the terms of the Modified BSD License.
#
#The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
#Imports
#-----------------------------------------------------------------------------

# Stdlib imports
from __future__ import print_function
import sys
import os
import glob

# From IPython
from IPython.core.application import BaseIPythonApplication, base_aliases, base_flags
from IPython.config import catch_config_error, Configurable
from IPython.utils.traitlets import (
    Unicode, List, Instance, DottedObjectName, Type, CaselessStrEnum,
)
from IPython.utils.importstring import import_item

from .exporters.export import export_by_name, get_export_names, ExporterNameError
from IPython.nbconvert import exporters, transformers, writers, post_processors
from .utils.base import NbConvertBase
from .utils.exceptions import ConversionException

#-----------------------------------------------------------------------------
#Classes and functions
#-----------------------------------------------------------------------------

class DottedOrNone(DottedObjectName):
    """
    A string holding a valid dotted object name in Python, such as A.b3._c
    Also allows for None type."""

    default_value = u''

    def validate(self, obj, value):
        if value is not None and len(value) > 0:
            return super(DottedOrNone, self).validate(obj, value)
        else:
            return value

nbconvert_aliases = {}
nbconvert_aliases.update(base_aliases)
nbconvert_aliases.update({
    'to' : 'NbConvertApp.export_format',
    'template' : 'Exporter.template_file',
    'notebooks' : 'NbConvertApp.notebooks',
    'writer' : 'NbConvertApp.writer_class',
    'post': 'NbConvertApp.post_processor_class'
})

nbconvert_flags = {}
nbconvert_flags.update(base_flags)
nbconvert_flags.update({
    'stdout' : (
        {'NbConvertApp' : {'writer_class' : "StdoutWriter"}},
        "Write notebook output to stdout instead of files."
        )
})

class NbConvertApp(BaseIPythonApplication):
    """Application used to convert to and from notebook file type (*.ipynb)"""

    name = 'ipython-nbconvert'
    aliases = nbconvert_aliases
    flags = nbconvert_flags

    def _classes_default(self):
        classes = [NbConvertBase]
        for pkg in (exporters, transformers, writers):
            for name in dir(pkg):
                cls = getattr(pkg, name)
                if isinstance(cls, type) and issubclass(cls, Configurable):
                    classes.append(cls)
        return classes

    description = Unicode(
        u"""This application is used to convert notebook files (*.ipynb)
        to various other formats.""")

    examples = Unicode(u"""
        The simplest way to use nbconvert is

        > ipython nbconvert mynotebook.ipynb

        which will convert mynotebook.ipynb to the default format (probably HTML).

        You can specify the export format with `--to`.
        Options include {0}

        > ipython nbconvert --to latex mynotebook.ipnynb

        Both HTML and LaTeX support multiple output templates. LaTeX includes
        'basic', 'book', and 'article'.  HTML includes 'basic' and 'full'.  You
        can specify the flavor of the format used.

        > ipython nbconvert --to html --template reveal mynotebook.ipnynb

        You can also pipe the output to stdout, rather than a file

        > ipython nbconvert mynotebook.ipynb --stdout

        A post-processor can be used to compile a PDF

        > ipython nbconvert mynotebook.ipynb --to latex --post PDF

        Multiple notebooks can be given at the command line in a couple of
        different ways:

        > ipython nbconvert notebook*.ipynb
        > ipython nbconvert notebook1.ipynb notebook2.ipynb

        or you can specify the notebooks list in a config file, containing::

            c.NbConvertApp.notebooks = ["my_notebook.ipynb"]

        > ipython nbconvert --config mycfg.py
        """.format(get_export_names()))

    # Writer specific variables
    writer = Instance('IPython.nbconvert.writers.base.WriterBase',
                      help="""Instance of the writer class used to write the
                      results of the conversion.""")
    writer_class = DottedObjectName('FilesWriter', config=True,
                                    help="""Writer class used to write the
                                    results of the conversion""")
    writer_aliases = {'FilesWriter': 'IPython.nbconvert.writers.files.FilesWriter',
                      'PDFWriter': 'IPython.nbconvert.writers.pdf.PDFWriter',
                      'DebugWriter': 'IPython.nbconvert.writers.debug.DebugWriter',
                      'StdoutWriter': 'IPython.nbconvert.writers.stdout.StdoutWriter'}
    writer_factory = Type()

    def _writer_class_changed(self, name, old, new):
        if new in self.writer_aliases:
            new = self.writer_aliases[new]
        self.writer_factory = import_item(new)

    # Post-processor specific variables
    post_processor = Instance('IPython.nbconvert.post_processors.base.PostProcessorBase',
                      help="""Instance of the PostProcessor class used to write the
                      results of the conversion.""")

    post_processor_class = DottedOrNone(config=True,
                                    help="""PostProcessor class used to write the
                                    results of the conversion""")
    post_processor_aliases = {'PDF': 'IPython.nbconvert.post_processors.pdf.PDFPostProcessor'}
    post_processor_factory = Type()

    def _post_processor_class_changed(self, name, old, new):
        if new in self.post_processor_aliases:
            new = self.post_processor_aliases[new]
        if new:
            self.post_processor_factory = import_item(new)

    # Other configurable variables
    export_format = CaselessStrEnum(get_export_names(),
        default_value="html",
        config=True,
        help="""The export format to be used."""
    )

    notebooks = List([], config=True, help="""List of notebooks to convert.
                     Wildcards are supported.
                     Filenames passed positionally will be added to the list.
                     """)

    @catch_config_error
    def initialize(self, argv=None):
        super(NbConvertApp, self).initialize(argv)
        self.init_syspath()
        self.init_notebooks()
        self.init_writer()
        self.init_post_processor()

    def init_syspath(self):
        """
        Add the cwd to the sys.path ($PYTHONPATH)
        """
        sys.path.insert(0, os.getcwd())

    def init_notebooks(self):
        """Construct the list of notebooks.
        If notebooks are passed on the command-line,
        they override notebooks specified in config files.
        Glob each notebook to replace notebook patterns with filenames.
        """

        # Specifying notebooks on the command-line overrides (rather than adds)
        # the notebook list
        if self.extra_args:
            patterns = self.extra_args
        else:
            patterns = self.notebooks

        # Use glob to replace all the notebook patterns with filenames.
        filenames = []
        for pattern in patterns:

            # Use glob to find matching filenames.  Allow the user to convert
            # notebooks without having to type the extension.
            globbed_files = glob.glob(pattern)
            globbed_files.extend(glob.glob(pattern + '.ipynb'))

            for filename in globbed_files:
                if not filename in filenames:
                    filenames.append(filename)
        self.notebooks = filenames

    def init_writer(self):
        """
        Initialize the writer (which is stateless)
        """
        self._writer_class_changed(None, self.writer_class, self.writer_class)
        self.writer = self.writer_factory(parent=self)

    def init_post_processor(self):
        """
        Initialize the post_processor (which is stateless)
        """
        self._post_processor_class_changed(None, self.post_processor_class,
            self.post_processor_class)
        if self.post_processor_factory:
            self.post_processor = self.post_processor_factory(parent=self)

    def start(self):
        """
        Ran after initialization completed
        """
        super(NbConvertApp, self).start()
        self.convert_notebooks()

    def convert_notebooks(self):
        """
        Convert the notebooks in the self.notebook traitlet
        """
        # Export each notebook
        conversion_success = 0
        for notebook_filename in self.notebooks:

            # Get a unique key for the notebook and set it in the resources object.
            basename = os.path.basename(notebook_filename)
            notebook_name = basename[:basename.rfind('.')]
            resources = {}
            resources['unique_key'] = notebook_name
            resources['output_files_dir'] = '%s_files' % notebook_name

            # Try to export
            try:
                output, resources = export_by_name(self.export_format,
                                              notebook_filename,
                                              resources=resources,
                                              config=self.config)
            except ExporterNameError as e:
                print("Error while converting '%s': '%s' exporter not found."
                      %(notebook_filename, self.export_format),
                      file=sys.stderr)
                print("Known exporters are:",
                      "\n\t" + "\n\t".join(get_export_names()),
                      file=sys.stderr)
                self.exit(1)
            except ConversionException as e:
                print("Error while converting '%s': %s" %(notebook_filename, e),
                      file=sys.stderr)
                self.exit(1)
            else:
                write_resultes = self.writer.write(output, resources, notebook_name=notebook_name)

                #Post-process if post processor has been defined.
                if hasattr(self, 'post_processor') and self.post_processor:
                    self.post_processor(write_resultes)
                conversion_success += 1

        # If nothing was converted successfully, help the user.
        if conversion_success == 0:
            self.print_help()
            sys.exit(-1)

#-----------------------------------------------------------------------------
# Main entry point
#-----------------------------------------------------------------------------

launch_new_instance = NbConvertApp.launch_instance
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""NBConvert is a utility for conversion of .ipynb files.

Command-line interface for the NbConvert conversion utility.
"""
#-----------------------------------------------------------------------------
#Copyright (c) 2013, the IPython Development Team.
#
#Distributed under the terms of the Modified BSD License.
#
#The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
#Imports
#-----------------------------------------------------------------------------

# Stdlib imports
from __future__ import print_function
import sys
import os
import glob

# From IPython
from IPython.core.application import BaseIPythonApplication, base_aliases, base_flags
from IPython.config import catch_config_error, Configurable
from IPython.utils.traitlets import (
    Unicode, List, Instance, DottedObjectName, Type, CaselessStrEnum,
)
from IPython.utils.importstring import import_item

from .exporters.export import export_by_name, get_export_names, ExporterNameError
from IPython.nbconvert import exporters, transformers, writers, post_processors
from .utils.base import NbConvertBase
from .utils.exceptions import ConversionException

#-----------------------------------------------------------------------------
#Classes and functions
#-----------------------------------------------------------------------------

class DottedOrNone(DottedObjectName):
    """
    A string holding a valid dotted object name in Python, such as A.b3._c
    Also allows for None type."""

    default_value = u''

    def validate(self, obj, value):
        if value is not None and len(value) > 0:
            return super(DottedOrNone, self).validate(obj, value)
        else:
            return value

nbconvert_aliases = {}
nbconvert_aliases.update(base_aliases)
nbconvert_aliases.update({
    'to' : 'NbConvertApp.export_format',
    'template' : 'Exporter.template_file',
    'notebooks' : 'NbConvertApp.notebooks',
    'writer' : 'NbConvertApp.writer_class',
    'post': 'NbConvertApp.post_processor_class'
})

nbconvert_flags = {}
nbconvert_flags.update(base_flags)
nbconvert_flags.update({
    'stdout' : (
        {'NbConvertApp' : {'writer_class' : "StdoutWriter"}},
        "Write notebook output to stdout instead of files."
        )
})

class NbConvertApp(BaseIPythonApplication):
    """Application used to convert to and from notebook file type (*.ipynb)"""

    name = 'ipython-nbconvert'
    aliases = nbconvert_aliases
    flags = nbconvert_flags

    def _classes_default(self):
        classes = [NbConvertBase]
        for pkg in (exporters, transformers, writers):
            for name in dir(pkg):
                cls = getattr(pkg, name)
                if isinstance(cls, type) and issubclass(cls, Configurable):
                    classes.append(cls)
        return classes

    description = Unicode(
        u"""This application is used to convert notebook files (*.ipynb)
        to various other formats.

        WARNING: THE COMMANDLINE INTERFACE MAY CHANGE IN FUTURE RELEASES.""")

    examples = Unicode(u"""
        The simplest way to use nbconvert is

        > ipython nbconvert mynotebook.ipynb

        which will convert mynotebook.ipynb to the default format (probably HTML).

        You can specify the export format with `--to`.
        Options include {0}

        > ipython nbconvert --to latex mynotebook.ipnynb

        Both HTML and LaTeX support multiple output templates. LaTeX includes
        'basic', 'book', and 'article'.  HTML includes 'basic' and 'full'.  You
        can specify the flavor of the format used.

        > ipython nbconvert --to html --template reveal mynotebook.ipnynb

        You can also pipe the output to stdout, rather than a file

        > ipython nbconvert mynotebook.ipynb --stdout

        A post-processor can be used to compile a PDF

        > ipython nbconvert mynotebook.ipynb --to latex --post PDF

        Multiple notebooks can be given at the command line in a couple of
        different ways:

        > ipython nbconvert notebook*.ipynb
        > ipython nbconvert notebook1.ipynb notebook2.ipynb

        or you can specify the notebooks list in a config file, containing::

            c.NbConvertApp.notebooks = ["my_notebook.ipynb"]

        > ipython nbconvert --config mycfg.py
        """.format(get_export_names()))

    # Writer specific variables
    writer = Instance('IPython.nbconvert.writers.base.WriterBase',
                      help="""Instance of the writer class used to write the
                      results of the conversion.""")
    writer_class = DottedObjectName('FilesWriter', config=True,
                                    help="""Writer class used to write the
                                    results of the conversion""")
    writer_aliases = {'FilesWriter': 'IPython.nbconvert.writers.files.FilesWriter',
                      'PDFWriter': 'IPython.nbconvert.writers.pdf.PDFWriter',
                      'DebugWriter': 'IPython.nbconvert.writers.debug.DebugWriter',
                      'StdoutWriter': 'IPython.nbconvert.writers.stdout.StdoutWriter'}
    writer_factory = Type()

    def _writer_class_changed(self, name, old, new):
        if new in self.writer_aliases:
            new = self.writer_aliases[new]
        self.writer_factory = import_item(new)

    # Post-processor specific variables
    post_processor = Instance('IPython.nbconvert.post_processors.base.PostProcessorBase',
                      help="""Instance of the PostProcessor class used to write the
                      results of the conversion.""")

    post_processor_class = DottedOrNone(config=True,
                                    help="""PostProcessor class used to write the
                                    results of the conversion""")
    post_processor_aliases = {'PDF': 'IPython.nbconvert.post_processors.pdf.PDFPostProcessor'}
    post_processor_factory = Type()

    def _post_processor_class_changed(self, name, old, new):
        if new in self.post_processor_aliases:
            new = self.post_processor_aliases[new]
        if new:
            self.post_processor_factory = import_item(new)

    # Other configurable variables
    export_format = CaselessStrEnum(get_export_names(),
        default_value="html",
        config=True,
        help="""The export format to be used."""
    )

    notebooks = List([], config=True, help="""List of notebooks to convert.
                     Wildcards are supported.
                     Filenames passed positionally will be added to the list.
                     """)

    @catch_config_error
    def initialize(self, argv=None):
        super(NbConvertApp, self).initialize(argv)
        self.init_syspath()
        self.init_notebooks()
        self.init_writer()
        self.init_post_processor()

    def init_syspath(self):
        """
        Add the cwd to the sys.path ($PYTHONPATH)
        """
        sys.path.insert(0, os.getcwd())

    def init_notebooks(self):
        """Construct the list of notebooks.
        If notebooks are passed on the command-line,
        they override notebooks specified in config files.
        Glob each notebook to replace notebook patterns with filenames.
        """

        # Specifying notebooks on the command-line overrides (rather than adds)
        # the notebook list
        if self.extra_args:
            patterns = self.extra_args
        else:
            patterns = self.notebooks

        # Use glob to replace all the notebook patterns with filenames.
        filenames = []
        for pattern in patterns:

            # Use glob to find matching filenames.  Allow the user to convert
            # notebooks without having to type the extension.
            globbed_files = glob.glob(pattern)
            globbed_files.extend(glob.glob(pattern + '.ipynb'))

            for filename in globbed_files:
                if not filename in filenames:
                    filenames.append(filename)
        self.notebooks = filenames

    def init_writer(self):
        """
        Initialize the writer (which is stateless)
        """
        self._writer_class_changed(None, self.writer_class, self.writer_class)
        self.writer = self.writer_factory(parent=self)

    def init_post_processor(self):
        """
        Initialize the post_processor (which is stateless)
        """
        self._post_processor_class_changed(None, self.post_processor_class,
            self.post_processor_class)
        if self.post_processor_factory:
            self.post_processor = self.post_processor_factory(parent=self)

    def start(self):
        """
        Ran after initialization completed
        """
        super(NbConvertApp, self).start()
        self.convert_notebooks()

    def convert_notebooks(self):
        """
        Convert the notebooks in the self.notebook traitlet
        """
        # Export each notebook
        conversion_success = 0
        for notebook_filename in self.notebooks:

            # Get a unique key for the notebook and set it in the resources object.
            basename = os.path.basename(notebook_filename)
            notebook_name = basename[:basename.rfind('.')]
            resources = {}
            resources['unique_key'] = notebook_name
            resources['output_files_dir'] = '%s_files' % notebook_name

            # Try to export
            try:
                output, resources = export_by_name(self.export_format,
                                              notebook_filename,
                                              resources=resources,
                                              config=self.config)
            except ExporterNameError as e:
                print("Error while converting '%s': '%s' exporter not found."
                      %(notebook_filename, self.export_format),
                      file=sys.stderr)
                print("Known exporters are:",
                      "\n\t" + "\n\t".join(get_export_names()),
                      file=sys.stderr)
                self.exit(1)
            except ConversionException as e:
                print("Error while converting '%s': %s" %(notebook_filename, e),
                      file=sys.stderr)
                self.exit(1)
            else:
                write_resultes = self.writer.write(output, resources, notebook_name=notebook_name)

                #Post-process if post processor has been defined.
                if hasattr(self, 'post_processor') and self.post_processor:
                    self.post_processor(write_resultes)
                conversion_success += 1

        # If nothing was converted successfully, help the user.
        if conversion_success == 0:
            self.print_help()
            sys.exit(-1)

#-----------------------------------------------------------------------------
# Main entry point
#-----------------------------------------------------------------------------

launch_new_instance = NbConvertApp.launch_instance
<commit_msg>Update comment<commit_after>#!/usr/bin/env python
"""NBConvert is a utility for conversion of .ipynb files.

Command-line interface for the NbConvert conversion utility.
"""
#-----------------------------------------------------------------------------
#Copyright (c) 2013, the IPython Development Team.
#
#Distributed under the terms of the Modified BSD License.
#
#The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
#Imports
#-----------------------------------------------------------------------------

# Stdlib imports
from __future__ import print_function
import sys
import os
import glob

# From IPython
from IPython.core.application import BaseIPythonApplication, base_aliases, base_flags
from IPython.config import catch_config_error, Configurable
from IPython.utils.traitlets import (
    Unicode, List, Instance, DottedObjectName, Type, CaselessStrEnum,
)
from IPython.utils.importstring import import_item

from .exporters.export import export_by_name, get_export_names, ExporterNameError
from IPython.nbconvert import exporters, transformers, writers, post_processors
from .utils.base import NbConvertBase
from .utils.exceptions import ConversionException

#-----------------------------------------------------------------------------
#Classes and functions
#-----------------------------------------------------------------------------

class DottedOrNone(DottedObjectName):
    """
    A string holding a valid dotted object name in Python, such as A.b3._c
    Also allows for None type."""

    default_value = u''

    def validate(self, obj, value):
        if value is not None and len(value) > 0:
            return super(DottedOrNone, self).validate(obj, value)
        else:
            return value

nbconvert_aliases = {}
nbconvert_aliases.update(base_aliases)
nbconvert_aliases.update({
    'to' : 'NbConvertApp.export_format',
    'template' : 'Exporter.template_file',
    'notebooks' : 'NbConvertApp.notebooks',
    'writer' : 'NbConvertApp.writer_class',
    'post': 'NbConvertApp.post_processor_class'
})

nbconvert_flags = {}
nbconvert_flags.update(base_flags)
nbconvert_flags.update({
    'stdout' : (
        {'NbConvertApp' : {'writer_class' : "StdoutWriter"}},
        "Write notebook output to stdout instead of files."
        )
})

class NbConvertApp(BaseIPythonApplication):
    """Application used to convert to and from notebook file type (*.ipynb)"""

    name = 'ipython-nbconvert'
    aliases = nbconvert_aliases
    flags = nbconvert_flags

    def _classes_default(self):
        classes = [NbConvertBase]
        for pkg in (exporters, transformers, writers):
            for name in dir(pkg):
                cls = getattr(pkg, name)
                if isinstance(cls, type) and issubclass(cls, Configurable):
                    classes.append(cls)
        return classes

    description = Unicode(
        u"""This application is used to convert notebook files (*.ipynb)
        to various other formats.

        WARNING: THE COMMANDLINE INTERFACE MAY CHANGE IN FUTURE RELEASES.""")

    examples = Unicode(u"""
        The simplest way to use nbconvert is

        > ipython nbconvert mynotebook.ipynb

        which will convert mynotebook.ipynb to the default format (probably HTML).

        You can specify the export format with `--to`.
        Options include {0}

        > ipython nbconvert --to latex mynotebook.ipnynb

        Both HTML and LaTeX support multiple output templates. LaTeX includes
        'basic', 'book', and 'article'.  HTML includes 'basic' and 'full'.  You
        can specify the flavor of the format used.

        > ipython nbconvert --to html --template basic mynotebook.ipynb

        You can also pipe the output to stdout, rather than a file

        > ipython nbconvert mynotebook.ipynb --stdout

        A post-processor can be used to compile a PDF

        > ipython nbconvert mynotebook.ipynb --to latex --post PDF

        Multiple notebooks can be given at the command line in a couple of
        different ways:

        > ipython nbconvert notebook*.ipynb
        > ipython nbconvert notebook1.ipynb notebook2.ipynb

        or you can specify the notebooks list in a config file, containing::

            c.NbConvertApp.notebooks = ["my_notebook.ipynb"]

        > ipython nbconvert --config mycfg.py
        """.format(get_export_names()))

    # Writer specific variables
    writer = Instance('IPython.nbconvert.writers.base.WriterBase',
                      help="""Instance of the writer class used to write the
                      results of the conversion.""")
    writer_class = DottedObjectName('FilesWriter', config=True,
                                    help="""Writer class used to write the
                                    results of the conversion""")
    writer_aliases = {'FilesWriter': 'IPython.nbconvert.writers.files.FilesWriter',
                      'PDFWriter': 'IPython.nbconvert.writers.pdf.PDFWriter',
                      'DebugWriter': 'IPython.nbconvert.writers.debug.DebugWriter',
                      'StdoutWriter': 'IPython.nbconvert.writers.stdout.StdoutWriter'}
    writer_factory = Type()

    def _writer_class_changed(self, name, old, new):
        if new in self.writer_aliases:
            new = self.writer_aliases[new]
        self.writer_factory = import_item(new)

    # Post-processor specific variables
    post_processor = Instance('IPython.nbconvert.post_processors.base.PostProcessorBase',
                      help="""Instance of the PostProcessor class used to write the
                      results of the conversion.""")

    post_processor_class = DottedOrNone(config=True,
                                    help="""PostProcessor class used to write the
                                    results of the conversion""")
    post_processor_aliases = {'PDF': 'IPython.nbconvert.post_processors.pdf.PDFPostProcessor'}
    post_processor_factory = Type()

    def _post_processor_class_changed(self, name, old, new):
        if new in self.post_processor_aliases:
            new = self.post_processor_aliases[new]
        if new:
            self.post_processor_factory = import_item(new)

    # Other configurable variables
    export_format = CaselessStrEnum(get_export_names(),
        default_value="html",
        config=True,
        help="""The export format to be used."""
    )

    notebooks = List([], config=True, help="""List of notebooks to convert.
                     Wildcards are supported.
                     Filenames passed positionally will be added to the list.
                     """)

    @catch_config_error
    def initialize(self, argv=None):
        super(NbConvertApp, self).initialize(argv)
        self.init_syspath()
        self.init_notebooks()
        self.init_writer()
        self.init_post_processor()

    def init_syspath(self):
        """
        Add the cwd to the sys.path ($PYTHONPATH)
        """
        sys.path.insert(0, os.getcwd())

    def init_notebooks(self):
        """Construct the list of notebooks.
        If notebooks are passed on the command-line,
        they override notebooks specified in config files.
        Glob each notebook to replace notebook patterns with filenames.
        """

        # Specifying notebooks on the command-line overrides (rather than adds)
        # the notebook list
        if self.extra_args:
            patterns = self.extra_args
        else:
            patterns = self.notebooks

        # Use glob to replace all the notebook patterns with filenames.
        filenames = []
        for pattern in patterns:

            # Use glob to find matching filenames.  Allow the user to convert
            # notebooks without having to type the extension.
            globbed_files = glob.glob(pattern)
            globbed_files.extend(glob.glob(pattern + '.ipynb'))

            for filename in globbed_files:
                if not filename in filenames:
                    filenames.append(filename)
        self.notebooks = filenames

    def init_writer(self):
        """
        Initialize the writer (which is stateless)
        """
        self._writer_class_changed(None, self.writer_class, self.writer_class)
        self.writer = self.writer_factory(parent=self)

    def init_post_processor(self):
        """
        Initialize the post_processor (which is stateless)
        """
        self._post_processor_class_changed(None, self.post_processor_class,
            self.post_processor_class)
        if self.post_processor_factory:
            self.post_processor = self.post_processor_factory(parent=self)

    def start(self):
        """
        Ran after initialization completed
        """
        super(NbConvertApp, self).start()
        self.convert_notebooks()

    def convert_notebooks(self):
        """
        Convert the notebooks in the self.notebook traitlet
        """
        # Export each notebook
        conversion_success = 0
        for notebook_filename in self.notebooks:

            # Get a unique key for the notebook and set it in the resources object.
            basename = os.path.basename(notebook_filename)
            notebook_name = basename[:basename.rfind('.')]
            resources = {}
            resources['unique_key'] = notebook_name
            resources['output_files_dir'] = '%s_files' % notebook_name

            # Try to export
            try:
                output, resources = export_by_name(self.export_format,
                                              notebook_filename,
                                              resources=resources,
                                              config=self.config)
            except ExporterNameError as e:
                print("Error while converting '%s': '%s' exporter not found."
                      %(notebook_filename, self.export_format),
                      file=sys.stderr)
                print("Known exporters are:",
                      "\n\t" + "\n\t".join(get_export_names()),
                      file=sys.stderr)
                self.exit(1)
            except ConversionException as e:
                print("Error while converting '%s': %s" %(notebook_filename, e),
                      file=sys.stderr)
                self.exit(1)
            else:
                write_resultes = self.writer.write(output, resources, notebook_name=notebook_name)

                #Post-process if post processor has been defined.
                if hasattr(self, 'post_processor') and self.post_processor:
                    self.post_processor(write_resultes)
                conversion_success += 1

        # If nothing was converted successfully, help the user.
        if conversion_success == 0:
            self.print_help()
            sys.exit(-1)

#-----------------------------------------------------------------------------
# Main entry point
#-----------------------------------------------------------------------------

launch_new_instance = NbConvertApp.launch_instance
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Defines a variety of Pygments lexers for highlighting IPython code.

This includes:

    IPythonLexer
    IPython3Lexer
        Lexers for pure IPython (python + magic/shell commands)

    IPythonPartialTracebackLexer
    IPythonTracebackLexer
        Supports 2.x and 3.x via keyword `python3`.  The partial traceback
        lexer reads everything but the Python code appearing in a traceback.
        The full lexer combines the partial lexer with an IPython lexer.

    IPythonConsoleLexer
        A lexer for IPython console sessions, with support for tracebacks.

    IPyLexer
        A friendly lexer which examines the first line of text and from it,
        decides whether to use an IPython lexer or an IPython console lexer.
        This is probably the only lexer that needs to be explicitly added
        to Pygments.

"""

# Standard library
import re

# Third party
from pygments.lexers import BashLexer, PythonLexer, Python3Lexer
from pygments.lexer import (
    Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
)
from pygments.token import (
    Comment, Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
)
from pygments.util import get_bool_opt

# Local
from IPython.testing.skipdoctest import skip_doctest

line_re = re.compile('.*?\n')

ipython_tokens = [
  (r'(\%+)(\w+)\s+(\.*)(\n)', bygroups(Operator, Keyword,
                                       using(BashLexer), Text)),
  (r'(\%+)(\w+)\b', bygroups(Operator, Keyword)),
  (r'^(!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
]

def build_ipy_lexer(python3):
    """Builds IPython lexers depending on the value of `python3`.

    The lexer inherits from an appropriate Python lexer and then adds
    information about IPython specific keywords (i.e. magic commands,
    shell commands, etc.)

    Parameters
    ----------
    python3 : bool
        If `True`, then build an IPython lexer from a Python 3 lexer.

    """
    # It would be nice to have a single IPython lexer class which takes
    # a boolean `python3`.  But since there are two Python lexer classes,
    # we will also have two IPython lexer classes.
    if python3:
        PyLexer = Python3Lexer
        clsname = 'IPython3Lexer'
        name = 'IPython3'
        aliases = ['ipython3']
        doc = """IPython3 Lexer"""
    else:
        PyLexer = PythonLexer
        clsname = 'IPythonLexer'
        name = 'IPython'
        aliases = ['ipython']
        doc = """IPython Lexer"""

    tokens = PyLexer.tokens.copy()
    tokens['root'] = ipython_tokens + tokens['root']

    attrs = {'name': name, 'aliases': aliases,
             '__doc__': doc, 'tokens': tokens}

    return type(name, (PyLexer,), attrs)

IPython3Lexer = build_ipy_lexer(python3=True)
IPythonLexer = build_ipy_lexer(python3=False)

class IPythonPartialTracebackLexer(RegexLexer):
    """
    Partial lexer for IPython tracebacks.

    Handles all the non-python output. This works for both Python 2.x and 3.x.

    """
    name = 'IPython Partial Traceback'

    tokens = {
        'root': [
            # Tracebacks for syntax errors have a different style.
            # For both types of tracebacks, we mark the first line with
            # Generic.Traceback.  For syntax errors, we mark the filename
            # as we mark the filenames for non-syntax tracebacks.
            #
            # These two regexps define how IPythonConsoleLexer finds a
            # traceback.
            #
            ## Non-syntax traceback
            (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
            ## Syntax traceback
            (r'^(  File)(.*)(, line )(\d+\n)',
             bygroups(Generic.Traceback, Name.Namespace,
                      Generic.Traceback, Literal.Number.Integer)),

            # (Exception Identifier)(Whitespace)(Traceback Message)
            (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
             bygroups(Name.Exception, Generic.Whitespace, Text)),
            # (Module/Filename)(Text)(Callee)(Function Signature)
            # Better options for callee and function signature?
            (r'(.*)( in )(.*)(\(.*\)\n)',
             bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
            # Regular line: (Whitespace)(Line Number)(Python Code)
            (r'(\s*?)(\d+)(.*?\n)',
             bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
            # Emphasized line: (Arrow)(Line Number)(Python Code)
            # Using Exception token so arrow color matches the Exception.
            (r'(-*>?\s?)(\d+)(.*?\n)',
             bygroups(Name.Exception, Literal.Number.Integer, Other)),
            # (Exception Identifier)(Message)
            (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
             bygroups(Name.Exception, Text)),
            # Tag everything else as Other, will be handled later.
            (r'.*\n', Other),
        ],
    }

class IPythonTracebackLexer(DelegatingLexer):
    """
    IPython traceback lexer.

    For doctests, the tracebacks can be snipped as much as desired with the
    exception to the lines that designate a traceback. For non-syntax error
    tracebacks, this is the line of hyphens. For syntax error tracebacks,
    this is the line which lists the File and line number.

    """
    # The lexer inherits from DelegatingLexer.  The "root" lexer is an
    # appropriate IPython lexer, which depends on the value of the boolean
    # `python3`.  First, we parse with the partial IPython traceback lexer.
    # Then, any code marked with the "Other" token is delegated to the root
    # lexer.
    #
    name = 'IPython Traceback'
    aliases = ['ipythontb']

    def __init__(self, **options):
        self.python3 = get_bool_opt(options, 'python3', False)

        if self.python3:
            IPyLexer = IPython3Lexer
        else:
            IPyLexer = IPythonLexer

        DelegatingLexer.__init__(self, IPyLexer,
                                 IPythonPartialTracebackLexer, **options)

@skip_doctest
class IPythonConsoleLexer(Lexer):
    """
    An IPython console lexer for IPython code-blocks and doctests, such as::

        .. sourcecode:: ipythoncon

            In [1]: a = 'foo'

            In [2]: a
            Out[2]: 'foo'

            In [3]: print a
            foo

            In [4]: 1 / 0

    Support is also provided for IPython exceptions. ::

        .. code-block:: ipythoncon

            In [1]: raise Exception
            ---------------------------------------------------------------------------
            Exception                                 Traceback (most recent call last)
            <ipython-input-1-fca2ab0ca76b> in <module>()
            ----> 1 raise Exception

            Exception:

    """
    name = 'IPython console session'
    aliases = ['ipythoncon']
    mimetypes = ['text/x-ipython-console']

    # The regexps used to determine what is input and what is output. The
    # input regex should be consistent with and also be the combination of
    # the values of the `in_template` and `in2_templates`. For example, the
    # defaults prompts are:
    #
    #     c.PromptManager.in_template  = 'In [\#]: '
    #     c.PromptManager.in2_template = '   .\D.: '
    #     c.PromptManager.out_template = 'Out[\#]: '
    #
    # Note, we do not include the trailing whitespace in the regex since
    # we want to allow blank prompts (and editors often remove trailing
    # whitespace).
    #
    in1_regex = r'In \[[0-9]+\]: '
    in2_regex = r'   \.\.+\.: '
    out_regex = r'Out\[[0-9]+\]: '

    #: The regex to determine when a traceback starts.
    ipytb_start = re.compile(r'^(\^C)?(-+\n)|^(  File)(.*)(, line )(\d+\n)')

    def __init__(self, **options):
        """Initialize the IPython console lexer.

        Parameters
        ----------
        python3 : bool
            If `True`, then the console inputs are parsed using a Python 3
            lexer. Otherwise, they are parsed using a Python 2 lexer.
        in1_regex : RegexObject
            The compiled regular expression used to detect the start
            of inputs. Although the IPython configuration setting may have a
            trailing whitespace, do not include it in the regex. If `None`,
            then the default input prompt is assumed.
        in2_regex : RegexObject
            The compiled regular expression used to detect the continuation
            of inputs. Although the IPython configuration setting may have a
            trailing whitespace, do not include it in the regex. If `None`,
            then the default input prompt is assumed.
        out_regex : RegexObject
            The compiled regular expression used to detect outputs. If `None`,
            then the default output prompt is assumed.

        """
        self.python3 = get_bool_opt(options, 'python3', False)

        in1_regex = options.get('in1_regex', self.in1_regex)
        in2_regex = options.get('in2_regex', self.in2_regex)
        out_regex = options.get('out_regex', self.out_regex)

        # So that we can work with input and output prompts which have been
        # rstrip'd (possibly by editors) we also need rstrip'd variants. If
        # we do not do this, then such prompts will be tagged as 'output'.
        # The reason can't just use the rstrip'd variants instead is because
        # we want any whitespace associated with the prompt to be inserted
        # with the token. This allows formatted code to be modified so as hide
        # the appearance of prompts.  For example, see copybutton.js.
        in1_regex_rstrip = in1_regex.rstrip() + '\n'
        in2_regex_rstrip = in2_regex.rstrip() + '\n'
        out_regex_rstrip = out_regex.rstrip() + '\n'

        # Compile and save them all.
        attrs = ['in1_regex', 'in2_regex', 'out_regex',
                 'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
        for attr in attrs:
            self.__setattr__(attr, re.compile(locals()[attr]))

        Lexer.__init__(self, **options)

        if self.python3:
            pylexer = IPython3Lexer
            tblexer = IPythonTracebackLexer
        else:
            pylexer = IPythonLexer
            tblexer = IPythonTracebackLexer

        self.pylexer = pylexer(**options)
        self.tblexer = tblexer(**options)

        self.reset()

    def reset(self):
        self.mode = 'output'
        self.index = 0
        self.buffer = u''
        self.insertions = []

    def buffered_tokens(self):
        """
        Generator of unprocessed tokens after doing insertions and before
        changing to a new state.

        """
        if self.mode == 'output':
            tokens = [(0, Generic.Output, self.buffer)]
        elif self.mode == 'input':
            tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
        else: # traceback
            tokens = self.tblexer.get_tokens_unprocessed(self.buffer)

        for i, t, v in do_insertions(self.insertions, tokens):
            # All token indexes are relative to the buffer.
            yield self.index + i, t, v

        # Clear it all
        self.index += len(self.buffer)
        self.buffer = u''
        self.insertions = []

    def get_modecode(self, line):
        """
        Returns the next mode and code to be added to the next mode's buffer.

        The next mode depends on current mode and contents of line.

        """
        # To reduce the number of regex match checks, we have multiple
        # 'if' blocks instead of 'if-elif' blocks.

        ### Check for possible end of input
        ###
        in2_match = self.in2_regex.match(line)
        in2_match_rstrip = self.in2_regex_rstrip.match(line)
        if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
           in2_match_rstrip:
            end_input = True
        else:
            end_input = False
        if end_input and self.mode != 'tb':
            # Only look for an end of input when not in tb mode.
            # An ellipsis could appear within the traceback.
            mode = 'output'
            code = u''
            insertion = (0, Generic.Prompt, line)
            return mode, code, insertion

        ### Check for output prompt
        ###
        out_match = self.out_regex.match(line)
        out_match_rstrip = self.out_regex_rstrip.match(line)
        if out_match or out_match_rstrip:
            mode = 'output'
            if out_match:
                idx = out_match.end()
            else:
                idx = out_match_rstrip.end()
            code = line[idx:]
            # Use the 'heading' token for output.  We cannot use Generic.Error
            # since it would conflict with exceptions.
            insertion = (0, Generic.Heading, line[:idx])
            return mode, code, insertion

        ### Check for input or continuation prompt (non stripped version)
        ###
        in1_match = self.in1_regex.match(line)
        if in1_match or (in2_match and self.mode != 'tb'):
            # New input or when not in tb, continued input.
            # We do not check for continued input when in tb since it is
            # allowable to replace a long stack with an ellipsis.
            mode = 'input'
            if in1_match:
                idx = in1_match.end()
            else: # in2_match
                idx = in2_match.end()
            code = line[idx:]
            insertion = (0, Generic.Prompt, line[:idx])
            return mode, code, insertion

        ### Check for input or continuation prompt (stripped version)
        ###
        in1_match_rstrip = self.in1_regex_rstrip.match(line)
        if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
            # New input or when not in tb, continued input.
            # We do not check for continued input when in tb since it is
            # allowable to replace a long stack with an ellipsis.
            mode = 'input'
            if in1_match_rstrip:
                idx = in1_match_rstrip.end()
            else: # in2_match
                idx = in2_match_rstrip.end()
            code = line[idx:]
            insertion = (0, Generic.Prompt, line[:idx])
            return mode, code, insertion

        ### Check for traceback
        ###
        if self.ipytb_start.match(line):
            mode = 'tb'
            code = line
            insertion = None
            return mode, code, insertion

        ### All other stuff...
        ###
        if self.mode in ('input', 'output'):
            # We assume all other text is output. Multiline input that
            # does not use the continuation marker cannot be detected.
            # For example, the 3 in the following is clearly output:
            #
            #    In [1]: print 3
            #    3
            #
            # But the following second line is part of the input:
            #
            #    In [2]: while True:
            #        print True
            #
            # In both cases, the 2nd line will be 'output'.
            #
            mode = 'output'
        else:
            mode = 'tb'

        code = line
        insertion = None

        return mode, code, insertion

    def get_tokens_unprocessed(self, text):
        self.reset()
        for match in line_re.finditer(text):
            line = match.group()
            mode, code, insertion = self.get_modecode(line)

            if mode != self.mode:
                # Yield buffered tokens before transitioning to new mode.
                for token in self.buffered_tokens():
                    yield token
                self.mode = mode

            if insertion:
                self.insertions.append((len(self.buffer), [insertion]))
            self.buffer += code
        else:
            for token in self.buffered_tokens():
                yield token

class IPyLexer(Lexer):
    """
    Primary lexer for all IPython-like code.

    This is a simple helper lexer.  If the first line of the text begins with
    "In \[[0-9]+\]:", then the entire text is parsed with an IPython console
    lexer. If not, then the entire text is parsed with an IPython lexer.

    The goal is to reduce the number of lexers that are registered
    with Pygments.

    """
    name = 'IPy session'
    aliases = ['ipy']

    def __init__(self, **options):
        self.python3 = get_bool_opt(options, 'python3', False)
        Lexer.__init__(self, **options)

        self.IPythonLexer = IPythonLexer(**options)
        self.IPythonConsoleLexer = IPythonConsoleLexer(**options)

    def get_tokens_unprocessed(self, text):
        if re.match(r'(In \[[0-9]+\]:)', text.strip()):
            lex = self.IPythonConsoleLexer
        else:
            lex = self.IPythonLexer
        for token in lex.get_tokens_unprocessed(text):
            yield token

<commit_msg>Update docstring for IPythonConsoleLexer.<commit_after># -*- coding: utf-8 -*-
"""
Defines a variety of Pygments lexers for highlighting IPython code.

This includes:

    IPythonLexer
    IPython3Lexer
        Lexers for pure IPython (python + magic/shell commands)

    IPythonPartialTracebackLexer
    IPythonTracebackLexer
        Supports 2.x and 3.x via keyword `python3`.  The partial traceback
        lexer reads everything but the Python code appearing in a traceback.
        The full lexer combines the partial lexer with an IPython lexer.

    IPythonConsoleLexer
        A lexer for IPython console sessions, with support for tracebacks.

    IPyLexer
        A friendly lexer which examines the first line of text and from it,
        decides whether to use an IPython lexer or an IPython console lexer.
        This is probably the only lexer that needs to be explicitly added
        to Pygments.

"""

# Standard library
import re

# Third party
from pygments.lexers import BashLexer, PythonLexer, Python3Lexer
from pygments.lexer import (
    Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
)
from pygments.token import (
    Comment, Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
)
from pygments.util import get_bool_opt

# Local
from IPython.testing.skipdoctest import skip_doctest

line_re = re.compile('.*?\n')

ipython_tokens = [
  (r'(\%+)(\w+)\s+(\.*)(\n)', bygroups(Operator, Keyword,
                                       using(BashLexer), Text)),
  (r'(\%+)(\w+)\b', bygroups(Operator, Keyword)),
  (r'^(!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
]

def build_ipy_lexer(python3):
    """Builds IPython lexers depending on the value of `python3`.

    The lexer inherits from an appropriate Python lexer and then adds
    information about IPython specific keywords (i.e. magic commands,
    shell commands, etc.)

    Parameters
    ----------
    python3 : bool
        If `True`, then build an IPython lexer from a Python 3 lexer.

    """
    # It would be nice to have a single IPython lexer class which takes
    # a boolean `python3`.  But since there are two Python lexer classes,
    # we will also have two IPython lexer classes.
    if python3:
        PyLexer = Python3Lexer
        clsname = 'IPython3Lexer'
        name = 'IPython3'
        aliases = ['ipython3']
        doc = """IPython3 Lexer"""
    else:
        PyLexer = PythonLexer
        clsname = 'IPythonLexer'
        name = 'IPython'
        aliases = ['ipython']
        doc = """IPython Lexer"""

    tokens = PyLexer.tokens.copy()
    tokens['root'] = ipython_tokens + tokens['root']

    attrs = {'name': name, 'aliases': aliases,
             '__doc__': doc, 'tokens': tokens}

    return type(name, (PyLexer,), attrs)

IPython3Lexer = build_ipy_lexer(python3=True)
IPythonLexer = build_ipy_lexer(python3=False)

class IPythonPartialTracebackLexer(RegexLexer):
    """
    Partial lexer for IPython tracebacks.

    Handles all the non-python output. This works for both Python 2.x and 3.x.

    """
    name = 'IPython Partial Traceback'

    tokens = {
        'root': [
            # Tracebacks for syntax errors have a different style.
            # For both types of tracebacks, we mark the first line with
            # Generic.Traceback.  For syntax errors, we mark the filename
            # as we mark the filenames for non-syntax tracebacks.
            #
            # These two regexps define how IPythonConsoleLexer finds a
            # traceback.
            #
            ## Non-syntax traceback
            (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
            ## Syntax traceback
            (r'^(  File)(.*)(, line )(\d+\n)',
             bygroups(Generic.Traceback, Name.Namespace,
                      Generic.Traceback, Literal.Number.Integer)),

            # (Exception Identifier)(Whitespace)(Traceback Message)
            (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
             bygroups(Name.Exception, Generic.Whitespace, Text)),
            # (Module/Filename)(Text)(Callee)(Function Signature)
            # Better options for callee and function signature?
            (r'(.*)( in )(.*)(\(.*\)\n)',
             bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
            # Regular line: (Whitespace)(Line Number)(Python Code)
            (r'(\s*?)(\d+)(.*?\n)',
             bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
            # Emphasized line: (Arrow)(Line Number)(Python Code)
            # Using Exception token so arrow color matches the Exception.
            (r'(-*>?\s?)(\d+)(.*?\n)',
             bygroups(Name.Exception, Literal.Number.Integer, Other)),
            # (Exception Identifier)(Message)
            (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
             bygroups(Name.Exception, Text)),
            # Tag everything else as Other, will be handled later.
            (r'.*\n', Other),
        ],
    }

class IPythonTracebackLexer(DelegatingLexer):
    """
    IPython traceback lexer.

    For doctests, the tracebacks can be snipped as much as desired with the
    exception to the lines that designate a traceback. For non-syntax error
    tracebacks, this is the line of hyphens. For syntax error tracebacks,
    this is the line which lists the File and line number.

    """
    # The lexer inherits from DelegatingLexer.  The "root" lexer is an
    # appropriate IPython lexer, which depends on the value of the boolean
    # `python3`.  First, we parse with the partial IPython traceback lexer.
    # Then, any code marked with the "Other" token is delegated to the root
    # lexer.
    #
    name = 'IPython Traceback'
    aliases = ['ipythontb']

    def __init__(self, **options):
        self.python3 = get_bool_opt(options, 'python3', False)

        if self.python3:
            IPyLexer = IPython3Lexer
        else:
            IPyLexer = IPythonLexer

        DelegatingLexer.__init__(self, IPyLexer,
                                 IPythonPartialTracebackLexer, **options)

@skip_doctest
class IPythonConsoleLexer(Lexer):
    """
    An IPython console lexer for IPython code-blocks and doctests, such as:

    .. code-block:: rst

        .. code-block:: ipythoncon

            In [1]: a = 'foo'

            In [2]: a
            Out[2]: 'foo'

            In [3]: print a
            foo

            In [4]: 1 / 0

    Support is also provided for IPython exceptions:

    .. code-block:: rst

        .. code-block:: ipythoncon

            In [1]: raise Exception
            ---------------------------------------------------------------------------
            Exception                                 Traceback (most recent call last)
            <ipython-input-1-fca2ab0ca76b> in <module>()
            ----> 1 raise Exception

            Exception:

    """
    name = 'IPython console session'
    aliases = ['ipythoncon']
    mimetypes = ['text/x-ipython-console']

    # The regexps used to determine what is input and what is output. The
    # input regex should be consistent with and also be the combination of
    # the values of the `in_template` and `in2_templates`. For example, the
    # defaults prompts are:
    #
    #     c.PromptManager.in_template  = 'In [\#]: '
    #     c.PromptManager.in2_template = '   .\D.: '
    #     c.PromptManager.out_template = 'Out[\#]: '
    #
    # Note, we do not include the trailing whitespace in the regex since
    # we want to allow blank prompts (and editors often remove trailing
    # whitespace).
    #
    in1_regex = r'In \[[0-9]+\]: '
    in2_regex = r'   \.\.+\.: '
    out_regex = r'Out\[[0-9]+\]: '

    #: The regex to determine when a traceback starts.
    ipytb_start = re.compile(r'^(\^C)?(-+\n)|^(  File)(.*)(, line )(\d+\n)')

    def __init__(self, **options):
        """Initialize the IPython console lexer.

        Parameters
        ----------
        python3 : bool
            If `True`, then the console inputs are parsed using a Python 3
            lexer. Otherwise, they are parsed using a Python 2 lexer.
        in1_regex : RegexObject
            The compiled regular expression used to detect the start
            of inputs. Although the IPython configuration setting may have a
            trailing whitespace, do not include it in the regex. If `None`,
            then the default input prompt is assumed.
        in2_regex : RegexObject
            The compiled regular expression used to detect the continuation
            of inputs. Although the IPython configuration setting may have a
            trailing whitespace, do not include it in the regex. If `None`,
            then the default input prompt is assumed.
        out_regex : RegexObject
            The compiled regular expression used to detect outputs. If `None`,
            then the default output prompt is assumed.

        """
        self.python3 = get_bool_opt(options, 'python3', False)

        in1_regex = options.get('in1_regex', self.in1_regex)
        in2_regex = options.get('in2_regex', self.in2_regex)
        out_regex = options.get('out_regex', self.out_regex)

        # So that we can work with input and output prompts which have been
        # rstrip'd (possibly by editors) we also need rstrip'd variants. If
        # we do not do this, then such prompts will be tagged as 'output'.
        # The reason can't just use the rstrip'd variants instead is because
        # we want any whitespace associated with the prompt to be inserted
        # with the token. This allows formatted code to be modified so as hide
        # the appearance of prompts.  For example, see copybutton.js.
        in1_regex_rstrip = in1_regex.rstrip() + '\n'
        in2_regex_rstrip = in2_regex.rstrip() + '\n'
        out_regex_rstrip = out_regex.rstrip() + '\n'

        # Compile and save them all.
        attrs = ['in1_regex', 'in2_regex', 'out_regex',
                 'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
        for attr in attrs:
            self.__setattr__(attr, re.compile(locals()[attr]))

        Lexer.__init__(self, **options)

        if self.python3:
            pylexer = IPython3Lexer
            tblexer = IPythonTracebackLexer
        else:
            pylexer = IPythonLexer
            tblexer = IPythonTracebackLexer

        self.pylexer = pylexer(**options)
        self.tblexer = tblexer(**options)

        self.reset()

    def reset(self):
        self.mode = 'output'
        self.index = 0
        self.buffer = u''
        self.insertions = []

    def buffered_tokens(self):
        """
        Generator of unprocessed tokens after doing insertions and before
        changing to a new state.

        """
        if self.mode == 'output':
            tokens = [(0, Generic.Output, self.buffer)]
        elif self.mode == 'input':
            tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
        else: # traceback
            tokens = self.tblexer.get_tokens_unprocessed(self.buffer)

        for i, t, v in do_insertions(self.insertions, tokens):
            # All token indexes are relative to the buffer.
            yield self.index + i, t, v

        # Clear it all
        self.index += len(self.buffer)
        self.buffer = u''
        self.insertions = []

    def get_modecode(self, line):
        """
        Returns the next mode and code to be added to the next mode's buffer.

        The next mode depends on current mode and contents of line.

        """
        # To reduce the number of regex match checks, we have multiple
        # 'if' blocks instead of 'if-elif' blocks.

        ### Check for possible end of input
        ###
        in2_match = self.in2_regex.match(line)
        in2_match_rstrip = self.in2_regex_rstrip.match(line)
        if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
           in2_match_rstrip:
            end_input = True
        else:
            end_input = False
        if end_input and self.mode != 'tb':
            # Only look for an end of input when not in tb mode.
            # An ellipsis could appear within the traceback.
            mode = 'output'
            code = u''
            insertion = (0, Generic.Prompt, line)
            return mode, code, insertion

        ### Check for output prompt
        ###
        out_match = self.out_regex.match(line)
        out_match_rstrip = self.out_regex_rstrip.match(line)
        if out_match or out_match_rstrip:
            mode = 'output'
            if out_match:
                idx = out_match.end()
            else:
                idx = out_match_rstrip.end()
            code = line[idx:]
            # Use the 'heading' token for output.  We cannot use Generic.Error
            # since it would conflict with exceptions.
            insertion = (0, Generic.Heading, line[:idx])
            return mode, code, insertion

        ### Check for input or continuation prompt (non stripped version)
        ###
        in1_match = self.in1_regex.match(line)
        if in1_match or (in2_match and self.mode != 'tb'):
            # New input or when not in tb, continued input.
            # We do not check for continued input when in tb since it is
            # allowable to replace a long stack with an ellipsis.
            mode = 'input'
            if in1_match:
                idx = in1_match.end()
            else: # in2_match
                idx = in2_match.end()
            code = line[idx:]
            insertion = (0, Generic.Prompt, line[:idx])
            return mode, code, insertion

        ### Check for input or continuation prompt (stripped version)
        ###
        in1_match_rstrip = self.in1_regex_rstrip.match(line)
        if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
            # New input or when not in tb, continued input.
            # We do not check for continued input when in tb since it is
            # allowable to replace a long stack with an ellipsis.
            mode = 'input'
            if in1_match_rstrip:
                idx = in1_match_rstrip.end()
            else: # in2_match
                idx = in2_match_rstrip.end()
            code = line[idx:]
            insertion = (0, Generic.Prompt, line[:idx])
            return mode, code, insertion

        ### Check for traceback
        ###
        if self.ipytb_start.match(line):
            mode = 'tb'
            code = line
            insertion = None
            return mode, code, insertion

        ### All other stuff...
        ###
        if self.mode in ('input', 'output'):
            # We assume all other text is output. Multiline input that
            # does not use the continuation marker cannot be detected.
            # For example, the 3 in the following is clearly output:
            #
            #    In [1]: print 3
            #    3
            #
            # But the following second line is part of the input:
            #
            #    In [2]: while True:
            #        print True
            #
            # In both cases, the 2nd line will be 'output'.
            #
            mode = 'output'
        else:
            mode = 'tb'

        code = line
        insertion = None

        return mode, code, insertion

    def get_tokens_unprocessed(self, text):
        self.reset()
        for match in line_re.finditer(text):
            line = match.group()
            mode, code, insertion = self.get_modecode(line)

            if mode != self.mode:
                # Yield buffered tokens before transitioning to new mode.
                for token in self.buffered_tokens():
                    yield token
                self.mode = mode

            if insertion:
                self.insertions.append((len(self.buffer), [insertion]))
            self.buffer += code
        else:
            for token in self.buffered_tokens():
                yield token

class IPyLexer(Lexer):
    """
    Primary lexer for all IPython-like code.

    This is a simple helper lexer.  If the first line of the text begins with
    "In \[[0-9]+\]:", then the entire text is parsed with an IPython console
    lexer. If not, then the entire text is parsed with an IPython lexer.

    The goal is to reduce the number of lexers that are registered
    with Pygments.

    """
    name = 'IPy session'
    aliases = ['ipy']

    def __init__(self, **options):
        self.python3 = get_bool_opt(options, 'python3', False)
        Lexer.__init__(self, **options)

        self.IPythonLexer = IPythonLexer(**options)
        self.IPythonConsoleLexer = IPythonConsoleLexer(**options)

    def get_tokens_unprocessed(self, text):
        if re.match(r'(In \[[0-9]+\]:)', text.strip()):
            lex = self.IPythonConsoleLexer
        else:
            lex = self.IPythonLexer
        for token in lex.get_tokens_unprocessed(text):
            yield token

<|endoftext|>


<commit_before>#!/usr/bin/env python2

"""Test texture properties for ROI search."""

import argparse
import glob
import re
import numpy as np
import skimage

import dwi.dataset
import dwi.util
import dwi.patient
import dwi.dwimage
import dwi.mask
import dwi.plot
import dwi.texture

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description = __doc__)
    p.add_argument('--verbose', '-v', action='count',
            help='increase verbosity')
    p.add_argument('--dir', '-d', required=True,
            help='image directory')
    p.add_argument('--cases', '-c', nargs='+', type=int, default=[],
            help='case numbers to draw')
    p.add_argument('--total', '-t', action='store_true',
            help='show total AUC')
    p.add_argument('--step', '-s', type=int, default=5,
            help='window step size')
    args = p.parse_args()
    return args

def get_roi_slices(data):
    for d in data:
        img = d['image']
        d['cancer_slice'] = d['cancer_mask'].get_masked_slice(img)
        d['normal_slice'] = d['normal_mask'].get_masked_slice(img)

def get_lbpf(img):
    lbp, lbp_freq, n_patterns = dwi.texture.get_lbp_freqs(img)
    return lbp_freq
    #return lbp_freq[...,1:] # Drop non-uniform patterns

def avg_lbpf_map(histogram_map):
    """Return the average of multidimensional LPB frequency histogram map."""
    histogram_map = np.asanyarray(histogram_map)
    n_patterns = histogram_map.shape[-1]
    histograms = histogram_map.reshape(-1, n_patterns)
    return np.mean(histograms, axis=0)

def get_lbpfs(data):
    """Get LBP frequency histograms."""
    for d in data:
        d['cancer_slice_lbpf'] = get_lbpf(d['cancer_slice'][...,0])
        d['cancer_slice_lbpf_avg'] = avg_lbpf_map(d['cancer_slice_lbpf'])
        d['cancer_lbpf_avg'] = avg_lbpf_map(get_lbpf(d['cancer_roi']))
        d['normal_lbpf_avg'] = avg_lbpf_map(get_lbpf(d['normal_roi']))

def get_distances(data, model):
    """Get LBP frequence histogram distances."""
    for d in data:
        sample = d['cancer_slice_lbpf']
        #model = d['cancer_lbpf_avg']
        dist = np.zeros_like(sample[...,0])
        for y, x in np.ndindex(dist.shape):
            dist[y,x] = dwi.texture.lbpf_dist(sample[y,x], model,
                    method='log-likelihood')
        d['distance_map'] = dist

def draw(data, param, filename):
    import matplotlib
    import matplotlib.pyplot as plt
    import pylab as pl

    plt.rcParams['image.cmap'] = 'gray'
    plt.rcParams['image.interpolation'] = 'nearest'
    n_cols, n_rows = 2, 1
    fig = plt.figure(figsize=(n_cols*6, n_rows*6))

    CANCER_COLOR = (1.0, 0.0, 0.0, 1.0)
    NORMAL_COLOR = (0.0, 1.0, 0.0, 1.0)
    AUTO_COLOR = (1.0, 1.0, 0.0, 1.0)

    pmap = data['cancer_slice'].copy()
    dwi.util.clip_pmap(pmap, [param])
    pmap = pmap[...,0]

    cancer_pos = data['cancer_mask'].where()[0][1:3]
    normal_pos = data['normal_mask'].where()[0][1:3]

    ax1 = fig.add_subplot(1, n_cols, 1)
    ax1.set_title(param)
    plt.imshow(pmap)

    ax3 = fig.add_subplot(1, n_cols, 2)
    ax3.set_title('LBP freq dist')
    view = np.zeros(pmap.shape + (3,), dtype=float)
    view[...,0] = pmap / pmap.max()
    view[...,1] = pmap / pmap.max()
    view[...,2] = pmap / pmap.max()
    #for i, a in enumerate(pmap):
    #    for j, v in enumerate(a):
    #        if v < dwi.autoroi.ADCM_MIN:
    #            view[i,j,:] = [0.5, 0, 0]
    #        elif v > dwi.autoroi.ADCM_MAX:
    #            view[i,j,:] = [0, 0.5, 0]
    plt.imshow(view)
    plt.imshow(get_roi_layer(pmap, cancer_pos, CANCER_COLOR), alpha=0.7)
    plt.imshow(get_roi_layer(pmap, normal_pos, NORMAL_COLOR), alpha=0.7)

    plt.tight_layout()
    print 'Writing figure:', filename
    plt.savefig(filename, bbox_inches='tight')
    plt.close()

args = parse_args()

print 'Reading dataset...'
data = dwi.dataset.dataset_read_samplelist('samples_train.txt', cases=args.cases, scans=['1a', '2a'])
dwi.dataset.dataset_read_patientinfo(data, 'patients.txt')
dwi.dataset.dataset_read_subregions(data, 'bounding_box_100_10pad')
dwi.dataset.dataset_read_pmaps(data, 'results_Mono_combinedDICOM', 'ADCm')
#dwi.dataset.dataset_read_prostate_masks(data, 'masks_prostate')
dwi.dataset.dataset_read_roi_masks(data, 'masks_rois', shape=(5,5))
get_roi_slices(data)
for d in data:
    print d['case'], d['scan'], d['score'], d['image'].shape
    #print d['subregion'], dwi.util.subwindow_shape(d['subregion'])

print 'Calculating texture properties...'
get_lbpfs(data)
model = avg_lbpf_map([d['cancer_lbpf_avg'] for d in data])
get_distances(data, model)

print 'Plotting...'
l = []
for d in data:
    imgs = [d['cancer_slice'][...,0], d['distance_map']]
    l.append(imgs)
imgs = [[d['cancer_slice'][...,0], d['distance_map']] for d in data]
ylabels=[d['case'] for d in data]
xlabels=['adc', 'dist']
dwi.plot.show_images(l, ylabels, xlabels, outfile='dist.png')
for d in data:
    draw(d, 'ADCm', 'dist_fig/dist_%s_%s.png' % (d['case'], d['scan']))
<commit_msg>Fixes<commit_after>#!/usr/bin/env python2

"""Test texture properties for ROI search."""

import argparse
import glob
import re
import numpy as np
import skimage

import dwi.dataset
import dwi.util
import dwi.patient
import dwi.dwimage
import dwi.mask
import dwi.plot
import dwi.texture

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description = __doc__)
    p.add_argument('--verbose', '-v', action='count',
            help='increase verbosity')
    p.add_argument('--dir', '-d', required=True,
            help='image directory')
    p.add_argument('--cases', '-c', nargs='+', type=int, default=[],
            help='case numbers to draw')
    p.add_argument('--total', '-t', action='store_true',
            help='show total AUC')
    p.add_argument('--step', '-s', type=int, default=5,
            help='window step size')
    args = p.parse_args()
    return args

def get_roi_slices(data):
    for d in data:
        img = d['image']
        d['cancer_slice'] = d['cancer_mask'].get_masked_slice(img)
        d['normal_slice'] = d['normal_mask'].get_masked_slice(img)

def get_lbpf(img):
    lbp, lbp_freq, n_patterns = dwi.texture.get_lbp_freqs(img)
    return lbp_freq
    #return lbp_freq[...,1:] # Drop non-uniform patterns

def avg_lbpf_map(histogram_map):
    """Return the average of multidimensional LPB frequency histogram map."""
    histogram_map = np.asanyarray(histogram_map)
    n_patterns = histogram_map.shape[-1]
    histograms = histogram_map.reshape(-1, n_patterns)
    return np.mean(histograms, axis=0)

def get_lbpfs(data):
    """Get LBP frequency histograms."""
    for d in data:
        d['cancer_slice_lbpf'] = get_lbpf(d['cancer_slice'][...,0])
        d['cancer_slice_lbpf_avg'] = avg_lbpf_map(d['cancer_slice_lbpf'])
        d['cancer_lbpf_avg'] = avg_lbpf_map(get_lbpf(d['cancer_roi']))
        d['normal_lbpf_avg'] = avg_lbpf_map(get_lbpf(d['normal_roi']))

def get_distances(data, model):
    """Get LBP frequence histogram distances."""
    for d in data:
        sample = d['cancer_slice_lbpf']
        #model = d['cancer_lbpf_avg']
        dist = np.zeros_like(sample[...,0])
        for y, x in np.ndindex(dist.shape):
            dist[y,x] = dwi.texture.lbpf_dist(sample[y,x], model,
                    method='log-likelihood')
        d['distance_map'] = dist

def draw_roi(img, pos, color):
    """Draw a rectangle ROI on a layer."""
    y, x = pos
    img[y:y+5, x:x+5] = color

def get_roi_layer(img, pos, color):
    """Get a layer with a rectangle ROI for drawing."""
    layer = np.zeros(img.shape + (4,))
    draw_roi(layer, pos, color)
    return layer

def draw(data, param, filename):
    import matplotlib
    import matplotlib.pyplot as plt
    import pylab as pl

    plt.rcParams['image.cmap'] = 'gray'
    plt.rcParams['image.interpolation'] = 'nearest'
    n_cols, n_rows = 2, 1
    fig = plt.figure(figsize=(n_cols*6, n_rows*6))

    CANCER_COLOR = (1.0, 0.0, 0.0, 1.0)
    NORMAL_COLOR = (0.0, 1.0, 0.0, 1.0)

    cancer_pos = data['cancer_mask'].where()[0][1:3]
    normal_pos = data['normal_mask'].where()[0][1:3]

    pmap = data['cancer_slice'].copy()
    dwi.util.clip_pmap(pmap, [param])
    pmap = pmap[...,0]

    dmap = data['distance_map'].copy()

    ax1 = fig.add_subplot(1, n_cols, 1)
    ax1.set_title(param)
    plt.imshow(pmap)

    ax2 = fig.add_subplot(1, n_cols, 2)
    ax2.set_title('LBP freq dist')
    view = np.zeros(dmap.shape + (3,), dtype=float)
    view[...,0] = dmap / dmap.max()
    view[...,1] = dmap / dmap.max()
    view[...,2] = dmap / dmap.max()
    #for i, a in enumerate(dmap):
    #    for j, v in enumerate(a):
    #        if v < dwi.autoroi.ADCM_MIN:
    #            view[i,j,:] = [0.5, 0, 0]
    #        elif v > dwi.autoroi.ADCM_MAX:
    #            view[i,j,:] = [0, 0.5, 0]
    plt.imshow(view)
    plt.imshow(get_roi_layer(dmap, cancer_pos, CANCER_COLOR), alpha=0.7)
    plt.imshow(get_roi_layer(dmap, normal_pos, NORMAL_COLOR), alpha=0.7)

    plt.tight_layout()
    print 'Writing figure:', filename
    plt.savefig(filename, bbox_inches='tight')
    plt.close()

args = parse_args()

print 'Reading dataset...'
data = dwi.dataset.dataset_read_samplelist('samples_train.txt', cases=args.cases, scans=['1a', '2a'])
dwi.dataset.dataset_read_patientinfo(data, 'patients.txt')
dwi.dataset.dataset_read_subregions(data, 'bounding_box_100_10pad')
dwi.dataset.dataset_read_pmaps(data, 'results_Mono_combinedDICOM', 'ADCm')
#dwi.dataset.dataset_read_prostate_masks(data, 'masks_prostate')
dwi.dataset.dataset_read_roi_masks(data, 'masks_rois', shape=(5,5))
get_roi_slices(data)
for d in data:
    print d['case'], d['scan'], d['score'], d['image'].shape
    #print d['subregion'], dwi.util.subwindow_shape(d['subregion'])

print 'Calculating texture properties...'
get_lbpfs(data)
model = avg_lbpf_map([d['cancer_lbpf_avg'] for d in data])
get_distances(data, model)

print 'Plotting...'
l = []
for d in data:
    imgs = [d['cancer_slice'][...,0], d['distance_map']]
    l.append(imgs)
imgs = [[d['cancer_slice'][...,0], d['distance_map']] for d in data]
ylabels=[d['case'] for d in data]
xlabels=['adc', 'dist']
dwi.plot.show_images(l, ylabels, xlabels, outfile='dist.png')
for d in data:
    draw(d, 'ADCm', 'dist_fig/dist_%s_%s.png' % (d['case'], d['scan']))
<|endoftext|>