<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Stingray documentation build configuration file, created by
# sphinx-quickstart on Thu Feb  1 17:02:36 2018.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.mathjax',
    'sphinx.ext.viewcode',
    'sphinx.ext.autodoc',
    'sphinx.ext.napoleon']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'Stingray'
copyright = '2018, the Stingray team'
author = 'the Stingray team'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = 'dev'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['stingray/astropy_helpers', '_build', 'Thumbs.db',
                    '.DS_Store', '**/.ipynb_checkpoints',
                    "**/README.rst", "**/_templates",
                    "**/CREDITS.rst", '**/validation/*.ipynb',
                    '**/docs/index.rst', '**/docs/simulator.rst']

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Napoleon Settings ----------------------------------------------------
napoleon_numpy_docstring = True
napoleon_include_init_with_doc = False
napoleon_include_private_with_doc = False
napoleon_include_special_with_doc = True
napoleon_use_admonition_for_examples = False
napoleon_use_admonition_for_notes = False
napoleon_use_admonition_for_references = False
napoleon_use_ivar = False
napoleon_use_param = True
napoleon_use_rtype = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#

#import astropy_sphinx_theme
#html_theme_path = astropy_sphinx_theme.get_html_theme_path()

#html_theme = 'bootstrap-astropy'
html_theme = 'sphinx_rtd_theme'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = '{0} v{1}'.format(project, release)

# Output file base name for HTML help builder.
#htmlhelp_basename = project + 'doc'

#html_theme_options = {
#    'logotext1': 'Sting',  # white,  semi-bold
#    'logotext2': 'ray',     # orange, light
#    'logotext3': ':docs'   # white,  light
#    }

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = ['_static']

# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# This is required for the alabaster theme
# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
html_sidebars = {
    '**': [
        'about.html',
        'navigation.html',
        'relations.html',  # needs 'show_related': True theme option to display
        'searchbox.html',
        'donate.html',
    ]
}

extensions += ['matplotlib.sphinxext.plot_directive','sphinx.ext.autodoc','nbsphinx']

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = 'Stingraydoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #
    # 'preamble': '',

    # Latex figure (float) alignment
    #
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'Stingray.tex', 'Stingray Documentation',
     'the Stingray team', 'manual'),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'stingray', 'Stingray Documentation',
     [author], 1)
]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'Stingray', 'Stingray Documentation',
     author, 'Stingray', 'One line description of project.',
     'Miscellaneous'),
]

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'https://docs.python.org/': None}
<commit_msg>Add ci-helpers to excluded patterns<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Stingray documentation build configuration file, created by
# sphinx-quickstart on Thu Feb  1 17:02:36 2018.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.autodoc',
    'sphinx.ext.doctest',
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.coverage',
    'sphinx.ext.mathjax',
    'sphinx.ext.viewcode',
    'sphinx.ext.autodoc',
    'sphinx.ext.napoleon']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'Stingray'
copyright = '2018, the Stingray team'
author = 'the Stingray team'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = 'dev'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['stingray/astropy_helpers', '_build', 'Thumbs.db',
                    '.DS_Store', '**/.ipynb_checkpoints',
                    "**/README.rst", "**/_templates",
                    "**/CREDITS.rst", '**/validation/*.ipynb',
                    '**/docs/index.rst', '**/docs/simulator.rst',
                    'ci-helpers']

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Napoleon Settings ----------------------------------------------------
napoleon_numpy_docstring = True
napoleon_include_init_with_doc = False
napoleon_include_private_with_doc = False
napoleon_include_special_with_doc = True
napoleon_use_admonition_for_examples = False
napoleon_use_admonition_for_notes = False
napoleon_use_admonition_for_references = False
napoleon_use_ivar = False
napoleon_use_param = True
napoleon_use_rtype = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#

#import astropy_sphinx_theme
#html_theme_path = astropy_sphinx_theme.get_html_theme_path()

#html_theme = 'bootstrap-astropy'
html_theme = 'sphinx_rtd_theme'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = '{0} v{1}'.format(project, release)

# Output file base name for HTML help builder.
#htmlhelp_basename = project + 'doc'

#html_theme_options = {
#    'logotext1': 'Sting',  # white,  semi-bold
#    'logotext2': 'ray',     # orange, light
#    'logotext3': ':docs'   # white,  light
#    }

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = ['_static']

# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# This is required for the alabaster theme
# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
html_sidebars = {
    '**': [
        'about.html',
        'navigation.html',
        'relations.html',  # needs 'show_related': True theme option to display
        'searchbox.html',
        'donate.html',
    ]
}

extensions += ['matplotlib.sphinxext.plot_directive','sphinx.ext.autodoc','nbsphinx']

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = 'Stingraydoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #
    # 'preamble': '',

    # Latex figure (float) alignment
    #
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'Stingray.tex', 'Stingray Documentation',
     'the Stingray team', 'manual'),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'stingray', 'Stingray Documentation',
     [author], 1)
]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'Stingray', 'Stingray Documentation',
     author, 'Stingray', 'One line description of project.',
     'Miscellaneous'),
]

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'https://docs.python.org/': None}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# CMPI Standard documentation build configuration file, created by
# sphinx-quickstart on Mon Aug  3 15:11:55 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

from __future__ import print_function
import sys
import os
import shlex
import subprocess, os

def print_err(*objs):
    print(*objs, file=sys.stderr)
    sys.stderr.flush()

def run(command, folder=None):
    """Run the command in the designated folder (default: current directory)."""
    if folder is not None:
        cmd = "cd %s; %s" % (folder, command)
    else:
        cmd = command
    print_err("Debug: Invoking command '%s'" % cmd)
    try:
        retcode = subprocess.call(cmd, shell=True)
        if retcode < 0:
            print_err("Error: Command '%s' terminated by signal %s" % \
                      (cmd, -retcode))
    except OSError as exc:
        print_err("Error: Command '%s' failed: %s" % (cmd, exc))
    except Exception as exc:
        print_err("Error: Command '%s' failed with exception %s" % \
                  (cmd, exc.type))
    else:
        print_err("Debug: Command '%s' succeeded" % cmd)

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.todo',
    'sphinx.ext.ifconfig',
    'sphinx.ext.viewcode',
    'sphinx.ext.pngmath', # added for Breathe
    'sphinx.ext.todo',    # added for Breathe
    'breathe'             # added for Breathe
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = ['.rst']

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'CMPI Standard'
copyright = u'2006-2015, The Open Group'
author = u'The Open Group'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '2.1'
# The full version, including alpha/beta/rc tags.
release = '2.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['doxygen', '_build', 'sphinx-ext']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'CMPIStandarddoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'CMPIStandard.tex', u'CMPI Standard Documentation',
   u'The Open Group', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'cmpistandard', u'CMPI Standard Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'CMPIStandard', u'CMPI Standard Documentation',
   author, 'CMPIStandard', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Breathe extension ----------------------------------------------------

# Download the Breathe 4.0.0 tarball from PyPi and expand it into ./sphinx-ext/,
# so that the 'breathe' module is found in the following path:
sys.path.append("sphinx-ext/breathe-4.0.0")

breathe_projects = {
    "cmpi-headers": "doxygen/xml/"
}
breathe_default_project = "cmpi-headers"

# -- Running Doxygen ------------------------------------------------------

print_err("Debug: In conf.py: CWD=%s" % os.getcwd())

read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True'

if read_the_docs_build:
    # We run on readthedocs.org.

    print_err("Debug: Calling Doxygen in ReadTheDocs environment")
    run('ls -l ../doxygen')
    run('which doxygen')
    run('doxygen', '../doxygen')

else:
    # We run locally.

    print_err("Debug: Calling Doxygen in local environment")
    run('which doxygen')
    run('doxygen')
<commit_msg>Improved debug prints and doxygen invocation for Sphinx/Breathe.<commit_after># -*- coding: utf-8 -*-
#
# CMPI Standard documentation build configuration file, created by
# sphinx-quickstart on Mon Aug  3 15:11:55 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

from __future__ import print_function
import sys
import os
import shlex
import subprocess, os

def print_err(*objs):
    print(*objs, file=sys.stderr)
    sys.stderr.flush()

def run(command, folder=None):
    """Run the command in the designated folder (default: current directory)."""
    if folder is not None:
        cmd = "cd %s; %s" % (folder, command)
    else:
        cmd = command
    print_err("Debug: Invoking command '%s'" % cmd)
    try:
        retcode = subprocess.call(cmd, shell=True)
        if retcode == 0:
            print_err("Debug: Command '%s' succeeded" % cmd)
        elif retcode < 0:
            # old way?
            print_err("Error: Command '%s' terminated by signal %s" % \
                      (cmd, -retcode))
        elif retcode > 128:
            # current Python 2.7 way: return 128+N
            print_err("Error: Command '%s' terminated by signal %s" % \
                      (cmd, retcode-128))
        else:
            print_err("Error: Command '%s' returns rc=%s" % (cmd, retcode))
    except OSError as exc:
        print_err("Error: Command '%s' failed: %s" % (cmd, exc))
    except Exception as exc:
        print_err("Error: subprocess.call for command '%s' raised %s: %s" % \
                  (cmd, exc.type, exc))

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.todo',
    'sphinx.ext.ifconfig',
    'sphinx.ext.viewcode',
    'sphinx.ext.pngmath', # added for Breathe
    'sphinx.ext.todo',    # added for Breathe
    'breathe'             # added for Breathe
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = ['.rst']

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'CMPI Standard'
copyright = u'2006-2015, The Open Group'
author = u'The Open Group'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '2.1'
# The full version, including alpha/beta/rc tags.
release = '2.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['doxygen', '_build', 'sphinx-ext']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'CMPIStandarddoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'CMPIStandard.tex', u'CMPI Standard Documentation',
   u'The Open Group', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'cmpistandard', u'CMPI Standard Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'CMPIStandard', u'CMPI Standard Documentation',
   author, 'CMPIStandard', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# -- Breathe extension ----------------------------------------------------

# Download the Breathe 4.0.0 tarball from PyPi and expand it into ./sphinx-ext/,
# so that the 'breathe' module is found in the following path:
sys.path.append("sphinx-ext/breathe-4.0.0")

breathe_projects = {
    "cmpi-headers": "doxygen/xml/"
}
breathe_default_project = "cmpi-headers"

# -- Running Doxygen ------------------------------------------------------

print_err("Debug: In conf.py: CWD=%s" % os.getcwd())
print_err("Debug: In conf.py: sys.path=%s" % sys.path)

read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True'

if read_the_docs_build:
    # We run on readthedocs.org.

    print_err("Debug: Calling Doxygen in ReadTheDocs environment")
    # run('ls -l ../doxygen')
    run('which doxygen')
    run('doxygen')

else:
    # We run locally.

    print_err("Debug: Calling Doxygen in local environment")
    run('which doxygen')
    run('doxygen')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals
import time

# Data about this site
BLOG_AUTHOR = "chronodekar"
BLOG_TITLE = "Note To Self"
SITE_URL = "https://note2self-abrahamvarricatt.github.io/"
BLOG_EMAIL = "no@email.here"
BLOG_DESCRIPTION = "Snippets of information"

# Multi-lingual settings (Not used - keeping them blank)
DEFAULT_LANG = "en"
TRANSLATIONS = {
    DEFAULT_LANG: "",
}

# Sidebar/Navigation
NAVIGATION_LINKS = {
    DEFAULT_LANG: (
        ("/archive.html", "Archives"),
        ("/categories/index.html", "Tags"),
        ("/rss.xml", "RSS feed"),
    ),
}

# Theme settings
THEME = "bootstrap3"
THEME_COLOR = '#5670d4'

# POST/PAGE tuples
POSTS = (
    ("posts/*.rst", "posts", "post.tmpl"),
    ("posts/*.html", "posts", "post.tmpl"),
)
PAGES = (
    ("stories/*.rst", "stories", "story.tmpl"),
    ("stories/*.html", "stories", "story.tmpl"),
)

TIMEZONE = "UTC+5:30"

# Mapping language with file extension
COMPILERS = {
    "rest": ('.rst', '.txt'),
    "html": ('.html', '.htm'),
}

WRITE_TAG_CLOUD = True
POSTS_SECTIONS = True
CATEGORY_ALLOW_HIERARCHIES = False
CATEGORY_OUTPUT_FLAT_HIERARCHY = False
FRONT_INDEX_HEADER = {
    DEFAULT_LANG: '',
}
REDIRECTIONS = []
GITHUB_COMMIT_SOURCE = True
OUTPUT_FOLDER = 'output'
IMAGE_FOLDERS = {'images': 'images'}
GLOBAL_CONTEXT = {}
GLOBAL_CONTEXT_FILLER = []

COMMENT_SYSTEM = ""
COMMENT_SYSTEM_ID = ""

CONTENT_FOOTER = 'Contents © {date}         {author} - Powered byNikola         {license}'
CONTENT_FOOTER_FORMATS = {
    DEFAULT_LANG: (
        (),
        {
            "email": BLOG_EMAIL,
            "author": BLOG_AUTHOR,
            "date": time.gmtime().tm_year,
            "license": LICENSE,
        }
    )
}

<commit_msg>adding missing LICENSE setting<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals
import time

# Data about this site
BLOG_AUTHOR = "chronodekar"
BLOG_TITLE = "Note To Self"
SITE_URL = "https://note2self-abrahamvarricatt.github.io/"
BLOG_EMAIL = "no@email.here"
BLOG_DESCRIPTION = "Snippets of information"

# Multi-lingual settings (Not used - keeping them blank)
DEFAULT_LANG = "en"
TRANSLATIONS = {
    DEFAULT_LANG: "",
}

# Sidebar/Navigation
NAVIGATION_LINKS = {
    DEFAULT_LANG: (
        ("/archive.html", "Archives"),
        ("/categories/index.html", "Tags"),
        ("/rss.xml", "RSS feed"),
    ),
}

# Theme settings
THEME = "bootstrap3"
THEME_COLOR = '#5670d4'

# POST/PAGE tuples
POSTS = (
    ("posts/*.rst", "posts", "post.tmpl"),
    ("posts/*.html", "posts", "post.tmpl"),
)
PAGES = (
    ("stories/*.rst", "stories", "story.tmpl"),
    ("stories/*.html", "stories", "story.tmpl"),
)

TIMEZONE = "UTC+5:30"

# Mapping language with file extension
COMPILERS = {
    "rest": ('.rst', '.txt'),
    "html": ('.html', '.htm'),
}

WRITE_TAG_CLOUD = True
POSTS_SECTIONS = True
CATEGORY_ALLOW_HIERARCHIES = False
CATEGORY_OUTPUT_FLAT_HIERARCHY = False
FRONT_INDEX_HEADER = {
    DEFAULT_LANG: '',
}
REDIRECTIONS = []
GITHUB_COMMIT_SOURCE = True
OUTPUT_FOLDER = 'output'
IMAGE_FOLDERS = {'images': 'images'}
GLOBAL_CONTEXT = {}
GLOBAL_CONTEXT_FILLER = []

COMMENT_SYSTEM = ""
COMMENT_SYSTEM_ID = ""

LICENSE = ""

CONTENT_FOOTER = 'Contents © {date}         {author} - Powered byNikola         {license}'
CONTENT_FOOTER_FORMATS = {
    DEFAULT_LANG: (
        (),
        {
            "email": BLOG_EMAIL,
            "author": BLOG_AUTHOR,
            "date": time.gmtime().tm_year,
            "license": LICENSE,
        }
    )
}

<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Lean Reference Manual documentation build configuration file, created by
# sphinx-quickstart on Mon Jul 17 21:46:05 2017.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.mathjax', 'sphinx.ext.githubpages', 'lean_sphinx']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = ['.rst']

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'The Lean Reference Manual'
copyright = '2017, Jeremy Avigad, Gabriel Ebner, Sebastian Ullrich'
author = 'Jeremy Avigad, Gabriel Ebner, Sebastian Ullrich'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = 'master'
# The full version, including alpha/beta/rc tags.
release = 'master'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '.venv']

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

source_parsers = {
}

# use numbering for section references with :numref:, e.g. 'Section 3.2'.
numfig = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = "alabaster"

html_sidebars = {
   '**': ['about.html', 'navigation_without_header.html', 'searchbox.html'],
}

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
html_theme_options = {
   # 'logo': 'lean_logo.svg',
   'logo_name': True,
   'font_family': 'Times New Roman, Times, serif',
   'head_font_family': 'Times New Roman, Times, serif',
   # 'font_family': 'Lato, Arial, Helvetica, sans',
   # 'head_font_family': 'Lato, Arial, Helvetica, sans',
   'extra_nav_links': {'PDF version':'lean_reference.pdf',
                       'Lean Home':'https://leanprover.github.io/',},
   # 'sidebar_width' : '200px',
   # 'page_width' : '960px'
}

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

html_favicon = '_static/favicon.ico'

html_output_encoding = 'ascii'

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = 'lean_reference'

# -- Options for LaTeX output ---------------------------------------------

latex_engine = 'xelatex'

latex_additional_files = ['unixode.sty']

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    # load packages and make box around code lighter
    'preamble': r'''
\usepackage{unixode}
\definecolor{VerbatimBorderColor}{rgb}{0.7,0.7,0.7}
''',

    # Latex figure (float) alignment
    #
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'lean_reference.tex', 'The Lean Reference Manual',
     'Jeremy Avigad, Gabriel Ebner, and Sebastian Ullrich', 'manual'),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'lean_reference', 'The Lean Reference Manual',
     [author], 1)
]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'lean_reference', 'The Lean Reference Manual',
     author, 'lean_reference', 'Reference for the Lean Theorem Prover.',
     'Miscellaneous'),
]

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''

# A unique identification for the text.
#
# epub_uid = ''

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']<commit_msg>refactor(conf.py): turn of grey background behind literals<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Lean Reference Manual documentation build configuration file, created by
# sphinx-quickstart on Mon Jul 17 21:46:05 2017.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.mathjax', 'sphinx.ext.githubpages', 'lean_sphinx']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = ['.rst']

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'The Lean Reference Manual'
copyright = '2017, Jeremy Avigad, Gabriel Ebner, Sebastian Ullrich'
author = 'Jeremy Avigad, Gabriel Ebner, Sebastian Ullrich'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = 'master'
# The full version, including alpha/beta/rc tags.
release = 'master'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '.venv']

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

source_parsers = {
}

# use numbering for section references with :numref:, e.g. 'Section 3.2'.
numfig = True

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = "alabaster"

html_sidebars = {
   '**': ['about.html', 'navigation_without_header.html', 'searchbox.html'],
}

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
html_theme_options = {
   # 'logo': 'lean_logo.svg',
   'logo_name': True,
   'font_family': 'Times New Roman, Times, serif',
   'head_font_family': 'Times New Roman, Times, serif',
   'code_bg': 'white',
   # 'font_family': 'Lato, Arial, Helvetica, sans',
   # 'head_font_family': 'Lato, Arial, Helvetica, sans',
   'extra_nav_links': {'PDF version':'lean_reference.pdf',
                       'Lean Home':'https://leanprover.github.io/',},
   # 'sidebar_width' : '200px',
   # 'page_width' : '960px'
}

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

html_favicon = '_static/favicon.ico'

html_output_encoding = 'ascii'

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = 'lean_reference'

# -- Options for LaTeX output ---------------------------------------------

latex_engine = 'xelatex'

latex_additional_files = ['unixode.sty']

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    # load packages and make box around code lighter
    'preamble': r'''
\usepackage{unixode}
\definecolor{VerbatimBorderColor}{rgb}{0.7,0.7,0.7}
''',

    # Latex figure (float) alignment
    #
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'lean_reference.tex', 'The Lean Reference Manual',
     'Jeremy Avigad, Gabriel Ebner, and Sebastian Ullrich', 'manual'),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'lean_reference', 'The Lean Reference Manual',
     [author], 1)
]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'lean_reference', 'The Lean Reference Manual',
     author, 'lean_reference', 'Reference for the Lean Theorem Prover.',
     'Miscellaneous'),
]

# -- Options for Epub output ----------------------------------------------

# Bibliographic Dublin Core info.
epub_title = project
epub_author = author
epub_publisher = author
epub_copyright = copyright

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#
# epub_identifier = ''

# A unique identification for the text.
#
# epub_uid = ''

# A list of files that should not be packed into the epub file.
epub_exclude_files = ['search.html']<|endoftext|>


<commit_before># License stuff

import logging

import pafy

from audio import AudioDL

core_logger = logging.getLogger("PlaylistDL.core")

def var_def():
    core_logger.debug("core.var_def() started.")

    download_url = str(input("\nEnter the URL of download (can be playlist URL): "))
    if "https://www.youtube.com/" not in download_url:
        core_logger.error("Incorrect URL entered, must be either video URL or playlist URL from Youtube.")
        return()

    is_playlist = False
    # Catch playlists
    if "playlist" in download_url:
        logging.info("Playlist URL recognised, verifying integrity.")
        try:
            playlist_to_dl = pafy.get_playlist(download_url)
            print("Playlist: '{}' containing {} videos in found.".format(playlist_to_dl["title"],
                                                                         len(playlist_to_dl["items"])))
            is_playlist = True
        except ValueError:
            core_logger.error("pafy.get_playlist failed. URL entered is not found on Youtube, check for mistakes.")
            print("Failed to find playlist, check the URL")
            return()

    if is_playlist is False:
        try:
            video = pafy.new(download_url)
        except ValueError:
            print("URL incorrect, try again.")
            return()

    download_quality = str(input("Quality of download (arbitrary High/Low): "))
    if download_quality != "High" and download_quality != "Low":
        print("Enter download quality correctly.")
        return()

    download_type = str(input("File-form to download from youtube (audio/video): "))
    if download_type.lower() == "audio":
        pref_audio_type = str(input("Preferred audio file (ogg/m4a/webm): "))
        audiodl = AudioDL()
        core_logger.debug("audioDL initialised.")
        if is_playlist is True:
            core_logger.debug("is_playlist = True")
            audiodl.audio_playlist_gather(set=playlist_to_dl, fformat=pref_audio_type,
                                          quality=download_quality)
        audiodl.audio_single_gather(video, fformat=pref_audio_type, quality=download_quality)
    # elif download_type.lower() == "video":
    #     dl_video(...)
<commit_msg>Sorted some silly lack of else.<commit_after># License stuff

import logging

import pafy

from audio import AudioDL

core_logger = logging.getLogger("PlaylistDL.core")

def var_def():
    core_logger.debug("core.var_def() started.")

    download_url = str(input("\nEnter the URL of download (can be playlist URL): "))
    if "https://www.youtube.com/" not in download_url:
        core_logger.error("Incorrect URL entered, must be either video URL or playlist URL from Youtube.")
        return()

    is_playlist = False
    # Catch playlists
    if "playlist" in download_url:
        logging.info("Playlist URL recognised, verifying integrity.")
        try:
            playlist_to_dl = pafy.get_playlist(download_url)
            print("Playlist: '{}' containing {} videos in found.".format(playlist_to_dl["title"],
                                                                         len(playlist_to_dl["items"])))
            is_playlist = True
        except ValueError:
            core_logger.error("pafy.get_playlist failed. URL entered is not found on Youtube, check for mistakes.")
            print("Failed to find playlist, check the URL")
            return()

    if is_playlist is False:
        try:
            video = pafy.new(download_url)
        except ValueError:
            print("URL incorrect, try again.")
            return()

    download_quality = str(input("Quality of download (arbitrary High/Low): "))
    if download_quality != "High" and download_quality != "Low":
        print("Enter download quality correctly.")
        return()

    download_type = str(input("File-form to download from youtube (audio/video): "))
    if download_type.lower() == "audio":
        pref_audio_type = str(input("Preferred audio file (ogg/m4a/webm): "))
        audiodl = AudioDL()
        core_logger.debug("audioDL initialised.")
        if is_playlist is True:
            core_logger.debug("is_playlist = True")
            audiodl.audio_playlist_gather(set=playlist_to_dl, fformat=pref_audio_type,
                                          quality=download_quality)
        else:
            audiodl.audio_single_gather(video, fformat=pref_audio_type, quality=download_quality)
    # elif download_type.lower() == "video":
    #     dl_video(...)
<|endoftext|>


<commit_before>import json
import random
from pathlib import Path

import numpy as np
from tqdm import tqdm
from utils import read_img, sample, split

def gen_image_npy(video_dirs, target_dir, n_samples):
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        imgs = sorted((video_dir / 'frames/').iterdir())
        label = json.load((video_dir / 'label.json').open())['label']
        x_all.extend(imgs)
        y_all.extend(label)

    x_use, y_use = sample(x_all, y_all, k=n_samples)

    parts = split(x_use, y_use, k=1000)
    for (idx, x_part, y_part) in tqdm(parts):
        n = len(x_part)
        xs = np.zeros((n, 224, 224, 3), dtype=np.float32)
        ys = np.zeros((n, 1), dtype=np.uint8)
        for i in range(n):
            xs[i] = read_img(x_part[i])
            ys[i] = y_part[i]
        np.save(str(target_dir / 'x_{:05d}.npy'.format(idx)), xs)
        np.save(str(target_dir / 'y_{:05d}.npy'.format(idx)), ys)
        del xs, ys

def gen_window_npy(video_dirs, target_dir, n_samples, timesteps):
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        n_frames = len((video_dir / 'frames/').iterdir())
        labels = read_json(video_dir / 'label.json')['label']
        windows = [(video_dir, i, i + timesteps) for i in range(n_frames - timesteps + 1)]
        x_all.extend(windows)
        y_all.extend([labels[e - 1] for (s, e) in windows])

    x_use, y_use = sample(x_all, y_all, k=n_samples)

    for idx, x_part, y_part in split(x_use, y_use, k=1000):
        n = len(x_part)
        xs = np.zeros((n, timesteps, 224, 224, 3), dtype=np.float32)
        ys = np.zeros((n, 1), dtype=np.uint8)
        for i in range(n):
            (video_dir, s, e) = x_part[i]
            for f in range(s, e):
                path = video_dir / 'frames' / '{:08d}'.format(f)
                xs[i][f - s] = read_img(path)
            ys[i] = y_part[i]

        np.save(str(target_dir / 'x_{:05d}.npy'.format(idx)), xs)
        np.save(str(target_dir / 'y_{:05d}.npy'.format(idx)), ys)
        del xs, ys

def image_generator(npy_dir, batch_size):
    x_paths = sorted(npy_dir.glob('x_*.npy'))
    y_paths = sorted(npy_dir.glob('y_*.npy'))

    idx = 0
    x_batch = np.zeros((batch_size, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    while True:
        for x_part, y_part in zip(x_paths, y_paths):
            for x, y in zip(x_part, y_part):
                x_batch[idx] = x
                y_batch[idx] = y
                if idx + 1 == batch_size:
                    yield x_batch, y_batch
                idx = (idx + 1) % batch_size

if __name__ == '__main__':
    dataset = Path('~/dataset/').expanduser()
    # gen_image_npy([dataset / 'video00'], Path('npy/image_train'), 25000)
    gen_image_npy([dataset / 'video01'], Path('npy/image_val'), 5000)

# def image_generator(video_dirs, batch_size):
#     # Get all paths
#     x_all = []
#     y_all = []
#     for video_dir in video_dirs:
#         imgs = sorted((video_dir / 'frames/').iterdir())
#         label = json.load((video_dir / 'label.json').open())['label']
#         x_all.extend(imgs)
#         y_all.extend(label)

#     indices = np.random.permutation(len(x_all))
#     x_use = [x_all[i] for i in indices]
#     y_use = [y_all[i] for i in indices]

#     x_batch = np.zeros((batch_size, 224, 224, 3), dtype=np.float32)
#     y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

#     while True:
#         for i, img_path in enumerate(x_use):
#             idx = i % batch_size
#             x_batch[idx] = read_img(img_path)
#             y_batch[idx] = y_use[i]

#             if idx == batch_size - 1:
#                 yield (x_batch, y_batch)

# def window_generator(video_dirs, batch_size, timesteps):
#     x_batch = np.zeros((batch_size, timesteps, 224, 224, 3), dtype=np.float32)
#     y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

#     idx = 0
#     while True:
#         for video_dir in video_dirs:
#             xs = sorted((video_dir / 'frames/').iterdir())
#             ys = json.load((video_dir / 'label.json').open())['label']

#             # [i, i + timesteps)
#             windows = [(i, i + timesteps)
#                        for i in range(len(xs) - timesteps + 1)]
#             random.shuffle(windows)

#             for (s, e) in windows:
#                 for f in range(s, e):
#                     x_batch[idx][f - s] = read_img(xs[f])
#                 y_batch[idx] = ys[e - 1]

#                 if idx == batch_size - 1:
#                     yield x_batch, y_batch
#                 idx = (idx + 1) % batch_size

# def window_npy_generator(video_dirs, batch_size, timesteps):
#     x_batch = np.zeros((batch_size, timesteps, 224, 224, 3), dtype=np.float32)
#     y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

#     idx = 0
#     while True:
#         for video_dir in video_dirs:
#             xs = np.load(str(video_dir / 'imgs.npy'))
#             ys = json.load((video_dir / 'label.json').open())['label']

#             # [i, i + timesteps)
#             windows = [(i, i + timesteps)
#                        for i in range(len(xs) - timesteps + 1)]
#             random.shuffle(windows)

#             for (s, e) in windows:
#                 for f in range(s, e):
#                     x_batch[idx][f - s] = xs[f]
#                 y_batch[idx] = ys[e - 1]

#                 if idx == batch_size - 1:
#                     yield x_batch, y_batch
#                 idx = (idx + 1) % batch_size

#             xs = None
<commit_msg>Update at 2017-07-15 01-09-16<commit_after>import json
import random
from pathlib import Path

import numpy as np
from tqdm import tqdm
from utils import read_img, sample, split

def gen_image_npy(video_dirs, target_dir, n_samples):
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        imgs = sorted((video_dir / 'frames/').iterdir())
        label = json.load((video_dir / 'label.json').open())['label']
        x_all.extend(imgs)
        y_all.extend(label)

    x_use, y_use = sample(x_all, y_all, k=n_samples)

    parts = split(x_use, y_use, k=1000)
    for (idx, x_part, y_part) in tqdm(parts):
        n = len(x_part)
        xs = np.zeros((n, 224, 224, 3), dtype=np.float32)
        ys = np.zeros((n, 1), dtype=np.uint8)
        for i in range(n):
            xs[i] = read_img(x_part[i])
            ys[i] = y_part[i]
        np.save(str(target_dir / 'x_{:05d}.npy'.format(idx)), xs)
        np.save(str(target_dir / 'y_{:05d}.npy'.format(idx)), ys)
        del xs, ys

def gen_window_npy(video_dirs, target_dir, n_samples, timesteps):
    x_all = []
    y_all = []
    for video_dir in video_dirs:
        n_frames = len((video_dir / 'frames/').iterdir())
        labels = read_json(video_dir / 'label.json')['label']
        windows = [(video_dir, i, i + timesteps) for i in range(n_frames - timesteps + 1)]
        x_all.extend(windows)
        y_all.extend([labels[e - 1] for (s, e) in windows])

    x_use, y_use = sample(x_all, y_all, k=n_samples)

    for idx, x_part, y_part in split(x_use, y_use, k=1000):
        n = len(x_part)
        xs = np.zeros((n, timesteps, 224, 224, 3), dtype=np.float32)
        ys = np.zeros((n, 1), dtype=np.uint8)
        for i in range(n):
            (video_dir, s, e) = x_part[i]
            for f in range(s, e):
                path = video_dir / 'frames' / '{:08d}'.format(f)
                xs[i][f - s] = read_img(path)
            ys[i] = y_part[i]

        np.save(str(target_dir / 'x_{:05d}.npy'.format(idx)), xs)
        np.save(str(target_dir / 'y_{:05d}.npy'.format(idx)), ys)
        del xs, ys

def image_generator(npy_dir, batch_size):
    x_paths = sorted(npy_dir.glob('x_*.npy'))
    y_paths = sorted(npy_dir.glob('y_*.npy'))

    idx = 0
    x_batch = np.zeros((batch_size, 224, 224, 3), dtype=np.float32)
    y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

    while True:
        for x_part, y_part in zip(x_paths, y_paths):
            for x, y in zip(x_part, y_part):
                x_batch[idx] = x
                y_batch[idx] = y
                if idx + 1 == batch_size:
                    yield x_batch, y_batch
                idx = (idx + 1) % batch_size

if __name__ == '__main__':
    dataset = Path('~/dataset/').expanduser()
    gen_image_npy([dataset / 'video00'], Path('npy/image_train'), 25000)
    gen_image_npy([dataset / 'video01'], Path('npy/image_val'), 5000)

# def image_generator(video_dirs, batch_size):
#     # Get all paths
#     x_all = []
#     y_all = []
#     for video_dir in video_dirs:
#         imgs = sorted((video_dir / 'frames/').iterdir())
#         label = json.load((video_dir / 'label.json').open())['label']
#         x_all.extend(imgs)
#         y_all.extend(label)

#     indices = np.random.permutation(len(x_all))
#     x_use = [x_all[i] for i in indices]
#     y_use = [y_all[i] for i in indices]

#     x_batch = np.zeros((batch_size, 224, 224, 3), dtype=np.float32)
#     y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

#     while True:
#         for i, img_path in enumerate(x_use):
#             idx = i % batch_size
#             x_batch[idx] = read_img(img_path)
#             y_batch[idx] = y_use[i]

#             if idx == batch_size - 1:
#                 yield (x_batch, y_batch)

# def window_generator(video_dirs, batch_size, timesteps):
#     x_batch = np.zeros((batch_size, timesteps, 224, 224, 3), dtype=np.float32)
#     y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

#     idx = 0
#     while True:
#         for video_dir in video_dirs:
#             xs = sorted((video_dir / 'frames/').iterdir())
#             ys = json.load((video_dir / 'label.json').open())['label']

#             # [i, i + timesteps)
#             windows = [(i, i + timesteps)
#                        for i in range(len(xs) - timesteps + 1)]
#             random.shuffle(windows)

#             for (s, e) in windows:
#                 for f in range(s, e):
#                     x_batch[idx][f - s] = read_img(xs[f])
#                 y_batch[idx] = ys[e - 1]

#                 if idx == batch_size - 1:
#                     yield x_batch, y_batch
#                 idx = (idx + 1) % batch_size

# def window_npy_generator(video_dirs, batch_size, timesteps):
#     x_batch = np.zeros((batch_size, timesteps, 224, 224, 3), dtype=np.float32)
#     y_batch = np.zeros((batch_size, 1), dtype=np.uint8)

#     idx = 0
#     while True:
#         for video_dir in video_dirs:
#             xs = np.load(str(video_dir / 'imgs.npy'))
#             ys = json.load((video_dir / 'label.json').open())['label']

#             # [i, i + timesteps)
#             windows = [(i, i + timesteps)
#                        for i in range(len(xs) - timesteps + 1)]
#             random.shuffle(windows)

#             for (s, e) in windows:
#                 for f in range(s, e):
#                     x_batch[idx][f - s] = xs[f]
#                 y_batch[idx] = ys[e - 1]

#                 if idx == batch_size - 1:
#                     yield x_batch, y_batch
#                 idx = (idx + 1) % batch_size

#             xs = None
<|endoftext|>


<commit_before># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
# Modifications Copyright 2017 Abigail See
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""This file contains code to read the train/eval/test data from file and process it, and read the vocab data from file and process it"""

import glob
import random
import struct
import csv
import six
import click
import codecs
from tensorflow.core.example import example_pb2

# <s> and </s> are used in the data files to segment the abstracts into sentences. They don't receive vocab ids.
SENTENCE_START = '<s>'
SENTENCE_END = '</s>'

PAD_TOKEN = '[PAD]' # This has a vocab id, which is used to pad the encoder input, decoder input and target sequence
UNKNOWN_TOKEN = '[UNK]' # This has a vocab id, which is used to represent out-of-vocabulary words
START_DECODING = '[START]' # This has a vocab id, which is used at the start of every decoder input sequence
STOP_DECODING = '[STOP]' # This has a vocab id, which is used at the end of untruncated target sequences

# Note: none of <s>, </s>, [PAD], [UNK], [START], [STOP] should appear in the vocab file.

class Vocab(object):
  """Vocabulary class for mapping between words and ids (integers)"""

  def __init__(self, vocab_file, max_size):
    """Creates a vocab of up to max_size words, reading from the vocab_file. If max_size is 0, reads the entire vocab file.

    Args:
      vocab_file: path to the vocab file, which is assumed to contain "<word> <frequency>" on each line, sorted with most frequent word first. This code doesn't actually use the frequencies, though.
      max_size: integer. The maximum size of the resulting Vocabulary."""
    self._word_to_id = {}
    self._id_to_word = {}
    self._count = 0 # keeps track of total number of words in the Vocab

    # [UNK], [PAD], [START] and [STOP] get the ids 0,1,2,3.
    for w in [UNKNOWN_TOKEN, PAD_TOKEN, START_DECODING, STOP_DECODING]:
      self._word_to_id[w] = self._count
      self._id_to_word[self._count] = w
      self._count += 1

    # Read the vocab file and add words up to max_size
    with codecs.open(vocab_file, 'r', "utf-8") as vocab_f:
      for line in vocab_f:
        pieces = line.split()
        if len(pieces) != 2:
          print('Warning: incorrectly formatted line in vocabulary file: %s\n' % line)
          continue
        w = pieces[0]
        if w in [SENTENCE_START, SENTENCE_END, UNKNOWN_TOKEN, PAD_TOKEN, START_DECODING, STOP_DECODING]:
          raise Exception('<s>, </s>, [UNK], [PAD], [START] and [STOP] shouldn\'t be in the vocab file, but %s is' % w)
        if w in self._word_to_id:
          raise Exception('Duplicated word in vocabulary file: %s' % w)
        self._word_to_id[w] = self._count
        self._id_to_word[self._count] = w
        self._count += 1
        if max_size != 0 and self._count >= max_size:
          print("max_size of vocab was specified as %i; we now have %i words. Stopping reading." % (max_size, self._count))
          break

    print("Finished constructing vocabulary of {} total words. Last word added: {}".format(self._count, self._id_to_word[self._count-1]))

  def word2id(self, word):
    """Returns the id (integer) of a word (string). Returns [UNK] id if word is OOV."""
    if word not in self._word_to_id:
      return self._word_to_id[UNKNOWN_TOKEN]
    return self._word_to_id[word]

  def id2word(self, word_id):
    """Returns the word (string) corresponding to an id (integer)."""
    if word_id not in self._id_to_word:
      raise ValueError('Id not found in vocab: %d' % word_id)
    return self._id_to_word[word_id]

  def size(self):
    """Returns the total size of the vocabulary"""
    return self._count

  def write_metadata(self, fpath):
    """Writes metadata file for Tensorboard word embedding visualizer as described here:
      https://www.tensorflow.org/get_started/embedding_viz

    Args:
      fpath: place to write the metadata file
    """
    print("Writing word embedding metadata file to %s..." % (fpath))
    with codecs.open(fpath, "w", "utf-8") as f:
      fieldnames = ['word']
      writer = csv.DictWriter(f, delimiter="\t", fieldnames=fieldnames)
      for i in range(self.size()):
        word = self._id_to_word[i]
        f.write(word+"\n")

def example_generator(data_path, single_pass):
  """Generates tf.Examples from data files.

    Binary data format: <length><blob>. <length> represents the byte size
    of <blob>. <blob> is serialized tf.Example proto. The tf.Example contains
    the tokenized article text and summary.

  Args:
    data_path:
      Path to tf.Example data files. Can include wildcards, e.g. if you have several training data chunk files train_001.bin, train_002.bin, etc, then pass data_path=train_* to access them all.
    single_pass:
      Boolean. If True, go through the dataset exactly once, generating examples in the order they appear, then return. Otherwise, generate random examples indefinitely.

  Yields:
    Deserialized tf.Example.
  """
  while True:
    filelist = glob.glob(data_path) # get the list of datafiles
    assert filelist, ('Error: Empty filelist at %s' % data_path) # check filelist isn't empty
    if single_pass:
      filelist = sorted(filelist)
    else:
      random.shuffle(filelist)
    for f in filelist:
      reader = open(f, 'rb')
      while True:
        len_bytes = reader.read(8)
        if not len_bytes: break # finished reading this file
        str_len = struct.unpack('q', len_bytes)[0]
        example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0]
        yield example_pb2.Example.FromString(example_str)
    if single_pass:
      print("example_generator completed reading all datafiles. No more data.")
      break

def article2ids(article_words, vocab):
  """Map the article words to their ids. Also return a list of OOVs in the article.

  Args:
    article_words: list of words (strings)
    vocab: Vocabulary object

  Returns:
    ids:
      A list of word ids (integers); OOVs are represented by their temporary article OOV number. If the vocabulary size is 50k and the article has 3 OOVs, then these temporary OOV numbers will be 50000, 50001, 50002.
    oovs:
      A list of the OOV words in the article (strings), in the order corresponding to their temporary article OOV numbers."""
  ids = []
  oovs = []
  unk_id = vocab.word2id(UNKNOWN_TOKEN)
  for w in article_words:
    i = vocab.word2id(w)
    if i == unk_id: # If w is OOV
      if w not in oovs: # Add to list of OOVs
        oovs.append(w)
      oov_num = oovs.index(w) # This is 0 for the first article OOV, 1 for the second article OOV...
      ids.append(vocab.size() + oov_num) # This is e.g. 50000 for the first article OOV, 50001 for the second...
    else:
      ids.append(i)
  return ids, oovs

def abstract2ids(abstract_words, vocab, article_oovs):
  """Map the abstract words to their ids. In-article OOVs are mapped to their temporary OOV numbers.

  Args:
    abstract_words: list of words (strings)
    vocab: Vocabulary object
    article_oovs: list of in-article OOV words (strings), in the order corresponding to their temporary article OOV numbers

  Returns:
    ids: List of ids (integers). In-article OOV words are mapped to their temporary OOV numbers. Out-of-article OOV words are mapped to the UNK token id."""
  ids = []
  unk_id = vocab.word2id(UNKNOWN_TOKEN)
  for w in abstract_words:
    i = vocab.word2id(w)
    if i == unk_id: # If w is an OOV word
      if w in article_oovs: # If w is an in-article OOV
        vocab_idx = vocab.size() + article_oovs.index(w) # Map to its temporary article OOV number
        ids.append(vocab_idx)
      else: # If w is an out-of-article OOV
        ids.append(unk_id) # Map to the UNK token id
    else:
      ids.append(i)
  return ids

def outputids2words(id_list, vocab, article_oovs):
  """Maps output ids to words, including mapping in-article OOVs from their temporary ids to the original OOV string (applicable in pointer-generator mode).

  Args:
    id_list: list of ids (integers)
    vocab: Vocabulary object
    article_oovs: list of OOV words (strings) in the order corresponding to their temporary article OOV ids (that have been assigned in pointer-generator mode), or None (in baseline mode)

  Returns:
    words: list of words (strings)
  """
  words = []
  for i in id_list:
    try:
      w = vocab.id2word(i) # might be [UNK]
    except ValueError as e: # w is OOV
      assert article_oovs is not None, "Error: model produced a word ID that isn't in the vocabulary. This should not happen in baseline (no pointer-generator) mode"
      article_oov_idx = i - vocab.size()
      try:
        w = article_oovs[article_oov_idx]
      except ValueError as e: # i doesn't correspond to an article oov
        raise ValueError('Error: model produced word ID %i which corresponds to article OOV %i but this example only has %i article OOVs' % (i, article_oov_idx, len(article_oovs)))
    words.append(w)
  return words

def abstract2sents(abstract):
  """Splits abstract text from datafile into list of sentences.

  Args:
    abstract: string containing <s> and </s> tags for starts and ends of sentences

  Returns:
    sents: List of sentence strings (no tags)"""
  cur = 0
  sents = []
  while True:
    try:
      start_p = abstract.index(SENTENCE_START, cur)
      end_p = abstract.index(SENTENCE_END, start_p + 1)
      cur = end_p + len(SENTENCE_END)
      sents.append(abstract[start_p+len(SENTENCE_START):end_p])
    except ValueError as e: # no more sentences
      return sents

def show_art_oovs(article, vocab):
  """Returns the article string, highlighting the OOVs by placing __underscores__ around them"""
  unk_token = vocab.word2id(UNKNOWN_TOKEN)
  words = article.split(' ')
  words = [("__%s__" % w) if vocab.word2id(w)==unk_token else w for w in words]
  out_str = ' '.join(words)
  return out_str

def show_abs_oovs(abstract, vocab, article_oovs):
  """Returns the abstract string, highlighting the article OOVs with __underscores__.

  If a list of article_oovs is provided, non-article OOVs are differentiated like !!__this__!!.

  Args:
    abstract: string
    vocab: Vocabulary object
    article_oovs: list of words (strings), or None (in baseline mode)
  """
  unk_token = vocab.word2id(UNKNOWN_TOKEN)
  words = abstract.split(' ')
  new_words = []
  for w in words:
    if vocab.word2id(w) == unk_token: # w is oov
      if article_oovs is None: # baseline mode
        new_words.append("__%s__" % w)
      else: # pointer-generator mode
        if w in article_oovs:
          new_words.append("__%s__" % w)
        else:
          new_words.append("!!__%s__!!" % w)
    else: # w is in-vocab word
      new_words.append(w)
  out_str = ' '.join(new_words)
  return out_str
<commit_msg>last word encode<commit_after># Copyright 2016 The TensorFlow Authors. All Rights Reserved.
# Modifications Copyright 2017 Abigail See
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""This file contains code to read the train/eval/test data from file and process it, and read the vocab data from file and process it"""

import glob
import random
import struct
import csv
import six
import click
import codecs
from tensorflow.core.example import example_pb2

# <s> and </s> are used in the data files to segment the abstracts into sentences. They don't receive vocab ids.
SENTENCE_START = '<s>'
SENTENCE_END = '</s>'

PAD_TOKEN = '[PAD]' # This has a vocab id, which is used to pad the encoder input, decoder input and target sequence
UNKNOWN_TOKEN = '[UNK]' # This has a vocab id, which is used to represent out-of-vocabulary words
START_DECODING = '[START]' # This has a vocab id, which is used at the start of every decoder input sequence
STOP_DECODING = '[STOP]' # This has a vocab id, which is used at the end of untruncated target sequences

# Note: none of <s>, </s>, [PAD], [UNK], [START], [STOP] should appear in the vocab file.

class Vocab(object):
  """Vocabulary class for mapping between words and ids (integers)"""

  def __init__(self, vocab_file, max_size):
    """Creates a vocab of up to max_size words, reading from the vocab_file. If max_size is 0, reads the entire vocab file.

    Args:
      vocab_file: path to the vocab file, which is assumed to contain "<word> <frequency>" on each line, sorted with most frequent word first. This code doesn't actually use the frequencies, though.
      max_size: integer. The maximum size of the resulting Vocabulary."""
    self._word_to_id = {}
    self._id_to_word = {}
    self._count = 0 # keeps track of total number of words in the Vocab

    # [UNK], [PAD], [START] and [STOP] get the ids 0,1,2,3.
    for w in [UNKNOWN_TOKEN, PAD_TOKEN, START_DECODING, STOP_DECODING]:
      self._word_to_id[w] = self._count
      self._id_to_word[self._count] = w
      self._count += 1

    # Read the vocab file and add words up to max_size
    with codecs.open(vocab_file, 'r', "utf-8") as vocab_f:
      for line in vocab_f:
        pieces = line.split()
        if len(pieces) != 2:
          print('Warning: incorrectly formatted line in vocabulary file: %s\n' % line)
          continue
        w = pieces[0]
        if w in [SENTENCE_START, SENTENCE_END, UNKNOWN_TOKEN, PAD_TOKEN, START_DECODING, STOP_DECODING]:
          raise Exception('<s>, </s>, [UNK], [PAD], [START] and [STOP] shouldn\'t be in the vocab file, but %s is' % w)
        if w in self._word_to_id:
          raise Exception('Duplicated word in vocabulary file: %s' % w)
        self._word_to_id[w] = self._count
        self._id_to_word[self._count] = w
        self._count += 1
        if max_size != 0 and self._count >= max_size:
          print("max_size of vocab was specified as %i; we now have %i words. Stopping reading." % (max_size, self._count))
          break
    last_word = self._id_to_word[self._count-1]
    if six.PY2:
      last_word = last_word.encode("utf-8")
    print("Finished constructing vocabulary of {} total words. Last word added: {}".format(self._count, last_word))

  def word2id(self, word):
    """Returns the id (integer) of a word (string). Returns [UNK] id if word is OOV."""
    if word not in self._word_to_id:
      return self._word_to_id[UNKNOWN_TOKEN]
    return self._word_to_id[word]

  def id2word(self, word_id):
    """Returns the word (string) corresponding to an id (integer)."""
    if word_id not in self._id_to_word:
      raise ValueError('Id not found in vocab: %d' % word_id)
    return self._id_to_word[word_id]

  def size(self):
    """Returns the total size of the vocabulary"""
    return self._count

  def write_metadata(self, fpath):
    """Writes metadata file for Tensorboard word embedding visualizer as described here:
      https://www.tensorflow.org/get_started/embedding_viz

    Args:
      fpath: place to write the metadata file
    """
    print("Writing word embedding metadata file to %s..." % (fpath))
    with codecs.open(fpath, "w", "utf-8") as f:
      fieldnames = ['word']
      writer = csv.DictWriter(f, delimiter="\t", fieldnames=fieldnames)
      for i in range(self.size()):
        word = self._id_to_word[i]
        f.write(word+"\n")

def example_generator(data_path, single_pass):
  """Generates tf.Examples from data files.

    Binary data format: <length><blob>. <length> represents the byte size
    of <blob>. <blob> is serialized tf.Example proto. The tf.Example contains
    the tokenized article text and summary.

  Args:
    data_path:
      Path to tf.Example data files. Can include wildcards, e.g. if you have several training data chunk files train_001.bin, train_002.bin, etc, then pass data_path=train_* to access them all.
    single_pass:
      Boolean. If True, go through the dataset exactly once, generating examples in the order they appear, then return. Otherwise, generate random examples indefinitely.

  Yields:
    Deserialized tf.Example.
  """
  while True:
    filelist = glob.glob(data_path) # get the list of datafiles
    assert filelist, ('Error: Empty filelist at %s' % data_path) # check filelist isn't empty
    if single_pass:
      filelist = sorted(filelist)
    else:
      random.shuffle(filelist)
    for f in filelist:
      reader = open(f, 'rb')
      while True:
        len_bytes = reader.read(8)
        if not len_bytes: break # finished reading this file
        str_len = struct.unpack('q', len_bytes)[0]
        example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0]
        yield example_pb2.Example.FromString(example_str)
    if single_pass:
      print("example_generator completed reading all datafiles. No more data.")
      break

def article2ids(article_words, vocab):
  """Map the article words to their ids. Also return a list of OOVs in the article.

  Args:
    article_words: list of words (strings)
    vocab: Vocabulary object

  Returns:
    ids:
      A list of word ids (integers); OOVs are represented by their temporary article OOV number. If the vocabulary size is 50k and the article has 3 OOVs, then these temporary OOV numbers will be 50000, 50001, 50002.
    oovs:
      A list of the OOV words in the article (strings), in the order corresponding to their temporary article OOV numbers."""
  ids = []
  oovs = []
  unk_id = vocab.word2id(UNKNOWN_TOKEN)
  for w in article_words:
    i = vocab.word2id(w)
    if i == unk_id: # If w is OOV
      if w not in oovs: # Add to list of OOVs
        oovs.append(w)
      oov_num = oovs.index(w) # This is 0 for the first article OOV, 1 for the second article OOV...
      ids.append(vocab.size() + oov_num) # This is e.g. 50000 for the first article OOV, 50001 for the second...
    else:
      ids.append(i)
  return ids, oovs

def abstract2ids(abstract_words, vocab, article_oovs):
  """Map the abstract words to their ids. In-article OOVs are mapped to their temporary OOV numbers.

  Args:
    abstract_words: list of words (strings)
    vocab: Vocabulary object
    article_oovs: list of in-article OOV words (strings), in the order corresponding to their temporary article OOV numbers

  Returns:
    ids: List of ids (integers). In-article OOV words are mapped to their temporary OOV numbers. Out-of-article OOV words are mapped to the UNK token id."""
  ids = []
  unk_id = vocab.word2id(UNKNOWN_TOKEN)
  for w in abstract_words:
    i = vocab.word2id(w)
    if i == unk_id: # If w is an OOV word
      if w in article_oovs: # If w is an in-article OOV
        vocab_idx = vocab.size() + article_oovs.index(w) # Map to its temporary article OOV number
        ids.append(vocab_idx)
      else: # If w is an out-of-article OOV
        ids.append(unk_id) # Map to the UNK token id
    else:
      ids.append(i)
  return ids

def outputids2words(id_list, vocab, article_oovs):
  """Maps output ids to words, including mapping in-article OOVs from their temporary ids to the original OOV string (applicable in pointer-generator mode).

  Args:
    id_list: list of ids (integers)
    vocab: Vocabulary object
    article_oovs: list of OOV words (strings) in the order corresponding to their temporary article OOV ids (that have been assigned in pointer-generator mode), or None (in baseline mode)

  Returns:
    words: list of words (strings)
  """
  words = []
  for i in id_list:
    try:
      w = vocab.id2word(i) # might be [UNK]
    except ValueError as e: # w is OOV
      assert article_oovs is not None, "Error: model produced a word ID that isn't in the vocabulary. This should not happen in baseline (no pointer-generator) mode"
      article_oov_idx = i - vocab.size()
      try:
        w = article_oovs[article_oov_idx]
      except ValueError as e: # i doesn't correspond to an article oov
        raise ValueError('Error: model produced word ID %i which corresponds to article OOV %i but this example only has %i article OOVs' % (i, article_oov_idx, len(article_oovs)))
    words.append(w)
  return words

def abstract2sents(abstract):
  """Splits abstract text from datafile into list of sentences.

  Args:
    abstract: string containing <s> and </s> tags for starts and ends of sentences

  Returns:
    sents: List of sentence strings (no tags)"""
  cur = 0
  sents = []
  while True:
    try:
      start_p = abstract.index(SENTENCE_START, cur)
      end_p = abstract.index(SENTENCE_END, start_p + 1)
      cur = end_p + len(SENTENCE_END)
      sents.append(abstract[start_p+len(SENTENCE_START):end_p])
    except ValueError as e: # no more sentences
      return sents

def show_art_oovs(article, vocab):
  """Returns the article string, highlighting the OOVs by placing __underscores__ around them"""
  unk_token = vocab.word2id(UNKNOWN_TOKEN)
  words = article.split(' ')
  words = [("__%s__" % w) if vocab.word2id(w)==unk_token else w for w in words]
  out_str = ' '.join(words)
  return out_str

def show_abs_oovs(abstract, vocab, article_oovs):
  """Returns the abstract string, highlighting the article OOVs with __underscores__.

  If a list of article_oovs is provided, non-article OOVs are differentiated like !!__this__!!.

  Args:
    abstract: string
    vocab: Vocabulary object
    article_oovs: list of words (strings), or None (in baseline mode)
  """
  unk_token = vocab.word2id(UNKNOWN_TOKEN)
  words = abstract.split(' ')
  new_words = []
  for w in words:
    if vocab.word2id(w) == unk_token: # w is oov
      if article_oovs is None: # baseline mode
        new_words.append("__%s__" % w)
      else: # pointer-generator mode
        if w in article_oovs:
          new_words.append("__%s__" % w)
        else:
          new_words.append("!!__%s__!!" % w)
    else: # w is in-vocab word
      new_words.append(w)
  out_str = ' '.join(new_words)
  return out_str
<|endoftext|>